// SPDX-FileCopyrightText: 2026 Coffey Labs // SPDX-License-Identifier: GPL-3.0-or-later package backup import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "os" "os/exec" "regexp" "sort" "strconv" "strings" "time" ) // DefaultMigrationScriptURL is Stalwart's own v0.15->v0.16 settings // converter, referenced directly from UPGRADING/v0_16.md. It's an external, // Stalwart-owned dependency this tool doesn't vendor a copy of - see the // pinning discussion on DownloadFile and ARCHITECTURE.md §8. const DefaultMigrationScriptURL = "https://raw.githubusercontent.com/stalwartlabs/stalwart/main/resources/scripts/migrate_v016.py" // ProvideFile puts the migration script at destPath, from srcPath when one // is given and from url otherwise, and returns its SHA256. // // A local copy is not only a convenience for testing. A mail server with no // route to the internet - an air-gapped host, or a clone deliberately cut // off so it cannot renew certificates or deliver queued mail for the domains // it was copied from - cannot fetch anything, and could not be migrated at // all without this. func ProvideFile(ctx context.Context, httpClient *http.Client, srcPath, url, destPath, expectedSHA256 string) (sha256Hex string, err error) { if srcPath == "" { return DownloadFile(ctx, httpClient, url, destPath, expectedSHA256) } data, err := os.ReadFile(srcPath) if err != nil { return "", fmt.Errorf("backup: read %s: %w", srcPath, err) } sum := fmt.Sprintf("%x", sha256.Sum256(data)) if expectedSHA256 != "" && !strings.EqualFold(sum, expectedSHA256) { return "", fmt.Errorf("backup: %s has sha256 %s, expected %s", srcPath, sum, expectedSHA256) } if err := os.WriteFile(destPath, data, 0o640); err != nil { return "", fmt.Errorf("backup: write %s: %w", destPath, err) } return sum, nil } // DownloadFile fetches url to destPath and returns its SHA256. If // expectedSHA256 is non-empty, a mismatching download is rejected (and the // partial file removed) - this is how a pinned migration-script hash is // enforced, so a run never silently executes a different version of a // script than the one it was reviewed against. If expectedSHA256 is empty, // the download is accepted unconditionally and its hash is returned so the // caller can record it as the pin for next time; first-run trust-on-first-use // is a known gap, flagged in ARCHITECTURE.md §8. func DownloadFile(ctx context.Context, httpClient *http.Client, url, destPath, expectedSHA256 string) (sha256Hex string, err error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return "", err } if httpClient == nil { httpClient = &http.Client{Timeout: 60 * time.Second} } resp, err := httpClient.Do(req) if err != nil { return "", fmt.Errorf("backup: fetch %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("backup: fetch %s: unexpected status %s", url, resp.Status) } f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) if err != nil { return "", fmt.Errorf("backup: create %s: %w", destPath, err) } h := sha256.New() _, copyErr := io.Copy(io.MultiWriter(f, h), resp.Body) closeErr := f.Close() if copyErr != nil { os.Remove(destPath) return "", fmt.Errorf("backup: download %s: %w", url, copyErr) } if closeErr != nil { os.Remove(destPath) return "", fmt.Errorf("backup: close %s: %w", destPath, closeErr) } sha256Hex = hex.EncodeToString(h.Sum(nil)) if expectedSHA256 != "" && sha256Hex != expectedSHA256 { os.Remove(destPath) return "", fmt.Errorf( "backup: %s checksum mismatch: got %s, want %s (refusing to run an unexpected version of a script that irreversibly wipes settings on first v0.16 start)", url, sha256Hex, expectedSHA256, ) } return sha256Hex, nil } // SettingsDumpOptions configures a migrate_v016.py `dump` invocation // against a live v0.15.x instance - see UPGRADING/v0_16.md. type SettingsDumpOptions struct { PythonPath string // defaults to "python3" ScriptPath string // local path to the already-downloaded, checksum-verified script URL string // the running v0.15.x instance's base URL Username string Password string SettingsPath string PrincipalsPath string } // RunSettingsDump runs migrate_v016.py's dump subcommand, which reads the // live v0.15.x server's settings and principals over its admin API and // writes them to SettingsPath/PrincipalsPath for the later convert step // (ARCHITECTURE.md §4.3). This step is read-only against the server, so // it's safe to run well before cutover - ARCHITECTURE.md §4.2 calls for it // both at preflight time and again immediately before cutover, since the // live settings may have changed in between. func RunSettingsDump(ctx context.Context, o SettingsDumpOptions) error { python := o.PythonPath if python == "" { python = "python3" } args := []string{ o.ScriptPath, "dump", "--url", o.URL, "--username", o.Username, "--password", o.Password, "--settings", o.SettingsPath, "--principals", o.PrincipalsPath, } cmd := exec.CommandContext(ctx, python, args...) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("backup: migrate_v016.py dump failed: %w (output: %s)", err, out) } return nil } // SettingsConvertOptions configures a migrate_v016.py `convert` invocation, // which turns the settings/principals dump into the v0.16 config.json and // export.json that recovery mode consumes - see UPGRADING/v0_16.md. type SettingsConvertOptions struct { PythonPath string // defaults to "python3" ScriptPath string SettingsPath string PrincipalsPath string ConfigPath string // output: config.json for the new binary's --config flag OutputPath string // output: export.json for `stalwart-cli apply` // PatchPaths rewrites path prefixes in the generated config (documented // for Docker deployments as "--patch-paths /opt/stalwart=/var/lib/stalwart", // e.g. old-path -> new-path). This is the officially documented // mechanism a dry-run relies on to point the generated config at a // sandbox data directory instead of the production one - see // ARCHITECTURE.md's dry-run design - rather than this tool editing // config.json's contents directly, which would require depending on its // exact schema. PatchPaths map[string]string // WorkDir is where the script runs. It matters more than it looks: // migrate_v016.py writes its unmigrated.txt report into the current // working directory, so without this the convert either fails outright // (an unwritable CWD - which is what happens running as a service from // /) or silently drops the single most important output of the whole // migration wherever the operator happened to be standing. WorkDir string } // RunSettingsConvert runs migrate_v016.py's convert subcommand. func RunSettingsConvert(ctx context.Context, o SettingsConvertOptions) error { python := o.PythonPath if python == "" { python = "python3" } args := []string{ o.ScriptPath, "convert", "--settings", o.SettingsPath, "--principals", o.PrincipalsPath, "--config", o.ConfigPath, "--output", o.OutputPath, } if len(o.PatchPaths) > 0 { pairs := make([]string, 0, len(o.PatchPaths)) for old, new := range o.PatchPaths { pairs = append(pairs, old+"="+new) } sort.Strings(pairs) // deterministic argv, easier to test and to log args = append(args, "--patch-paths", strings.Join(pairs, ",")) } cmd := exec.CommandContext(ctx, python, args...) if o.WorkDir != "" { if err := os.MkdirAll(o.WorkDir, 0o750); err != nil { return fmt.Errorf("backup: create convert working directory %s: %w", o.WorkDir, err) } cmd.Dir = o.WorkDir } out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("backup: migrate_v016.py convert failed: %w (output: %s)", err, out) } return nil } // UnmigratedPrefix is one group of v0.15 settings the conversion did not // carry over, as reported by migrate_v016.py's unmigrated.txt. type UnmigratedPrefix struct { Prefix string Keys int } // UnmigratedReport summarizes what a conversion left behind. // // This is not a footnote. Against a real production instance - 12,401 // settings - Stalwart's own converter migrated 219 of them, 1.8%, and left // 12,182 for the operator to recreate by hand: spam-filter rules, DNSBLs, // trusted-domain and URL-redirector lookups, queue scheduling and TLS // settings, and server.listener itself, which is why a freshly migrated // instance answers on none of the ports the old one did. A migration that // reported success while silently discarding this would be worse than one // that failed. type UnmigratedReport struct { Path string TotalKeys int Prefixes []UnmigratedPrefix } // Summary renders the report for an operator, largest groups first. func (r *UnmigratedReport) Summary(maxPrefixes int) string { if r == nil || r.TotalKeys == 0 { return "no unmigrated settings were reported" } var b strings.Builder fmt.Fprintf(&b, "%d v0.15 setting(s) were NOT migrated and must be recreated by hand (full list: %s)", r.TotalKeys, r.Path) prefixes := r.Prefixes if len(prefixes) > maxPrefixes { prefixes = prefixes[:maxPrefixes] } for _, p := range prefixes { fmt.Fprintf(&b, "\n %-32s %d keys", p.Prefix, p.Keys) } if len(r.Prefixes) > len(prefixes) { fmt.Fprintf(&b, "\n ... and %d more prefix(es)", len(r.Prefixes)-len(prefixes)) } return b.String() } // unmigratedPattern matches the report's per-prefix lines, e.g. // " spam-filter.rule 424 keys". var unmigratedPattern = regexp.MustCompile(`^\s+(\S+)\s+(\d+) keys\s*$`) // ReadUnmigratedReport parses the unmigrated.txt migrate_v016.py writes // beside its output. A missing file is not an error - an older script, or // a conversion with nothing left over, simply won't produce one - so // callers get a nil report rather than a failure. func ReadUnmigratedReport(path string) (*UnmigratedReport, error) { data, err := os.ReadFile(path) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, fmt.Errorf("backup: read %s: %w", path, err) } report := &UnmigratedReport{Path: path} for _, line := range strings.Split(string(data), "\n") { if m := unmigratedPattern.FindStringSubmatch(line); m != nil { n, convErr := strconv.Atoi(m[2]) if convErr != nil { continue } report.Prefixes = append(report.Prefixes, UnmigratedPrefix{Prefix: m[1], Keys: n}) continue } if strings.HasPrefix(strings.TrimSpace(line), "Total unmigrated keys:") { fields := strings.Fields(line) if len(fields) >= 4 { if n, convErr := strconv.Atoi(fields[3]); convErr == nil { report.TotalKeys = n } } } } sort.Slice(report.Prefixes, func(i, j int) bool { return report.Prefixes[i].Keys > report.Prefixes[j].Keys }) return report, nil } // ReadSettingsDump loads the flat {key: value} settings map // migrate_v016.py's dump step writes. func ReadSettingsDump(path string) (map[string]string, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("backup: read settings dump %s: %w", path, err) } var raw map[string]any if err := json.Unmarshal(data, &raw); err != nil { return nil, fmt.Errorf("backup: parse settings dump %s: %w", path, err) } settings := make(map[string]string, len(raw)) for k, v := range raw { if s, ok := v.(string); ok { settings[k] = s continue } settings[k] = fmt.Sprint(v) } return settings, nil } // ReadUnmigratedKeys returns the set of settings keys migrate_v016.py // reported it did not carry over. // // unmigrated.txt lists prefixes and counts rather than individual keys, so // this expands those prefixes against the settings dump. That is why it // needs both files: the report says "spam-filter.rule: 424 keys", and only // the dump knows which 424. func ReadUnmigratedKeys(reportPath string, settings map[string]string) (map[string]bool, error) { report, err := ReadUnmigratedReport(reportPath) if err != nil { return nil, err } keys := map[string]bool{} if report == nil { return keys, nil } for _, p := range report.Prefixes { for k := range settings { if k == p.Prefix || strings.HasPrefix(k, p.Prefix+".") { keys[k] = true } } } return keys, nil } // Principal is the subset of a v0.15 principals dump this tool needs. The // shape is what the REST management API returns and what // migrate_v016.py's dump step writes out verbatim. type Principal struct { ID int `json:"id"` Type string `json:"type"` // "individual", "group", "domain", ... Name string `json:"name"` Emails []string `json:"emails"` Roles []string `json:"roles"` } // ReadPrincipalsDump loads the principals dump written alongside the // settings dump. func ReadPrincipalsDump(path string) ([]Principal, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("backup: read principals dump %s: %w", path, err) } var principals []Principal if err := json.Unmarshal(data, &principals); err != nil { return nil, fmt.Errorf("backup: parse principals dump %s: %w", path, err) } return principals, nil }