diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml new file mode 100644 index 0000000..a7aca19 --- /dev/null +++ b/.gitea/workflows/upstream-watch.yml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Watch upstream for releases not imported yet, and open an issue for each one +# so it waits in the tracker until someone takes it in. +# +# Reads metadata only: the releases list from GitHub's API and the head of +# this repository's `upstream` branch from Gitea's. Importing is by hand, with +# scripts/upstream-import.py, which commits the release, renamed, onto +# `upstream` for main to merge. +# +# The imported release is the version in the `upstream` branch's head commit +# subject ("Import upstream v1.0.13, renamed"). Drafts and pre-releases are +# ignored. An issue is opened once per release: an existing one with the same +# title, open or closed, stops a second. +# +# Daily 06:23 UTC; run it by hand with workflow_dispatch. +name: upstream-watch + +on: + schedule: + - cron: '23 6 * * *' + workflow_dispatch: + +concurrency: + group: upstream-watch + cancel-in-progress: false + +jobs: + upstream-watch: + runs-on: light + container: + image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + steps: + - shell: bash + run: | + python3 - <<'PY' + import json, os, re, sys, urllib.request + + api = f"{os.environ['CI_SERVER_INTERNAL']}/api/v1/repos/{os.environ['REPO']}" + def call(method, url, body=None, token=os.environ["TOKEN"]): + headers = {"Content-Type": "application/json", "User-Agent": "inbuxa-migrate-upstream-watch"} + if token: + headers["Authorization"] = f"token {token}" + req = urllib.request.Request(url, method=method, headers=headers, + data=json.dumps(body).encode() if body is not None else None) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + SEMVER = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + def key(tag): + return tuple(int(x) for x in SEMVER.match(tag).groups()) + + subject = call("GET", f"{api}/branches/upstream")["commit"]["message"].splitlines()[0] + m = re.search(r"\b(v\d+\.\d+\.\d+)\b", subject) + if not m: + print(f"Can't read the imported release from the upstream branch: {subject!r}", file=sys.stderr); sys.exit(1) + base = m.group(1) + + # Unauthenticated: a public repository, once a day, well inside the limit. + rels = call("GET", "https://api.github.com/repos/stalwartlabs/vandelay/releases?per_page=30", token=None) + newer = sorted((r for r in rels + if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"]) + and key(r["tag_name"]) > key(base)), + key=lambda r: key(r["tag_name"])) + if not newer: + print(f"Up to date: {base} is the newest upstream release."); sys.exit(0) + + existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=Import+upstream&limit=50")} + for r in newer: + tag = r["tag_name"] + title = f"Import upstream {tag}" + if title in existing: + print(f"{tag}: issue already exists."); continue + body = (f"Upstream published {tag} on {r['published_at'][:10]}. " + f"The last import is {base}.\n\n" + "Take it in as scripts/upstream-import.py describes:\n\n" + "```bash\n" + "git fetch origin upstream:upstream\n" + f"scripts/upstream-import.py {tag}\n" + f"git switch -c import/upstream-{tag} origin/main\n" + "git merge upstream\n" + "```\n\n" + "Before reading the changes, scan them for hidden characters and text aimed at " + "automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.") + issue = call("POST", f"{api}/issues", {"title": title, "body": body}) + print(f"{tag}: opened #{issue['number']}.") + PY diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 422394d..9b184c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,9 @@ name: ci on: push: - branches: ['**'] + # `upstream` holds upstream's releases, renamed, for main to merge + # (scripts/upstream-import.py); main tests them once merged. + branches: ['**', '!upstream'] tags: ['**'] # For a run GitHub queued and then orphaned. Run it from a tag to redo that # tag's release; attaching skips any file already on the Release. diff --git a/scripts/upstream-import.py b/scripts/upstream-import.py new file mode 100755 index 0000000..7f27c2d --- /dev/null +++ b/scripts/upstream-import.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Take an upstream release in: commit it, renamed, onto the `upstream` branch, +# ready to merge into main. +# +# Usage: scripts/upstream-import.py REF +# scripts/upstream-import.py v1.0.13 +# scripts/upstream-import.py --first 486a384 +# +# REF is an upstream tag or commit. The tag is fetched from the fetch-only +# `upstream` remote into refs/upstream-tags/, never refs/tags/: a pushed +# upstream tag would start a release here. +# +# The snapshot is upstream's tree with the fork's rename applied (the crate, +# binary, variables and test names, as in "Rename to inbuxa-migrate"), the +# fork's own files in place of upstream's (OWNED) and upstream's release +# tooling left out (DROPPED). Its parent is the branch's last snapshot, so +# merging the branch into main brings in only what upstream changed between +# the two releases, already renamed. Nothing in the working tree is touched. +# +# --first starts the branch. Give it the upstream commit the fork was made +# from; the snapshot's parent is that commit. Merge it into main once with +# `git merge -s ours upstream`, which records that main already has it, and +# later imports merge cleanly against it. +# +# After an import: +# git switch -c import/upstream-REF origin/main +# git merge upstream +# (resolve; for Cargo.lock take main's and let `cargo build` add what's new) +# then the usual checks and a PR. Push `upstream` to origin only once that PR +# has landed. The upstream-watch workflow reads the imported release from the +# branch's head subject. + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +REMOTE = "upstream" +BRANCH = "upstream" +REPO_URL = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" + +# The fork's own copies always win: upstream's versions of these are dropped +# and main's are put in their place. +OWNED = ["README.md", "CHANGELOG.md", ".gitignore", ".github", ".gitea"] +# Upstream's release tooling, which the fork replaced. +DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg"] + +# Order matters: the specific forms before the plain name. +RENAMES = [ + (re.compile(r"https://github\.com/stalwartlabs/vandelay"), REPO_URL), + (re.compile(r"\bvandelay(?=::|_)"), "inbuxa_migrate"), + (re.compile(r"\b(use|extern crate) vandelay\b"), r"\1 inbuxa_migrate"), + (re.compile(r"VANDELAY_"), "INBUXA_MIGRATE_"), + (re.compile(r"VANDELAY"), "INBUXA"), + (re.compile(r"\ba Vandelay\b"), "an inbuxa-migrate"), + (re.compile(r"Vandelay"), "Inbuxa"), + (re.compile(r"vandelay"), "inbuxa-migrate"), +] +OLD_NAME = re.compile(r"vandelay", re.I) + + +def git(*args, capture=True, env=None, check=True): + r = subprocess.run(["git", *args], capture_output=capture, text=True, env=env) + if check and r.returncode != 0: + sys.exit(f"git {' '.join(args)}: {r.stderr.strip()}") + return r.stdout.strip() if capture else "" + + +def extract(rev, dest, paths=()): + """Write the tree of `rev` (or only `paths` in it) into `dest`.""" + present = [p for p in paths + if subprocess.run(["git", "cat-file", "-e", f"{rev}:{p}"], capture_output=True).returncode == 0] + if paths and not present: + return + archive = subprocess.run(["git", "archive", "--format=tar", rev, *present], + capture_output=True, check=True).stdout + with tempfile.TemporaryFile() as f: + f.write(archive) + f.seek(0) + with tarfile.open(fileobj=f) as tar: + tar.extractall(dest, filter="tar") + + +def remove(root, rel): + path = os.path.join(root, rel) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + elif os.path.lexists(path): + os.remove(path) + + +def rename_text(text): + for pattern, new in RENAMES: + text = pattern.sub(new, text) + return text + + +def fork_cargo_toml(upstream_text, fork_text): + """Upstream's Cargo.toml with the fork's [package] table, and without the + tables the fork's release tooling dropped.""" + head = fork_text[: fork_text.index("[[bin]]")] + body = upstream_text[upstream_text.index("[[bin]]"):] + for table in ("[profile.dist]", "[workspace.metadata.dist]"): + body = re.sub(r"(?ms)^" + re.escape(table) + r"\n.*?(?=^\[|\Z)", "", body) + return head + body.rstrip("\n") + "\n" + + +def apply(root, fork): + """Rename in place; returns the files the rename changed and the paths + still naming the old project afterwards.""" + changed, left = [], [] + for dirpath, dirnames, filenames in os.walk(root): + for name in filenames: + path = os.path.join(dirpath, name) + rel = os.path.relpath(path, root) + if os.path.islink(path) or any(rel == o or rel.startswith(o + "/") for o in OWNED): + continue + try: + with open(path, encoding="utf-8", newline="") as f: + text = f.read() + except UnicodeDecodeError: + continue + new = rename_text(text) + if rel == "Cargo.toml": + new = fork_cargo_toml(new, git("show", f"{fork}:Cargo.toml")) + if new != text: + with open(path, "w", encoding="utf-8", newline="") as f: + f.write(new) + changed.append(rel) + if OLD_NAME.search(new): + left.append(rel) + for dirpath, dirnames, filenames in os.walk(root, topdown=False): + for name in filenames + dirnames: + if OLD_NAME.search(name): + new = rename_text(name) + os.rename(os.path.join(dirpath, name), os.path.join(dirpath, new)) + changed.append(os.path.relpath(os.path.join(dirpath, new), root)) + return sorted(changed), sorted(left) + + +def main(): + ap = argparse.ArgumentParser(description="Commit an upstream release, renamed, onto the upstream branch.") + ap.add_argument("ref", help="upstream tag (v1.0.13) or commit") + ap.add_argument("--first", action="store_true", help="start the branch at this upstream commit") + ap.add_argument("--fork", default="origin/main", help="where the fork's own files come from (origin/main)") + args = ap.parse_args() + + os.chdir(git("rev-parse", "--show-toplevel")) + if git("remote", "get-url", "--push", REMOTE, check=False) not in ("DISABLED", ""): + sys.exit(f"the `{REMOTE}` remote must be fetch-only (push URL DISABLED)") + + if re.fullmatch(r"v\d+\.\d+\.\d+", args.ref): + git("fetch", "--no-tags", REMOTE, f"+refs/tags/{args.ref}:refs/upstream-tags/{args.ref}") + rev = git("rev-parse", f"refs/upstream-tags/{args.ref}^{{commit}}") + else: + git("fetch", "--no-tags", REMOTE, "+refs/tags/*:refs/upstream-tags/*") + rev = git("rev-parse", f"{args.ref}^{{commit}}") + label = args.ref if args.ref.startswith("v") else rev[:7] + + have = git("rev-parse", "--verify", "--quiet", f"refs/heads/{BRANCH}", check=False) + if args.first == bool(have): + sys.exit(f"`{BRANCH}` already exists; leave out --first" if have + else f"no `{BRANCH}` branch yet; start it with --first") + parent = rev if args.first else have + + with tempfile.TemporaryDirectory(prefix="upstream-import-") as tmp: + tree_dir = os.path.join(tmp, "tree") + os.mkdir(tree_dir) + extract(rev, tree_dir) + for rel in OWNED + DROPPED: + remove(tree_dir, rel) + extract(args.fork, tree_dir, OWNED) + changed, left = apply(tree_dir, args.fork) + # The rename moves imports and line breaks; main is kept formatted. + rust = [os.path.join(tree_dir, rel) for rel in changed if rel.endswith(".rs")] + if rust and subprocess.run(["rustfmt", "--edition", "2024", *rust]).returncode != 0: + sys.exit("rustfmt failed on the renamed files") + + env = dict(os.environ, GIT_INDEX_FILE=os.path.join(tmp, "index")) + git("--work-tree", tree_dir, "add", "--all", "--force", ".", env=env) + tree = git("write-tree", env=env) + + if not args.first and git("rev-parse", f"{parent}^{{tree}}") == tree: + print(f"{label}: nothing changed since the last import.") + return + if args.first: + # The newest release the commit includes, which the watch counts from. + tags = git("for-each-ref", "--merged", rev, "--sort=-version:refname", + "--format=%(refname:lstrip=2)", "refs/upstream-tags/v*").split() + label = f"{label} ({tags[0] if tags else 'untagged'} and later)" + message = (f"Import upstream {label}, renamed\n\n" + f"Upstream commit {rev}, renamed by scripts/upstream-import.py.\n") + commit = git("commit-tree", "-S", tree, "-p", parent, "-m", message) + git("update-ref", f"refs/heads/{BRANCH}", commit, *([] if args.first else [parent])) + + print(f"{BRANCH} is now {commit[:7]}: upstream {label}, renamed.") + print(f"The rename changed {len(changed)} file(s).") + if left: + print("Still naming the old project, to look at before merging:") + for rel in left: + print(f" {rel}") + + +if __name__ == "__main__": + main()