From c570a91ad5a27d1dc0823772d59a44c42e75f1b6 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 15:51:52 -0700 Subject: [PATCH] Keep up with upstream: an import script and a daily watch scripts/upstream-import.py commits an upstream release onto the `upstream` branch with the fork's rename applied (rustfmt'd), the fork's own README, changelog, .gitignore and workflows in place of upstream's, and upstream's release tooling left out. Each snapshot's parent is the last, so merging the branch into main brings in only what upstream changed, already renamed. Upstream tags go to refs/upstream-tags/, never refs/tags/. The upstream-watch workflow reads the imported release from the branch's head subject and opens an "Import upstream vX.Y.Z" issue for each newer release, daily, from metadata only. GitHub's CI skips the `upstream` branch. --- .gitea/workflows/upstream-watch.yml | 90 ++++++++++++ .github/workflows/ci.yml | 4 +- scripts/upstream-import.py | 212 ++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/upstream-watch.yml create mode 100755 scripts/upstream-import.py diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml new file mode 100644 index 0000000..a7aca19 --- /dev/null +++ b/.gitea/workflows/upstream-watch.yml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Watch upstream for releases not imported yet, and open an issue for each one +# so it waits in the tracker until someone takes it in. +# +# Reads metadata only: the releases list from GitHub's API and the head of +# this repository's `upstream` branch from Gitea's. Importing is by hand, with +# scripts/upstream-import.py, which commits the release, renamed, onto +# `upstream` for main to merge. +# +# The imported release is the version in the `upstream` branch's head commit +# subject ("Import upstream v1.0.13, renamed"). Drafts and pre-releases are +# ignored. An issue is opened once per release: an existing one with the same +# title, open or closed, stops a second. +# +# Daily 06:23 UTC; run it by hand with workflow_dispatch. +name: upstream-watch + +on: + schedule: + - cron: '23 6 * * *' + workflow_dispatch: + +concurrency: + group: upstream-watch + cancel-in-progress: false + +jobs: + upstream-watch: + runs-on: light + container: + image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + steps: + - shell: bash + run: | + python3 - <<'PY' + import json, os, re, sys, urllib.request + + api = f"{os.environ['CI_SERVER_INTERNAL']}/api/v1/repos/{os.environ['REPO']}" + def call(method, url, body=None, token=os.environ["TOKEN"]): + headers = {"Content-Type": "application/json", "User-Agent": "inbuxa-migrate-upstream-watch"} + if token: + headers["Authorization"] = f"token {token}" + req = urllib.request.Request(url, method=method, headers=headers, + data=json.dumps(body).encode() if body is not None else None) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + SEMVER = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + def key(tag): + return tuple(int(x) for x in SEMVER.match(tag).groups()) + + subject = call("GET", f"{api}/branches/upstream")["commit"]["message"].splitlines()[0] + m = re.search(r"\b(v\d+\.\d+\.\d+)\b", subject) + if not m: + print(f"Can't read the imported release from the upstream branch: {subject!r}", file=sys.stderr); sys.exit(1) + base = m.group(1) + + # Unauthenticated: a public repository, once a day, well inside the limit. + rels = call("GET", "https://api.github.com/repos/stalwartlabs/vandelay/releases?per_page=30", token=None) + newer = sorted((r for r in rels + if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"]) + and key(r["tag_name"]) > key(base)), + key=lambda r: key(r["tag_name"])) + if not newer: + print(f"Up to date: {base} is the newest upstream release."); sys.exit(0) + + existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=Import+upstream&limit=50")} + for r in newer: + tag = r["tag_name"] + title = f"Import upstream {tag}" + if title in existing: + print(f"{tag}: issue already exists."); continue + body = (f"Upstream published {tag} on {r['published_at'][:10]}. " + f"The last import is {base}.\n\n" + "Take it in as scripts/upstream-import.py describes:\n\n" + "```bash\n" + "git fetch origin upstream:upstream\n" + f"scripts/upstream-import.py {tag}\n" + f"git switch -c import/upstream-{tag} origin/main\n" + "git merge upstream\n" + "```\n\n" + "Before reading the changes, scan them for hidden characters and text aimed at " + "automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.") + issue = call("POST", f"{api}/issues", {"title": title, "body": body}) + print(f"{tag}: opened #{issue['number']}.") + PY diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 422394d..9b184c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,9 @@ name: ci on: push: - branches: ['**'] + # `upstream` holds upstream's releases, renamed, for main to merge + # (scripts/upstream-import.py); main tests them once merged. + branches: ['**', '!upstream'] tags: ['**'] # For a run GitHub queued and then orphaned. Run it from a tag to redo that # tag's release; attaching skips any file already on the Release. diff --git a/scripts/upstream-import.py b/scripts/upstream-import.py new file mode 100755 index 0000000..7f27c2d --- /dev/null +++ b/scripts/upstream-import.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Take an upstream release in: commit it, renamed, onto the `upstream` branch, +# ready to merge into main. +# +# Usage: scripts/upstream-import.py REF +# scripts/upstream-import.py v1.0.13 +# scripts/upstream-import.py --first 486a384 +# +# REF is an upstream tag or commit. The tag is fetched from the fetch-only +# `upstream` remote into refs/upstream-tags/, never refs/tags/: a pushed +# upstream tag would start a release here. +# +# The snapshot is upstream's tree with the fork's rename applied (the crate, +# binary, variables and test names, as in "Rename to inbuxa-migrate"), the +# fork's own files in place of upstream's (OWNED) and upstream's release +# tooling left out (DROPPED). Its parent is the branch's last snapshot, so +# merging the branch into main brings in only what upstream changed between +# the two releases, already renamed. Nothing in the working tree is touched. +# +# --first starts the branch. Give it the upstream commit the fork was made +# from; the snapshot's parent is that commit. Merge it into main once with +# `git merge -s ours upstream`, which records that main already has it, and +# later imports merge cleanly against it. +# +# After an import: +# git switch -c import/upstream-REF origin/main +# git merge upstream +# (resolve; for Cargo.lock take main's and let `cargo build` add what's new) +# then the usual checks and a PR. Push `upstream` to origin only once that PR +# has landed. The upstream-watch workflow reads the imported release from the +# branch's head subject. + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +REMOTE = "upstream" +BRANCH = "upstream" +REPO_URL = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" + +# The fork's own copies always win: upstream's versions of these are dropped +# and main's are put in their place. +OWNED = ["README.md", "CHANGELOG.md", ".gitignore", ".github", ".gitea"] +# Upstream's release tooling, which the fork replaced. +DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg"] + +# Order matters: the specific forms before the plain name. +RENAMES = [ + (re.compile(r"https://github\.com/stalwartlabs/vandelay"), REPO_URL), + (re.compile(r"\bvandelay(?=::|_)"), "inbuxa_migrate"), + (re.compile(r"\b(use|extern crate) vandelay\b"), r"\1 inbuxa_migrate"), + (re.compile(r"VANDELAY_"), "INBUXA_MIGRATE_"), + (re.compile(r"VANDELAY"), "INBUXA"), + (re.compile(r"\ba Vandelay\b"), "an inbuxa-migrate"), + (re.compile(r"Vandelay"), "Inbuxa"), + (re.compile(r"vandelay"), "inbuxa-migrate"), +] +OLD_NAME = re.compile(r"vandelay", re.I) + + +def git(*args, capture=True, env=None, check=True): + r = subprocess.run(["git", *args], capture_output=capture, text=True, env=env) + if check and r.returncode != 0: + sys.exit(f"git {' '.join(args)}: {r.stderr.strip()}") + return r.stdout.strip() if capture else "" + + +def extract(rev, dest, paths=()): + """Write the tree of `rev` (or only `paths` in it) into `dest`.""" + present = [p for p in paths + if subprocess.run(["git", "cat-file", "-e", f"{rev}:{p}"], capture_output=True).returncode == 0] + if paths and not present: + return + archive = subprocess.run(["git", "archive", "--format=tar", rev, *present], + capture_output=True, check=True).stdout + with tempfile.TemporaryFile() as f: + f.write(archive) + f.seek(0) + with tarfile.open(fileobj=f) as tar: + tar.extractall(dest, filter="tar") + + +def remove(root, rel): + path = os.path.join(root, rel) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + elif os.path.lexists(path): + os.remove(path) + + +def rename_text(text): + for pattern, new in RENAMES: + text = pattern.sub(new, text) + return text + + +def fork_cargo_toml(upstream_text, fork_text): + """Upstream's Cargo.toml with the fork's [package] table, and without the + tables the fork's release tooling dropped.""" + head = fork_text[: fork_text.index("[[bin]]")] + body = upstream_text[upstream_text.index("[[bin]]"):] + for table in ("[profile.dist]", "[workspace.metadata.dist]"): + body = re.sub(r"(?ms)^" + re.escape(table) + r"\n.*?(?=^\[|\Z)", "", body) + return head + body.rstrip("\n") + "\n" + + +def apply(root, fork): + """Rename in place; returns the files the rename changed and the paths + still naming the old project afterwards.""" + changed, left = [], [] + for dirpath, dirnames, filenames in os.walk(root): + for name in filenames: + path = os.path.join(dirpath, name) + rel = os.path.relpath(path, root) + if os.path.islink(path) or any(rel == o or rel.startswith(o + "/") for o in OWNED): + continue + try: + with open(path, encoding="utf-8", newline="") as f: + text = f.read() + except UnicodeDecodeError: + continue + new = rename_text(text) + if rel == "Cargo.toml": + new = fork_cargo_toml(new, git("show", f"{fork}:Cargo.toml")) + if new != text: + with open(path, "w", encoding="utf-8", newline="") as f: + f.write(new) + changed.append(rel) + if OLD_NAME.search(new): + left.append(rel) + for dirpath, dirnames, filenames in os.walk(root, topdown=False): + for name in filenames + dirnames: + if OLD_NAME.search(name): + new = rename_text(name) + os.rename(os.path.join(dirpath, name), os.path.join(dirpath, new)) + changed.append(os.path.relpath(os.path.join(dirpath, new), root)) + return sorted(changed), sorted(left) + + +def main(): + ap = argparse.ArgumentParser(description="Commit an upstream release, renamed, onto the upstream branch.") + ap.add_argument("ref", help="upstream tag (v1.0.13) or commit") + ap.add_argument("--first", action="store_true", help="start the branch at this upstream commit") + ap.add_argument("--fork", default="origin/main", help="where the fork's own files come from (origin/main)") + args = ap.parse_args() + + os.chdir(git("rev-parse", "--show-toplevel")) + if git("remote", "get-url", "--push", REMOTE, check=False) not in ("DISABLED", ""): + sys.exit(f"the `{REMOTE}` remote must be fetch-only (push URL DISABLED)") + + if re.fullmatch(r"v\d+\.\d+\.\d+", args.ref): + git("fetch", "--no-tags", REMOTE, f"+refs/tags/{args.ref}:refs/upstream-tags/{args.ref}") + rev = git("rev-parse", f"refs/upstream-tags/{args.ref}^{{commit}}") + else: + git("fetch", "--no-tags", REMOTE, "+refs/tags/*:refs/upstream-tags/*") + rev = git("rev-parse", f"{args.ref}^{{commit}}") + label = args.ref if args.ref.startswith("v") else rev[:7] + + have = git("rev-parse", "--verify", "--quiet", f"refs/heads/{BRANCH}", check=False) + if args.first == bool(have): + sys.exit(f"`{BRANCH}` already exists; leave out --first" if have + else f"no `{BRANCH}` branch yet; start it with --first") + parent = rev if args.first else have + + with tempfile.TemporaryDirectory(prefix="upstream-import-") as tmp: + tree_dir = os.path.join(tmp, "tree") + os.mkdir(tree_dir) + extract(rev, tree_dir) + for rel in OWNED + DROPPED: + remove(tree_dir, rel) + extract(args.fork, tree_dir, OWNED) + changed, left = apply(tree_dir, args.fork) + # The rename moves imports and line breaks; main is kept formatted. + rust = [os.path.join(tree_dir, rel) for rel in changed if rel.endswith(".rs")] + if rust and subprocess.run(["rustfmt", "--edition", "2024", *rust]).returncode != 0: + sys.exit("rustfmt failed on the renamed files") + + env = dict(os.environ, GIT_INDEX_FILE=os.path.join(tmp, "index")) + git("--work-tree", tree_dir, "add", "--all", "--force", ".", env=env) + tree = git("write-tree", env=env) + + if not args.first and git("rev-parse", f"{parent}^{{tree}}") == tree: + print(f"{label}: nothing changed since the last import.") + return + if args.first: + # The newest release the commit includes, which the watch counts from. + tags = git("for-each-ref", "--merged", rev, "--sort=-version:refname", + "--format=%(refname:lstrip=2)", "refs/upstream-tags/v*").split() + label = f"{label} ({tags[0] if tags else 'untagged'} and later)" + message = (f"Import upstream {label}, renamed\n\n" + f"Upstream commit {rev}, renamed by scripts/upstream-import.py.\n") + commit = git("commit-tree", "-S", tree, "-p", parent, "-m", message) + git("update-ref", f"refs/heads/{BRANCH}", commit, *([] if args.first else [parent])) + + print(f"{BRANCH} is now {commit[:7]}: upstream {label}, renamed.") + print(f"The rename changed {len(changed)} file(s).") + if left: + print("Still naming the old project, to look at before merging:") + for rel in left: + print(f" {rel}") + + +if __name__ == "__main__": + main() -- 2.54.0