Keep up with upstream: an import script and a daily watch #48

Merged
jcoffey-dev merged 1 commits from feat/upstream-import into main 2026-10-30 22:56:40 +00:00
3 changed files with 305 additions and 1 deletions

No files matched your search

+90
View File
@@ -0,0 +1,90 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
# Watch upstream for releases not imported yet, and open an issue for each one
# so it waits in the tracker until someone takes it in.
#
# Reads metadata only: the releases list from GitHub's API and the head of
# this repository's `upstream` branch from Gitea's. Importing is by hand, with
# scripts/upstream-import.py, which commits the release, renamed, onto
# `upstream` for main to merge.
#
# The imported release is the version in the `upstream` branch's head commit
# subject ("Import upstream v1.0.13, renamed"). Drafts and pre-releases are
# ignored. An issue is opened once per release: an existing one with the same
# title, open or closed, stops a second.
#
# Daily 06:23 UTC; run it by hand with workflow_dispatch.
name: upstream-watch
on:
schedule:
- cron: '23 6 * * *'
workflow_dispatch:
concurrency:
group: upstream-watch
cancel-in-progress: false
jobs:
upstream-watch:
runs-on: light
container:
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
env:
TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: ${{ github.repository }}
steps:
- shell: bash
run: |
python3 - <<'PY'
import json, os, re, sys, urllib.request
api = f"{os.environ['CI_SERVER_INTERNAL']}/api/v1/repos/{os.environ['REPO']}"
def call(method, url, body=None, token=os.environ["TOKEN"]):
headers = {"Content-Type": "application/json", "User-Agent": "inbuxa-migrate-upstream-watch"}
if token:
headers["Authorization"] = f"token {token}"
req = urllib.request.Request(url, method=method, headers=headers,
data=json.dumps(body).encode() if body is not None else None)
with urllib.request.urlopen(req, timeout=30) as r:
return json.load(r)
SEMVER = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
def key(tag):
return tuple(int(x) for x in SEMVER.match(tag).groups())
subject = call("GET", f"{api}/branches/upstream")["commit"]["message"].splitlines()[0]
m = re.search(r"\b(v\d+\.\d+\.\d+)\b", subject)
if not m:
print(f"Can't read the imported release from the upstream branch: {subject!r}", file=sys.stderr); sys.exit(1)
base = m.group(1)
# Unauthenticated: a public repository, once a day, well inside the limit.
rels = call("GET", "https://api.github.com/repos/stalwartlabs/vandelay/releases?per_page=30", token=None)
newer = sorted((r for r in rels
if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"])
and key(r["tag_name"]) > key(base)),
key=lambda r: key(r["tag_name"]))
if not newer:
print(f"Up to date: {base} is the newest upstream release."); sys.exit(0)
existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=Import+upstream&limit=50")}
for r in newer:
tag = r["tag_name"]
title = f"Import upstream {tag}"
if title in existing:
print(f"{tag}: issue already exists."); continue
body = (f"Upstream published {tag} on {r['published_at'][:10]}. "
f"The last import is {base}.\n\n"
"Take it in as scripts/upstream-import.py describes:\n\n"
"```bash\n"
"git fetch origin upstream:upstream\n"
f"scripts/upstream-import.py {tag}\n"
f"git switch -c import/upstream-{tag} origin/main\n"
"git merge upstream\n"
"```\n\n"
"Before reading the changes, scan them for hidden characters and text aimed at "
"automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.")
issue = call("POST", f"{api}/issues", {"title": title, "body": body})
print(f"{tag}: opened #{issue['number']}.")
PY
+3 -1
View File
@@ -40,7 +40,9 @@ name: ci
on:
push:
branches: ['**']
# `upstream` holds upstream's releases, renamed, for main to merge
# (scripts/upstream-import.py); main tests them once merged.
branches: ['**', '!upstream']
tags: ['**']
# For a run GitHub queued and then orphaned. Run it from a tag to redo that
# tag's release; attaching skips any file already on the Release.
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
# Take an upstream release in: commit it, renamed, onto the `upstream` branch,
# ready to merge into main.
#
# Usage: scripts/upstream-import.py REF
# scripts/upstream-import.py v1.0.13
# scripts/upstream-import.py --first 486a384
#
# REF is an upstream tag or commit. The tag is fetched from the fetch-only
# `upstream` remote into refs/upstream-tags/, never refs/tags/: a pushed
# upstream tag would start a release here.
#
# The snapshot is upstream's tree with the fork's rename applied (the crate,
# binary, variables and test names, as in "Rename to inbuxa-migrate"), the
# fork's own files in place of upstream's (OWNED) and upstream's release
# tooling left out (DROPPED). Its parent is the branch's last snapshot, so
# merging the branch into main brings in only what upstream changed between
# the two releases, already renamed. Nothing in the working tree is touched.
#
# --first starts the branch. Give it the upstream commit the fork was made
# from; the snapshot's parent is that commit. Merge it into main once with
# `git merge -s ours upstream`, which records that main already has it, and
# later imports merge cleanly against it.
#
# After an import:
# git switch -c import/upstream-REF origin/main
# git merge upstream
# (resolve; for Cargo.lock take main's and let `cargo build` add what's new)
# then the usual checks and a PR. Push `upstream` to origin only once that PR
# has landed. The upstream-watch workflow reads the imported release from the
# branch's head subject.
import argparse
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
REMOTE = "upstream"
BRANCH = "upstream"
REPO_URL = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate"
# The fork's own copies always win: upstream's versions of these are dropped
# and main's are put in their place.
OWNED = ["README.md", "CHANGELOG.md", ".gitignore", ".github", ".gitea"]
# Upstream's release tooling, which the fork replaced.
DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg"]
# Order matters: the specific forms before the plain name.
RENAMES = [
(re.compile(r"https://github\.com/stalwartlabs/vandelay"), REPO_URL),
(re.compile(r"\bvandelay(?=::|_)"), "inbuxa_migrate"),
(re.compile(r"\b(use|extern crate) vandelay\b"), r"\1 inbuxa_migrate"),
(re.compile(r"VANDELAY_"), "INBUXA_MIGRATE_"),
(re.compile(r"VANDELAY"), "INBUXA"),
(re.compile(r"\ba Vandelay\b"), "an inbuxa-migrate"),
(re.compile(r"Vandelay"), "Inbuxa"),
(re.compile(r"vandelay"), "inbuxa-migrate"),
]
OLD_NAME = re.compile(r"vandelay", re.I)
def git(*args, capture=True, env=None, check=True):
r = subprocess.run(["git", *args], capture_output=capture, text=True, env=env)
if check and r.returncode != 0:
sys.exit(f"git {' '.join(args)}: {r.stderr.strip()}")
return r.stdout.strip() if capture else ""
def extract(rev, dest, paths=()):
"""Write the tree of `rev` (or only `paths` in it) into `dest`."""
present = [p for p in paths
if subprocess.run(["git", "cat-file", "-e", f"{rev}:{p}"], capture_output=True).returncode == 0]
if paths and not present:
return
archive = subprocess.run(["git", "archive", "--format=tar", rev, *present],
capture_output=True, check=True).stdout
with tempfile.TemporaryFile() as f:
f.write(archive)
f.seek(0)
with tarfile.open(fileobj=f) as tar:
tar.extractall(dest, filter="tar")
def remove(root, rel):
path = os.path.join(root, rel)
if os.path.isdir(path) and not os.path.islink(path):
shutil.rmtree(path)
elif os.path.lexists(path):
os.remove(path)
def rename_text(text):
for pattern, new in RENAMES:
text = pattern.sub(new, text)
return text
def fork_cargo_toml(upstream_text, fork_text):
"""Upstream's Cargo.toml with the fork's [package] table, and without the
tables the fork's release tooling dropped."""
head = fork_text[: fork_text.index("[[bin]]")]
body = upstream_text[upstream_text.index("[[bin]]"):]
for table in ("[profile.dist]", "[workspace.metadata.dist]"):
body = re.sub(r"(?ms)^" + re.escape(table) + r"\n.*?(?=^\[|\Z)", "", body)
return head + body.rstrip("\n") + "\n"
def apply(root, fork):
"""Rename in place; returns the files the rename changed and the paths
still naming the old project afterwards."""
changed, left = [], []
for dirpath, dirnames, filenames in os.walk(root):
for name in filenames:
path = os.path.join(dirpath, name)
rel = os.path.relpath(path, root)
if os.path.islink(path) or any(rel == o or rel.startswith(o + "/") for o in OWNED):
continue
try:
with open(path, encoding="utf-8", newline="") as f:
text = f.read()
except UnicodeDecodeError:
continue
new = rename_text(text)
if rel == "Cargo.toml":
new = fork_cargo_toml(new, git("show", f"{fork}:Cargo.toml"))
if new != text:
with open(path, "w", encoding="utf-8", newline="") as f:
f.write(new)
changed.append(rel)
if OLD_NAME.search(new):
left.append(rel)
for dirpath, dirnames, filenames in os.walk(root, topdown=False):
for name in filenames + dirnames:
if OLD_NAME.search(name):
new = rename_text(name)
os.rename(os.path.join(dirpath, name), os.path.join(dirpath, new))
changed.append(os.path.relpath(os.path.join(dirpath, new), root))
return sorted(changed), sorted(left)
def main():
ap = argparse.ArgumentParser(description="Commit an upstream release, renamed, onto the upstream branch.")
ap.add_argument("ref", help="upstream tag (v1.0.13) or commit")
ap.add_argument("--first", action="store_true", help="start the branch at this upstream commit")
ap.add_argument("--fork", default="origin/main", help="where the fork's own files come from (origin/main)")
args = ap.parse_args()
os.chdir(git("rev-parse", "--show-toplevel"))
if git("remote", "get-url", "--push", REMOTE, check=False) not in ("DISABLED", ""):
sys.exit(f"the `{REMOTE}` remote must be fetch-only (push URL DISABLED)")
if re.fullmatch(r"v\d+\.\d+\.\d+", args.ref):
git("fetch", "--no-tags", REMOTE, f"+refs/tags/{args.ref}:refs/upstream-tags/{args.ref}")
rev = git("rev-parse", f"refs/upstream-tags/{args.ref}^{{commit}}")
else:
git("fetch", "--no-tags", REMOTE, "+refs/tags/*:refs/upstream-tags/*")
rev = git("rev-parse", f"{args.ref}^{{commit}}")
label = args.ref if args.ref.startswith("v") else rev[:7]
have = git("rev-parse", "--verify", "--quiet", f"refs/heads/{BRANCH}", check=False)
if args.first == bool(have):
sys.exit(f"`{BRANCH}` already exists; leave out --first" if have
else f"no `{BRANCH}` branch yet; start it with --first")
parent = rev if args.first else have
with tempfile.TemporaryDirectory(prefix="upstream-import-") as tmp:
tree_dir = os.path.join(tmp, "tree")
os.mkdir(tree_dir)
extract(rev, tree_dir)
for rel in OWNED + DROPPED:
remove(tree_dir, rel)
extract(args.fork, tree_dir, OWNED)
changed, left = apply(tree_dir, args.fork)
# The rename moves imports and line breaks; main is kept formatted.
rust = [os.path.join(tree_dir, rel) for rel in changed if rel.endswith(".rs")]
if rust and subprocess.run(["rustfmt", "--edition", "2024", *rust]).returncode != 0:
sys.exit("rustfmt failed on the renamed files")
env = dict(os.environ, GIT_INDEX_FILE=os.path.join(tmp, "index"))
git("--work-tree", tree_dir, "add", "--all", "--force", ".", env=env)
tree = git("write-tree", env=env)
if not args.first and git("rev-parse", f"{parent}^{{tree}}") == tree:
print(f"{label}: nothing changed since the last import.")
return
if args.first:
# The newest release the commit includes, which the watch counts from.
tags = git("for-each-ref", "--merged", rev, "--sort=-version:refname",
"--format=%(refname:lstrip=2)", "refs/upstream-tags/v*").split()
label = f"{label} ({tags[0] if tags else 'untagged'} and later)"
message = (f"Import upstream {label}, renamed\n\n"
f"Upstream commit {rev}, renamed by scripts/upstream-import.py.\n")
commit = git("commit-tree", "-S", tree, "-p", parent, "-m", message)
git("update-ref", f"refs/heads/{BRANCH}", commit, *([] if args.first else [parent]))
print(f"{BRANCH} is now {commit[:7]}: upstream {label}, renamed.")
print(f"The rename changed {len(changed)} file(s).")
if left:
print("Still naming the old project, to look at before merging:")
for rel in left:
print(f" {rel}")
if __name__ == "__main__":
main()