From fc2e6f86456558c7099a1979fe9f94ee528a5501 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Sat, 26 Sep 2026 13:15:10 +0000 Subject: [PATCH 1/3] Push each build to openipc.org once; stop uploading size sidecars to releases The per-device sizes.*.json files uploaded to each dated release existed for the firmware explorer to poll. Releases are for firmware, and openipc.org no longer polls GitHub. - publish uploads the device images only, to the dated release as to nightly and latest. The size reports still travel as fw-* artifacts. With no images, no dated release is created. - A new report job, after publish and outside ci-gate, pushes the build to https://openipc.org/api/v1/builds once: the images with sha256 and each device's size report. It authenticates with its GitHub OIDC token (audience https://openipc.org); no secret is stored. The push is idempotent by build id, so the job can be re-run by itself. - push_build.py, soc_aliases.py and test_push_build.py are copies of OpenIPC/firmware's. lint.yml runs the tests. Contract: OpenIPC/website service/internal/builds/PUSH.md. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/ci-matrix.py | 3 +- .github/scripts/push_build.py | 213 +++++++++++++++++++++++++++++ .github/scripts/soc_aliases.py | 59 ++++++++ .github/scripts/test_push_build.py | 200 +++++++++++++++++++++++++++ .github/workflows/lint.yml | 8 ++ .github/workflows/master.yml | 93 ++++++++++--- 6 files changed, 558 insertions(+), 18 deletions(-) create mode 100644 .github/scripts/push_build.py create mode 100644 .github/scripts/soc_aliases.py create mode 100644 .github/scripts/test_push_build.py diff --git a/.github/scripts/ci-matrix.py b/.github/scripts/ci-matrix.py index 03a9cf1b3..80c49d1b3 100644 --- a/.github/scripts/ci-matrix.py +++ b/.github/scripts/ci-matrix.py @@ -80,7 +80,8 @@ NO_BUILD_WORKFLOWS = {"build-one.yml", "cleanup.yml", "lint.yml", "manifest.yml", "firmware-drift.yml"} NO_BUILD_SCRIPTS = {"enrich_manifest.py", "lint-workflow-shell.py", - "check-firmware-drift.py"} + "check-firmware-drift.py", "push_build.py", "soc_aliases.py", + "test_push_build.py"} NO_BUILD_FILES = { ".github/CODEOWNERS", ".gitignore", "LICENSE", "package.sh", "repack.sh", # The drift checker's config. It records what has been reconciled against diff --git a/.github/scripts/push_build.py b/.github/scripts/push_build.py new file mode 100644 index 000000000..c30d633ce --- /dev/null +++ b/.github/scripts/push_build.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Tell openipc.org about a published build, once. + +A copy of OpenIPC/firmware's .github/scripts/push_build.py, with +soc_aliases.py and test_push_build.py beside it. Change them there first and +copy them here. The builder pushes no aliases, because its devices build from +firmware's defconfigs, which firmware already reports. + +openipc.org learns about a build from this push, sent once when the build has +published. It never polls GitHub for it, and no metadata files are uploaded +to the release for it to find. The contract lives with the receiver: +https://github.com/OpenIPC/website/blob/master/service/internal/builds/PUSH.md + +Authorization is the job's GitHub Actions OIDC token, requested with audience +https://openipc.org. There is no shared secret. The job needs +`permissions: id-token: write`. + + push_build.py --source firmware --build-id nightly-20260925-230295e \ + --sha <40 hex> --built-at 2026-09-25T17:48:37Z --dist dist \ + --assets 'dist/openipc.*.tgz' --aliases-root . + + push_build.py ... --out payload.json assemble and write, do not send + +Stdlib only: the report job installs nothing. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import glob +import gzip +import hashlib +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +AUDIENCE = "https://openipc.org" +DEFAULT_URL = "https://openipc.org" +ATTEMPTS = 5 +FIRST_DELAY = 5 # seconds, doubling + +# openipc.--.tgz: the platform is -, +# the same name size_report.py and kconfig_graph.py give their files. +FIRMWARE_ASSET_RE = re.compile(r"^openipc\.([^.]+?)-(nor|nand|emmc|sd)-([a-z0-9]+)\.tgz$") +SIDECAR_RE = re.compile(r"^(sizes|kconfig-graph|kconfig-help)\.(.+)\.json$") + + +def sha256_of(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def collect_assets(patterns: list[str]) -> list[dict]: + """Every file matching the patterns, once, sorted by name.""" + seen: dict[str, Path] = {} + for pattern in patterns: + for p in glob.glob(pattern): + path = Path(p) + if path.is_file(): + seen.setdefault(path.name, path) + return [ + {"name": name, "size": path.stat().st_size, "sha256": sha256_of(path)} + for name, path in sorted(seen.items()) + ] + + +def collect_platforms(dist: Path | None, assets: list[dict]) -> list[dict]: + """One entry per platform that built. + + A platform is named by its size report and kconfig files and, for + firmware, by its tarballs. A board whose size report failed still has a + tarball, so it is still listed, without `sizes`. + """ + docs: dict[str, dict] = {} + if dist is not None and dist.is_dir(): + for path in sorted(dist.iterdir()): + m = SIDECAR_RE.match(path.name) + if not m or not path.is_file(): + continue + kind, plat = m.groups() + key = {"sizes": "sizes", "kconfig-graph": "kconfig_graph", + "kconfig-help": "kconfig_help"}[kind] + try: + docs.setdefault(plat, {})[key] = json.loads(path.read_text()) + except (OSError, ValueError) as e: + # A broken sidecar costs that platform its detail, never the push. + print(f"::warning::{path.name} is unreadable ({e}); pushing {plat} without it") + for a in assets: + m = FIRMWARE_ASSET_RE.match(a["name"]) + if m: + board, _storage, edition = m.groups() + docs.setdefault(f"{board}-{edition}", {}) + return [{"name": plat, **docs[plat]} for plat in sorted(docs)] + + +def build_payload(args: argparse.Namespace, now: dt.datetime) -> dict: + assets = collect_assets(args.assets) + if not assets: + raise SystemExit("::error::no assets matched; nothing published, nothing to push") + payload: dict = { + "schema": 1, + "source": args.source, + "build": { + "id": args.build_id, + "release": args.release or args.build_id, + "sha": args.sha, + "built_at": args.built_at, + "published_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + }, + "assets": assets, + } + if args.webui_digest: + payload["build"]["webui_digest"] = args.webui_digest + if args.aliases_root: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import soc_aliases # beside this script + + aliases = soc_aliases.scan(Path(args.aliases_root)) + if aliases: + payload["aliases"] = aliases + if args.source != "uboot": + payload["platforms"] = collect_platforms( + Path(args.dist) if args.dist else None, assets) + return payload + + +def oidc_token(env=os.environ, opener=urllib.request.urlopen) -> str: + """The job's OIDC token for openipc.org, from the Actions token service.""" + url = env.get("ACTIONS_ID_TOKEN_REQUEST_URL") + bearer = env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if not url or not bearer: + raise SystemExit("::error::no OIDC token service; the job needs `permissions: id-token: write`") + sep = "&" if "?" in url else "?" + req = urllib.request.Request( + f"{url}{sep}audience={urllib.parse.quote(AUDIENCE, safe='')}", + headers={"Authorization": f"bearer {bearer}", "Accept": "application/json"}, + ) + with opener(req, timeout=30) as resp: + return json.load(resp)["value"] + + +def push(payload: dict, base_url: str, env=os.environ, + opener=urllib.request.urlopen, sleep=time.sleep) -> int: + """POST the build. Retries network errors and 5xx; a 4xx is final.""" + body = gzip.compress(json.dumps(payload, separators=(",", ":")).encode()) + url = base_url.rstrip("/") + "/api/v1/builds" + delay = FIRST_DELAY + for attempt in range(1, ATTEMPTS + 1): + try: + req = urllib.request.Request(url, data=body, method="POST", headers={ + "Authorization": f"Bearer {oidc_token(env, opener)}", + "Content-Type": "application/json", + "Content-Encoding": "gzip", + "User-Agent": "OpenIPC build push", + }) + with opener(req, timeout=120) as resp: + print(f"pushed {payload['build']['id']}: HTTP {resp.status} {resp.read().decode(errors='replace')}") + return 0 + except urllib.error.HTTPError as e: + text = e.read().decode(errors="replace") + if 400 <= e.code < 500: + print(f"::error::openipc.org refused the push: HTTP {e.code} {text}") + return 1 + reason = f"HTTP {e.code} {text}" + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + reason = str(e) + if attempt == ATTEMPTS: + print(f"::error::push failed after {ATTEMPTS} attempts: {reason}") + return 1 + print(f"::warning::push attempt {attempt} failed ({reason}); retrying in {delay}s") + sleep(delay) + delay *= 2 + return 1 + + +def parse_args(argv: list[str]) -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--source", required=True, choices=["firmware", "builder", "uboot"]) + p.add_argument("--build-id", required=True) + p.add_argument("--release", help="release tag the assets download from (default: the build id)") + p.add_argument("--sha", required=True) + p.add_argument("--built-at", required=True) + p.add_argument("--webui-digest", default="") + p.add_argument("--dist", help="directory holding the size report and kconfig files") + p.add_argument("--assets", action="append", default=[], help="glob of published files; repeatable") + p.add_argument("--aliases-root", help="repository root to scan for BR2_OPENIPC_SOC_ALIASES") + p.add_argument("--url", default=os.environ.get("OPENIPC_ORG_URL") or DEFAULT_URL) + p.add_argument("--out", help="write the payload here instead of pushing it") + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + payload = build_payload(args, dt.datetime.now(dt.timezone.utc)) + print(f"{payload['build']['id']}: {len(payload['assets'])} asset(s), " + f"{len(payload.get('platforms', []))} platform(s), {len(payload.get('aliases', {}))} alias(es)") + if args.out: + Path(args.out).write_text(json.dumps(payload, indent=1)) + return 0 + return push(payload, args.url) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/soc_aliases.py b/.github/scripts/soc_aliases.py new file mode 100644 index 000000000..43f38a36f --- /dev/null +++ b/.github/scripts/soc_aliases.py @@ -0,0 +1,59 @@ +"""SoC aliases from the defconfigs: BR2_OPENIPC_SOC_ALIASES. + +A published image's BR2_OPENIPC_SOC_MODEL also serves the space-separated +retired or compatible chip ids listed in BR2_OPENIPC_SOC_ALIASES. That lets a +camera still reporting the old id (xm550, gk7205v210, hi3516cv610, ...) be +sent to the image that exists. + +Two readers share this module: + - enrich_manifest.py, for the gh-pages manifest that on-device sysupgrade + reads; + - push_build.py, for the build facts pushed to openipc.org. +They read the aliases the same way, so the two cannot disagree. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +SOC_MODEL_RE = re.compile(r'^BR2_OPENIPC_SOC_MODEL\s*=\s*"?([A-Za-z0-9]+)"?\s*$') +SOC_ALIASES_RE = re.compile(r'^BR2_OPENIPC_SOC_ALIASES\s*=\s*"?([^"\n]*)"?\s*$') + + +def scan(root: Path) -> dict[str, str]: + """Map each alias chip id to the SOC_MODEL it is published under. + + `root` is the repository root, holding br-ext-chip-*/configs. A chip + claimed by two models keeps the first, in sorted defconfig order, and the + conflict is reported on stderr. A missing tree yields {}. + """ + aliases: dict[str, str] = {} + for cfg in sorted(root.glob("br-ext-chip-*/configs/*_defconfig")): + try: + text = cfg.read_text() + except OSError: + continue + model = "" + alias_field = "" + for line in text.splitlines(): + m = SOC_MODEL_RE.match(line) + if m: + model = m.group(1) + continue + a = SOC_ALIASES_RE.match(line) + if a: + alias_field = a.group(1) + if not model or not alias_field.strip(): + continue + for chip in alias_field.split(): + if not chip or chip == model: + continue + prev = aliases.get(chip) + if prev and prev != model: + sys.stderr.write( + f"alias conflict: {chip} -> {prev} and {model}; keeping {prev}\n" + ) + continue + aliases[chip] = model + return dict(sorted(aliases.items())) diff --git a/.github/scripts/test_push_build.py b/.github/scripts/test_push_build.py new file mode 100644 index 000000000..c0a9a0516 --- /dev/null +++ b/.github/scripts/test_push_build.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Tests for push_build.py: what it pushes, and how it talks to openipc.org. + + python3 .github/scripts/test_push_build.py +""" +from __future__ import annotations + +import datetime as dt +import gzip +import hashlib +import io +import json +import sys +import tempfile +import unittest +import urllib.error +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import push_build # noqa: E402 +import soc_aliases # noqa: E402 + +NOW = dt.datetime(2026, 9, 26, 18, 51, 28, tzinfo=dt.timezone.utc) + + +def args(tmp: Path, *extra: str): + return push_build.parse_args([ + "--source", "firmware", "--build-id", "nightly-20260925-230295e", + "--sha", "230295e494013e17a9802633a58b30ed7c937f8c", + "--built-at", "2026-09-25T17:48:37Z", "--dist", str(tmp), + "--assets", str(tmp / "openipc.*.tgz"), *extra, + ]) + + +class Payload(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.dist = Path(self._tmp.name) + (self.dist / "openipc.gk7205v200-nor-lite.tgz").write_bytes(b"lite image") + (self.dist / "openipc.gk7205v200-nor-ultimate.tgz").write_bytes(b"ultimate") + (self.dist / "openipc.ssc338q-nand-lite.tgz").write_bytes(b"nand") + (self.dist / "sizes.gk7205v200-lite.json").write_text( + json.dumps({"schema": 1, "board": "gk7205v200", "variant": "lite", "flash_mb": 8})) + (self.dist / "kconfig-graph.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "symbols": {}})) + (self.dist / "kconfig-help.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "help": {}})) + + def tearDown(self): + self._tmp.cleanup() + + def test_build_and_assets(self): + p = push_build.build_payload(args(self.dist), NOW) + self.assertEqual(p["schema"], 1) + self.assertEqual(p["source"], "firmware") + self.assertEqual(p["build"], { + "id": "nightly-20260925-230295e", "release": "nightly-20260925-230295e", + "sha": "230295e494013e17a9802633a58b30ed7c937f8c", + "built_at": "2026-09-25T17:48:37Z", "published_at": "2026-09-26T18:51:28Z"}) + names = [a["name"] for a in p["assets"]] + self.assertEqual(names, sorted(names)) + lite = next(a for a in p["assets"] if a["name"] == "openipc.gk7205v200-nor-lite.tgz") + self.assertEqual(lite["size"], len(b"lite image")) + self.assertEqual(lite["sha256"], hashlib.sha256(b"lite image").hexdigest()) + # The sidecars are pushed as platform detail, never as assets. + self.assertFalse(any(n.endswith(".json") for n in names)) + + def test_platforms_carry_their_documents_and_survive_a_missing_report(self): + p = push_build.build_payload(args(self.dist), NOW) + plats = {x["name"]: x for x in p["platforms"]} + self.assertEqual(sorted(plats), ["gk7205v200-lite", "gk7205v200-ultimate", "ssc338q-lite"]) + self.assertEqual(plats["gk7205v200-lite"]["sizes"]["flash_mb"], 8) + self.assertIn("kconfig_graph", plats["gk7205v200-lite"]) + self.assertIn("kconfig_help", plats["gk7205v200-lite"]) + # No size report for these two: still listed, without detail. + self.assertEqual(plats["gk7205v200-ultimate"], {"name": "gk7205v200-ultimate"}) + self.assertEqual(plats["ssc338q-lite"], {"name": "ssc338q-lite"}) + + def test_an_unreadable_report_costs_its_detail_not_the_push(self): + (self.dist / "sizes.gk7205v200-ultimate.json").write_text("{not json") + p = push_build.build_payload(args(self.dist), NOW) + plats = {x["name"]: x for x in p["platforms"]} + self.assertNotIn("sizes", plats["gk7205v200-ultimate"]) + + def test_nothing_published_is_an_error(self): + empty = Path(self._tmp.name) / "empty" + empty.mkdir() + with self.assertRaises(SystemExit): + push_build.build_payload(args(empty), NOW) + + def test_webui_digest_and_release(self): + p = push_build.build_payload(args(self.dist, "--webui-digest", "sha256:ab", "--release", "latest"), NOW) + self.assertEqual(p["build"]["webui_digest"], "sha256:ab") + self.assertEqual(p["build"]["release"], "latest") + + def test_uboot_has_no_platforms(self): + (self.dist / "u-boot-t31-universal-nor.bin").write_bytes(b"u-boot") + a = push_build.parse_args([ + "--source", "uboot", "--build-id", "uboot-20260926T120000Z-abcdef0", "--release", "latest", + "--sha", "a" * 40, "--built-at", "2026-09-26T12:00:00Z", + "--assets", str(self.dist / "u-boot-*.bin")]) + p = push_build.build_payload(a, NOW) + self.assertNotIn("platforms", p) + self.assertEqual([x["name"] for x in p["assets"]], ["u-boot-t31-universal-nor.bin"]) + + +class Aliases(unittest.TestCase): + def test_scan(self): + with tempfile.TemporaryDirectory() as t: + root = Path(t) + cfgs = root / "br-ext-chip-goke" / "configs" + cfgs.mkdir(parents=True) + (cfgs / "gk7205v200_lite_defconfig").write_text( + 'BR2_OPENIPC_SOC_MODEL="gk7205v200"\nBR2_OPENIPC_SOC_ALIASES="gk7205v210 gk7205v200"\n') + (cfgs / "gk7205v300_lite_defconfig").write_text( + 'BR2_OPENIPC_SOC_MODEL="gk7205v300"\nBR2_OPENIPC_SOC_ALIASES="gk7205v210"\n') + (cfgs / "gk7202v300_lite_defconfig").write_text('BR2_OPENIPC_SOC_MODEL="gk7202v300"\n') + # First model in sorted order keeps a contested chip; a model is + # never its own alias; a defconfig without aliases adds nothing. + self.assertEqual(soc_aliases.scan(root), {"gk7205v210": "gk7205v200"}) + + def test_the_real_tree_parses(self): + root = Path(__file__).resolve().parents[2] + self.assertTrue(all(k and v for k, v in soc_aliases.scan(root).items())) + + +class Response(io.BytesIO): + def __init__(self, status: int, body: bytes): + super().__init__(body) + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +class Transport(unittest.TestCase): + ENV = {"ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.example/req?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "runner-bearer"} + PAYLOAD = {"build": {"id": "nightly-20260925-230295e"}, "assets": []} + + def opener(self, post_results): + calls = [] + results = list(post_results) + + def open_(req, timeout=None): + calls.append(req) + if req.get_method() == "GET": + return Response(200, json.dumps({"value": "oidc.jwt"}).encode()) + r = results.pop(0) + if isinstance(r, Exception): + raise r + return Response(r, b'{"build":"nightly-20260925-230295e"}') + return open_, calls + + def test_token_request_names_openipc_org_as_audience(self): + open_, calls = self.opener([201]) + self.assertEqual(push_build.oidc_token(self.ENV, open_), "oidc.jwt") + req = calls[0] + self.assertEqual(req.full_url, + "https://token.example/req?api-version=2.0&audience=https%3A%2F%2Fopenipc.org") + self.assertEqual(req.get_header("Authorization"), "bearer runner-bearer") + + def test_no_token_service_is_a_clear_error(self): + with self.assertRaises(SystemExit): + push_build.oidc_token({}, None) + + def test_post_is_gzip_json_with_the_bearer(self): + open_, calls = self.opener([201]) + self.assertEqual(push_build.push(self.PAYLOAD, "https://openipc.org/", self.ENV, open_, lambda s: None), 0) + post = [c for c in calls if c.get_method() == "POST"][0] + self.assertEqual(post.full_url, "https://openipc.org/api/v1/builds") + self.assertEqual(post.get_header("Authorization"), "Bearer oidc.jwt") + self.assertEqual(post.get_header("Content-encoding"), "gzip") + self.assertEqual(json.loads(gzip.decompress(post.data)), self.PAYLOAD) + + def test_5xx_and_network_errors_are_retried_with_backoff(self): + err = urllib.error.HTTPError("u", 502, "bad gateway", {}, io.BytesIO(b"upstream")) + open_, calls = self.opener([err, urllib.error.URLError("reset"), 201]) + slept = [] + self.assertEqual(push_build.push(self.PAYLOAD, "https://openipc.org", self.ENV, open_, slept.append), 0) + self.assertEqual(slept, [5, 10]) + self.assertEqual(sum(c.get_method() == "POST" for c in calls), 3) + + def test_4xx_is_final(self): + err = urllib.error.HTTPError("u", 403, "forbidden", {}, io.BytesIO(b'{"error":"workflow not allowed"}')) + open_, calls = self.opener([err, 201]) + slept = [] + self.assertEqual(push_build.push(self.PAYLOAD, "https://openipc.org", self.ENV, open_, slept.append), 1) + self.assertEqual(slept, []) + + def test_gives_up_after_five_attempts(self): + open_, calls = self.opener([urllib.error.URLError("down")] * 5) + slept = [] + self.assertEqual(push_build.push(self.PAYLOAD, "https://openipc.org", self.ENV, open_, slept.append), 1) + self.assertEqual(slept, [5, 10, 20, 40]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 731c31b62..55fec08c5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,6 +17,9 @@ on: - '.github/workflows/**' - '.github/scripts/lint-workflow-shell.py' - '.github/scripts/lint-cli-paths.py' + - '.github/scripts/push_build.py' + - '.github/scripts/soc_aliases.py' + - '.github/scripts/test_push_build.py' workflow_dispatch: # Reads the tree and reports; writes nothing. Declared rather than inherited so @@ -82,3 +85,8 @@ jobs: run: python3 .github/scripts/lint-cli-paths.py --self-test - name: Check every shipped cli setting path run: python3 .github/scripts/lint-cli-paths.py + # The nightly's report job is the only way openipc.org learns about a + # build, and it runs once, after publish, where nobody watches it. What + # it sends and how it authenticates are checked here instead. + - name: Test the build push + run: python3 .github/scripts/test_push_build.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f2ebdab93..0bb9a0fa1 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -599,27 +599,24 @@ jobs: NOTES=$(printf 'sha=%s\nshort=%s\nbuilt_at=%s\n' "$HEAD_SHA" "$SHORT_SHA" "$BUILT_AT") - # Full asset set (images + size sidecars) -> the dated release, which - # is the only one enrich_manifest.py reads. - mapfile -t DATED < <(find dist -maxdepth 1 -type f | sort) - # nightly/latest are firmware-delivery aliases for flashers, so ship - # only the images there. Re-uploading the size sidecars to two more - # releases is pure rate-limit cost: the manifest never reads them from - # these tags, and nothing else does either. + # Releases carry firmware and nothing else. The per-device size + # reports that used to ride along as sizes.*.json sidecars reach + # openipc.org once, from the report job below, and stay in dist/ + # only as workflow artifacts. mapfile -t IMAGES < <(find dist -maxdepth 1 -type f -name '*.tgz' | sort) - # --- dated: immutable per-build history --- - ensure_release "$BUILD_ID" --prerelease --title "$BUILD_ID" - upload_paced "$BUILD_ID" "${DATED[@]}" - - # nightly and latest only ever move when there is firmware to move - # them to. dist/ can hold sidecars and no images -- one device that - # produced a size report and no .tgz is enough -- and pointing the two - # tags users flash from at a build with no images behind them is worse - # than leaving them on yesterday's. + # Nothing moves when there is no firmware to move it to. dist/ can + # hold size reports and no images -- one device that produced a + # report and no .tgz is enough -- and a dated release with nothing + # in it, or the two tags users flash from pointing at one, is worse + # than leaving everything on yesterday's. if [ "${#IMAGES[@]}" -eq 0 ]; then - echo "::warning::no images in this build; leaving nightly and latest where they are" + echo "::warning::no images in this build; no dated release, and nightly and latest stay where they are" else + # --- dated: immutable per-build history --- + ensure_release "$BUILD_ID" --prerelease --title "$BUILD_ID" + upload_paced "$BUILD_ID" "${IMAGES[@]}" + # --- rolling nightly: move tag + body to this build, refresh images --- ensure_release nightly gh_retry gh release edit nightly --notes "$NOTES" @@ -632,6 +629,68 @@ jobs: upload_paced latest "${IMAGES[@]}" fi + # openipc.org learns about the build here, once: which device images were + # published and their digests, and each device's size report. It never polls + # GitHub for any of it, and none of it is uploaded to a release. The + # contract, and who may push, is + # https://github.com/OpenIPC/website/blob/master/service/internal/builds/PUSH.md + # + # Authorization is this job's OIDC token (audience https://openipc.org), + # which openipc.org checks against the repository, this workflow and + # master. There is no secret to store or rotate. + # + # A job of its own, after publish and outside ci-gate, so that openipc.org + # being down costs a published nightly nothing, and the push can be re-run by + # itself: it is idempotent by build id. The fw-* artifacts are kept one day, + # which bounds how late that re-run can be. + report: + name: Report build to openipc.org + needs: [preflight, publish] + if: >- + !cancelled() && + github.event_name != 'pull_request' && + needs.publish.result == 'success' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + timeout-minutes: 20 + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + pattern: fw-* + path: dist + merge-multiple: true + + - uses: actions/checkout@v4 + with: + ref: ${{ needs.preflight.outputs.head_sha }} + sparse-checkout: .github/scripts + + - name: Push the build + env: + BUILD_ID: ${{ needs.preflight.outputs.build_id }} + HEAD_SHA: ${{ needs.preflight.outputs.head_sha }} + BUILT_AT: ${{ needs.preflight.outputs.built_at }} + run: | + set -euo pipefail + shopt -s nullglob + images=(dist/*.tgz) + # publish wrote no release when there were no images; there is + # nothing to report either. + if [ "${#images[@]}" -eq 0 ]; then + echo "::notice::no images in this build; nothing to report" + exit 0 + fi + python3 .github/scripts/push_build.py \ + --source builder \ + --build-id "$BUILD_ID" \ + --sha "$HEAD_SHA" \ + --built-at "$BUILT_AT" \ + --dist dist \ + --assets 'dist/*.tgz' + # Single umbrella status check covering the whole run, so branch protection # can require one context instead of a hardcoded "Build ()" per # device -- the matrix is now dynamic, and its job names change per PR. From 2da0a8226c28e538eda1141877fb64dcc7735526 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Sat, 26 Sep 2026 14:05:19 +0000 Subject: [PATCH 2/3] Describe published images from the release, and keep the reports a week The report job no longer needs the one-day fw-* artifacts: - Each device image's size and sha256 come from the dated release after publish (`gh release view --json assets`; GitHub computes the digest on upload). An image with no digest fails the push. - The size report travels in its own reports- artifact, kept 7 days; fw-* carries only the image and keeps its retention. A push that could not reach openipc.org can be re-run for a week. push_build.py and its tests are refreshed from OpenIPC/firmware. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/push_build.py | 75 +++++++++++++++---- .github/scripts/test_push_build.py | 114 +++++++++++++++++++++-------- .github/workflows/master.yml | 67 +++++++++++------ 3 files changed, 188 insertions(+), 68 deletions(-) diff --git a/.github/scripts/push_build.py b/.github/scripts/push_build.py index c30d633ce..bf89bd8e7 100644 --- a/.github/scripts/push_build.py +++ b/.github/scripts/push_build.py @@ -15,9 +15,10 @@ https://openipc.org. There is no shared secret. The job needs `permissions: id-token: write`. + gh release view "$BUILD_ID" --json assets > release.json push_build.py --source firmware --build-id nightly-20260925-230295e \ - --sha <40 hex> --built-at 2026-09-25T17:48:37Z --dist dist \ - --assets 'dist/openipc.*.tgz' --aliases-root . + --sha <40 hex> --built-at 2026-09-25T17:48:37Z --reports reports \ + --release-json release.json --match '^openipc\..*\.tgz$' --aliases-root . push_build.py ... --out payload.json assemble and write, do not send @@ -59,18 +60,60 @@ def sha256_of(path: Path) -> str: return h.hexdigest() -def collect_assets(patterns: list[str]) -> list[dict]: - """Every file matching the patterns, once, sorted by name.""" - seen: dict[str, Path] = {} +def release_assets(path: str | None) -> dict[str, dict]: + """name -> {size, digest} from the release as GitHub reports it. + + Accepts `gh release view --json assets` output or a REST release + object; both carry `assets[].{name,size,digest}`. GitHub computes the + digest ("sha256:") itself when an asset is uploaded, so the push + describes exactly the bytes a download returns, and nothing has to be kept + around to hash afterwards. + """ + if not path: + return {} + doc = json.loads(Path(path).read_text()) + out = {} + for a in doc.get("assets", []): + digest = a.get("digest") or "" + out[a["name"]] = { + "size": int(a["size"]), + "sha256": digest[len("sha256:"):] if digest.startswith("sha256:") else "", + } + return out + + +def collect_assets(published: dict[str, dict], match: str | None, + patterns: list[str]) -> list[dict]: + """The assets to push, sorted by name. + + - Every release asset whose name matches `match`, with GitHub's size and + digest. An asset GitHub has no digest for is an error: the push would + otherwise describe a file nobody hashed. + - Every local file matching `patterns`, with the release's size and digest + when the release has them, and its own otherwise (uboot uploads to + `latest` and hashes locally only as a fallback). + """ + out: dict[str, dict] = {} + if match: + rx = re.compile(match) + for name, a in published.items(): + if not rx.search(name): + continue + if not a["sha256"]: + raise SystemExit(f"::error::GitHub reports no digest for {name}") + out[name] = {"name": name, **a} for pattern in patterns: for p in glob.glob(pattern): path = Path(p) - if path.is_file(): - seen.setdefault(path.name, path) - return [ - {"name": name, "size": path.stat().st_size, "sha256": sha256_of(path)} - for name, path in sorted(seen.items()) - ] + if not path.is_file() or path.name in out: + continue + a = published.get(path.name) + if a and a["sha256"]: + out[path.name] = {"name": path.name, **a} + else: + out[path.name] = {"name": path.name, "size": path.stat().st_size, + "sha256": sha256_of(path)} + return [out[n] for n in sorted(out)] def collect_platforms(dist: Path | None, assets: list[dict]) -> list[dict]: @@ -103,7 +146,7 @@ def collect_platforms(dist: Path | None, assets: list[dict]) -> list[dict]: def build_payload(args: argparse.Namespace, now: dt.datetime) -> dict: - assets = collect_assets(args.assets) + assets = collect_assets(release_assets(args.release_json), args.match, args.assets) if not assets: raise SystemExit("::error::no assets matched; nothing published, nothing to push") payload: dict = { @@ -190,8 +233,12 @@ def parse_args(argv: list[str]) -> argparse.Namespace: p.add_argument("--sha", required=True) p.add_argument("--built-at", required=True) p.add_argument("--webui-digest", default="") - p.add_argument("--dist", help="directory holding the size report and kconfig files") - p.add_argument("--assets", action="append", default=[], help="glob of published files; repeatable") + p.add_argument("--reports", "--dist", dest="dist", + help="directory holding the size report and kconfig files") + p.add_argument("--release-json", help="`gh release view --json assets` output") + p.add_argument("--match", help="regex selecting the release assets to push") + p.add_argument("--assets", action="append", default=[], + help="glob of local published files; repeatable (uboot)") p.add_argument("--aliases-root", help="repository root to scan for BR2_OPENIPC_SOC_ALIASES") p.add_argument("--url", default=os.environ.get("OPENIPC_ORG_URL") or DEFAULT_URL) p.add_argument("--out", help="write the payload here instead of pushing it") diff --git a/.github/scripts/test_push_build.py b/.github/scripts/test_push_build.py index c0a9a0516..5dff3b60b 100644 --- a/.github/scripts/test_push_build.py +++ b/.github/scripts/test_push_build.py @@ -23,48 +23,75 @@ NOW = dt.datetime(2026, 9, 26, 18, 51, 28, tzinfo=dt.timezone.utc) +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +IMAGES = { + "openipc.gk7205v200-nor-lite.tgz": b"lite image", + "openipc.gk7205v200-nor-ultimate.tgz": b"ultimate", + "openipc.ssc338q-nand-lite.tgz": b"nand", +} + + def args(tmp: Path, *extra: str): return push_build.parse_args([ "--source", "firmware", "--build-id", "nightly-20260925-230295e", "--sha", "230295e494013e17a9802633a58b30ed7c937f8c", - "--built-at", "2026-09-25T17:48:37Z", "--dist", str(tmp), - "--assets", str(tmp / "openipc.*.tgz"), *extra, + "--built-at", "2026-09-25T17:48:37Z", "--reports", str(tmp / "reports"), + "--release-json", str(tmp / "release.json"), "--match", r"^openipc\..*\.tgz$", + *extra, ]) class Payload(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory() - self.dist = Path(self._tmp.name) - (self.dist / "openipc.gk7205v200-nor-lite.tgz").write_bytes(b"lite image") - (self.dist / "openipc.gk7205v200-nor-ultimate.tgz").write_bytes(b"ultimate") - (self.dist / "openipc.ssc338q-nand-lite.tgz").write_bytes(b"nand") - (self.dist / "sizes.gk7205v200-lite.json").write_text( + self.tmp = Path(self._tmp.name) + # The release as `gh release view --json assets` reports it: the + # images with GitHub's digests, plus files the push must not pick up. + assets = [{"name": n, "size": len(b), "digest": "sha256:" + digest(b)} for n, b in IMAGES.items()] + assets.append({"name": "sizes.gk7205v200-lite.json", "size": 10, "digest": "sha256:" + "0" * 64}) + self.write_release(assets) + reports = self.tmp / "reports" + reports.mkdir() + (reports / "sizes.gk7205v200-lite.json").write_text( json.dumps({"schema": 1, "board": "gk7205v200", "variant": "lite", "flash_mb": 8})) - (self.dist / "kconfig-graph.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "symbols": {}})) - (self.dist / "kconfig-help.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "help": {}})) + (reports / "kconfig-graph.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "symbols": {}})) + (reports / "kconfig-help.gk7205v200-lite.json").write_text(json.dumps({"schema": 1, "help": {}})) + + def write_release(self, assets): + (self.tmp / "release.json").write_text(json.dumps({"assets": assets})) def tearDown(self): self._tmp.cleanup() - def test_build_and_assets(self): - p = push_build.build_payload(args(self.dist), NOW) + def test_build_and_assets_come_from_the_release(self): + p = push_build.build_payload(args(self.tmp), NOW) self.assertEqual(p["schema"], 1) self.assertEqual(p["source"], "firmware") self.assertEqual(p["build"], { "id": "nightly-20260925-230295e", "release": "nightly-20260925-230295e", "sha": "230295e494013e17a9802633a58b30ed7c937f8c", "built_at": "2026-09-25T17:48:37Z", "published_at": "2026-09-26T18:51:28Z"}) - names = [a["name"] for a in p["assets"]] - self.assertEqual(names, sorted(names)) - lite = next(a for a in p["assets"] if a["name"] == "openipc.gk7205v200-nor-lite.tgz") - self.assertEqual(lite["size"], len(b"lite image")) - self.assertEqual(lite["sha256"], hashlib.sha256(b"lite image").hexdigest()) - # The sidecars are pushed as platform detail, never as assets. - self.assertFalse(any(n.endswith(".json") for n in names)) + self.assertEqual(p["assets"], [ + {"name": n, "size": len(IMAGES[n]), "sha256": digest(IMAGES[n])} for n in sorted(IMAGES)]) + + def test_a_release_asset_without_a_digest_is_refused(self): + self.write_release([{"name": "openipc.gk7205v200-nor-lite.tgz", "size": 3, "digest": None}]) + with self.assertRaises(SystemExit): + push_build.build_payload(args(self.tmp), NOW) + + def test_rest_release_objects_work_too(self): + # The REST release object has the same assets[].{name,size,digest}. + rest = {"tag_name": "nightly-20260925-230295e", "assets": [ + {"name": "openipc.x-nor-lite.tgz", "size": 1, "digest": "sha256:" + "a" * 64, "id": 7}]} + (self.tmp / "release.json").write_text(json.dumps(rest)) + p = push_build.build_payload(args(self.tmp), NOW) + self.assertEqual(p["assets"], [{"name": "openipc.x-nor-lite.tgz", "size": 1, "sha256": "a" * 64}]) def test_platforms_carry_their_documents_and_survive_a_missing_report(self): - p = push_build.build_payload(args(self.dist), NOW) + p = push_build.build_payload(args(self.tmp), NOW) plats = {x["name"]: x for x in p["platforms"]} self.assertEqual(sorted(plats), ["gk7205v200-lite", "gk7205v200-ultimate", "ssc338q-lite"]) self.assertEqual(plats["gk7205v200-lite"]["sizes"]["flash_mb"], 8) @@ -74,32 +101,57 @@ def test_platforms_carry_their_documents_and_survive_a_missing_report(self): self.assertEqual(plats["gk7205v200-ultimate"], {"name": "gk7205v200-ultimate"}) self.assertEqual(plats["ssc338q-lite"], {"name": "ssc338q-lite"}) + def test_no_reports_at_all_still_lists_every_platform(self): + p = push_build.build_payload(args(self.tmp, "--reports", str(self.tmp / "missing")), NOW) + self.assertEqual(len(p["platforms"]), 3) + def test_an_unreadable_report_costs_its_detail_not_the_push(self): - (self.dist / "sizes.gk7205v200-ultimate.json").write_text("{not json") - p = push_build.build_payload(args(self.dist), NOW) + (self.tmp / "reports" / "sizes.gk7205v200-ultimate.json").write_text("{not json") + p = push_build.build_payload(args(self.tmp), NOW) plats = {x["name"]: x for x in p["platforms"]} self.assertNotIn("sizes", plats["gk7205v200-ultimate"]) def test_nothing_published_is_an_error(self): - empty = Path(self._tmp.name) / "empty" - empty.mkdir() + self.write_release([]) with self.assertRaises(SystemExit): - push_build.build_payload(args(empty), NOW) + push_build.build_payload(args(self.tmp), NOW) def test_webui_digest_and_release(self): - p = push_build.build_payload(args(self.dist, "--webui-digest", "sha256:ab", "--release", "latest"), NOW) + p = push_build.build_payload(args(self.tmp, "--webui-digest", "sha256:ab", "--release", "latest"), NOW) self.assertEqual(p["build"]["webui_digest"], "sha256:ab") self.assertEqual(p["build"]["release"], "latest") - def test_uboot_has_no_platforms(self): - (self.dist / "u-boot-t31-universal-nor.bin").write_bytes(b"u-boot") - a = push_build.parse_args([ + def uboot(self, *extra): + return push_build.parse_args([ "--source", "uboot", "--build-id", "uboot-20260926T120000Z-abcdef0", "--release", "latest", "--sha", "a" * 40, "--built-at", "2026-09-26T12:00:00Z", - "--assets", str(self.dist / "u-boot-*.bin")]) - p = push_build.build_payload(a, NOW) + "--assets", str(self.tmp / "out" / "*.bin"), *extra]) + + def test_uboot_takes_the_release_digest_for_what_it_uploaded(self): + out = self.tmp / "out" + out.mkdir() + (out / "u-boot-t31-nor.bin").write_bytes(b"u-boot") + (out / "u-boot-t40-nor.bin").write_bytes(b"local only") + # latest has the first with GitHub's digest (deliberately not the local + # file's, to prove which one is used) and the second with none yet. + self.write_release([ + {"name": "u-boot-t31-nor.bin", "size": 6, "digest": "sha256:" + "b" * 64}, + {"name": "u-boot-t40-nor.bin", "size": 10, "digest": None}, + {"name": "u-boot-other-nor.bin", "size": 1, "digest": "sha256:" + "c" * 64}, + ]) + p = push_build.build_payload(self.uboot("--release-json", str(self.tmp / "release.json")), NOW) self.assertNotIn("platforms", p) - self.assertEqual([x["name"] for x in p["assets"]], ["u-boot-t31-universal-nor.bin"]) + self.assertEqual(p["assets"], [ + {"name": "u-boot-t31-nor.bin", "size": 6, "sha256": "b" * 64}, + {"name": "u-boot-t40-nor.bin", "size": 10, "sha256": digest(b"local only")}, + ]) + + def test_uboot_without_the_release_hashes_locally(self): + out = self.tmp / "out" + out.mkdir() + (out / "u-boot-t31-nor.bin").write_bytes(b"u-boot") + p = push_build.build_payload(self.uboot(), NOW) + self.assertEqual(p["assets"], [{"name": "u-boot-t31-nor.bin", "size": 6, "sha256": digest(b"u-boot")}]) class Aliases(unittest.TestCase): diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0bb9a0fa1..218c41987 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -402,8 +402,12 @@ jobs: - name: Stage artifacts if: env.NORFW || env.NANDFW run: | - mkdir -p dist - for f in "${NORFW:-}" "${NANDFW:-}" "${SIZES:-}"; do + mkdir -p dist reports + # The size report goes to its own artifact below, not with the image. + if [ -n "${SIZES:-}" ] && [ -e "${SIZES}" ]; then + cp -a "${SIZES}" reports/ + fi + for f in "${NORFW:-}" "${NANDFW:-}"; do # An `if` rather than a `&&` chain: the step runs under `bash -e`, # where a trailing `&&` that tests false is only harmless because # something after it resets the status. @@ -411,7 +415,7 @@ jobs: cp -a "$f" dist/ fi done - ls -la dist + ls -la dist reports # Hand the images to the single `publish` job instead of writing releases # from here. 107 matrix jobs each deleting and re-uploading assets on the @@ -440,6 +444,20 @@ jobs: retention-days: ${{ github.event_name == 'pull_request' && 7 || 1 }} path: dist/* + # The size report, for the `report` job that pushes it to openipc.org. + # A few tens of KB, kept a week -- unlike the image, which publish + # consumes within the run -- so a push that could not reach openipc.org + # can be re-run days later. Optional, so is this. + - name: Upload report + if: github.event_name != 'pull_request' && (env.NORFW || env.NANDFW) + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reports-${{ matrix.platform }} + if-no-files-found: ignore + retention-days: 7 + path: reports/* + - name: Send binary if: github.event_name != 'pull_request' && env.NORFW run: | @@ -601,15 +619,13 @@ jobs: # Releases carry firmware and nothing else. The per-device size # reports that used to ride along as sizes.*.json sidecars reach - # openipc.org once, from the report job below, and stay in dist/ - # only as workflow artifacts. + # openipc.org once, from the report job below, out of their own + # reports-* artifacts. mapfile -t IMAGES < <(find dist -maxdepth 1 -type f -name '*.tgz' | sort) - # Nothing moves when there is no firmware to move it to. dist/ can - # hold size reports and no images -- one device that produced a - # report and no .tgz is enough -- and a dated release with nothing - # in it, or the two tags users flash from pointing at one, is worse - # than leaving everything on yesterday's. + # Nothing moves when there is no firmware to move it to. A dated + # release with nothing in it, or the two tags users flash from + # pointing at one, is worse than leaving everything on yesterday's. if [ "${#IMAGES[@]}" -eq 0 ]; then echo "::warning::no images in this build; no dated release, and nightly and latest stay where they are" else @@ -641,8 +657,8 @@ jobs: # # A job of its own, after publish and outside ci-gate, so that openipc.org # being down costs a published nightly nothing, and the push can be re-run by - # itself: it is idempotent by build id. The fw-* artifacts are kept one day, - # which bounds how late that re-run can be. + # itself: it is idempotent by build id. It reads the release and the + # reports-* artifacts, which are kept a week. report: name: Report build to openipc.org needs: [preflight, publish] @@ -656,11 +672,14 @@ jobs: contents: read timeout-minutes: 20 steps: - - name: Download build artifacts + # The reports only. The images are described from the release itself, + # so nothing here depends on the one-day fw-* artifacts. + - name: Download reports uses: actions/download-artifact@v4 + continue-on-error: true with: - pattern: fw-* - path: dist + pattern: reports-* + path: reports merge-multiple: true - uses: actions/checkout@v4 @@ -668,19 +687,20 @@ jobs: ref: ${{ needs.preflight.outputs.head_sha }} sparse-checkout: .github/scripts + # What was published, as GitHub has it: each device image's size and the + # sha256 GitHub computed on upload. publish creates no dated release when + # there were no images, and then there is nothing to report. - name: Push the build env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} BUILD_ID: ${{ needs.preflight.outputs.build_id }} HEAD_SHA: ${{ needs.preflight.outputs.head_sha }} BUILT_AT: ${{ needs.preflight.outputs.built_at }} run: | set -euo pipefail - shopt -s nullglob - images=(dist/*.tgz) - # publish wrote no release when there were no images; there is - # nothing to report either. - if [ "${#images[@]}" -eq 0 ]; then - echo "::notice::no images in this build; nothing to report" + if ! gh release view "$BUILD_ID" --json assets > release.json; then + echo "::notice::no release $BUILD_ID; nothing to report" exit 0 fi python3 .github/scripts/push_build.py \ @@ -688,8 +708,9 @@ jobs: --build-id "$BUILD_ID" \ --sha "$HEAD_SHA" \ --built-at "$BUILT_AT" \ - --dist dist \ - --assets 'dist/*.tgz' + --reports reports \ + --release-json release.json \ + --match '\.tgz$' # Single umbrella status check covering the whole run, so branch protection # can require one context instead of a hardcoded "Build ()" per From fea4a9934900af8dae34b5a48f057e93155ae0d7 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Sat, 26 Sep 2026 14:39:46 +0000 Subject: [PATCH 3/3] List compound devices without a report, and stop losing the reports From Qodo's review of #170: - Compound devices publish -.tgz, with no openipc. prefix, so push_build.py dropped any whose size report was missing. It now recognises both naming schemes; tests cover each, with and without report files, and a report and its tarball being one device. - The report job downloaded reports-* before actions/checkout, which cleans the workspace, so every push would have lost its reports. Checkout now comes first and the reports land in runner.temp. - A guard tells "no reports produced" from "reports lost": if the matrix uploaded reports-* artifacts and none arrived, the push fails instead of sending devices without their detail (actions: read, to count them). - The job runs only this repository's push script; the artifacts are read as data. push_build.py and its tests are refreshed from OpenIPC/firmware. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/push_build.py | 10 ++++++- .github/scripts/test_push_build.py | 45 +++++++++++++++++++++++++++++ .github/workflows/master.yml | 46 ++++++++++++++++++++++++------ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/.github/scripts/push_build.py b/.github/scripts/push_build.py index bf89bd8e7..cae57de99 100644 --- a/.github/scripts/push_build.py +++ b/.github/scripts/push_build.py @@ -49,6 +49,10 @@ # openipc.--.tgz: the platform is -, # the same name size_report.py and kconfig_graph.py give their files. FIRMWARE_ASSET_RE = re.compile(r"^openipc\.([^.]+?)-(nor|nand|emmc|sd)-([a-z0-9]+)\.tgz$") +# OpenIPC/builder's compound devices publish -.tgz, +# and their size report is sizes..json, so the platform is +# the name before the storage suffix. +DEVICE_ASSET_RE = re.compile(r"^(?!openipc\.)([A-Za-z0-9._-]+?)-(nor|nand|emmc|sd)\.tgz$") SIDECAR_RE = re.compile(r"^(sizes|kconfig-graph|kconfig-help)\.(.+)\.json$") @@ -103,7 +107,7 @@ def collect_assets(published: dict[str, dict], match: str | None, raise SystemExit(f"::error::GitHub reports no digest for {name}") out[name] = {"name": name, **a} for pattern in patterns: - for p in glob.glob(pattern): + for p in glob.glob(pattern, recursive=True): path = Path(p) if not path.is_file() or path.name in out: continue @@ -142,6 +146,10 @@ def collect_platforms(dist: Path | None, assets: list[dict]) -> list[dict]: if m: board, _storage, edition = m.groups() docs.setdefault(f"{board}-{edition}", {}) + continue + m = DEVICE_ASSET_RE.match(a["name"]) + if m: + docs.setdefault(m.group(1), {}) return [{"name": plat, **docs[plat]} for plat in sorted(docs)] diff --git a/.github/scripts/test_push_build.py b/.github/scripts/test_push_build.py index 5dff3b60b..9d2d1a295 100644 --- a/.github/scripts/test_push_build.py +++ b/.github/scripts/test_push_build.py @@ -146,6 +146,18 @@ def test_uboot_takes_the_release_digest_for_what_it_uploaded(self): {"name": "u-boot-t40-nor.bin", "size": 10, "sha256": digest(b"local only")}, ]) + def test_uboot_artifact_layout_is_searched_at_any_depth(self): + # upload-artifact keeps u-boot-/output/ under its root. + nested = self.tmp / "art" / "u-boot-ingenic" / "output" + nested.mkdir(parents=True) + (nested / "u-boot-t31-nor.bin").write_bytes(b"deep") + a = push_build.parse_args([ + "--source", "uboot", "--build-id", "uboot-20260926T120000Z-abcdef0", "--release", "latest", + "--sha", "a" * 40, "--built-at", "2026-09-26T12:00:00Z", + "--assets", str(self.tmp / "art" / "**" / "*-nor.bin")]) + p = push_build.build_payload(a, NOW) + self.assertEqual([x["name"] for x in p["assets"]], ["u-boot-t31-nor.bin"]) + def test_uboot_without_the_release_hashes_locally(self): out = self.tmp / "out" out.mkdir() @@ -154,6 +166,39 @@ def test_uboot_without_the_release_hashes_locally(self): self.assertEqual(p["assets"], [{"name": "u-boot-t31-nor.bin", "size": 6, "sha256": digest(b"u-boot")}]) +class Platforms(unittest.TestCase): + """Both publishing schemes name a platform, with or without its report.""" + + def assets(self, *names): + return [{"name": n, "size": 1, "sha256": "0" * 64} for n in names] + + def test_builder_compound_devices_without_reports(self): + plats = push_build.collect_platforms(None, self.assets( + "gk7205v200_lite_hisilicon-ipc-a-nor.tgz", + "ssc338q_fpv_emax-wyvern-link-nand.tgz", + "openipc.t31-nor-lite.tgz")) + self.assertEqual([p["name"] for p in plats], [ + "gk7205v200_lite_hisilicon-ipc-a", "ssc338q_fpv_emax-wyvern-link", "t31-lite"]) + + def test_builder_compound_devices_with_reports(self): + with tempfile.TemporaryDirectory() as t: + reports = Path(t) + (reports / "sizes.gk7205v200_lite_hisilicon-ipc-a.json").write_text(json.dumps({"flash_mb": 8})) + (reports / "sizes.t31-lite.json").write_text(json.dumps({"flash_mb": 16})) + plats = {p["name"]: p for p in push_build.collect_platforms(reports, self.assets( + "gk7205v200_lite_hisilicon-ipc-a-nor.tgz", "openipc.t31-nor-lite.tgz"))} + self.assertEqual(sorted(plats), ["gk7205v200_lite_hisilicon-ipc-a", "t31-lite"]) + self.assertEqual(plats["gk7205v200_lite_hisilicon-ipc-a"]["sizes"]["flash_mb"], 8) + self.assertEqual(plats["t31-lite"]["sizes"]["flash_mb"], 16) + + def test_a_report_and_its_tarball_are_one_platform(self): + with tempfile.TemporaryDirectory() as t: + reports = Path(t) + (reports / "sizes.ssc338q_fpv_emax-wyvern-link.json").write_text("{}") + plats = push_build.collect_platforms(reports, self.assets("ssc338q_fpv_emax-wyvern-link-nor.tgz")) + self.assertEqual(len(plats), 1) + + class Aliases(unittest.TestCase): def test_scan(self): with tempfile.TemporaryDirectory() as t: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 218c41987..cfe97793b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -667,33 +667,49 @@ jobs: github.event_name != 'pull_request' && needs.publish.result == 'success' runs-on: ubuntu-latest + # id-token is what authorizes the push, so nothing but this repository's + # own push script, at the built commit, runs here: the artifacts are read + # as data, never executed. actions: read is for counting the reports-* + # artifacts the matrix uploaded (the guard below). permissions: id-token: write contents: read + actions: read timeout-minutes: 20 steps: - # The reports only. The images are described from the release itself, - # so nothing here depends on the one-day fw-* artifacts. + # First: checkout cleans the workspace, so anything downloaded before it + # would be deleted. + - uses: actions/checkout@v4 + with: + ref: ${{ needs.preflight.outputs.head_sha }} + sparse-checkout: .github/scripts + + # The reports only, outside the checkout. The images are described from + # the release itself, so nothing here depends on the one-day fw-* + # artifacts. - name: Download reports + id: reports uses: actions/download-artifact@v4 continue-on-error: true with: pattern: reports-* - path: reports + path: ${{ runner.temp }}/reports merge-multiple: true - - uses: actions/checkout@v4 - with: - ref: ${{ needs.preflight.outputs.head_sha }} - sparse-checkout: .github/scripts - # What was published, as GitHub has it: each device image's size and the # sha256 GitHub computed on upload. publish creates no dated release when # there were no images, and then there is nothing to report. + # + # "No reports produced" and "reports lost" look the same to the push, and + # only the first is acceptable: if the matrix uploaded any reports-* + # artifact, some report has to be here. - name: Push the build env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + REPORTS: ${{ runner.temp }}/reports + DOWNLOAD: ${{ steps.reports.outcome }} BUILD_ID: ${{ needs.preflight.outputs.build_id }} HEAD_SHA: ${{ needs.preflight.outputs.head_sha }} BUILT_AT: ${{ needs.preflight.outputs.built_at }} @@ -703,12 +719,24 @@ jobs: echo "::notice::no release $BUILD_ID; nothing to report" exit 0 fi + uploaded=$(gh api "repos/${GH_REPO}/actions/runs/${RUN_ID}/artifacts" --paginate \ + --jq '.artifacts[] | select(.name | startswith("reports-")) | select(.expired | not) | .name' | wc -l) + # A missing directory is zero reports, not a failed step. + found=0 + if [ -d "$REPORTS" ]; then + found=$(find "$REPORTS" -maxdepth 1 -name '*.json' | wc -l) + fi + echo "reports-* artifacts uploaded: ${uploaded}; report files here: ${found}" + if [ "$uploaded" -gt 0 ] && { [ "$DOWNLOAD" != success ] || [ "$found" -eq 0 ]; }; then + echo "::error::the matrix uploaded ${uploaded} reports-* artifact(s) but none reached this job (download: ${DOWNLOAD}); refusing to push devices without their reports" + exit 1 + fi python3 .github/scripts/push_build.py \ --source builder \ --build-id "$BUILD_ID" \ --sha "$HEAD_SHA" \ --built-at "$BUILT_AT" \ - --reports reports \ + --reports "$REPORTS" \ --release-json release.json \ --match '\.tgz$'