From aac7de42e1afc4cfbacb9e478b670828a36fcf1f Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 18 Sep 2026 10:00:34 +0900 Subject: [PATCH 1/2] feat(health): mention the maintainer when the problem set changes Editing an issue body notifies nobody, so a report nobody opens is a report nobody reads. When the set of problems changes, post a comment that @-mentions the configured user. Only on change: the same known failure every morning trains people to ignore the ping. The fingerprint is where + result, not the run link, so a workflow failing again tomorrow does not count as new. All clear never pings. --- machine/health.py | 37 ++++++++++++++++++++++++++++---- machine/repos.json | 50 +++++++++++++++++++++++++++++++++++++------- tests/test_health.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 11 deletions(-) diff --git a/machine/health.py b/machine/health.py index 754fb0c..2601125 100644 --- a/machine/health.py +++ b/machine/health.py @@ -126,7 +126,23 @@ def render(findings: list[Finding], config: dict[str, Any]) -> str: return "\n".join(lines) + "\n" -def publish(body: str, token: str, repo: str) -> None: +def fingerprint(findings: list[Finding]) -> str: + """Stable identity of a problem set: what is broken, not when it was checked.""" + return ",".join(sorted(f"{f.where}={f.what}" for f in findings)) + + +def alert_needed(previous_body: str, current: str) -> bool: + """Alert only when the set of problems changed — and never for all clear. + + A daily ping about the same known failure trains people to ignore the + ping; the point is to hear about the *new* one. + """ + if not current: + return False + return f"" not in previous_body + + +def publish(body: str, token: str, repo: str, findings: list[Finding], mention: str) -> None: """Keep one open issue up to date instead of opening one per run.""" def call(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: data = json.dumps(payload).encode("utf-8") if payload is not None else None @@ -138,12 +154,24 @@ def call(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: with urllib.request.urlopen(request, timeout=30) as response: return json.loads(response.read().decode("utf-8") or "null") + current = fingerprint(findings) + stamped = f"{body}\n\n" + issues = call("GET", f"/repos/{repo}/issues?state=open&per_page=100") existing = next((i for i in issues if i.get("title") == ISSUE_TITLE), None) if existing: - call("PATCH", f"/repos/{repo}/issues/{existing['number']}", {"body": body}) + number = existing["number"] + previous = existing.get("body") or "" + call("PATCH", f"/repos/{repo}/issues/{number}", {"body": stamped}) else: - call("POST", f"/repos/{repo}/issues", {"title": ISSUE_TITLE, "body": body}) + number = call("POST", f"/repos/{repo}/issues", {"title": ISSUE_TITLE, "body": stamped})["number"] + previous = "" + + # A comment, not an edit: editing an issue body notifies nobody. + if mention and alert_needed(previous, current): + lines = "\n".join(f"- {f.where}: **{f.what}**" for f in findings) + call("POST", f"/repos/{repo}/issues/{number}/comments", + {"body": f"@{mention} the org health report changed:\n\n{lines}"}) def main() -> int: @@ -161,7 +189,8 @@ def main() -> int: if summary: Path(summary).write_text(report, encoding="utf-8") if args.issue and token: - publish(report, token, os.environ.get("GITHUB_REPOSITORY", "GetTechAPI/TechMachine")) + publish(report, token, os.environ.get("GITHUB_REPOSITORY", "GetTechAPI/TechMachine"), + findings, config.get("notify", "")) # The report is the output; a red run would only add a second alert. return 0 diff --git a/machine/repos.json b/machine/repos.json index 7ff93da..659e22b 100644 --- a/machine/repos.json +++ b/machine/repos.json @@ -1,13 +1,49 @@ { + "notify": "Seungpyo1007", "repos": [ - {"name": "GetTechAPI/TechAPI", "branches": ["develop", "main"]}, - {"name": "GetTechAPI/TechEngine", "branches": ["main"]}, - {"name": "GetTechAPI/game-catalog", "branches": ["develop", "main"]}, - {"name": "GetTechAPI/cpu-engineering-samples", "branches": ["develop", "main"]} + { + "name": "GetTechAPI/TechAPI", + "branches": [ + "develop", + "main" + ] + }, + { + "name": "GetTechAPI/TechEngine", + "branches": [ + "main" + ] + }, + { + "name": "GetTechAPI/game-catalog", + "branches": [ + "develop", + "main" + ] + }, + { + "name": "GetTechAPI/cpu-engineering-samples", + "branches": [ + "develop", + "main" + ] + } ], "endpoints": [ - {"name": "TechAPI manifest", "url": "https://gettechapi.github.io/TechAPI/v1/index.json", "expect": "collections"}, - {"name": "game-catalog summary", "url": "https://gettechapi.github.io/game-catalog/summary.json", "expect": "count"}, - {"name": "cpu-engineering-samples summary", "url": "https://gettechapi.github.io/cpu-engineering-samples/summary.json", "expect": "count"} + { + "name": "TechAPI manifest", + "url": "https://gettechapi.github.io/TechAPI/v1/index.json", + "expect": "collections" + }, + { + "name": "game-catalog summary", + "url": "https://gettechapi.github.io/game-catalog/summary.json", + "expect": "count" + }, + { + "name": "cpu-engineering-samples summary", + "url": "https://gettechapi.github.io/cpu-engineering-samples/summary.json", + "expect": "count" + } ] } diff --git a/tests/test_health.py b/tests/test_health.py index 943d225..053d825 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -51,3 +51,36 @@ def test_render_all_clear_and_problem_table(): assert "all clear" in render([], config) problems = render(run_findings("org/repo", "main", [_run("deploy", "cancelled")]), config) assert "1 problem" in problems and "| cancelled |" in problems + + +# --- alerts ----------------------------------------------------------------- + +from machine.health import Finding, alert_needed, fingerprint # noqa: E402 + +BROKEN = [Finding("TechEngine@main · weekly-refresh", "cancelled")] + + +def test_new_problem_alerts(): + assert alert_needed("", fingerprint(BROKEN)) + + +def test_same_problem_twice_does_not_alert_again(): + body = f"report\n\n" + assert not alert_needed(body, fingerprint(BROKEN)) + + +def test_a_different_problem_alerts(): + body = f"report\n\n" + worse = BROKEN + [Finding("TechAPI@main · deploy-pages", "failure")] + assert alert_needed(body, fingerprint(worse)) + + +def test_all_clear_never_alerts(): + body = f"report\n\n" + assert not alert_needed(body, fingerprint([])) + + +def test_fingerprint_ignores_order_and_links(): + a = [Finding("x", "failure", "https://run/1"), Finding("y", "cancelled", "https://run/2")] + b = [Finding("y", "cancelled", "https://run/9"), Finding("x", "failure", "https://run/8")] + assert fingerprint(a) == fingerprint(b) From 3f13af796740110f36c21a2ef0151da566f4901f Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 18 Sep 2026 10:04:03 +0900 Subject: [PATCH 2/2] feat: scaffold satellite repositories from the game-catalog layout game-catalog took a day to get right: a validator that streams instead of loading ~1M records, a site that publishes counts rather than a listing, and a Pages environment that silently rejected main. new_satellite writes that layout from a handful of flags and prints the org-level steps it does not perform. The tests generate a repository and run its own suite and validator, so a template change that breaks generated repos fails here, not in a new repo. --- README.md | 29 ++++ machine/new_satellite.py | 153 ++++++++++++++++++ .../.github/workflows/deploy-pages.yml | 42 +++++ .../.github/workflows/validate-data.yml | 22 +++ machine/satellite_template/DATA_LICENSE.md | 8 + machine/satellite_template/LICENSE | 21 +++ machine/satellite_template/app/__init__.py | 0 machine/satellite_template/app/validate.py | 106 ++++++++++++ machine/satellite_template/gitignore | 11 ++ machine/satellite_template/pyproject.toml | 12 ++ machine/satellite_template/site/.nojekyll | 0 machine/satellite_template/site/build.py | 91 +++++++++++ machine/satellite_template/site/index.html | 32 ++++ .../satellite_template/tests_test_validate.py | 36 +++++ tests/test_new_satellite.py | 60 +++++++ 15 files changed, 623 insertions(+) create mode 100644 machine/new_satellite.py create mode 100644 machine/satellite_template/.github/workflows/deploy-pages.yml create mode 100644 machine/satellite_template/.github/workflows/validate-data.yml create mode 100644 machine/satellite_template/DATA_LICENSE.md create mode 100644 machine/satellite_template/LICENSE create mode 100644 machine/satellite_template/app/__init__.py create mode 100644 machine/satellite_template/app/validate.py create mode 100644 machine/satellite_template/gitignore create mode 100644 machine/satellite_template/pyproject.toml create mode 100644 machine/satellite_template/site/.nojekyll create mode 100644 machine/satellite_template/site/build.py create mode 100644 machine/satellite_template/site/index.html create mode 100644 machine/satellite_template/tests_test_validate.py create mode 100644 tests/test_new_satellite.py diff --git a/README.md b/README.md index 7820751..e1f6e16 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,35 @@ python -m pytest -q Adding a repository or endpoint is one entry in `machine/repos.json`. +When the set of problems changes, the check comments on the issue and +@-mentions `notify` from that file — editing an issue body notifies nobody. +It stays quiet when the same failure recurs (the fingerprint is +workflow + result, not the run link) and never pings for an all-clear. + +## Satellite repositories + +Categories that do not belong in TechAPI live in their own repository (games: +[game-catalog](https://github.com/GetTechAPI/game-catalog)). The split +criterion is identity, not size — software and websites are tech data and +stay in TechAPI. + +`machine/new_satellite.py` writes a new one from the layout game-catalog +proved out: a streaming validator, a site build that publishes +`summary.json` + `history.json` (never a listing of every record), CI, and +licences. + +```bash +python -m machine.new_satellite --repo game-catalog --category game --title "Game catalog" --plural games --date-field release_date --range rating:0:5 --range metacritic:0:100 --out ../game-catalog +``` + +It only writes files. Creating the repository changes the organisation, so +that is left to a person; the remaining steps are printed at the end, +including adding `main` to the Pages environment's deployment branches — +without it every deploy fails and leaves no log. + +The tests generate a repository and run *its* test suite and validator, so a +template change that breaks generated repos fails here. + ## Branching `develop` is the default branch; `main` is the released state. Pull requests diff --git a/machine/new_satellite.py b/machine/new_satellite.py new file mode 100644 index 0000000..725b534 --- /dev/null +++ b/machine/new_satellite.py @@ -0,0 +1,153 @@ +"""Scaffold a satellite data repository from the proven game-catalog layout. + +Writes the files only. Creating the GitHub repository is left to a person — +it is an org-level change — and the remaining one-off steps are printed at +the end, including the one that silently broke game-catalog's first deploys. + +Example: + python -m machine.new_satellite --repo game-catalog --category game \ + --title "Game catalog" --plural games --date-field release_date \ + --range rating:0:5 --range metacritic:0:100 --out ../game-catalog +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +TEMPLATE = Path(__file__).with_name("satellite_template") +PLACEHOLDER = re.compile(r"\{\{(\w+)\}\}") +# Template files stored under a name git or pytest would otherwise act on. +RENAMES = {"gitignore": ".gitignore", "tests_test_validate.py": "tests/test_validate.py"} + + +def parse_range(text: str) -> tuple[str, float, float]: + field, low, high = text.split(":") + return field, float(low), float(high) + + +def render(text: str, values: dict[str, str]) -> str: + def replace(match: re.Match[str]) -> str: + key = match.group(1) + if key not in values: + raise KeyError(f"template placeholder {{{{{key}}}}} has no value") + return values[key] + return PLACEHOLDER.sub(replace, text) + + +def values_from(args: argparse.Namespace) -> dict[str, str]: + ranges = {field: (low, high) for field, low, high in args.range} + return { + "repo": args.repo, + "category": args.category, + "title": args.title, + "plural": args.plural, + "description": args.description, + "date_fields": repr(tuple(args.date_field)), + "ranges": repr({k: (int(a) if a.is_integer() else a, int(b) if b.is_integer() else b) + for k, (a, b) in ranges.items()}), + } + + +def scaffold(out: Path, values: dict[str, str]) -> list[Path]: + if out.exists() and any(out.iterdir()): + raise SystemExit(f"{out} is not empty; refusing to overwrite") + written = [] + for source in sorted(TEMPLATE.rglob("*")): + if source.is_dir(): + continue + rel = source.relative_to(TEMPLATE).as_posix() + target = out / RENAMES.get(rel, rel) + target.parent.mkdir(parents=True, exist_ok=True) + if source.suffix in {".py", ".md", ".toml", ".yml", ".html", ""} or source.name == "gitignore": + target.write_text(render(source.read_text(encoding="utf-8"), values), + encoding="utf-8", newline="\n") + else: + target.write_bytes(source.read_bytes()) + written.append(target) + (out / "data" / values["category"]).mkdir(parents=True, exist_ok=True) + (out / "README.md").write_text(readme(values), encoding="utf-8", newline="\n") + written.append(out / "README.md") + return written + + +def readme(v: dict[str, str]) -> str: + return f"""# {v['repo']} + +[![validate-data](https://github.com/GetTechAPI/{v['repo']}/actions/workflows/validate-data.yml/badge.svg)](https://github.com/GetTechAPI/{v['repo']}/actions/workflows/validate-data.yml) + +{v['description']} + +Code is MIT; the records under `data/` are CC BY-SA 4.0 ([DATA_LICENSE.md](DATA_LICENSE.md)). + +## Layout + +``` +data/{v['category']}//.json # bucket = first two slug characters +app/validate.py # schema / slug / date / range checks +site/build.py # summary.json + history.json +``` + +A record needs `slug`, `name`, `source_urls` and `verified`. + +## Self-check + +```bash +python -m app.validate +python -m pytest -q +``` + +## Site + +`python site/build.py` writes `summary.json` (`{{"count": N}}`) and +`history.json` (one point per data commit). The TechAPI homepage reads these +to count this catalog; there is deliberately no listing of every record. + +## Branching (git-flow) + +`develop` is the default branch; `main` is the released state and deploys the +site. Pull requests target `develop`; a release is a PR from `develop` to `main`. +""" + + +def checklist(v: dict[str, str]) -> str: + repo = f"GetTechAPI/{v['repo']}" + return f""" +Next steps (not automated — each changes the org): + + 1. gh repo create {repo} --public + 2. push the scaffold to develop, then develop:main + 3. gh api -X POST repos/{repo}/pages -f build_type=workflow + 4. gh api -X POST repos/{repo}/environments/github-pages/deployment-branch-policies -f name=main + (skip this and every deploy fails with no log — game-catalog, 2026-09-17) + 5. add {{"name": "{repo}", "branches": ["develop", "main"]}} to TechMachine machine/repos.json + and the summary.json URL to its endpoints + 6. add {{ key: "{v['plural']}", label: "{v['plural']}", base: "https://gettechapi.github.io/{v['repo']}/" }} + to SATELLITES in TechAPI site/src/scripts/techapi.js +""" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", required=True, help="repository name, e.g. game-catalog") + parser.add_argument("--category", required=True, help="data directory, e.g. game") + parser.add_argument("--title", required=True, help='page title, e.g. "Game catalog"') + parser.add_argument("--plural", required=True, help="count label, e.g. games") + parser.add_argument("--description", default="Split out of TechAPI.") + parser.add_argument("--date-field", action="append", default=[], help="YYYY-MM-DD field, repeatable") + parser.add_argument("--range", action="append", default=[], type=parse_range, + help="field:low:high, repeatable") + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + + values = values_from(args) + written = scaffold(args.out, values) + print(f"wrote {len(written)} files to {args.out}") + print(checklist(values)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/machine/satellite_template/.github/workflows/deploy-pages.yml b/machine/satellite_template/.github/workflows/deploy-pages.yml new file mode 100644 index 0000000..06421b4 --- /dev/null +++ b/machine/satellite_template/.github/workflows/deploy-pages.yml @@ -0,0 +1,42 @@ +name: deploy-pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # history.json replays every data commit + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build summary + history + run: python site/build.py + - uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/machine/satellite_template/.github/workflows/validate-data.yml b/machine/satellite_template/.github/workflows/validate-data.yml new file mode 100644 index 0000000..02763cc --- /dev/null +++ b/machine/satellite_template/.github/workflows/validate-data.yml @@ -0,0 +1,22 @@ +name: validate-data + +on: + pull_request: + push: + branches: [develop, main] + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 90 # 962k files; measure a real run before trimming + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Validate {{title}} + run: python -m app.validate + - name: Tests + run: | + pip install pytest + python -m pytest -q diff --git a/machine/satellite_template/DATA_LICENSE.md b/machine/satellite_template/DATA_LICENSE.md new file mode 100644 index 0000000..b1a5684 --- /dev/null +++ b/machine/satellite_template/DATA_LICENSE.md @@ -0,0 +1,8 @@ +# Data license + +JSON records under `data/` are licensed under +[Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/). + +Attribute **"Data from GetTechAPI / {{repo}}"** and share alike. + +Validator, site, and workflow code remain MIT (see `LICENSE`). diff --git a/machine/satellite_template/LICENSE b/machine/satellite_template/LICENSE new file mode 100644 index 0000000..0d88fe6 --- /dev/null +++ b/machine/satellite_template/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 GTA Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/machine/satellite_template/app/__init__.py b/machine/satellite_template/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/machine/satellite_template/app/validate.py b/machine/satellite_template/app/validate.py new file mode 100644 index 0000000..a8bb295 --- /dev/null +++ b/machine/satellite_template/app/validate.py @@ -0,0 +1,106 @@ +"""Validate the {{title}}. + +Generated by TechMachine's satellite template. Records are checked one at a +time rather than loaded into a list first, so the check scales to catalogs of +any size (game-catalog validates ~1M records in about 90 seconds on CI). + +Run with: python -m app.validate +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +DATA = ROOT / "data" / "{{category}}" + +REQUIRED = {"slug", "name", "source_urls", "verified"} +DATE_FIELDS = {{date_fields}} +RANGES = {{ranges}} # field -> (low, high), inclusive +SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +MAX_ERRORS = 200 # a wall of identical errors helps nobody + + +def _check(rel: str, rec: dict[str, Any], errors: list[str]) -> None: + missing = REQUIRED - rec.keys() + if missing: + errors.append(f"{rel}: missing required field(s) {sorted(missing)}") + + slug = rec.get("slug") + if isinstance(slug, str) and not SLUG_RE.match(slug): + errors.append(f"{rel}: slug '{slug}' is not kebab-case") + + urls = rec.get("source_urls") + if not isinstance(urls, list) or not urls: + errors.append(f"{rel}: source_urls must be a non-empty list") + elif not all(isinstance(u, str) and u.startswith("http") for u in urls): + errors.append(f"{rel}: every source_url must be an http(s) string") + + for field in DATE_FIELDS: + value = rec.get(field) + if value is not None and not (isinstance(value, str) and DATE_RE.match(value)): + errors.append(f"{rel}: {field} '{value}' is not YYYY-MM-DD") + + for field, (low, high) in RANGES.items(): + value = rec.get(field) + if value is None or isinstance(value, bool): + continue + if not isinstance(value, (int, float)) or not low <= value <= high: + errors.append(f"{rel}: {field} {value!r} outside {low}-{high}") + + +def validate(data_dir: Path = DATA) -> list[str]: + errors: list[str] = [] + seen: dict[str, str] = {} + count = 0 + + for path in sorted(data_dir.rglob("*.json")): + rel = path.relative_to(data_dir.parent).as_posix() + count += 1 + try: + rec = json.loads(path.read_text(encoding="utf-8-sig")) + except json.JSONDecodeError as exc: + errors.append(f"{rel}: invalid JSON ({exc})") + continue + if not isinstance(rec, dict): + errors.append(f"{rel}: top level must be an object") + continue + + _check(rel, rec, errors) + + slug = rec.get("slug") + if isinstance(slug, str): + if slug in seen: + errors.append(f"{rel}: duplicate slug '{slug}' (first seen in {seen[slug]})") + else: + seen[slug] = rel + if path.stem != slug: + errors.append(f"{rel}: filename does not match slug '{slug}'") + + if len(errors) >= MAX_ERRORS: + errors.append(f"... stopped after {MAX_ERRORS} errors") + break + + if count == 0: + errors.append(f"no {{category}} records found under {data_dir}") + return errors + + +def main() -> int: + errors = validate() + for error in errors: + print(error) + if errors: + print(f"FAIL: {len(errors)} problem(s)") + return 1 + print("OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/machine/satellite_template/gitignore b/machine/satellite_template/gitignore new file mode 100644 index 0000000..efa6d4b --- /dev/null +++ b/machine/satellite_template/gitignore @@ -0,0 +1,11 @@ +__pycache__/ +*.py[cod] +.venv/ +venv/ +.idea/ +.vscode/ +.DS_Store + +# Generated site outputs (rebuilt in CI) +site/summary.json +site/history.json diff --git a/machine/satellite_template/pyproject.toml b/machine/satellite_template/pyproject.toml new file mode 100644 index 0000000..0b29dda --- /dev/null +++ b/machine/satellite_template/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "{{repo}}" +version = "0.1.0" +description = "{{title}} data, validator and static site" +requires-python = ">=3.11" + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/machine/satellite_template/site/.nojekyll b/machine/satellite_template/site/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/machine/satellite_template/site/build.py b/machine/satellite_template/site/build.py new file mode 100644 index 0000000..40a1f12 --- /dev/null +++ b/machine/satellite_template/site/build.py @@ -0,0 +1,91 @@ +"""Build the static site payload: summary.json and history.json. + +Deliberately NOT a catalog.json of every record. TechAPI's homepage counts a +satellite by downloading its catalog and reading `.length`; at ~1M games that +would be hundreds of MB fetched twice on page load. It reads `summary.json` +instead, and the records themselves stay in `data/` and the API dump. + +History is built in ONE git pass over add/delete name-status output, not one +`ls-tree` per commit (which is O(commits x records) — fine for 147 records, +hopeless for a million). + +Run with: python site/build.py +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DATA = ROOT / "data" / "{{category}}" +OUT = Path(__file__).resolve().parent +TRACKED = "data/{{category}}" + + +def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout + + +def count_records() -> int: + return sum(1 for _ in DATA.rglob("*.json")) + + +def build_history() -> list[dict[str, object]]: + """Record count after every commit that touched the data, one git pass.""" + log = git( + "log", + "--reverse", + "--no-renames", + "--format=%x00%H %cI", + "--name-status", + "--diff-filter=AD", + "--", + TRACKED, + ) + points: list[dict[str, object]] = [] + running = 0 + sha = date = "" + for line in log.splitlines(): + if line.startswith("\x00"): + if sha: + points.append({"sha": sha, "date": date, "count": running}) + sha, date = line[1:].split(" ", 1) + continue + if not line.strip() or not line.endswith(".json"): + continue + running += 1 if line[0] == "A" else -1 + if sha: + points.append({"sha": sha, "date": date, "count": running}) + return points + + +def main() -> int: + count = count_records() + summary = { + "count": count, + "generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + } + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(f"wrote summary.json ({count} {{category}} records)") + + history = build_history() + (OUT / "history.json").write_text( + json.dumps({"points": history}, indent=2) + "\n", encoding="utf-8" + ) + print(f"wrote history.json ({len(history)} points)") + + if history and history[-1]["count"] != count: + # The working tree and the replayed history disagree — usually an + # uncommitted change locally, but in CI it means the replay is wrong. + print(f"WARNING: history ends at {history[-1]['count']}, tree has {count}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/machine/satellite_template/site/index.html b/machine/satellite_template/site/index.html new file mode 100644 index 0000000..6864356 --- /dev/null +++ b/machine/satellite_template/site/index.html @@ -0,0 +1,32 @@ + + +{{title}} — GetTechAPI + + +
+

{{title}}

+

{{description}}

+

—

+

+

+ Records live in data/{{category}}/ in this repository. Machine-readable + counts: summary.json, history.json. +

+

Repository · TechAPI

+
+ diff --git a/machine/satellite_template/tests_test_validate.py b/machine/satellite_template/tests_test_validate.py new file mode 100644 index 0000000..2d7eb51 --- /dev/null +++ b/machine/satellite_template/tests_test_validate.py @@ -0,0 +1,36 @@ +"""Generated smoke tests: a well-formed record passes, broken ones do not.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from app.validate import validate + +GOOD = {"slug": "example-record", "name": "Example", "verified": False, + "source_urls": ["https://example.org/record"]} + + +def _write(tmp_path: Path, rec: dict, name: str | None = None) -> Path: + folder = tmp_path / "{{category}}" / "ex" + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{name or rec['slug']}.json").write_text(json.dumps(rec), encoding="utf-8") + return tmp_path / "{{category}}" + + +def test_good_record_passes(tmp_path): + assert validate(_write(tmp_path, GOOD)) == [] + + +def test_empty_catalog_is_an_error(tmp_path): + (tmp_path / "{{category}}").mkdir() + assert validate(tmp_path / "{{category}}") + + +def test_missing_source_urls(tmp_path): + rec = {k: v for k, v in GOOD.items() if k != "source_urls"} + assert validate(_write(tmp_path, rec)) + + +def test_filename_must_match_slug(tmp_path): + assert any("filename" in e for e in validate(_write(tmp_path, GOOD, "other"))) diff --git a/tests/test_new_satellite.py b/tests/test_new_satellite.py new file mode 100644 index 0000000..b3c6ac4 --- /dev/null +++ b/tests/test_new_satellite.py @@ -0,0 +1,60 @@ +"""A generated satellite must pass its own checks, not just render.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from machine.new_satellite import PLACEHOLDER, main, render + +ARGS = ["--repo", "demo-catalog", "--category", "demo", "--title", "Demo catalog", + "--plural", "demos", "--date-field", "release_date", "--range", "rating:0:5"] + + +@pytest.fixture +def generated(tmp_path: Path) -> Path: + out = tmp_path / "demo-catalog" + assert main([*ARGS, "--out", str(out)]) == 0 + return out + + +def test_no_placeholder_survives(generated: Path): + for path in generated.rglob("*"): + if path.is_file() and path.suffix in {".py", ".md", ".toml", ".yml", ".html"}: + # Actions expressions (${{ ... }}) are legitimate; template ones are not. + assert not PLACEHOLDER.search(path.read_text(encoding="utf-8")), path + + +def test_generated_repo_passes_its_own_tests(generated: Path, tmp_path: Path): + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", + "--basetemp", str(tmp_path / "inner")], + cwd=generated, capture_output=True, text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_validator_enforces_configured_range(generated: Path): + record = {"slug": "x", "name": "X", "verified": False, + "source_urls": ["https://example.org"], "rating": 9, "release_date": "2020"} + folder = generated / "data" / "demo" / "x_" + folder.mkdir(parents=True) + (folder / "x.json").write_text(json.dumps(record), encoding="utf-8") + result = subprocess.run([sys.executable, "-m", "app.validate"], cwd=generated, + capture_output=True, text=True) + assert result.returncode == 1 + assert "rating" in result.stdout and "release_date" in result.stdout + + +def test_refuses_to_overwrite(generated: Path): + with pytest.raises(SystemExit): + main([*ARGS, "--out", str(generated)]) + + +def test_unknown_placeholder_is_an_error(): + with pytest.raises(KeyError): + render("{{nope}}", {})