Skip to content

Commit 6abd4ed

Browse files
committed
feat(site): read every satellite's count from summary.json
The homepage counted cpu-engineering-samples by downloading its whole catalog and reading `.length`. game-catalog is ~1M records, so that had to change before games could count at all: satellites now publish `summary.json` and the page reads `count` from it. The satellite list is data rather than three hard-coded places, so the next split is one line. `/v1/games/index.json` keeps a small pointer to the new home instead of 404-ing; its .gitignore exception (the 125MB list file) is no longer needed. Refs #19
1 parent 6e2195a commit 6abd4ed

3 files changed

Lines changed: 61 additions & 27 deletions

File tree

.gitignore

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,3 @@ coverage.xml
4646
.vscode/
4747
.DS_Store
4848

49-
# Oversized game list index (962k games = 125MB > GitHub 100MB limit); detail pages are committed
50-
site/public/v1/games/index.json

site/public/v1/games/index.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"moved": true,
3+
"message": "The game catalog moved to its own repository. This path is no longer generated by the TechAPI dump.",
4+
"repository": "https://github.com/GetTechAPI/game-catalog",
5+
"site": "https://gettechapi.github.io/game-catalog/",
6+
"summary_url": "https://gettechapi.github.io/game-catalog/summary.json",
7+
"moved_on": "2026-09-17"
8+
}

site/src/scripts/techapi.js

Lines changed: 53 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,31 @@ async function getJSON(path) {
5151
return r.json();
5252
}
5353

54+
/* ============================================================
55+
SATELLITE DATA REPOS
56+
Categories too large (or too differently sourced) to live in TechAPI keep
57+
their own repository, but still count on this page. Each publishes
58+
`summary.json` ({count}) and `history.json` ({points:[{date,count}]}).
59+
Counts come from summary.json, never from a record listing: game-catalog
60+
alone is ~1M records, hundreds of MB nobody should download to read a
61+
`.length`.
62+
============================================================ */
63+
const SATELLITES = [
64+
{ key: "games", label: "games", base: "https://gettechapi.github.io/game-catalog/" },
65+
{ key: "cpu_es", label: "cpu eng. samples", base: "https://gettechapi.github.io/cpu-engineering-samples/" },
66+
];
67+
68+
const satelliteJSON = (sat, file) =>
69+
fetch(sat.base + file).then((r) => (r.ok ? r.json() : Promise.reject(new Error(String(r.status)))));
70+
71+
// One entry per satellite that answered; an unreachable one is simply absent.
72+
const satelliteCounts = () =>
73+
Promise.all(SATELLITES.map((sat) =>
74+
satelliteJSON(sat, "summary.json")
75+
.then((d) => (typeof d.count === "number" ? { sat, count: d.count } : null))
76+
.catch(() => null)))
77+
.then((rows) => rows.filter(Boolean));
78+
5479
const listCache = {};
5580
async function loadList(resource) {
5681
if (!listCache[resource]) {
@@ -182,15 +207,14 @@ async function loadList(resource) {
182207
// The hero is already in view at first paint. Animate immediately so the
183208
// counters do not remain at zero when IntersectionObserver is delayed.
184209
el.querySelectorAll(".num").forEach((n) => countUp(n, +n.dataset.n));
185-
// Satellite data repos publish their own catalog; count them alongside.
186-
fetch("https://gettechapi.github.io/cpu-engineering-samples/catalog.json")
187-
.then((r) => (r.ok ? r.json() : Promise.reject()))
188-
.then((list) => {
210+
// Satellites are not in the dump manifest; count them alongside it.
211+
satelliteCounts().then((rows) => {
212+
for (const { sat, count } of rows) {
189213
el.insertAdjacentHTML("beforeend",
190-
`<div class="stat"><div class="n"><span class="num">0</span></div><div class="l">cpu eng. samples</div></div>`);
191-
countUp(el.lastElementChild.querySelector(".num"), list.length);
192-
})
193-
.catch(() => {});
214+
`<div class="stat"><div class="n"><span class="num">0</span></div><div class="l">${sat.label}</div></div>`);
215+
countUp(el.lastElementChild.querySelector(".num"), count);
216+
}
217+
});
194218
}).catch(() => {
195219
el.innerHTML = '<div class="stat"><div class="n">—</div><div class="l">build data first</div></div>';
196220
});
@@ -445,15 +469,21 @@ function countUp(node, target, opts = {}) {
445469
if (!points.length) throw new Error("empty history");
446470

447471
// Add each satellite's count as of every point's date (its own history.json).
448-
const es = await fetch("https://gettechapi.github.io/cpu-engineering-samples/history.json")
449-
.then((r) => (r.ok ? r.json() : Promise.reject()))
450-
.then((d) => (d.points || []).map((p) => ({ t: new Date(p.date).getTime(), count: p.count })))
451-
.catch(() => []);
452-
if (es.length) {
472+
const histories = await Promise.all(SATELLITES.map((sat) =>
473+
satelliteJSON(sat, "history.json")
474+
.then((d) => ({
475+
sat,
476+
points: (d.points || [])
477+
.map((p) => ({ t: new Date(p.date).getTime(), count: p.count }))
478+
.sort((a, b) => a.t - b.t),
479+
}))
480+
.catch(() => null)));
481+
for (const history of histories) {
482+
if (!history || !history.points.length) continue;
453483
for (const point of points) {
454-
const asOf = es.filter((p) => p.t <= point.dateValue).pop();
484+
const asOf = history.points.filter((p) => p.t <= point.dateValue).pop();
455485
if (!asOf) continue;
456-
point.rows = [...point.rows, { key: "cpu_es", count: asOf.count }];
486+
point.rows = [...point.rows, { key: history.sat.key, count: asOf.count }];
457487
point.total += asOf.count;
458488
}
459489
}
@@ -481,17 +511,15 @@ function countUp(node, target, opts = {}) {
481511
renderHistory(points);
482512
}
483513

484-
// Satellite repos are not in the dump; fold their current count into the
485-
// latest snapshot so the total and the newest point include them.
486-
// ponytail: current count only, no back-history until satellites publish one.
514+
// Satellite repos are not in the dump; fold their current counts into the
515+
// snapshot so the total and the newest point include them.
487516
const withSatellites = (manifest) =>
488-
fetch("https://gettechapi.github.io/cpu-engineering-samples/catalog.json")
489-
.then((r) => (r.ok ? r.json() : Promise.reject()))
490-
.then((list) => {
491-
manifest.collections = { ...manifest.collections, cpu_es: { count: list.length } };
492-
return manifest;
493-
})
494-
.catch(() => manifest);
517+
satelliteCounts().then((rows) => {
518+
const collections = { ...manifest.collections };
519+
for (const { sat, count } of rows) collections[sat.key] = { count };
520+
manifest.collections = collections;
521+
return manifest;
522+
}).catch(() => manifest);
495523

496524
getJSON("v1/index.json").then(withSatellites).then((manifest) => {
497525
renderSnapshot(manifest);

0 commit comments

Comments
 (0)