diff --git a/backend/apps/ifc_validation/management/commands/refresh_adoption_metrics.py b/backend/apps/ifc_validation/management/commands/refresh_adoption_metrics.py new file mode 100644 index 00000000..0761a886 --- /dev/null +++ b/backend/apps/ifc_validation/management/commands/refresh_adoption_metrics.py @@ -0,0 +1,164 @@ +"""Refresh the fact table behind the "Implementer Adoption" Grafana dashboard. + +Why a fact table: the capability panel ("which functional parts do tools +actually produce") aggregates ifc_validation_outcome, a 150M-row table with no +index on `created`. One month is seconds, twelve months in one statement times +out. So this command computes one month per statement -- translating the month +into a validation_task id range, which the index can use -- and stores the +result at a granularity the dashboard can still aggregate over any window: + + month x functional_part x tool_stem x company_id -> number of models + +Distinct counts over a range are then taken by the panel itself (a distinct +over 12 months is NOT the sum of 12 monthly distincts). The table is a few +hundred rows per month. + +"Activated" means the outcome severity is anything but N/A (executed, passed, +warning or error): a failed alignment rule still proves the tool writes +alignment. This is a lower bound - a rule only activates when its precondition +is met. + +tool_stem is the language-neutral tool name (canonical_name, IVS-884 migration +0035; falls back to name when empty) cut at the first digit: "Revit 26.4 (ENU)" +-> "Revit". So every version and language package of a tool counts once. + +Each month runs in its own transaction, so a timeout on one month does not +lose the others. Safe to re-run; months are deleted and re-inserted. + +Cron (manager node, nightly), see docker/prometheus/prod-crons/refresh_adoption_metrics.sh: + 30 2 * * * /home/prd-root/validation-service/docker/prometheus/prod-crons/refresh_adoption_metrics.sh +""" +import json +import time + +from django.core.management.base import BaseCommand +from django.db import connection, transaction + +TABLE = "vs_adoption_capability" + +DDL = [ + f""" + CREATE TABLE IF NOT EXISTS {TABLE} ( + month date NOT NULL, + functional_part text NOT NULL, + tool_stem text NOT NULL, + company_id integer, + models integer NOT NULL, + computed_at timestamptz NOT NULL DEFAULT now() + )""", + f"CREATE INDEX IF NOT EXISTS {TABLE}_month_idx ON {TABLE} (month)", +] + +# Month boundaries expressed as validation_task id ranges (ids are monotonic). +SQL_BOUNDS = """ +SELECT date_trunc('month', created)::date AS month, min(id) AS lo, max(id) AS hi +FROM ifc_validation_task +WHERE created >= date_trunc('month', now()) - make_interval(months => %s) +GROUP BY 1 +ORDER BY 1 +""" + +TOOL_STEM = (r"COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), " + r"'\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)')") + +SQL_FACTS = f""" +SELECT %s::date AS month, + left(vo.feature, 3) AS functional_part, + {TOOL_STEM} AS tool_stem, + at.company_id AS company_id, + COUNT(DISTINCT m.id) AS models +FROM ifc_validation_outcome vo +JOIN ifc_validation_task vt ON vt.id = vo.validation_task_id +JOIN ifc_validation_request vr ON vr.id = vt.request_id +JOIN ifc_model m ON m.id = vr.model_id +LEFT JOIN ifc_authoring_tool at ON at.id = m.produced_by_id +WHERE vo.validation_task_id >= %s + AND vo.validation_task_id < %s + AND vo.severity > 0 + AND vo.feature ~ '^[A-Z]{{3}}[0-9]{{3}}' +GROUP BY 1, 2, 3, 4 +""" + + +class Command(BaseCommand): + + help = ( + f"Recompute the last N months of {TABLE}, the fact table behind the " + "'Implementer Adoption' Grafana dashboard. One statement per month.\n" + "\n" + " python manage.py refresh_adoption_metrics # last 13 months\n" + " python manage.py refresh_adoption_metrics --months 24\n" + " python manage.py refresh_adoption_metrics --dry-run # compute, print, write nothing\n" + " python manage.py refresh_adoption_metrics --out /tmp/adoption.json\n" + ) + + def add_arguments(self, parser): + parser.add_argument("--months", type=int, default=13, + help="How many months back to (re)compute, current month included (default 13).") + parser.add_argument("--statement-timeout", type=int, default=900, + help="Per-month statement timeout in seconds (default 900).") + parser.add_argument("--dry-run", action="store_true", + help="Run the month queries and print row counts, but create/write nothing.") + parser.add_argument("--out", default=None, + help="Also write the computed facts to this JSON file.") + + def handle(self, *args, **options): + months = options["months"] + timeout_ms = options["statement_timeout"] * 1000 + dry_run = options["dry_run"] + out_path = options["out"] + + with connection.cursor() as c: + if dry_run: + c.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY") + else: + for stmt in DDL: + c.execute(stmt) + c.execute(SQL_BOUNDS, [months - 1]) + bounds = c.fetchall() + + self.stdout.write(f"{'DRY RUN - ' if dry_run else ''}{len(bounds)} month(s), " + f"timeout {options['statement_timeout']}s per month") + + collected = [] + failed = [] + t_all = time.time() + for month, lo, hi in bounds: + t0 = time.time() + try: + with transaction.atomic(): + with connection.cursor() as c: + c.execute("SET LOCAL statement_timeout = %s", [timeout_ms]) + if dry_run: + c.execute(SQL_FACTS, [month, lo, hi + 1]) + rows = c.fetchall() + n = len(rows) + else: + c.execute(f"DELETE FROM {TABLE} WHERE month = %s", [month]) + c.execute( + f"INSERT INTO {TABLE} (month, functional_part, tool_stem, company_id, models) " + + SQL_FACTS, [month, lo, hi + 1]) + n = c.rowcount + rows = [] + if out_path: + c.execute(f"SELECT month, functional_part, tool_stem, company_id, models " + f"FROM {TABLE} WHERE month = %s", [month]) + rows = c.fetchall() + self.stdout.write(f" {month} tasks {lo}-{hi} {n:5d} fact rows {time.time() - t0:6.1f}s") + collected.extend(rows) + except Exception as err: # timeout or SQL error: report and continue with the next month + failed.append(str(month)) + self.stderr.write(f" {month} FAILED after {time.time() - t0:.1f}s: {str(err).splitlines()[0]}") + + self.stdout.write(f"done in {time.time() - t_all:.0f}s" + + (f", FAILED months: {', '.join(failed)}" if failed else "")) + + if out_path: + with open(out_path, "w") as f: + json.dump([{"month": str(m), "functional_part": fp, "tool_stem": ts, + "company_id": cid, "models": n} + for m, fp, ts, cid, n in collected], f, indent=1) + self.stdout.write(f"written: {out_path} ({len(collected)} rows)") + + if failed: + raise SystemExit(1) diff --git a/backend/apps/ifc_validation/management/commands/top_failing_rules.py b/backend/apps/ifc_validation/management/commands/top_failing_rules.py new file mode 100644 index 00000000..8ad24004 --- /dev/null +++ b/backend/apps/ifc_validation/management/commands/top_failing_rules.py @@ -0,0 +1,187 @@ +import json +import os +from datetime import datetime, timedelta +from decimal import Decimal + +from django.core.management.base import BaseCommand +from django.db import connection + + +SQL_WEEK = """\ +WITH base_models AS ( + SELECT DISTINCT m.id + FROM ifc_model m + LEFT JOIN ifc_user_additional_info uai + ON uai.user_id = m.uploaded_by_id + WHERE + m.schema ILIKE %(schema)s + AND m.created >= %(from_date)s + AND m.created < %(to_date)s + AND COALESCE(uai.is_vendor, FALSE) = FALSE +), +failing_features AS ( + SELECT + SPLIT_PART(vo.feature, ' ', 1) AS rule_code, + COUNT(DISTINCT vr.model_id) AS models_failing + FROM ifc_validation_outcome vo + JOIN ifc_validation_task vt + ON vt.id = vo.validation_task_id + JOIN ifc_validation_request vr + ON vr.id = vt.request_id + WHERE + vr.model_id IN (SELECT id FROM base_models) + AND vo.severity = 4 + AND vo.feature IS NOT NULL + AND SPLIT_PART(vo.feature, ' ', 1) ~ '^[A-Z]{3}[0-9]{3}$' + GROUP BY SPLIT_PART(vo.feature, ' ', 1) +) +SELECT + ff.rule_code, + ff.models_failing, + (SELECT COUNT(*) FROM base_models) AS total_models +FROM failing_features ff +ORDER BY ff.models_failing DESC; +""" + + +class Command(BaseCommand): + + help = ( + 'Compute top failing validation rules per schema, ' + 'batched in weekly windows to avoid overloading the database.\n' + '\n' + 'Examples:\n' + '\n' + ' # Default: IFC4X3, weekly batches, top 20, from 2024-07-01 to today\n' + ' docker compose exec backend python manage.py top_failing_rules\n' + '\n' + ' # Custom date range\n' + ' docker compose exec backend python manage.py top_failing_rules --start 2025-01-01 --end 2026-01-01\n' + '\n' + ' # Different schema\n' + ' docker compose exec backend python manage.py top_failing_rules --schema \'%%IFC2X3%%\'\n' + '\n' + ' # Larger batch windows if it\'s slow\n' + ' docker compose exec backend python manage.py top_failing_rules --window 14\n' + '\n' + ' # Save results to a JSON file\n' + ' docker compose exec backend python manage.py top_failing_rules --out /tmp/results.json\n' + ) + + def add_arguments(self, parser): + + parser.add_argument( + '--schema', + type=str, + default='%IFC4X3%', + help='Schema filter (SQL ILIKE pattern). Default: %%IFC4X3%%', + ) + parser.add_argument( + '--start', + type=str, + default='2024-07-01', + help='Start date (YYYY-MM-DD). Default: 2024-07-01', + ) + parser.add_argument( + '--end', + type=str, + default=None, + help='End date exclusive (YYYY-MM-DD). Default: today.', + ) + parser.add_argument( + '--window', + type=int, + default=7, + help='Batch window size in days. Default: 7', + ) + parser.add_argument( + '--top', + type=int, + default=20, + help='Number of top rules to show. Default: 20', + ) + parser.add_argument( + '--out', + type=str, + default=None, + help='Output JSON file path (optional). If not set, prints to stdout.', + ) + + def handle(self, *args, **options): + schema = options['schema'] + start = datetime.strptime(options['start'], '%Y-%m-%d').date() + end = ( + datetime.strptime(options['end'], '%Y-%m-%d').date() + if options['end'] + else datetime.now().date() + ) + window = timedelta(days=options['window']) + top_n = options['top'] + out_path = options['out'] + + rule_failing = {} # rule_code -> total models failing + total_models = 0 + weeks_processed = 0 + + current = start + while current < end: + next_date = min(current + window, end) + + self.stdout.write(f" {current} -> {next_date} ...", ending="") + + with connection.cursor() as cursor: + cursor.execute(SQL_WEEK, { + 'schema': schema, + 'from_date': current.isoformat(), + 'to_date': next_date.isoformat(), + }) + rows = cursor.fetchall() + + week_total = 0 + for rule_code, models_failing, week_models in rows: + rule_failing[rule_code] = rule_failing.get(rule_code, 0) + models_failing + week_total = max(week_total, week_models) + + total_models += week_total + weeks_processed += 1 + self.stdout.write(f" {week_total} models, {len(rows)} rules") + + current = next_date + + # Build ranked result + ranked = [] + for code, failing in sorted(rule_failing.items(), key=lambda x: -x[1]): + pct = float(round( + Decimal(100) * Decimal(failing) / Decimal(max(total_models, 1)), + 1 + )) + ranked.append({ + 'rule_code': code, + 'models_failing': failing, + 'total_models': total_models, + 'failure_rate_pct': pct, + }) + + ranked = ranked[:top_n] + + self.stdout.write("") + self.stdout.write( + f"Processed {weeks_processed} windows, " + f"{total_models} total models, " + f"{len(rule_failing)} distinct failing rules." + ) + self.stdout.write("") + + # Table output + self.stdout.write(f"{'#':<4} {'Rule':<10} {'Failing':>8} {'Total':>8} {'Rate':>8}") + self.stdout.write("-" * 42) + for i, r in enumerate(ranked, 1): + self.stdout.write( + f"{i:<4} {r['rule_code']:<10} {r['models_failing']:>8} " + f"{r['total_models']:>8} {r['failure_rate_pct']:>7.1f}%" + ) + + if out_path: + with open(out_path, 'w', encoding='utf-8') as f: + json.dump(ranked, f, ensure_ascii=False, indent=2) + self.stdout.write(self.style.SUCCESS(f"\nWrote {out_path}")) diff --git a/docker/grafana/dashboards/vs-implementer-adoption.json b/docker/grafana/dashboards/vs-implementer-adoption.json new file mode 100644 index 00000000..2e31775b --- /dev/null +++ b/docker/grafana/dashboards/vs-implementer-adoption.json @@ -0,0 +1,609 @@ +{ + "uid": "vs-implementer-adoption", + "title": "Validation Service — Implementer Adoption", + "description": "Who uses the Validation Service, which IFC schemas and functional parts (alignment!) their tools actually produce, and which MVDs they declare. Built for the Implementers Assembly.", + "tags": [ + "adoption", + "implementers" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "", + "time": { + "from": "now-12M", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1h", + "6h", + "1d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "stat", + "title": "Companies", + "description": "How many companies use the service? Derived from the authoring tool named in the file header; 'in range' = at least one upload in the selected period.", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT (SELECT COUNT(DISTINCT at.company_id) FROM ifc_model m\n JOIN ifc_authoring_tool at ON at.id = m.produced_by_id WHERE $__timeFilter(m.created)) AS \"in range\",\n (SELECT COUNT(*) FROM ifc_company) AS \"all-time\"" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none", + "orientation": "horizontal", + "justifyMode": "center" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "id": 1 + }, + { + "type": "stat", + "title": "Tools", + "description": "How many distinct authoring tools upload here? Counted by name stem: canonical_name (language package stripped, IVS-884) cut at the first digit, so all versions of 'Revit' count once.", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT (SELECT COUNT(DISTINCT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)')) FROM ifc_model m\n JOIN ifc_authoring_tool at ON at.id = m.produced_by_id WHERE $__timeFilter(m.created)) AS \"in range\",\n (SELECT COUNT(DISTINCT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)')) FROM ifc_authoring_tool at) AS \"all-time\"" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none", + "orientation": "horizontal", + "justifyMode": "center" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "id": 2 + }, + { + "type": "stat", + "title": "Tool versions", + "description": "How many distinct tool versions do we see? A version is name + version as written in the header; language packages of the same version count once (canonical_name, IVS-884).", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT (SELECT COUNT(DISTINCT (COALESCE(NULLIF(at.canonical_name, ''), at.name), at.version)) FROM ifc_model m\n JOIN ifc_authoring_tool at ON at.id = m.produced_by_id WHERE $__timeFilter(m.created)) AS \"in range\",\n (SELECT COUNT(DISTINCT (COALESCE(NULLIF(canonical_name, ''), name), version)) FROM ifc_authoring_tool) AS \"all-time\"" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none", + "orientation": "horizontal", + "justifyMode": "center" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "id": 3 + }, + { + "type": "stat", + "title": "Models validated", + "description": "How many models were validated?", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT (SELECT COUNT(*) FROM ifc_model m WHERE $__timeFilter(m.created)) AS \"in range\",\n (SELECT COUNT(*) FROM ifc_model) AS \"all-time\"" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none", + "orientation": "horizontal", + "justifyMode": "center" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "type": "timeseries", + "title": "Schema adoption per month", + "description": "Is IFC 4.3 taking over from IFC4 and IFC2X3? Uploads per month by schema family; '(empty)' = no schema in the header.", + "gridPos": { + "h": 9, + "w": 14, + "x": 0, + "y": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH s AS (SELECT date_trunc('month', created) AS time, upper(replace(replace(COALESCE(schema, ''), ' ', ''), '_', '')) AS raw\n FROM ifc_model WHERE $__timeFilter(created))\nSELECT time, CASE WHEN raw ~ 'IFC4X3|IFC4\\.3' THEN 'IFC4X3' WHEN raw ~ 'IFC4X[12]' THEN 'IFC4X1/4X2' WHEN raw ~ '^IFC4' THEN 'IFC4' WHEN raw ~ 'IFC2X3|^2X3' THEN 'IFC2X3' WHEN raw = '' THEN '(empty)' ELSE 'other' END AS metric, COUNT(*)::float AS value\nFROM s GROUP BY 1, 2 ORDER BY 1, 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "id": 5 + }, + { + "type": "timeseries", + "title": "Who produces IFC 4.3: distinct tools and companies per month", + "description": "How many tools and companies actually ship IFC 4.3 files each month?", + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date_trunc('month', m.created) AS time, 'tools' AS metric, COUNT(DISTINCT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)'))::float AS value\nFROM ifc_model m JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE $__timeFilter(m.created) AND upper(replace(replace(COALESCE(m.schema, ''), ' ', ''), '_', '')) ~ 'IFC4X3|IFC4\\.3' GROUP BY 1\nUNION ALL\nSELECT date_trunc('month', m.created) AS time, 'companies' AS metric, COUNT(DISTINCT at.company_id)::float AS value\nFROM ifc_model m JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE $__timeFilter(m.created) AND upper(replace(replace(COALESCE(m.schema, ''), ' ', ''), '_', '')) ~ 'IFC4X3|IFC4\\.3' GROUP BY 1\nORDER BY 1, 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "id": 6 + }, + { + "type": "table", + "title": "Capability: functional parts produced, by distinct tools and companies", + "description": "Which parts of IFC do tools really produce, and by how many tools and companies? A part counts when one of its rules activated (any result other than N/A). Lower bound; refreshed nightly.", + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT functional_part AS \"functional part\",\n COUNT(DISTINCT tool_stem) AS tools,\n COUNT(DISTINCT company_id) AS companies,\n SUM(models) AS \"model-months\"\nFROM vs_adoption_capability\nWHERE $__timeFilter(month)\nGROUP BY 1 ORDER BY 2 DESC, 3 DESC" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "align": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "tools" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Alignment (ALA/ALB/ALS) per month: distinct tools and companies", + "description": "How many tools and companies produce alignment each month?", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT month::timestamptz AS time, 'tools' AS metric, COUNT(DISTINCT tool_stem)::float AS value\nFROM vs_adoption_capability WHERE functional_part LIKE 'AL_' AND $__timeFilter(month) GROUP BY 1\nUNION ALL\nSELECT month::timestamptz AS time, 'companies' AS metric, COUNT(DISTINCT company_id)::float AS value\nFROM vs_adoption_capability WHERE functional_part LIKE 'AL_' AND $__timeFilter(month) GROUP BY 1\nORDER BY 1, 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "id": 8 + }, + { + "type": "table", + "title": "MVD declared per tool (top 20)", + "description": "Which Model View Definitions do tools declare - and who declares Alignment-basedView?", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)') AS tool, COALESCE(NULLIF(m.mvd, ''), '(none)') AS mvd, COUNT(*) AS models\nFROM ifc_model m JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE $__timeFilter(m.created)\nGROUP BY 1, 2 ORDER BY 3 DESC LIMIT 20" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 9 + }, + { + "type": "table", + "title": "Capability, last 2 months - LIVE (slow, ~15 s; ignores time picker)", + "description": "Same question as the capability table, computed live for the last 2 months. Works before the nightly job has ever run; slow.", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "table", + "rawQuery": true, + "rawSql": "WITH w AS (SELECT min(id) AS tid FROM ifc_validation_task WHERE created >= now() - interval '2 months')\nSELECT left(vo.feature, 3) AS \"functional part\",\n COUNT(DISTINCT m.id) AS models,\n COUNT(DISTINCT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)')) AS tools,\n COUNT(DISTINCT at.company_id) AS companies\nFROM ifc_validation_outcome vo\nJOIN ifc_validation_task vt ON vt.id = vo.validation_task_id\nJOIN ifc_validation_request vr ON vr.id = vt.request_id\nJOIN ifc_model m ON m.id = vr.model_id\nLEFT JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE vo.validation_task_id >= (SELECT tid FROM w)\n AND vo.severity > 0 AND vo.feature ~ '^[A-Z]{3}[0-9]{3}'\nGROUP BY 1 ORDER BY 3 DESC, 4 DESC" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 10 + }, + { + "type": "timeseries", + "title": "Alignment per month - LIVE (slow, ~10 s)", + "description": "Same question as the alignment chart, computed live; slow.", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 25 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date_trunc('month', vo.created) AS time, 'tools' AS metric, COUNT(DISTINCT COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), '\\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)'))::float AS value\nFROM ifc_validation_outcome vo\nJOIN ifc_validation_task vt ON vt.id = vo.validation_task_id\nJOIN ifc_validation_request vr ON vr.id = vt.request_id\nJOIN ifc_model m ON m.id = vr.model_id\nLEFT JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE vo.feature >= 'ALA' AND vo.feature < 'ALC' AND vo.severity > 0 AND $__timeFilter(vo.created)\nGROUP BY 1\nUNION ALL\nSELECT date_trunc('month', vo.created) AS time, 'companies' AS metric, COUNT(DISTINCT at.company_id)::float AS value\nFROM ifc_validation_outcome vo\nJOIN ifc_validation_task vt ON vt.id = vo.validation_task_id\nJOIN ifc_validation_request vr ON vr.id = vt.request_id\nJOIN ifc_model m ON m.id = vr.model_id\nLEFT JOIN ifc_authoring_tool at ON at.id = m.produced_by_id\nWHERE vo.feature >= 'ALA' AND vo.feature < 'ALC' AND vo.severity > 0 AND $__timeFilter(vo.created)\nGROUP BY 1\nORDER BY 1, 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "id": 11 + } + ] +} \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-platform-usage.json b/docker/grafana/dashboards/vs-platform-usage.json index b9f24b8d..1b34bccb 100644 --- a/docker/grafana/dashboards/vs-platform-usage.json +++ b/docker/grafana/dashboards/vs-platform-usage.json @@ -22,7 +22,17 @@ ], "title": "Validation requests per day", "type": "timeseries", - "description": "Number of files submitted per day (fixed 30-day window, independent of the time range above). Includes requests that were soft-deleted later." + "description": "Number of files submitted per day (fixed 30-day window, independent of the time range above). Includes requests that were soft-deleted later.", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "points", + "showPoints": "always", + "pointSize": 6, + "spanNulls": false + } + } + } }, { "datasource": { @@ -45,7 +55,12 @@ ], "title": "Duration per task type: p50 / p95 (seconds)", "type": "table", - "description": "Median (p50) and slow-tail (p95) duration in seconds per validation step, over the selected time range. From ifc_validation_task.ended - started." + "description": "Median (p50) and slow-tail (p95) duration in seconds per validation step, over the selected time range. From ifc_validation_task.ended - started.", + "fieldConfig": { + "defaults": { + "custom": {} + } + } }, { "datasource": { @@ -68,7 +83,38 @@ ], "title": "Queue wait time p95 per day (s)", "type": "timeseries", - "description": "Wait time between submission and the start of the first task. NOTE: on DEV, tasks are sometimes re-run manually on old requests, which inflates this to hours. Read it as a trend, not an absolute." + "description": "Daily p95 of time between upload and processing start. Green < 1 min, yellow > 1 min, red > 10 min (queue is backing up).", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "points", + "showPoints": "always", + "pointSize": 6, + "spanNulls": false + }, + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 600 + } + ] + }, + "color": { + "mode": "thresholds" + } + } + } }, { "datasource": { @@ -78,7 +124,8 @@ "fieldConfig": { "defaults": { "max": 100, - "unit": "percent" + "unit": "percent", + "custom": {} }, "overrides": [] }, @@ -121,7 +168,12 @@ ], "title": "Activity by hour of day", "type": "barchart", - "description": "Which hour of the day the platform is used (UTC), over the selected time range." + "description": "Which hour of the day the platform is used (UTC), over the selected time range.", + "fieldConfig": { + "defaults": { + "custom": {} + } + } }, { "datasource": { @@ -142,7 +194,8 @@ "value": 1 } ] - } + }, + "custom": {} }, "overrides": [] }, @@ -185,13 +238,18 @@ ], "title": "Largest files (top 10)", "type": "table", - "description": "The ten largest files in the selected time range. Odd-looking file names are non-Latin names exactly as stored in the database." + "description": "The ten largest files in the selected time range. Odd-looking file names are non-Latin names exactly as stored in the database.", + "fieldConfig": { + "defaults": { + "custom": {} + } + } }, { "id": 8, "type": "timeseries", "title": "Uploads per day by channel (API vs WEBUI)", - "description": "NOTE: all rows before 2025-07-24 were written as WEBUI during a migration, so API figures are only reliable from Aug 2025 onwards. Counts uploads only — status polling and other GET traffic exist solely in the nginx logs.", + "description": "Daily upload totals, stacked by channel. Each bar sits at the START of its day-bucket (midnight UTC = 02:00 local) and covers the whole day — a bar \"at 02:00\" is simply \"that day's total so far\".", "gridPos": { "h": 8, "w": 12, @@ -207,7 +265,10 @@ "custom": { "stacking": { "mode": "normal" - } + }, + "drawStyle": "bars", + "fillOpacity": 60, + "lineWidth": 0 } }, "overrides": [] @@ -241,13 +302,18 @@ "format": "table", "rawSql": "SELECT u.username, COUNT(*) AS uploads, ROUND(SUM(r.size)/1024.0/1024.0,1) AS total_mb, ROUND(AVG(r.size)/1024.0/1024.0,2) AS avg_mb, MAX(r.created)::date AS last_upload FROM ifc_validation_request r JOIN auth_user u ON u.id = r.created_by_id WHERE r.channel='API' AND $__timeFilter(r.created) GROUP BY 1 ORDER BY uploads DESC LIMIT 15" } - ] + ], + "fieldConfig": { + "defaults": { + "custom": {} + } + } }, { "id": 10, "type": "table", "title": "Crash causes: why tasks FAIL (not validation errors)", - "description": "These are system failures, not 'the model is invalid'. The first line of status_reason is used as the category. Four recurring types: a duplicate-key race on concurrent requests, NUL bytes that PostgreSQL rejects, and two code bugs (TaskContext missing proc, NoneType has no id).", + "description": "Grouped technical task failures (not validation findings). latest_file/request_id/task_id point at the most recent example of each cause. Exit code -9 = killed by the kernel OOM killer (memory limit).", "gridPos": { "h": 9, "w": 24, @@ -262,9 +328,14 @@ { "refId": "A", "format": "table", - "rawSql": "SELECT CASE WHEN status_reason LIKE '%duplicate key%' THEN 'duplicate key (race on concurrent requests)' WHEN status_reason LIKE '%NUL (0x00)%' THEN 'NUL bytes in text (PostgreSQL rejects)' WHEN status_reason LIKE '%TaskContext%' THEN 'code bug: TaskContext missing proc' WHEN status_reason LIKE '%NoneType%' THEN 'code bug: NoneType has no id' WHEN status_reason IS NULL OR status_reason='' THEN '(no reason recorded)' ELSE split_part(status_reason, E'\\n', 1) END AS cause, COUNT(*) AS count, COUNT(DISTINCT type) AS task_types, MAX(created)::date AS last_seen FROM ifc_validation_task WHERE status='FAILED' AND $__timeFilter(created) GROUP BY 1 ORDER BY 2 DESC LIMIT 15" + "rawSql": "WITH failed AS (SELECT CASE WHEN t.status_reason LIKE '%duplicate key%' THEN 'duplicate key (race on concurrent requests)' WHEN t.status_reason LIKE '%NUL (0x00)%' THEN 'NUL bytes in text (PostgreSQL rejects)' WHEN t.status_reason LIKE '%TaskContext%' THEN 'code bug: TaskContext missing proc' WHEN t.status_reason LIKE '%NoneType%' THEN 'code bug: NoneType has no id' WHEN t.status_reason IS NULL OR t.status_reason='' THEN '(no reason recorded)' ELSE split_part(t.status_reason, E'\\n', 1) END AS cause, t.type, t.id AS task_id, t.request_id, t.created, r.file_name FROM ifc_validation_task t JOIN ifc_validation_request r ON r.id = t.request_id WHERE t.status='FAILED' AND $__timeFilter(t.created)) SELECT cause, COUNT(*) AS count, COUNT(DISTINCT type) AS task_types, MAX(created)::date AS last_seen, (ARRAY_AGG(file_name ORDER BY created DESC))[1] AS latest_file, (ARRAY_AGG(request_id ORDER BY created DESC))[1] AS request_id, (ARRAY_AGG(task_id ORDER BY created DESC))[1] AS task_id FROM failed GROUP BY cause ORDER BY count DESC LIMIT 15" } - ] + ], + "fieldConfig": { + "defaults": { + "custom": {} + } + } }, { "datasource": { @@ -291,7 +362,16 @@ ], "title": "API uploads per week", "type": "timeseries", - "description": "Weekly API-channel uploads and unique API users. Context: the external API user programme runs on DEV; on PROD this shows only internal usage until the API is opened up. Channel field is only reliable after Jul 2025 (migration wrote everything before that as WEBUI)." + "description": "Uploads bucketed per calendar week (bars). The rightmost bar is the current, incomplete week.", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "lineWidth": 0 + } + } + } }, { "datasource": { @@ -308,7 +388,7 @@ "targets": [ { "refId": "A", - "expr": "sum by (status) (rate(django_http_responses_total_by_status_total[5m]))", + "expr": "round(sum by (status) (increase(django_http_responses_total_by_status_total[$__interval])))", "legendFormat": "{{status}}", "datasource": { "type": "prometheus", @@ -316,9 +396,174 @@ } } ], - "title": "HTTP responses by status (incl. 429)", + "title": "HTTP responses by status (count per interval)", "type": "timeseries", - "description": "HTTP responses per status code straight from Django - including 429 rate-limit rejections, which never reach the database and were invisible until now." + "description": "Responses per time bucket, stacked by status. 200 = OK · 201 = upload accepted · 301/302 = redirect · 400 = bad request · 404 = not found · 429 = rate limited · 5xx = server error. Green/blue is healthy traffic; red (429) means the rate limiter refused requests; dark red (5xx) means the service itself failed.", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "stacking": { + "mode": "normal" + }, + "fillOpacity": 70, + "lineWidth": 0 + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "200" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "201" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "semi-dark-green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "301" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "302" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "400" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "404" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "429" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "500" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "502" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "503" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + } + ] + } }, { "datasource": { @@ -336,31 +581,78 @@ { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket[5m])))", - "legendFormat": "p95", + "legendFormat": "p95 (slow-but-real user)", "datasource": { "type": "prometheus", "uid": "prometheus" } + }, + { + "refId": "B", + "expr": "histogram_quantile(0.50, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket[5m])))", + "legendFormat": "p50 (median user)" } ], - "title": "HTTP p95 latency (django)", + "title": "HTTP latency (django): p50 vs p95", "type": "timeseries", - "description": "95th percentile response time of the Django backend, measured inside the app." + "description": "Green = median request, orange = 95th percentile. Diverging lines = outlier problem (a few slow requests); rising together = everything is slow.", + "fieldConfig": { + "defaults": { + "custom": { + "showPoints": "always", + "pointSize": 5, + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p95 (slow-but-real user)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50 (median user)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + } + ] + } } ], "refresh": "5m", "tags": [ - "observability", - "validation-service" + "Are users affected?" ], "time": { "from": "now-30d", "to": "now" }, - "title": "Validation Service — Platform Usage (DB)", + "title": "Validation Service — Platform Usage", "uid": "vs-platform-usage", "version": 1, "schemaVersion": 39, "editable": true, - "timezone": "browser" -} + "timezone": "browser", + "description": "Are users affected?" +} \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-system-health.json b/docker/grafana/dashboards/vs-system-health.json index bc614667..6a8e6015 100644 --- a/docker/grafana/dashboards/vs-system-health.json +++ b/docker/grafana/dashboards/vs-system-health.json @@ -295,7 +295,8 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes" + "unit": "bytes", + "min": 0 }, "overrides": [] }, @@ -315,7 +316,7 @@ ], "title": "Memory available (per node)", "type": "timeseries", - "description": "Memory the kernel can still hand out, per node. These VMs have no swap, so hitting zero freezes the node outright (as happened 28 Jul). Bands match the alert tile: orange below 4 GB, red below 2 GB." + "description": "Node has 62 GiB total. Dips are validation subtasks holding models in RAM; uploads themselves use disk, not memory." }, { "datasource": { @@ -429,7 +430,7 @@ "targets": [ { "expr": "celery_active_process_count", - "legendFormat": "{{hostname}}", + "legendFormat": "{{queue_name}}", "refId": "A" } ], @@ -516,9 +517,9 @@ }, { "id": 13, - "type": "timeseries", - "title": "Inode usage % (per node, /)", - "description": "Percentage of inodes used on the root filesystem. A disk can run 'full' with gigabytes still free: every file costs an inode, and /srv/nfs (the NFS export with thousands of gherkin log files) lives on this filesystem.", + "type": "gauge", + "title": "Inode usage (/, worst node)", + "description": "Inodes are the filesystem's bookkeeping slots: every file or directory uses exactly one, regardless of size. A disk can run out of inodes (millions of tiny files) while df still shows free space — the error is the same \"No space left on device\". Only worth attention above ~80%.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -534,16 +535,6 @@ "unit": "percent", "min": 0, "max": 100, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "showPoints": "auto", - "spanNulls": true, - "thresholdsStyle": { - "mode": "line" - } - }, "thresholds": { "mode": "absolute", "steps": [ @@ -552,12 +543,12 @@ "value": null }, { - "color": "orange", - "value": 85 + "color": "yellow", + "value": 70 }, { "color": "red", - "value": 95 + "value": 85 } ] } @@ -565,21 +556,19 @@ "overrides": [] }, "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "showThresholdMarkers": true }, "targets": [ { - "expr": "100 - (node_filesystem_files_free{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_files{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100)", - "legendFormat": "{{instance}}", - "refId": "A" + "expr": "max(100 * (1 - node_filesystem_files_free{mountpoint=\"/\"} / node_filesystem_files{mountpoint=\"/\"}))", + "legendFormat": "inodes used", + "refId": "A", + "instant": true } ] }, @@ -632,7 +621,12 @@ ], "title": "Swarm: replica deficit per service (0 = healthy)", "type": "timeseries", - "description": "Desired minus running replicas per Swarm service. Zero is healthy; anything above zero for more than a few minutes means Swarm is not delivering what was asked - the signature of the silent rollback of 3 Aug (CI reported success while the worker service had quietly reverted to the old image). Requires the swarm-state cron on the manager (docker/prometheus/swarm-state-textfile.sh) plus the textfile exporter. Alert candidate: deficit > 0 for 5 minutes." + "description": "Flatline at 0% means every service runs its desired replica count. A line only rises when a service loses replicas — this panel is supposed to look empty.", + "options": { + "legend": { + "showLegend": false + } + } }, { "datasource": { @@ -642,7 +636,14 @@ "fieldConfig": { "defaults": { "decimals": 0, - "unit": "short" + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 35, + "lineWidth": 1 + } }, "overrides": [] }, @@ -660,24 +661,195 @@ "refId": "A" } ], - "title": "Celery: busy slots per worker", + "title": "Celery: busy task slots (stacked, capacity 14)", "type": "timeseries", - "description": "Bezette pool-slots per celery-worker (celery_worker_tasks_active uit de celery-exporter). validate_worker heeft 4 slots per replica (CELERY_CONCURRENCY); een vlakke lijn op 4 = verzadigd, nieuwe checks wachten in de queue. Dit is de live-weergave van het slots-diagram uit de architectuurdocs." + "description": "Each band is one worker process (label = container id): 2 validation workers (4 slots each), scheduler (4), antivirus (2). Total height = tasks running right now; capacity is 14 slots." + }, + { + "id": 16, + "type": "bargauge", + "title": "Peak RSS per subtask (max observed)", + "description": "Highest peak RSS seen per subtask process, harvested from worker logs (persistent across redeploys). Worker memory limit is 16 GB.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(peak_rss_subtask_max_kb)", + "instant": true, + "legendFormat": "{{subtask}}" + } + ], + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "deckbytes", + "max": 16777216, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 17, + "type": "bargauge", + "title": "Peak RSS per subtask (average)", + "description": "Average of all harvested peak-RSS samples per subtask type.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(peak_rss_subtask_avg_kb)", + "instant": true, + "legendFormat": "{{subtask}}" + } + ], + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "deckbytes", + "max": 16777216, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 18, + "type": "bargauge", + "title": "Peak RSS per subtask (latest run)", + "description": "Peak RSS of the most recent completed run per subtask type. Live per-process memory needs a process exporter; container totals are in the worker memory panel.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(peak_rss_subtask_last_kb)", + "instant": true, + "legendFormat": "{{subtask}}" + } + ], + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "deckbytes", + "max": 16777216, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + } } ], "refresh": "30s", "tags": [ - "observability", - "validation-service" + "Is something breaking right now?" ], "time": { "from": "now-3h", "to": "now" }, "timezone": "browser", - "title": "Validation Service — System Health (DEV)", + "title": "Validation Service — System Health", "uid": "vs-system-health", "version": 2, "schemaVersion": 39, - "editable": true -} + "editable": true, + "description": "Is something breaking right now?" +} \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-validation-perf.json b/docker/grafana/dashboards/vs-validation-perf.json index 0193d9cb..1ae4fb15 100644 --- a/docker/grafana/dashboards/vs-validation-perf.json +++ b/docker/grafana/dashboards/vs-validation-perf.json @@ -3,9 +3,7 @@ "uid": "vs-validation-perf", "title": "Validation Service — Performance & Load", "tags": [ - "observability", - "performance", - "postgres" + "Why was this slow?" ], "editable": true, "schemaVersion": 39, @@ -16,7 +14,7 @@ "to": "now" }, "timezone": "browser", - "description": "Prestaties en belasting van de Validation Service, rechtstreeks uit de applicatiedatabase (datasource: DEV Postgres, uid devpg). Doorlooptijd = klok van aanmelden tot klaar (inclusief wachten in de wachtrij). Verwerkingstijd = som van de taakduren (alleen echt rekenwerk).", + "description": "Why was this slow?", "panels": [ { "id": 1, @@ -38,15 +36,63 @@ "unit": "s", "min": 0, "custom": { - "drawStyle": "line", + "drawStyle": "points", "lineWidth": 2, - "showPoints": "auto", - "pointSize": 5, - "spanNulls": true, + "showPoints": "always", + "pointSize": 6, + "spanNulls": false, "fillOpacity": 0 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "validations that day" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "bars" + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#888888" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisLabel", + "value": "validations/day" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": false + } + } + ] + } + ] }, "options": { "legend": { @@ -69,6 +115,15 @@ "format": "time_series", "rawQuery": true, "rawSql": "SELECT\n $__timeGroupAlias(t.started, '1d'),\n t.type AS metric,\n ROUND(AVG(EXTRACT(EPOCH FROM (t.ended - t.started)))::numeric, 1) AS duration_s\nFROM ifc_validation_task t\nWHERE $__timeFilter(t.started)\n AND t.started IS NOT NULL\n AND t.ended IS NOT NULL\nGROUP BY 1, 2\nORDER BY 1" + }, + { + "refId": "N", + "format": "time_series", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "rawSql": "SELECT $__timeGroupAlias(t.started, '1d'), 'validations that day' AS metric, COUNT(DISTINCT t.request_id) AS n FROM ifc_validation_task t WHERE $__timeFilter(t.started) AND t.started IS NOT NULL GROUP BY 1, 2 ORDER BY 1" } ] }, @@ -76,7 +131,7 @@ "id": 2, "type": "timeseries", "title": "Validation requests per day (by final status)", - "description": "Volume: how many files are submitted per day, and how they ended (COMPLETED / FAILED / PENDING / INITIATED). Source: ifc_validation_request.created.", + "description": "Daily validations stacked by final status — bar height = total that day, red band = FAILED share.", "datasource": { "type": "grafana-postgresql-datasource", "uid": "devpg" @@ -93,12 +148,11 @@ "min": 0, "custom": { "drawStyle": "bars", - "fillOpacity": 80, - "lineWidth": 1, + "fillOpacity": 70, + "lineWidth": 0, "barAlignment": 0, "stacking": { - "mode": "normal", - "group": "A" + "mode": "normal" } } }, @@ -506,8 +560,8 @@ { "id": 8, "type": "table", - "title": "Rule cost: total, average and longest run", - "description": "Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", + "title": "Rule cost: CPU and memory per rule", + "description": "CPU columns from 17 months of logs; RSS columns from the per-rule VmHWM measurement (since Aug 5). peak RSS = total process high-water during the rule (includes the model itself); avg RSS delta = memory growth attributable to the rule.", "gridPos": { "h": 9, "w": 12, @@ -525,6 +579,37 @@ "byField": "rule", "mode": "outer" } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(rule|Value #[A-E])$" + } + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "Value #A": "total CPU time", + "Value #B": "average per run", + "Value #C": "longest single run", + "Value #D": "peak RSS (worst run)", + "Value #E": "avg RSS delta" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "total CPU time", + "desc": true + } + ] + } } ], "targets": [ @@ -551,12 +636,69 @@ }, { "refId": "D", - "expr": "gherkin_rule_runs_total", + "expr": "gherkin_rule_peak_rss_mb_max", "format": "table", - "instant": true, - "legendFormat": "runs" + "instant": true + }, + { + "refId": "E", + "expr": "gherkin_rule_rss_delta_mb_avg", + "format": "table", + "instant": true } - ] + ], + "fieldConfig": { + "defaults": { + "unit": "dtdurations", + "decimals": 1 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rule" + }, + "properties": [ + { + "id": "unit", + "value": "string" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "peak RSS (worst run)" + }, + "properties": [ + { + "id": "unit", + "value": "mbytes" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "avg RSS delta" + }, + "properties": [ + { + "id": "unit", + "value": "mbytes" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + } }, { "id": 9, @@ -599,6 +741,243 @@ "rawSql": "SELECT\n CASE WHEN rule ~ '^[A-Z]{2,5}[0-9]{2,3}$' THEN rule\n ELSE '(SCHEMA check, not a gherkin rule)' END AS rule,\n SUM(warnings)::bigint AS warnings,\n SUM(errors)::bigint AS errors,\n SUM(total)::bigint AS total\nFROM (\n SELECT split_part(feature, ' ', 1) AS rule,\n SUM((severity = 3)::int) AS warnings,\n SUM((severity = 4)::int) AS errors,\n COUNT(*) AS total\n FROM ifc_validation_outcome\n WHERE severity >= 3 AND feature IS NOT NULL\n GROUP BY 1\n) sub\nGROUP BY 1\nORDER BY total DESC\nLIMIT 15" } ] + }, + { + "id": 10, + "type": "table", + "title": "Peak RSS per subtask (validation subprocesses)", + "description": "How much memory does each validation subprocess peak at, and is anything near the worker limit? Peaks cluster per file, not per subtask type - read 'worst run' as one heavy model.", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "subtask", + "mode": "outer" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(subtask|Value #[A-D])$" + } + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "Value #A": "peak RSS (worst run)", + "Value #B": "average peak RSS", + "Value #C": "most recent run", + "Value #D": "samples" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "peak RSS (worst run)", + "desc": true + } + ] + } + } + ], + "targets": [ + { + "refId": "A", + "expr": "peak_rss_subtask_max_kb", + "format": "table", + "instant": true + }, + { + "refId": "B", + "expr": "peak_rss_subtask_avg_kb", + "format": "table", + "instant": true + }, + { + "refId": "C", + "expr": "peak_rss_subtask_last_kb", + "format": "table", + "instant": true + }, + { + "refId": "D", + "expr": "peak_rss_subtask_samples", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "kbytes", + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "subtask" + }, + "properties": [ + { + "id": "unit", + "value": "string" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "samples" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + } + ] + } + }, + { + "id": 11, + "type": "barchart", + "title": "Rule cost per month — did our efforts help? ($rule)", + "description": "Did our optimisations make a rule cheaper over time? Pick a rule and compare average CPU per run with the number of runs, so volume noise is visible.", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 56 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "gherkin_rule_monthly_cpu_seconds_avg{rule=\"$rule\"}", + "format": "table", + "instant": true + }, + { + "refId": "B", + "expr": "gherkin_rule_monthly_runs{rule=\"$rule\"}", + "format": "table", + "instant": true + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "month", + "mode": "outer" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(month|Value #[AB])$" + } + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "Value #A": "avg CPU s per run", + "Value #B": "runs" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "month", + "desc": false + } + ] + } + } + ], + "options": { + "orientation": "vertical", + "xField": "month" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "runs" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + } } - ] -} + ], + "templating": { + "list": [ + { + "name": "rule", + "label": "gherkin rule", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(gherkin_rule_monthly_cpu_seconds_avg, rule)", + "refId": "rule_var" + }, + "refresh": 2, + "sort": 1, + "includeAll": false, + "multi": false, + "current": { + "text": "OJP001", + "value": "OJP001" + } + } + ] + } +} \ No newline at end of file diff --git a/docker/prometheus/prod-crons/db_health_metrics.sh b/docker/prometheus/prod-crons/db_health_metrics.sh index fb0fbb0f..1f958381 100755 --- a/docker/prometheus/prod-crons/db_health_metrics.sh +++ b/docker/prometheus/prod-crons/db_health_metrics.sh @@ -16,11 +16,11 @@ set -uo pipefail GRAFANA_URL=${GRAFANA_URL:-http://127.0.0.1:3000} -# Fallback-wachtwoord uit GRAFANA-CREDENTIALS.txt (sinds 31/7 is admin/admin uit; -# de oude default hier brak de cron stilletjes — scrape_success stond op 0, fix 11/8). -CRED_FILE=/home/geert/runbooks/observability/GRAFANA-CREDENTIALS.txt -GRAFANA_AUTH=${GRAFANA_AUTH:-admin:$(sed -n 's/^Wachtwoord: //p' "$CRED_FILE" 2>/dev/null)} -TEXTFILE_DIR=${TEXTFILE_DIR:-/home/geert/runbooks/observability/textfile} +# No password fallback on purpose: a wrong default here once broke the cron +# silently (scrape_success stayed 0). Pass GRAFANA_AUTH=user:password in the +# cron line, or better an env file the cron sources. +GRAFANA_AUTH=${GRAFANA_AUTH:?set GRAFANA_AUTH to user:password for the Grafana API} +TEXTFILE_DIR=${TEXTFILE_DIR:?set TEXTFILE_DIR to the node_exporter textfile directory, e.g. TEXTFILE_DIR=/data/srv/textfile} OUT="$TEXTFILE_DIR/vs_db_health.prom" TMP="$OUT.$$.tmp" NOW=$(date +%s) diff --git a/docker/prometheus/prod-crons/gherkin_rule_timings.sh b/docker/prometheus/prod-crons/gherkin_rule_timings.sh index 3a91d147..2ddc8059 100755 --- a/docker/prometheus/prod-crons/gherkin_rule_timings.sh +++ b/docker/prometheus/prod-crons/gherkin_rule_timings.sh @@ -10,32 +10,32 @@ set -uo pipefail LOG_DIR=${LOG_DIR:-/srv/nfs/gherkin_logs} -TEXTFILE_DIR=${TEXTFILE_DIR:-/home/geert/runbooks/observability/textfile} +TEXTFILE_DIR=${TEXTFILE_DIR:?set TEXTFILE_DIR to the node_exporter textfile directory, e.g. TEXTFILE_DIR=/data/srv/textfile} OUT="$TEXTFILE_DIR/gherkin_rules.prom" TMP="$OUT.$$.tmp" mkdir -p "$TEXTFILE_DIR" { - echo "# HELP gherkin_rule_cpu_seconds_total Totale CPU-tijd per gherkin-regel (uit de logs, cumulatief over alle runs)." + echo "# HELP gherkin_rule_cpu_seconds_total Total CPU time per gherkin rule (from the logs, cumulative over all runs)." echo "# TYPE gherkin_rule_cpu_seconds_total counter" - echo "# HELP gherkin_rule_runs_total Aantal keren dat de regel is uitgevoerd." + echo "# HELP gherkin_rule_runs_total Number of times the rule was executed." echo "# TYPE gherkin_rule_runs_total counter" - echo "# HELP gherkin_rule_cpu_seconds_max Langste enkele run van deze regel (CPU-seconden)." + echo "# HELP gherkin_rule_cpu_seconds_max Longest single run of this rule (CPU seconds)." echo "# TYPE gherkin_rule_cpu_seconds_max gauge" - echo "# HELP gherkin_rule_cpu_seconds_avg Gemiddelde CPU-tijd per run." + echo "# HELP gherkin_rule_cpu_seconds_avg Average CPU time per run." echo "# TYPE gherkin_rule_cpu_seconds_avg gauge" - # B1-uitbreiding (2/8): de logregel kan sinds de per-regel-geheugenmeting eindigen op - # " Peak RSS: MB (delta <+/-n> MB)." — veld 3/4 zijn dan gevuld, anders leeg. - # find|xargs i.p.v. glob: bij ~100k logbestanden overschrijdt "$LOG_DIR"/*.log - # de ARG_MAX-limiet (~2 MB) en faalt grep met "Argument list too long". + # Since the per-rule memory measurement (B1, Aug 2) a log line may end with + # " Peak RSS: MB (delta <+/-n> MB)." - fields 3/4 are then filled, else empty. + # find|xargs instead of a glob: with ~100k log files "$LOG_DIR"/*.log exceeds + # the ARG_MAX limit (~2 MB) and grep fails with "Argument list too long". find "$LOG_DIR" -maxdepth 1 -name '*.log' -print0 2>/dev/null \ | xargs -0 -r grep -h "Elapsed process time" \ | sed -E "s/.*Feature '([^']+)'.*time: ([0-9.]+) seconds\.( Peak RSS: ([0-9]+) MB \(delta ([+-][0-9]+) MB\)\.)?.*/\2\t\1\t\4\t\5/" \ | awk -F'\t' ' { - # regelcode = eerste woord vóór de spatie-streepje-spatie (bv. "CTX000 - ...") + # rule code = first word before the " - " separator (e.g. "CTX000 - ...") split($2, parts, " "); rule = parts[1]; gsub(/[^A-Za-z0-9_]/, "", rule); @@ -61,11 +61,50 @@ mkdir -p "$TEXTFILE_DIR" } }' - echo "# HELP gherkin_rule_timings_logfiles Aantal logbestanden dat is ingelezen." + # --- Per-month series (trend: "did our efforts help?") ------------------- + # The time axis cannot go into the TSDB as real history (the textfile + # collector rejects client-side timestamps), so the month is a label. + # Month = mtime of the log file: the moment of the run, not of the code + # change. Cardinality: ~150 rules x number of months; only months with runs + # get a series. When reading: _avg is the honest measure for trends, + # _cpu_seconds mostly follows upload volume. + echo "# HELP gherkin_rule_monthly_cpu_seconds Total CPU seconds per rule per calendar month (month = log file mtime)." + echo "# TYPE gherkin_rule_monthly_cpu_seconds gauge" + echo "# HELP gherkin_rule_monthly_runs Number of runs per rule per calendar month." + echo "# TYPE gherkin_rule_monthly_runs gauge" + echo "# HELP gherkin_rule_monthly_cpu_seconds_avg Average CPU seconds per run, per rule per calendar month." + echo "# TYPE gherkin_rule_monthly_cpu_seconds_avg gauge" + + MONTHMAP=$(mktemp) + find "$LOG_DIR" -maxdepth 1 -name '*.log' -printf '%p\t%TY-%Tm\n' 2>/dev/null > "$MONTHMAP" + find "$LOG_DIR" -maxdepth 1 -name '*.log' -print0 2>/dev/null \ + | xargs -0 -r grep -H "Elapsed process time" \ + | sed -E "s/^([^:]+):.*Feature '([^']+)'.*time: ([0-9.]+) seconds\..*/\1\t\2\t\3/" \ + | awk -F'\t' -v monthmap="$MONTHMAP" ' + BEGIN { while ((getline line < monthmap) > 0) { split(line, a, "\t"); mm[a[1]] = a[2] } } + { + split($2, parts, " "); + rule = parts[1]; + gsub(/[^A-Za-z0-9_]/, "", rule); + if (rule == "" || !($1 in mm)) next; + key = rule SUBSEP mm[$1]; + sum[key] += $3; n[key]++; + } + END { + for (k in sum) { + split(k, p, SUBSEP); + printf "gherkin_rule_monthly_cpu_seconds{rule=\"%s\",month=\"%s\"} %.2f\n", p[1], p[2], sum[k]; + printf "gherkin_rule_monthly_runs{rule=\"%s\",month=\"%s\"} %d\n", p[1], p[2], n[k]; + printf "gherkin_rule_monthly_cpu_seconds_avg{rule=\"%s\",month=\"%s\"} %.3f\n", p[1], p[2], sum[k]/n[k]; + } + }' + rm -f "$MONTHMAP" + + echo "# HELP gherkin_rule_timings_logfiles Number of log files read." echo "# TYPE gherkin_rule_timings_logfiles gauge" echo "gherkin_rule_timings_logfiles $(find "$LOG_DIR" -maxdepth 1 -name '*.log' 2>/dev/null | wc -l)" } > "$TMP" -# atomisch vervangen, zodat node_exporter nooit een half bestand leest +# replace atomically, so node_exporter never reads a half-written file mv "$TMP" "$OUT" chmod 644 "$OUT" diff --git a/docker/prometheus/prod-crons/harvest_peak_rss.sh b/docker/prometheus/prod-crons/harvest_peak_rss.sh index 25d98a0b..44634ab7 100755 --- a/docker/prometheus/prod-crons/harvest_peak_rss.sh +++ b/docker/prometheus/prod-crons/harvest_peak_rss.sh @@ -3,8 +3,72 @@ # persistent, deduplicated file -- container logs are lost on redeploy, this # file is not. Cron: every 10 min. Override OUT via env prefix in the cron # line (PROD: OUT=/data/srv/perf-collected/peak_rss.log). -OUT=${OUT:-/home/geert/runbooks/observability/perf-metrics/collected/peak_rss.log} +# +# The harvested file is also aggregated into Prometheus metrics for the +# node_exporter textfile collector, so peak RSS is graphable and alertable +# instead of only greppable. Set TEXTFILE_DIR in the cron line to enable it +# (PROD: TEXTFILE_DIR=/data/srv/textfile), same as gherkin_rule_timings.sh. +OUT=${OUT:?set OUT to the harvested log file, e.g. OUT=/data/srv/perf-collected/peak_rss.log} mkdir -p "$(dirname "$OUT")" TMP=$(mktemp) timeout 100 docker service logs validate_worker --since 30m 2>&1 | grep "Peak RSS for" >> "$OUT" 2>/dev/null sort -u "$OUT" > "$TMP" && mv "$TMP" "$OUT" + +TEXTFILE_DIR=${TEXTFILE_DIR:?set TEXTFILE_DIR to the node_exporter textfile directory, e.g. TEXTFILE_DIR=/data/srv/textfile} +PROM="$TEXTFILE_DIR/peak_rss.prom" +PROM_TMP="$PROM.$$.tmp" +mkdir -p "$TEXTFILE_DIR" + +# Source line (check_programs.py): +# Peak RSS for subprocess (task #): kB (min MemAvailable ...) +# Every run recomputes from the full harvested file -- no incremental state, so +# a one-off manual run backfills the whole history. "Last" is keyed on the task +# id rather than file order, because the dedup above sorts lexically and the +# newest line is therefore not the last one. +{ + sed -nE 's/.*Peak RSS for ([A-Z_]+) subprocess \(task #([0-9]+)\): ([0-9]+) kB.*/\1\t\2\t\3/p' "$OUT" \ + | awk -F'\t' ' + { + t = $1; id = $2 + 0; v = $3 + 0; + if (!(t in n)) order[++k] = t; + n[t]++; sum[t] += v; + if (v > mx[t]) mx[t] = v; + if (id > lastid[t]) { lastid[t] = id; last[t] = v } + if (v > gmax) gmax = v; + total++; + } + END { + print "# HELP peak_rss_subtask_max_kb Highest peak RSS observed per subtask type (from harvested worker logs)."; + print "# TYPE peak_rss_subtask_max_kb gauge"; + for (i = 1; i <= k; i++) printf "peak_rss_subtask_max_kb{subtask=\"%s\"} %d\n", order[i], mx[order[i]]; + + print "# HELP peak_rss_subtask_avg_kb Average peak RSS per subtask type over all harvested samples."; + print "# TYPE peak_rss_subtask_avg_kb gauge"; + for (i = 1; i <= k; i++) printf "peak_rss_subtask_avg_kb{subtask=\"%s\"} %.0f\n", order[i], sum[order[i]] / n[order[i]]; + + print "# HELP peak_rss_subtask_last_kb Peak RSS of the most recent observed run per subtask type (highest task id)."; + print "# TYPE peak_rss_subtask_last_kb gauge"; + for (i = 1; i <= k; i++) printf "peak_rss_subtask_last_kb{subtask=\"%s\"} %d\n", order[i], last[order[i]]; + + print "# HELP peak_rss_subtask_samples Number of harvested samples per subtask type."; + print "# TYPE peak_rss_subtask_samples gauge"; + for (i = 1; i <= k; i++) printf "peak_rss_subtask_samples{subtask=\"%s\"} %d\n", order[i], n[order[i]]; + + print "# HELP peak_rss_observed_max_kb Highest peak RSS observed across all subtask types."; + print "# TYPE peak_rss_observed_max_kb gauge"; + printf "peak_rss_observed_max_kb %d\n", gmax; + + print "# HELP peak_rss_samples Total number of harvested samples."; + print "# TYPE peak_rss_samples gauge"; + printf "peak_rss_samples %d\n", total + 0; + }' + + # Lets a panel or alert tell "quiet" apart from "the harvester stopped running". + echo "# HELP peak_rss_harvest_timestamp_seconds Unix time of the last harvest run." + echo "# TYPE peak_rss_harvest_timestamp_seconds gauge" + echo "peak_rss_harvest_timestamp_seconds $(date +%s)" +} > "$PROM_TMP" + +# atomically replace, so node_exporter never reads a half-written file +mv "$PROM_TMP" "$PROM" +chmod 644 "$PROM" diff --git a/docker/prometheus/prod-crons/refresh_adoption_metrics.sh b/docker/prometheus/prod-crons/refresh_adoption_metrics.sh new file mode 100755 index 00000000..5f22a808 --- /dev/null +++ b/docker/prometheus/prod-crons/refresh_adoption_metrics.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Nightly refresh of the vs_adoption_capability fact table behind the +# "Implementer Adoption" Grafana dashboard. Runs the management command inside +# the running backend container, so it uses the service's own DB credentials. +# +# Cron on the MANAGER node, e.g.: +# 30 2 * * * /home/prd-root/validation-service/docker/prometheus/prod-crons/refresh_adoption_metrics.sh +# +# Takes minutes, not seconds: one statement per month over the outcomes table. +# Exit code is the command's (1 if any month failed), so cron mail carries it. +set -uo pipefail + +SERVICE=${SERVICE:-validate_backend} +MONTHS=${MONTHS:-13} + +CID=$(docker ps --filter "name=${SERVICE}" --format '{{.ID}}' | head -1) +if [ -z "$CID" ]; then + echo "refresh_adoption_metrics: no running container for service ${SERVICE}" >&2 + exit 1 +fi + +exec docker exec -w /app/backend "$CID" python manage.py refresh_adoption_metrics --months "$MONTHS"