Monitoring/observability followup

backend/apps/ifc_validation/management/commands/refresh_adoption_metrics.py

@@ -0,0 +1,164 @@
+"""Refresh the fact table behind the "Implementer Adoption" Grafana dashboard.
+
+Why a fact table: the capability panel ("which functional parts do tools
+actually produce") aggregates ifc_validation_outcome, a 150M-row table with no
+index on `created`. One month is seconds, twelve months in one statement times
+out. So this command computes one month per statement -- translating the month
+into a validation_task id range, which the index can use -- and stores the
+result at a granularity the dashboard can still aggregate over any window:
+
+    month x functional_part x tool_stem x company_id -> number of models
+
+Distinct counts over a range are then taken by the panel itself (a distinct
+over 12 months is NOT the sum of 12 monthly distincts). The table is a few
+hundred rows per month.
+
+"Activated" means the outcome severity is anything but N/A (executed, passed,
+warning or error): a failed alignment rule still proves the tool writes
+alignment. This is a lower bound - a rule only activates when its precondition
+is met.
+
+tool_stem is the language-neutral tool name (canonical_name, IVS-884 migration
+0035; falls back to name when empty) cut at the first digit: "Revit 26.4 (ENU)"
+-> "Revit". So every version and language package of a tool counts once.
+
+Each month runs in its own transaction, so a timeout on one month does not
+lose the others. Safe to re-run; months are deleted and re-inserted.
+
+Cron (manager node, nightly), see docker/prometheus/prod-crons/refresh_adoption_metrics.sh:
+    30 2 * * *  /home/prd-root/validation-service/docker/prometheus/prod-crons/refresh_adoption_metrics.sh
+"""
+import json
+import time
+
+from django.core.management.base import BaseCommand
+from django.db import connection, transaction
+
+TABLE = "vs_adoption_capability"
+
+DDL = [
+    f"""
+    CREATE TABLE IF NOT EXISTS {TABLE} (
+        month           date        NOT NULL,
+        functional_part text        NOT NULL,
+        tool_stem       text        NOT NULL,
+        company_id      integer,
+        models          integer     NOT NULL,
+        computed_at     timestamptz NOT NULL DEFAULT now()
+    )""",
+    f"CREATE INDEX IF NOT EXISTS {TABLE}_month_idx ON {TABLE} (month)",
+]
+
+# Month boundaries expressed as validation_task id ranges (ids are monotonic).
+SQL_BOUNDS = """
+SELECT date_trunc('month', created)::date AS month, min(id) AS lo, max(id) AS hi
+FROM ifc_validation_task
+WHERE created >= date_trunc('month', now()) - make_interval(months => %s)
+GROUP BY 1
+ORDER BY 1
+"""
+
+TOOL_STEM = (r"COALESCE(NULLIF(regexp_replace(COALESCE(NULLIF(at.canonical_name, ''), at.name), "
+             r"'\s*[0-9][0-9.]*.*$', ''), ''), '(unknown)')")
+
+SQL_FACTS = f"""
+SELECT %s::date                     AS month,
+       left(vo.feature, 3)          AS functional_part,
+       {TOOL_STEM}                  AS tool_stem,
+       at.company_id                AS company_id,
+       COUNT(DISTINCT m.id)         AS models
+FROM ifc_validation_outcome vo
+JOIN ifc_validation_task    vt ON vt.id = vo.validation_task_id
+JOIN ifc_validation_request vr ON vr.id = vt.request_id
+JOIN ifc_model              m  ON m.id  = vr.model_id
+LEFT JOIN ifc_authoring_tool at ON at.id = m.produced_by_id
+WHERE vo.validation_task_id >= %s
+  AND vo.validation_task_id <  %s
+  AND vo.severity > 0
+  AND vo.feature ~ '^[A-Z]{{3}}[0-9]{{3}}'
+GROUP BY 1, 2, 3, 4
+"""
+
+
+class Command(BaseCommand):
+
+    help = (
+        f"Recompute the last N months of {TABLE}, the fact table behind the "
+        "'Implementer Adoption' Grafana dashboard. One statement per month.\n"
+        "\n"
+        "  python manage.py refresh_adoption_metrics                 # last 13 months\n"
+        "  python manage.py refresh_adoption_metrics --months 24\n"
+        "  python manage.py refresh_adoption_metrics --dry-run       # compute, print, write nothing\n"
+        "  python manage.py refresh_adoption_metrics --out /tmp/adoption.json\n"
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("--months", type=int, default=13,
+                            help="How many months back to (re)compute, current month included (default 13).")
+        parser.add_argument("--statement-timeout", type=int, default=900,
+                            help="Per-month statement timeout in seconds (default 900).")
+        parser.add_argument("--dry-run", action="store_true",
+                            help="Run the month queries and print row counts, but create/write nothing.")
+        parser.add_argument("--out", default=None,
+                            help="Also write the computed facts to this JSON file.")
+
+    def handle(self, *args, **options):
+        months = options["months"]
+        timeout_ms = options["statement_timeout"] * 1000
+        dry_run = options["dry_run"]
+        out_path = options["out"]
+
+        with connection.cursor() as c:
+            if dry_run:
+                c.execute("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY")
+            else:
+                for stmt in DDL:
+                    c.execute(stmt)
+            c.execute(SQL_BOUNDS, [months - 1])
+            bounds = c.fetchall()
+
+        self.stdout.write(f"{'DRY RUN - ' if dry_run else ''}{len(bounds)} month(s), "
+                          f"timeout {options['statement_timeout']}s per month")
+
+        collected = []
+        failed = []
+        t_all = time.time()
+        for month, lo, hi in bounds:
+            t0 = time.time()
+            try:
+                with transaction.atomic():
+                    with connection.cursor() as c:
+                        c.execute("SET LOCAL statement_timeout = %s", [timeout_ms])
+                        if dry_run:
+                            c.execute(SQL_FACTS, [month, lo, hi + 1])
+                            rows = c.fetchall()
+                            n = len(rows)
+                        else:
+                            c.execute(f"DELETE FROM {TABLE} WHERE month = %s", [month])
+                            c.execute(
+                                f"INSERT INTO {TABLE} (month, functional_part, tool_stem, company_id, models) "
+                                + SQL_FACTS, [month, lo, hi + 1])
+                            n = c.rowcount
+                            rows = []
+                            if out_path:
+                                c.execute(f"SELECT month, functional_part, tool_stem, company_id, models "
+                                          f"FROM {TABLE} WHERE month = %s", [month])
+                                rows = c.fetchall()
+                self.stdout.write(f"  {month}  tasks {lo}-{hi}  {n:5d} fact rows  {time.time() - t0:6.1f}s")
+                collected.extend(rows)
+            except Exception as err:  # timeout or SQL error: report and continue with the next month
+                failed.append(str(month))
+                self.stderr.write(f"  {month}  FAILED after {time.time() - t0:.1f}s: {str(err).splitlines()[0]}")
+
+        self.stdout.write(f"done in {time.time() - t_all:.0f}s"
+                          + (f", FAILED months: {', '.join(failed)}" if failed else ""))
+
+        if out_path:
+            with open(out_path, "w") as f:
+                json.dump([{"month": str(m), "functional_part": fp, "tool_stem": ts,
+                            "company_id": cid, "models": n}
+                           for m, fp, ts, cid, n in collected], f, indent=1)
+            self.stdout.write(f"written: {out_path} ({len(collected)} rows)")
+
+        if failed:
+            raise SystemExit(1)

backend/apps/ifc_validation/management/commands/top_failing_rules.py

@@ -0,0 +1,187 @@
+import json
+import os
+from datetime import datetime, timedelta
+from decimal import Decimal
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+SQL_WEEK = """\
+WITH base_models AS (
+  SELECT DISTINCT m.id
+  FROM ifc_model m
+  LEFT JOIN ifc_user_additional_info uai
+    ON uai.user_id = m.uploaded_by_id
+  WHERE
+    m.schema ILIKE %(schema)s
+    AND m.created >= %(from_date)s
+    AND m.created <  %(to_date)s
+    AND COALESCE(uai.is_vendor, FALSE) = FALSE
+),
+failing_features AS (
+  SELECT
+    SPLIT_PART(vo.feature, ' ', 1) AS rule_code,
+    COUNT(DISTINCT vr.model_id) AS models_failing
+  FROM ifc_validation_outcome vo
+  JOIN ifc_validation_task vt
+    ON vt.id = vo.validation_task_id
+  JOIN ifc_validation_request vr
+    ON vr.id = vt.request_id
+  WHERE
+    vr.model_id IN (SELECT id FROM base_models)
+    AND vo.severity = 4
+    AND vo.feature IS NOT NULL
+    AND SPLIT_PART(vo.feature, ' ', 1) ~ '^[A-Z]{3}[0-9]{3}$'
+  GROUP BY SPLIT_PART(vo.feature, ' ', 1)
+)
+SELECT
+  ff.rule_code,
+  ff.models_failing,
+  (SELECT COUNT(*) FROM base_models) AS total_models
+FROM failing_features ff
+ORDER BY ff.models_failing DESC;
+"""
+
+
+class Command(BaseCommand):
+
+    help = (
+        'Compute top failing validation rules per schema, '
+        'batched in weekly windows to avoid overloading the database.\n'
+        '\n'
+        'Examples:\n'
+        '\n'
+        '  # Default: IFC4X3, weekly batches, top 20, from 2024-07-01 to today\n'
+        '  docker compose exec backend python manage.py top_failing_rules\n'
+        '\n'
+        '  # Custom date range\n'
+        '  docker compose exec backend python manage.py top_failing_rules --start 2025-01-01 --end 2026-01-01\n'
+        '\n'
+        '  # Different schema\n'
+        '  docker compose exec backend python manage.py top_failing_rules --schema \'%%IFC2X3%%\'\n'
+        '\n'
+        '  # Larger batch windows if it\'s slow\n'
+        '  docker compose exec backend python manage.py top_failing_rules --window 14\n'
+        '\n'
+        '  # Save results to a JSON file\n'
+        '  docker compose exec backend python manage.py top_failing_rules --out /tmp/results.json\n'
+    )
+
+    def add_arguments(self, parser):
+
+        parser.add_argument(
+            '--schema',
+            type=str,
+            default='%IFC4X3%',
+            help='Schema filter (SQL ILIKE pattern). Default: %%IFC4X3%%',
+        )
+        parser.add_argument(
+            '--start',
+            type=str,
+            default='2024-07-01',
+            help='Start date (YYYY-MM-DD). Default: 2024-07-01',
+        )
+        parser.add_argument(
+            '--end',
+            type=str,
+            default=None,
+            help='End date exclusive (YYYY-MM-DD). Default: today.',
+        )
+        parser.add_argument(
+            '--window',
+            type=int,
+            default=7,
+            help='Batch window size in days. Default: 7',
+        )
+        parser.add_argument(
+            '--top',
+            type=int,
+            default=20,
+            help='Number of top rules to show. Default: 20',
+        )
+        parser.add_argument(
+            '--out',
+            type=str,
+            default=None,
+            help='Output JSON file path (optional). If not set, prints to stdout.',
+        )
+
+    def handle(self, *args, **options):
+        schema = options['schema']
+        start = datetime.strptime(options['start'], '%Y-%m-%d').date()
+        end = (
+            datetime.strptime(options['end'], '%Y-%m-%d').date()
+            if options['end']
+            else datetime.now().date()
+        )
+        window = timedelta(days=options['window'])
+        top_n = options['top']
+        out_path = options['out']
+
+        rule_failing = {}   # rule_code -> total models failing
+        total_models = 0
+        weeks_processed = 0
+
+        current = start
+        while current < end:
+            next_date = min(current + window, end)
+
+            self.stdout.write(f"  {current} -> {next_date} ...", ending="")
+
+            with connection.cursor() as cursor:
+                cursor.execute(SQL_WEEK, {
+                    'schema': schema,
+                    'from_date': current.isoformat(),
+                    'to_date': next_date.isoformat(),
+                })
+                rows = cursor.fetchall()
+
+            week_total = 0
+            for rule_code, models_failing, week_models in rows:
+                rule_failing[rule_code] = rule_failing.get(rule_code, 0) + models_failing
+                week_total = max(week_total, week_models)
+
+            total_models += week_total
+            weeks_processed += 1
+            self.stdout.write(f" {week_total} models, {len(rows)} rules")
+
+            current = next_date
+
+        # Build ranked result
+        ranked = []
+        for code, failing in sorted(rule_failing.items(), key=lambda x: -x[1]):
+            pct = float(round(
+                Decimal(100) * Decimal(failing) / Decimal(max(total_models, 1)),
+                1
+            ))
+            ranked.append({
+                'rule_code': code,
+                'models_failing': failing,
+                'total_models': total_models,
+                'failure_rate_pct': pct,
+            })
+
+        ranked = ranked[:top_n]
+
+        self.stdout.write("")
+        self.stdout.write(
+            f"Processed {weeks_processed} windows, "
+            f"{total_models} total models, "
+            f"{len(rule_failing)} distinct failing rules."
+        )
+        self.stdout.write("")
+
+        # Table output
+        self.stdout.write(f"{'#':<4} {'Rule':<10} {'Failing':>8} {'Total':>8} {'Rate':>8}")
+        self.stdout.write("-" * 42)
+        for i, r in enumerate(ranked, 1):
+            self.stdout.write(
+                f"{i:<4} {r['rule_code']:<10} {r['models_failing']:>8} "
+                f"{r['total_models']:>8} {r['failure_rate_pct']:>7.1f}%"
+            )
+
+        if out_path:
+            with open(out_path, 'w', encoding='utf-8') as f:
+                json.dump(ranked, f, ensure_ascii=False, indent=2)
+            self.stdout.write(self.style.SUCCESS(f"\nWrote {out_path}"))