name: Nightly Backend Perf Regression

on:
  schedule:
    - cron: '0 2 * * *'

  release:
    types: [published]

  workflow_dispatch:
    inputs:
      fork_filter:
        description: 'Comma-separated fork_ids (blank = all tracked)'
        required: false
        default: ''
      model_filter:
        description: 'Comma-separated model names (blank = default set)'
        required: false
        default: ''

# Only contents:write is needed — the report job pushes results to the
# benchmark-data branch. No issues/PRs are created (regressions are tracked in
# benchmark-data/regressions.json for the dashboard, not via GitHub issues).
permissions:
  contents: write

# Never let two runs overlap: parallel benches on the same box would share the
# GPU and corrupt perf numbers, and both would race to push benchmark history.
concurrency:
  group: benchmark-regression
  cancel-in-progress: false

env:
  LEMONADE_CI_MODE: "True"
  PYTHONIOENCODING: utf-8
  HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  # Branch that accumulates nightly results + leaderboard. This is what makes
  # day-over-day regression detection possible at all.
  HISTORY_BRANCH: benchmark-data
  # Empty = benchmark the full "hot" llama.cpp model set from the registry
  # (validate_backend_bench.py fetches it live from /api/v1/models). Override per
  # run via the workflow_dispatch model_filter input.
  # NOTE: gpt-oss-20b-mxfp4-GGUF is known to segfault llama-server (exit 139 —
  # its checkpoint resolves to the eagle3 draft); it's excluded from the hot set
  # or will simply be skipped (its crash no longer fails other models).
  DEFAULT_MODEL_FILTER: ""

jobs:
  # =========================================================================
  # Setup — build the fork matrix dynamically from src/cpp/resources/benchmark_forks.json.
  # Single source of truth: adding a fork (llamacpp fork, vllm fork, ...) to
  # the JSON is enough; no workflow edit needed. Honors the fork_filter
  # workflow_dispatch input.
  # =========================================================================
  setup:
    name: Build fork matrix
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      forks: ${{ steps.matrix.outputs.forks }}
    steps:
      - uses: actions/checkout@v5

      - name: Read tracked forks
        id: matrix
        env:
          FORK_FILTER: ${{ inputs.fork_filter }}
        run: |
          python3 - << 'PY'
          import json, os, sys
          from pathlib import Path
          forks = json.loads(Path("src/cpp/resources/benchmark_forks.json").read_text())["forks"]
          # tracked:true forks; optional per-fork opt-out via {"enabled": false}
          ids = [f["fork_id"] for f in forks
                 if f.get("tracked", True) and f.get("enabled", True)]
          flt = [s.strip() for s in os.environ.get("FORK_FILTER", "").split(",") if s.strip()]
          if flt:
            unknown = [s for s in flt if s not in ids]
            if unknown:
              print(f"ERROR: fork_filter entries not in benchmark_forks.json: {unknown}")
              sys.exit(1)
            ids = [i for i in ids if i in flt]
          if not ids:
            print("ERROR: no forks selected")
            sys.exit(1)
          print(f"Benching forks: {ids}")
          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
            f.write(f"forks={json.dumps(ids)}\n")
          PY

  # =========================================================================
  # Get lemonade — download embeddable release package, no build needed.
  # Uses curl + GitHub API only (no gh CLI, no sudo, no cmake).
  # =========================================================================
  # Bench — one job per fork.
  # Runner: [self-hosted, stx-halo, Linux, lemon-prod]
  # max-parallel: 1 is load-bearing: benches must run one at a time on the
  # shared box or the perf numbers are meaningless.
  # =========================================================================
  bench:
    name: Bench ${{ matrix.fork_id }}
    needs: [setup]
    runs-on: [self-hosted, stx-halo, Linux, lemon-prod]
    timeout-minutes: 180
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        fork_id: ${{ fromJSON(needs.setup.outputs.forks) }}

    steps:
      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Download lemonade embeddable
        run: |
          set -euo pipefail
          RELEASE=$(curl -sf \
            -H "Authorization: Bearer $GH_TOKEN" \
            -H "Accept: application/vnd.github+json" \
            "https://api.github.com/repos/lemonade-sdk/lemonade/releases/latest")
          VERSION=$(echo "$RELEASE" | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])")
          echo "Latest lemonade: $VERSION"
          ASSET_URL=$(echo "$RELEASE" | python3 -c "import sys,json; assets=json.load(sys.stdin)['assets']; url=next((a['browser_download_url'] for a in assets if 'embeddable' in a['name'] and 'ubuntu' in a['name'] and 'x64' in a['name']),None); print(url or 'NOT_FOUND')")
          if [ "$ASSET_URL" = "NOT_FOUND" ]; then
            echo "ERROR: no ubuntu embeddable asset found. Available assets:"
            echo "$RELEASE" | python3 -c "import sys,json; [print(a['name']) for a in json.load(sys.stdin)['assets']]"
            exit 1
          fi
          echo "Downloading: $ASSET_URL"
          mkdir -p build
          curl -sfL "$ASSET_URL" | tar -xz --strip-components=1 -C build/
          test -f build/lemond   || { echo "ERROR: lemond missing from embeddable package"; ls -la build/; exit 1; }
          test -f build/lemonade || { echo "ERROR: lemonade missing from embeddable package"; ls -la build/; exit 1; }
          test -d build/resources || { echo "ERROR: resources/ missing — embeddable package incomplete"; ls -la build/; exit 1; }
          chmod +x build/lemond build/lemonade
          build/lemond --version
          echo "Lemonade $VERSION ready"
          echo "LEMONADE_VERSION=$VERSION" >> "$GITHUB_ENV"

      # validate_backend_bench.py is pure stdlib (no pip deps), so plain
      # setup-python + system python3 is enough — no venv/requirements needed.
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      # -------------------------------------------------------------------
      # Restore this fork's benchmark history from the benchmark-data branch
      # so validate_backend_bench.py has previous runs to compare against.
      # validate_backend_bench.py treats existing run-*.json files in --output/<fork>/
      # <model>/ as prior runs (find_previous_result), so restore into that
      # same layout.
      # -------------------------------------------------------------------
      - name: Restore benchmark history
        env:
          FORK_ID: ${{ matrix.fork_id }}
        run: |
          set -e
          mkdir -p ci/results
          # Read this fork's prior run-*.json from the benchmark-data branch
          # (git archive extracts the tree without checking it out / disturbing
          # the workspace). validate_backend_bench.py compares against these.
          if git fetch origin "$HISTORY_BRANCH" 2>/dev/null; then
            mkdir -p .bench-history
            git archive FETCH_HEAD | tar -x -C .bench-history
            if [ -d ".bench-history/results/$FORK_ID" ]; then
              cp -r ".bench-history/results/$FORK_ID" ci/results/
              echo "Restored $(find "ci/results/$FORK_ID" -name 'run-*.json' | wc -l) prior run(s) for $FORK_ID"
            else
              echo "No history for $FORK_ID yet (first run for this fork)"
            fi
          else
            echo "No $HISTORY_BRANCH branch yet (first run overall)"
          fi

      - name: Download fork binary
        env:
          FORK_ID: ${{ matrix.fork_id }}
        run: |
          set -e
          # Download (or reuse cached) the fork's prebuilt binary into
          # ../bench-binaries/<fork_id>/. No lemond needed for this step.
          python3 .github/scripts/validate_backend_bench.py \
            --forks        src/cpp/resources/benchmark_forks.json \
            --output       ci/results \
            --binaries-dir "${{ github.workspace }}/../bench-binaries" \
            --fork-filter  "$FORK_ID" \
            --download-only

          # Resolve the config target: the real llamacpp backend key + the
          # FULL PATH to the llama-server executable. This lemond build execs
          # the llamacpp.<key>_bin value VERBATIM — a directory value produces
          # EISDIR ("Failed to execute") and /load 500. It must be the file.
          # (lemond derives LD_LIBRARY_PATH from dirname(bin), so the file path
          # also puts the fork's bundled .so dir on the library path for free.)
          # "rocm-nightly"/"rocm-gfx11" are NOT backend keys — forks map to a
          # real key ("rocm","vulkan",...) via install_as, plus rocm_channel.
          python3 - << 'PY'
          import json, os, sys
          from pathlib import Path
          forks = json.loads(Path("src/cpp/resources/benchmark_forks.json").read_text())["forks"]
          fork_id = os.environ["FORK_ID"]
          fork = next((f for f in forks if f["fork_id"] == fork_id), None)
          if not fork:
            print(f"ERROR: fork {fork_id} not found"); sys.exit(1)
          cfg_backend  = fork.get("install_as", fork["backend"]).lower()
          rocm_channel = fork.get("rocm_channel", "")
          exe          = fork.get("binary_exe_linux", "llama-server")
          bindir       = Path(os.environ["GITHUB_WORKSPACE"]) / ".." / "bench-binaries" / fork_id
          binary       = next(bindir.rglob(exe), None)
          if not binary:
            print(f"ERROR: {exe} not found under {bindir}"); sys.exit(1)
          binary = binary.resolve()
          print(f"Config target: llamacpp.{cfg_backend}_bin = {binary}")
          with open(os.environ["GITHUB_ENV"], "a") as f:
            f.write(f"BENCH_BACKEND_KEY={cfg_backend}\n")
            f.write(f"BENCH_BIN_FILE={binary}\n")
            f.write(f"BENCH_BIN_DIR={binary.parent}\n")
            f.write(f"BENCH_ROCM_CHANNEL={rocm_channel}\n")
          PY
          echo "Fork binary ready"

      - name: Run validation with lemond
        env:
          HOME: ${{ github.workspace }}/.home
          MODEL_FILTER_INPUT: ${{ inputs.model_filter }}
        run: |
          set -e
          mkdir -p "$HOME"
          echo "HOME=$HOME" >> "$GITHUB_ENV"
          FORK_ID="${{ matrix.fork_id }}"

          # workflow_dispatch model_filter input wins; otherwise DEFAULT_MODEL_FILTER
          # (empty = full hot-model set fetched live from the registry)
          MODEL_FILTER="${MODEL_FILTER_INPUT:-$DEFAULT_MODEL_FILTER}"

          # Persistent caches OUTSIDE the workspace (checkout clean:true wipes
          # the workspace, RUNNER_TEMP is wiped between jobs). Without this the
          # 35B model re-downloads every night. Shared across forks — model
          # files are identical and max-parallel:1 prevents races.
          BENCH_ROOT="$(cd "${{ github.workspace }}/.." && pwd)"
          CACHE_DIR="$BENCH_ROOT/lemond-bench-cache"
          export HF_HOME="$BENCH_ROOT/hf-cache"
          mkdir -p "$CACHE_DIR" "$HF_HOME"

          LOGS_DIR="$(pwd)/server-logs-$FORK_ID"
          mkdir -p "$LOGS_DIR"
          mkdir -p "$RUNNER_TEMP/xdg-runtime"
          chmod 700 "$RUNNER_TEMP/xdg-runtime"
          export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"

          pick_free_port() {
            if command -v python3 >/dev/null 2>&1; then
              python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' && return 0
            fi
            for _ in $(seq 1 50); do
              local p=$(( (RANDOM % 20000) + 20000 ))
              if ! ss -ltn 2>/dev/null | grep -q ":$p "; then echo "$p"; return 0; fi
            done
            return 1
          }

          LEMOND_PID=""
          started=0
          for start_attempt in $(seq 1 5); do
            PORT=$(pick_free_port) || { echo "ERROR: could not find a free port"; exit 1; }
            echo "Attempt $start_attempt: starting lemond on 127.0.0.1:$PORT..."
            build/lemond "$CACHE_DIR" --port "$PORT" --host 127.0.0.1 \
              > "$LOGS_DIR/lemond.stdout.log" 2> "$LOGS_DIR/lemond.stderr.log" &
            LEMOND_PID=$!
            for i in $(seq 1 30); do
              if curl -sf "http://127.0.0.1:$PORT/api/v1/health" > /dev/null 2>&1; then
                echo "lemond is healthy on port $PORT (PID $LEMOND_PID)"
                started=1
                break
              fi
              if ! kill -0 "$LEMOND_PID" 2>/dev/null; then
                echo "lemond (PID $LEMOND_PID) exited prematurely on port $PORT"
                cat "$LOGS_DIR/lemond.stderr.log" 2>/dev/null || true
                break
              fi
              sleep 2
            done
            [ "$started" = "1" ] && break
            echo "Start attempt $start_attempt failed; retrying on a new port..."
            kill -9 "$LEMOND_PID" 2>/dev/null || true
            sleep 1
          done
          if [ "$started" != "1" ]; then
            echo "ERROR: lemond failed to start after multiple attempts"
            cat "$LOGS_DIR/lemond.stderr.log" 2>/dev/null || true
            exit 1
          fi

          LEMONADE="$(pwd)/build/lemonade"

          # Wait until lemond is responsive to a config read (small helper).
          wait_lemond() {
            for _ in $(seq 1 30); do
              if curl -sf "http://127.0.0.1:$PORT/api/v1/health" >/dev/null 2>&1; then return 0; fi
              sleep 1
            done
            return 1
          }

          # ---------------------------------------------------------------
          # Route the backend to the fork binary. CRITICAL: this lemond build
          # execs the llamacpp.<key>_bin value VERBATIM, so it must be the FULL
          # PATH TO THE llama-server FILE — a directory value produces EISDIR
          # ("Failed to execute" -> exit 1 -> /load 500). Do NOT set a dir.
          # (Detection alone can't catch this: system-info reports "installed"
          # for both a dir and a file, since it only checks the path exists.)
          #
          # Set rocm_channel AND the *_bin path in ONE `config set` call. Setting
          # *_bin applies live (lemond reloads the backend); doing channel then
          # bin as two back-to-back calls left lemond mid-reload and the second
          # call hit a dropped connection — the intermittent rocm-stable failure.
          # One atomic call = one reload; then wait for health + retry once.
          # `config set` takes multiple key=value pairs.
          # ---------------------------------------------------------------
          KEY="llamacpp.${BENCH_BACKEND_KEY}_bin"
          SET_ARGS=("$KEY=$BENCH_BIN_FILE")
          if [ -n "$BENCH_ROCM_CHANNEL" ]; then
            # rocm_channel is a TOP-LEVEL config key (not nested under llamacpp)
            SET_ARGS+=("rocm_channel=$BENCH_ROCM_CHANNEL")
          fi
          echo "Routing (atomic): config set ${SET_ARGS[*]}"
          if ! "$LEMONADE" --port "$PORT" config set "${SET_ARGS[@]}"; then
            echo "config set dropped the connection (live reload); waiting for lemond to settle and retrying..."
            wait_lemond || { echo "ERROR: lemond did not become healthy after config reload"; exit 1; }
            "$LEMONADE" --port "$PORT" config set "${SET_ARGS[@]}" \
              || { echo "ERROR: config set retry failed"; exit 1; }
          fi
          wait_lemond || { echo "ERROR: lemond not healthy after routing config"; exit 1; }

          # Confirm detection against /api/v1/system-info — the exact source
          # `lemonade bench` reads (discover_backends filters on state=
          # installed/update_*). Catches "No suitable backends found" up front.
          if curl -sf "http://127.0.0.1:$PORT/api/v1/system-info" 2>/dev/null \
              | BACKEND="$BENCH_BACKEND_KEY" python3 -c '
          import json, os, sys
          key = os.environ["BACKEND"]
          info = json.load(sys.stdin)
          b = info.get("recipes", {}).get("llamacpp", {}).get("backends", {}).get(key, {})
          state = b.get("state", "missing")
          print(f"  llamacpp.{key} state = {state}")
          sys.exit(0 if state in ("installed", "update_required", "update_available") else 1)
          '; then
            echo "Detection confirmed: bench will see --backend $BENCH_BACKEND_KEY (routed to $BENCH_BIN_FILE)"
          else
            echo "ERROR: after setting $KEY, system-info never reported '$BENCH_BACKEND_KEY' installed."
            echo "lemonade bench would fail with 'No suitable backends found'. Aborting before wasting a bench run."
            echo "--- system-info llamacpp backends ---"
            curl -sf "http://127.0.0.1:$PORT/api/v1/system-info" 2>/dev/null \
              | python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin).get("recipes",{}).get("llamacpp",{}).get("backends",{}), indent=2))' || true
            exit 1
          fi

          # ---------------------------------------------------------------
          # SMOKE TEST the fork binary directly, BEFORE the bench, so a broken
          # binary (missing shared libs, bad HIP init, wrong arch) surfaces its
          # REAL error here instead of as an opaque "/load 500" later. lemond
          # launches this same binary from its own dir with that dir on
          # LD_LIBRARY_PATH (llamacpp_server.cpp), so replicate that here.
          # ---------------------------------------------------------------
          FORK_EXE="$BENCH_BIN_FILE"
          echo "=== fork binary smoke test: $FORK_EXE ==="
          ls -l "$FORK_EXE" || true
          echo "--- ldd (missing shared libs show as 'not found') ---"
          LD_LIBRARY_PATH="$BENCH_BIN_DIR:$LD_LIBRARY_PATH" ldd "$FORK_EXE" 2>&1 | grep -iE "not found|=>" | head -n 40 || true
          echo "--- llama-server --version ---"
          SMOKE_RC=0
          LD_LIBRARY_PATH="$BENCH_BIN_DIR:$LD_LIBRARY_PATH" "$FORK_EXE" --version \
            > "$LOGS_DIR/fork-binary-version.txt" 2>&1 || SMOKE_RC=$?
          cat "$LOGS_DIR/fork-binary-version.txt" || true
          if [ "$SMOKE_RC" != "0" ]; then
            echo "ERROR: fork binary failed to run (exit $SMOKE_RC). This is the true cause of the /load 500."
            echo "See ldd output above for missing libs; the fork archive likely does not bundle its ROCm runtime."
            kill "$LEMOND_PID" 2>/dev/null || true
            exit 1
          fi
          echo "Smoke test passed — binary is runnable"

          # ---------------------------------------------------------------
          # Pull models FIRST, before the bench run, so the download is a
          # distinct, visible stage (not buried inside --auto-pull mid-bench)
          # and shared across every fork via the persistent cache. Idempotent:
          # a no-op once the weights are on disk.
          # ---------------------------------------------------------------
          if [ -n "$MODEL_FILTER" ]; then
            IFS=',' read -ra MODELS <<< "$MODEL_FILTER"
            for m in "${MODELS[@]}"; do
              m="$(echo "$m" | xargs)"   # trim
              [ -z "$m" ] && continue
              echo "Pulling model $m ..."
              "$LEMONADE" --port "$PORT" pull "$m" && echo "$m ready" \
                || echo "WARN: pull $m returned non-zero (bench --auto-pull will retry)"
            done
          fi

          # snapshot pre-existing (restored-history) result files so that, if
          # binary verification fails below, the unverified FRESH results can
          # be purged and never reach the results artifact / history branch
          find ci/results -name 'run-*.json' 2>/dev/null | sort > /tmp/pre-bench-files || true

          EXIT_CODE=0
          BENCH_ARGS=(
            .github/scripts/validate_backend_bench.py
            --forks        src/cpp/resources/benchmark_forks.json
            --output       ci/results
            --binaries-dir "${{ github.workspace }}/../bench-binaries"
            --fork-filter  "$FORK_ID"
            --port         "$PORT"
          )
          if [ -n "$MODEL_FILTER" ]; then
            BENCH_ARGS+=(--model-filter "$MODEL_FILTER")
          fi
          LEMONADE_EXE="$LEMONADE" python3 "${BENCH_ARGS[@]}" || EXIT_CODE=$?

          # On failure, dump lemond's stderr INLINE so the real llama-server
          # startup error (the cause behind an opaque "/load 500") is visible in
          # the job log without downloading the artifact. CI runs at debug level
          # (LEMONADE_CI_MODE), so llama-server's own output is inherited here.
          if [ "$EXIT_CODE" != "0" ]; then
            echo "=================== lemond.stderr.log (tail) ==================="
            tail -n 120 "$LOGS_DIR/lemond.stderr.log" 2>/dev/null || echo "(no stderr log)"
            echo "=================== lemond.stdout.log (tail) ==================="
            tail -n 60 "$LOGS_DIR/lemond.stdout.log" 2>/dev/null || echo "(no stdout log)"
            echo "==============================================================="
          fi

          curl -sf -X POST "http://127.0.0.1:$PORT/internal/shutdown" || true

          # ---------------------------------------------------------------
          # VERIFY the fork binary actually SERVED the bench — but per-model,
          # so one crashing model does NOT discard the other models' good data.
          #
          # Guard against the fatal case only: the fork binary was NEVER the
          # one that ran (silent fallback to the builtin => WRONG-build numbers).
          # A single model crashing at load (exit 139/EISDIR/missing op) is
          # expected for some checkpoints; those runs simply have no valid
          # scenarios and are skipped by the aggregator. We keep the good ones.
          # ---------------------------------------------------------------
          purge_fresh_results() {
            find ci/results -name 'run-*.json' 2>/dev/null | sort > /tmp/post-bench-files || true
            comm -13 /tmp/pre-bench-files /tmp/post-bench-files | xargs -r rm --
          }

          # (A) Did the fork binary launch at all? If its FILE path never appears
          # in the logs, lemond fell back to the builtin — that IS fatal.
          if ! grep -rqs "$BENCH_BIN_FILE" "$LOGS_DIR"; then
            echo "ERROR: no evidence in lemond logs that $BENCH_BIN_FILE was launched."
            echo "Bench likely ran on the builtin binary — refusing to publish these numbers."
            grep -ri "llama-server" "$LOGS_DIR" | head -n 20 || true
            purge_fresh_results
            exit 1
          fi
          echo "Verified: lemond launched the fork binary at $BENCH_BIN_FILE"

          # (B) Did at least one model produce a valid TPS number? If EVERY model
          # crashed, there is nothing worth publishing — fail so it's visible.
          VALID=$(find ci/results -name 'run-*.json' 2>/dev/null -newer /tmp/pre-bench-files -print 2>/dev/null | xargs -r grep -l '"tps"' 2>/dev/null | wc -l || echo 0)
          # (fallback count: any fresh file containing a tps block)
          if [ "${VALID:-0}" = "0" ]; then
            FRESH=$(comm -13 /tmp/pre-bench-files <(find ci/results -name 'run-*.json' 2>/dev/null | sort) 2>/dev/null || true)
            VALID=$(printf '%s\n' "$FRESH" | xargs -r grep -l '"tps"' 2>/dev/null | wc -l || echo 0)
          fi
          if [ "${VALID:-0}" = "0" ]; then
            echo "ERROR: no model produced a valid result (all loads failed)."
            grep -rniE "terminated with exit code|Failed to execute|llama-server failed to start" "$LOGS_DIR" | head -n 20 || true
            purge_fresh_results
            exit 1
          fi

          # Surface any per-model crashes as a NON-fatal note (data still kept).
          if grep -rqsE "terminated with exit code|llama-server failed to start" "$LOGS_DIR"; then
            echo "NOTE: some model(s) crashed at load and were skipped; other models' results are kept."
            grep -rniE "terminated with exit code" "$LOGS_DIR" | head -n 10 || true
          fi

          # Exit 0 as long as at least one model produced data. Per-model bench
          # failures already surfaced above and are excluded by the aggregator.
          exit 0

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: bench-results-${{ matrix.fork_id }}-${{ github.run_id }}
          path: ci/results/
          retention-days: 90
          if-no-files-found: warn

      - name: Upload lemond logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: lemond-logs-${{ matrix.fork_id }}-${{ github.run_id }}
          path: server-logs-${{ matrix.fork_id }}/
          retention-days: 14
          if-no-files-found: ignore

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-linux

      - name: Cleanup fork binaries processes
        if: always()
        run: |
          pkill -9 -f "bench-binaries/${{ matrix.fork_id }}" 2>/dev/null || true

  # =========================================================================
  # Report — merges results into the benchmark-data branch, regenerates the
  # leaderboard from full history, files/updates an issue on critical
  # regression.
  # =========================================================================
  report:
    name: Aggregate and report
    needs: bench
    if: always() && needs.bench.result != 'skipped'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v5

      # Each artifact lands in its own subdirectory (artifacts/bench-results-
      # <fork>-<runid>/). Deliberately NOT merge-multiple: forks may write
      # identically-named files (e.g. regression_report.json) and merging
      # would silently overwrite all but one.
      - name: Download all bench results
        uses: actions/download-artifact@v7
        with:
          pattern: bench-results-*
          path: artifacts

      - name: Merge results into history branch
        run: |
          set -euo pipefail
          git config user.name  "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          if ! git fetch origin "$HISTORY_BRANCH:$HISTORY_BRANCH" 2>/dev/null; then
            echo "Creating $HISTORY_BRANCH (root commit)"
            git branch "$HISTORY_BRANCH" \
              "$(git commit-tree "$(git hash-object -t tree /dev/null)" -m 'init benchmark history')"
          fi
          git worktree add bench-history "$HISTORY_BRANCH"
          RUN_ID="${{ github.run_id }}"
          shopt -s nullglob
          for d in artifacts/bench-results-*; do
            name="$(basename "$d")"
            fork="${name#bench-results-}"
            fork="${fork%-$RUN_ID}"
            mkdir -p "bench-history/results/$fork"
            # results are nested under <fork>/<model>/run-*.json — preserve that
            find "$d" -name 'run-*.json' ! -name '*regression*' -print0 \
              | while IFS= read -r -d '' f; do
                  rel="${f#"$d"/}"           # e.g. ci/results/<fork>/<model>/run-X.json or <fork>/<model>/run-X.json
                  rel="${rel#ci/results/}"
                  rel="${rel#"$fork"/}"      # strip leading <fork>/ so we can re-root
                  dest="bench-history/results/$fork/$rel"
                  mkdir -p "$(dirname "$dest")"
                  cp "$f" "$dest"
                done
            echo "$fork: $(find "bench-history/results/$fork" -name 'run-*.json' 2>/dev/null | wc -l) run file(s) in history"
          done

      - name: Aggregate leaderboard (full history)
        run: |
          python3 - << 'PY'
          import json, os
          from pathlib import Path
          from datetime import datetime, timezone
          # Full history, so the leaderboard keeps entries for forks that were
          # filtered out of (or failed in) today's run. Sorted iteration +
          # per-fork replacement means the newest run per fork wins.
          results_dir = Path("bench-history/results")
          leaderboard = {}
          lemonade_version = None
          for run_file in sorted(results_dir.rglob("run-*.json")):
            if "regression" in run_file.name:
              continue
            try:
              data = json.loads(run_file.read_text())
            except Exception:
              continue
            fork_id = data.get("fork_id", "unknown")
            version = data.get("fork_version", "unknown")
            if lemonade_version is None and data.get("lemonade_version"):
              lemonade_version = data["lemonade_version"]
            # The bench writes a top-level "models" array (one entry per model),
            # each with its own model name + results[]. Fall back to the flat
            # top-level shape for any legacy files.
            model_entries = data.get("models")
            if not isinstance(model_entries, list):
              model_entries = [data]
            for me in model_entries:
              model = me.get("model", data.get("model", "unknown"))
              for br in me.get("results", []):
                for sc in br.get("scenarios", []):
                  tps = sc.get("tps", {}).get("mean")
                  if tps is None:
                    continue
                  key = f"{model}|{sc['name']}"
                  entry = {
                    "model": model, "scenario": sc["name"],
                    "fork_id": fork_id, "fork_label": data.get("fork_label", fork_id),
                    "fork_repo": data.get("fork_repo", ""), "fork_version": version,
                    "backend": br.get("backend"),
                    "ctx_size": br.get("ctx_size"),
                    "tps_mean": round(tps, 2),
                    "tps_p95": round(sc.get("tps", {}).get("p95", 0), 2),
                    "ttft_ms_mean": round(sc.get("ttft_ms", {}).get("mean", 0), 2),
                    "input_tokens": sc.get("input_tokens"),
                    "vram_peak_gb": sc.get("vram_peak_gb"),
                    "timestamp": me.get("timestamp", data.get("timestamp")),
                    # provenance: which CI run produced this number
                    "run_id": data.get("run_id"),
                    "run_url": data.get("run_url"),
                    "commit": data.get("commit"),
                  }
                  leaderboard.setdefault(key, [])
                  leaderboard[key] = [e for e in leaderboard[key] if e["fork_id"] != fork_id]
                  leaderboard[key].append(entry)
                  leaderboard[key].sort(key=lambda e: e["tps_mean"], reverse=True)

          # Per-fork build-history series (for the trend/regression chart):
          # model -> scenario -> fork_id -> [ {version, timestamp, tps, run_url} ]
          history = {}
          for run_file in sorted(results_dir.rglob("run-*.json")):
            if "regression" in run_file.name:
              continue
            try:
              data = json.loads(run_file.read_text())
            except Exception:
              continue
            fid = data.get("fork_id", "unknown")
            ver = data.get("fork_version", "unknown")
            model_entries = data.get("models")
            if not isinstance(model_entries, list):
              model_entries = [data]
            for me in model_entries:
              mdl = me.get("model", data.get("model", "unknown"))
              for br in me.get("results", []):
                for sc in br.get("scenarios", []):
                  tps = sc.get("tps", {}).get("mean")
                  if tps is None:
                    continue
                  h = history.setdefault(mdl, {}).setdefault(sc["name"], {}).setdefault(fid, [])
                  h.append({
                    "version": ver, "timestamp": me.get("timestamp", data.get("timestamp")),
                    "tps_mean": round(tps, 2),
                    "ttft_ms_mean": round(sc.get("ttft_ms", {}).get("mean", 0), 2),
                    "input_tokens": sc.get("input_tokens"),
                    "run_id": data.get("run_id"), "run_url": data.get("run_url"),
                  })
          # sort each series chronologically and keep the last 30 points
          for mdl in history:
            for scn in history[mdl]:
              for fid in history[mdl][scn]:
                pts = sorted(history[mdl][scn][fid], key=lambda p: p.get("timestamp") or "")
                history[mdl][scn][fid] = pts[-30:]

          server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
          repo   = os.environ.get("GITHUB_REPOSITORY", "")
          out = {
            "generated_at": datetime.now(timezone.utc).isoformat(),
            "run_id": os.environ.get("GITHUB_RUN_ID"),
            "run_url": f"{server}/{repo}/actions/runs/{os.environ.get('GITHUB_RUN_ID')}" if repo else None,
            "repo": repo,
            "repo_url": f"{server}/{repo}" if repo else None,
            "actions_url": f"{server}/{repo}/actions/workflows/benchmark-regression.yml" if repo else None,
            "commit": os.environ.get("GITHUB_SHA"),
            "lemonade_version": lemonade_version,
            "by_model_scenario": leaderboard,
            "history": history,
          }
          Path("bench-history/leaderboard.json").write_text(json.dumps(out, indent=2))
          Path("ci/results").mkdir(parents=True, exist_ok=True)
          Path("ci/results/leaderboard.json").write_text(json.dumps(out, indent=2))
          entries = sum(len(v) for v in leaderboard.values())
          print(f"Leaderboard: {entries} entries across {len(leaderboard)} model/scenario pairs")
          print(f"History: {sum(len(s) for m in history.values() for s in m.values())} series")
          PY
        env:
          GITHUB_SERVER_URL: ${{ github.server_url }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          GITHUB_SHA: ${{ github.sha }}

      # Record this run's regression alerts into the benchmark-data branch so
      # the dashboard can surface them. No GitHub issues are created (that path
      # was flaky). regressions.json is a rolling log: append this run's alerts,
      # keep the last 200. Never fails the job.
      - name: Record regressions into history branch
        run: |
          python3 - << 'PY'
          import json, os
          from pathlib import Path
          from datetime import datetime, timezone
          alerts = []
          for f in Path("artifacts").rglob("regression_report.json"):
            try:
              alerts += json.loads(f.read_text()).get("alerts", [])
            except Exception:
              pass
          run_id = os.environ.get("GITHUB_RUN_ID","")
          server = os.environ.get("GITHUB_SERVER_URL","https://github.com")
          repo   = os.environ.get("GITHUB_REPOSITORY","")
          now = datetime.now(timezone.utc).isoformat()
          run_url = f"{server}/{repo}/actions/runs/{run_id}" if repo and run_id else None
          for a in alerts:
            a.setdefault("timestamp", now)
            a.setdefault("run_id", run_id)
            a.setdefault("run_url", run_url)

          log_path = Path("bench-history/regressions.json")
          log = {"generated_at": now, "regressions": []}
          if log_path.exists():
            try: log = json.loads(log_path.read_text())
            except Exception: pass
          hist = log.get("regressions", [])
          hist.extend(alerts)
          log = {"generated_at": now, "regressions": hist[-200:]}
          log_path.write_text(json.dumps(log, indent=2))
          # mirror to ci/results so it's also in the run artifact
          Path("ci/results").mkdir(parents=True, exist_ok=True)
          Path("ci/results/regressions.json").write_text(json.dumps(log, indent=2))
          crit = [a for a in alerts if a.get("severity")=="CRITICAL"]
          print(f"Recorded {len(alerts)} alert(s) this run ({len(crit)} CRITICAL); "
                f"{len(hist[-200:])} kept in benchmark-data/regressions.json")
          PY
        env:
          GITHUB_SERVER_URL: ${{ github.server_url }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          GITHUB_RUN_ID: ${{ github.run_id }}

      - name: Push history branch
        run: |
          set -e
          git -C bench-history add -A
          if git -C bench-history commit -m "bench results + regressions (run ${{ github.run_id }})"; then
            git -C bench-history push origin "$HISTORY_BRANCH"
          else
            echo "No new results to commit"
          fi

      - name: Regression summary (non-failing)
        run: |
          python3 - << 'PY'
          import json
          from pathlib import Path
          crit = []
          for f in Path("artifacts").rglob("regression_report.json"):
            try:
              crit += [a for a in json.loads(f.read_text()).get("alerts", [])
                       if a.get("severity")=="CRITICAL"]
            except Exception:
              pass
          if not crit:
            print("No critical regressions this run.")
          else:
            print(f"{len(crit)} CRITICAL regression(s) recorded to benchmark-data/regressions.json:")
            for a in crit:
              print(f"  {a.get('fork_id')} / {a.get('model')} / {a.get('scenario')} "
                    f"{a.get('metric')}: {a.get('delta_pct')}%")
          # Intentionally do NOT fail the job — regressions are tracked in the
          # dashboard, not via red CI / GitHub issues.
          PY

      - name: Upload leaderboard
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: leaderboard-${{ github.run_id }}
          path: ci/results/leaderboard.json
          retention-days: 90
          if-no-files-found: warn

      - name: Cleanup
        if: always()
        run: |
          git worktree remove --force bench-history 2>/dev/null || true
          rm -rf artifacts ci/results bench-history
