name: Validate New llama.cpp Release

on:
  workflow_dispatch:
    inputs:
      lite:
        description: 'Run in lite mode (smallest model only)'
        type: boolean
        default: false
  pull_request:
  schedule:
    # Sunday at 12:00 PM ET (17:00 UTC, 16:00 UTC during EDT)
    - cron: "0 16 * * 0"

permissions:
  contents: write
  pull-requests: write

env:
  LEMONADE_CI_MODE: "True"
  PYTHONIOENCODING: utf-8
  LITE_MODE: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.lite == true) }}

jobs:
  # ========================================================================
  # Discover the latest llama.cpp release for each upstream repo
  # ========================================================================
  get-latest-releases:
    name: Get latest releases
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    outputs:
      llamacpp_release: ${{ steps.llamacpp.outputs.release }}
      llamacpp_rocm_release: ${{ steps.llamacpp_rocm.outputs.release }}
      llamacpp_rocm_preview_release: ${{ steps.llamacpp_rocm_preview.outputs.release }}
    steps:
      - name: Get latest llama.cpp release
        id: llamacpp
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          RELEASE=$(gh api repos/ggml-org/llama.cpp/releases/latest --jq '.tag_name')
          echo "Latest llama.cpp release: $RELEASE"
          echo "release=$RELEASE" >> "$GITHUB_OUTPUT"

      - name: Get latest llamacpp-rocm release
        id: llamacpp_rocm
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          RELEASE=$(gh api repos/lemonade-sdk/llamacpp-rocm/releases/latest --jq '.tag_name')
          echo "Latest llamacpp-rocm release: $RELEASE"
          echo "release=$RELEASE" >> "$GITHUB_OUTPUT"

      - name: Get latest llamacpp-rocm-preview release
        id: llamacpp_rocm_preview
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          RELEASE=$(gh api repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name')
          echo "Latest llamacpp-rocm-preview release: $RELEASE"
          echo "release=$RELEASE" >> "$GITHUB_OUTPUT"

  # ========================================================================
  # Build binaries with updated backend versions
  # ========================================================================
  build:
    name: Build
    needs: get-latest-releases
    if: always() && !failure() && !cancelled()
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Update backend_versions.json
        if: needs.get-latest-releases.result == 'success'
        shell: PowerShell
        env:
          LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
          LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
        run: |
          $ErrorActionPreference = "Stop"
          $path = "src/cpp/resources/backend_versions.json"
          $data = Get-Content $path -Raw | ConvertFrom-Json

          $release = $env:LLAMACPP_RELEASE
          Write-Host "Updating llamacpp vulkan/cpu/metal to $release" -ForegroundColor Cyan
          foreach ($key in @("vulkan", "cpu", "metal")) {
              $old = $data.llamacpp.$key
              $data.llamacpp.$key = $release
              Write-Host "  llamacpp.${key}: $old -> $release"
          }

          $rocm = $env:LLAMACPP_ROCM_RELEASE
          Write-Host "Updating llamacpp rocm-nightly to $rocm" -ForegroundColor Cyan
          $old = $data.llamacpp.'rocm-nightly'
          $data.llamacpp.'rocm-nightly' = $rocm
          Write-Host "  llamacpp.rocm-nightly: $old -> $rocm"

          $rocmStable = $env:LLAMACPP_RELEASE
          Write-Host "Updating llamacpp rocm-stable to $rocmStable" -ForegroundColor Cyan
          $old = $data.llamacpp.'rocm-stable'
          $data.llamacpp.'rocm-stable' = $rocmStable
          Write-Host "  llamacpp.rocm-stable: $old -> $rocmStable"

          $rocmPreview = $env:LLAMACPP_ROCM_PREVIEW_RELEASE
          Write-Host "Updating llamacpp rocm-preview to $rocmPreview" -ForegroundColor Cyan
          $old = $data.llamacpp.'rocm-preview'
          $data.llamacpp.'rocm-preview' = $rocmPreview
          Write-Host "  llamacpp.rocm-preview: $old -> $rocmPreview"

          $data | ConvertTo-Json -Depth 10 | Set-Content $path -Encoding UTF8

      - name: Build C++ Server with CMake
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          Write-Host "Building Lemonade server binaries..." -ForegroundColor Cyan
          if (Test-Path "build") { Remove-Item -Recurse -Force "build" }
          cmake --preset windows
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          cmake --build build --config Release
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          if (-not (Test-Path "build\Release\lemond.exe")) {
              Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red
              exit 1
          }
          Write-Host "Build successful!" -ForegroundColor Green

      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: llamacpp-build
          path: |
            build/Release/
            build/resources/
          retention-days: 7

  # ========================================================================
  # Validate backends on self-hosted runner
  # ========================================================================
  validate:
    name: Validate ${{ matrix.backend }}${{ matrix.channel && format(' ({0})', matrix.channel) || '' }}
    needs: [get-latest-releases, build]
    if: always() && needs.build.result == 'success'
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - backend: vulkan
            runner: [self-hosted, Windows, 128gb, vulkan]
          - backend: rocm
            channel: stable
            runner: [self-hosted, Windows, 128gb, rocm]
          - backend: rocm
            channel: preview
            runner: [self-hosted, Windows, 128gb, rocm]
          - backend: rocm
            channel: nightly
            runner: [self-hosted, Windows, 128gb, rocm]
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-windows

      - name: Download build artifacts
        uses: actions/download-artifact@v7
        with:
          name: llamacpp-build
          path: build

      - name: Verify binaries
        shell: PowerShell
        run: |
          $lemondExe = "build\Release\lemond.exe"
          if (-not (Test-Path $lemondExe)) {
              Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red
              Get-ChildItem -Recurse build | Select-Object FullName
              exit 1
          }
          & $lemondExe --version
          Write-Host "Binaries verified!" -ForegroundColor Green

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run validation with lemond
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path
          $backend = "${{ matrix.backend }}"
          $channel = "${{ matrix.channel }}"
          $label = if ($channel) { "$backend-$channel" } else { $backend }
          $logsDir = "server-logs-$label"
          $cacheDir = Join-Path $PWD "ci-cache-$label"
          $venvPython = ".\.venv\Scripts\python.exe"

          New-Item -ItemType Directory -Force -Path $logsDir | Out-Null
          New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null

          $stdoutLog = Join-Path $PWD "$logsDir\lemond.stdout.log"
          $stderrLog = Join-Path $PWD "$logsDir\lemond.stderr.log"

          # Start lemond (Python script handles readiness polling)
          $proc = Start-Process `
            -FilePath $lemondExe `
            -ArgumentList @($cacheDir, "--port", "13305", "--host", "127.0.0.1") `
            -RedirectStandardOutput $stdoutLog `
            -RedirectStandardError $stderrLog `
            -PassThru

          Write-Host "Started lemond PID $($proc.Id)" -ForegroundColor Cyan

          try {
            $validationArgs = @(
              "test/validate_llamacpp.py",
              "--backend", $backend,
              "--output", "llamacpp_validation_$label.json",
              "--logs-dir", $logsDir
            )
            if ($channel) {
              $validationArgs += "--channel"
              $validationArgs += $channel
            }
            if ("${{ env.LITE_MODE }}" -eq "true") {
              $validationArgs += "--lite"
            }

            & $venvPython @validationArgs
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          } finally {
            try {
              Invoke-WebRequest -Uri "http://127.0.0.1:13305/internal/shutdown" `
                -Method POST -TimeoutSec 10 | Out-Null
              Start-Sleep -Seconds 2
            } catch {
              Write-Host "lemond shutdown not reachable; relying on cleanup step." -ForegroundColor Yellow
            }
          }

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: validation-results-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}
          path: llamacpp_validation_${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}.json
          retention-days: 30

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}
          path: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}/
          retention-days: 30
          if-no-files-found: ignore

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-windows

  # ========================================================================
  # Create PR if both validations passed (schedule or workflow_dispatch only)
  # ========================================================================
  create-pr:
    name: Create update PR
    needs: [get-latest-releases, validate]
    runs-on: ubuntu-latest
    if: >-
      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') &&
      needs.validate.result == 'success'
    steps:
      - uses: actions/checkout@v5

      - name: Download validation results
        uses: actions/download-artifact@v7
        with:
          pattern: validation-results-*
          merge-multiple: true

      - name: Update backend_versions.json with both releases
        shell: bash
        env:
          LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
          LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
        run: |
          python3 -c "
          import json, os
          path = 'src/cpp/resources/backend_versions.json'
          with open(path, 'r') as f:
              data = json.load(f)
          llamacpp = os.environ['LLAMACPP_RELEASE']
          rocm_preview = os.environ['LLAMACPP_ROCM_PREVIEW_RELEASE']
          rocm_nightly = os.environ['LLAMACPP_ROCM_RELEASE']
          data['llamacpp']['vulkan'] = llamacpp
          data['llamacpp']['cpu'] = llamacpp
          data['llamacpp']['metal'] = llamacpp
          data['llamacpp']['rocm-stable'] = llamacpp
          data['llamacpp']['rocm-nightly'] = rocm_nightly
          data['llamacpp']['rocm-preview'] = rocm_preview
          with open(path, 'w') as f:
              json.dump(data, f, indent=2)
              f.write('\n')
          print(f'Updated: vulkan/cpu/metal/rocm-stable={llamacpp}, rocm-nightly={rocm_nightly}, rocm-preview={rocm_preview}')
          "

      - name: Generate PR body
        shell: bash
        env:
          LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
          LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
        run: |
          python3 << 'PYEOF'
          import json, os

          body_lines = []
          llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown")
          rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown")
          rocm_preview = os.environ.get("LLAMACPP_ROCM_PREVIEW_RELEASE", "unknown")

          body_lines.append("## Auto-update llama.cpp backends")
          body_lines.append("")
          body_lines.append(f"- **llama.cpp** (vulkan/cpu/metal/rocm-stable): `{llamacpp}`")
          body_lines.append(f"- **llamacpp-rocm** (rocm-nightly): `{rocm_nightly}`")
          body_lines.append(f"- **llamacpp-rocm-preview**: `{rocm_preview}`")
          body_lines.append("")
          body_lines.append("## Validation Results")
          body_lines.append("")
          body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |")
          body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|")

          for backend in ["vulkan", "rocm-stable", "rocm-preview", "rocm-nightly"]:
              # Labels match the artifact names produced by the validate job
              results_file = f"llamacpp_validation_{backend}.json"
              if not os.path.isfile(results_file):
                  body_lines.append(f"| _{backend}: no results_ | | | | | | |")
                  continue
              with open(results_file, "r") as f:
                  results = json.load(f)
              for r in results:
                  status = "PASS" if r["pass"] else "FAIL"
                  # Truncate response for table readability
                  resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ")
                  ttft = r.get("time_to_first_token", "N/A")
                  if isinstance(ttft, float):
                      ttft = f"{ttft:.3f}s"
                  tps = r.get("tokens_per_second", "N/A")
                  if isinstance(tps, float):
                      tps = f"{tps:.1f}"
                  body_lines.append(
                      f"| {r['model']} ({backend}) | {status} | {resp} | "
                      f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | "
                      f"{ttft} | {tps} |"
                  )

          body_lines.append("")
          body_lines.append("---")
          body_lines.append("*Auto-generated by validate_llamacpp workflow*")

          body = "\n".join(body_lines)
          with open("pr_body.md", "w") as f:
              f.write(body)
          PYEOF

      - name: Create Pull Request
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
          LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
        run: |
          BRANCH="auto/llamacpp-update-${LLAMACPP_RELEASE}-${LLAMACPP_ROCM_RELEASE}-${LLAMACPP_ROCM_PREVIEW_RELEASE}"

          # Check if a PR already exists for this update
          EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || echo "")
          if [ -n "$EXISTING_PR" ]; then
            echo "PR #$EXISTING_PR already exists for branch $BRANCH. Skipping."
            exit 0
          fi

          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Skip if backend_versions.json is already up to date
          if git diff --quiet src/cpp/resources/backend_versions.json; then
            echo "backend_versions.json is unchanged — nothing to update."
            exit 0
          fi

          git checkout -b "$BRANCH"
          git add src/cpp/resources/backend_versions.json
          git commit -m "Update llama.cpp to ${LLAMACPP_RELEASE}, rocm-nightly to ${LLAMACPP_ROCM_RELEASE}, rocm-preview to ${LLAMACPP_ROCM_PREVIEW_RELEASE}"
          git push origin "$BRANCH"

          gh pr create \
            --title "Update llama.cpp to ${LLAMACPP_RELEASE}" \
            --body-file pr_body.md \
            --base main \
            --head "$BRANCH"
