name: C++ Server Build, Test, and Release 🚀

on:
  push:
    branches: ["main"]
    tags:
      - v*
  pull_request:
  merge_group:
  workflow_dispatch:
    inputs:
      enable_signing:
        description: 'Enable MSI signing with SignPath (for testing)'
        required: false
        default: false
        type: boolean

permissions:
  contents: write
  actions: read  # Required for SignPath to read workflow/job details

env:
  LEMONADE_DISABLE_SYSTEMD_JOURNAL: "1"

jobs:
  # ========================================================================
  # BUILD JOBS - Run on rai-160-sdk workers
  # ========================================================================

  build-lemonade-server-installer:
    name: Build Lemonade Server Installer
    runs-on: windows-latest
    outputs:
      unsigned-artifact-id: ${{ steps.upload-unsigned-msi.outputs.artifact-id }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        shell: PowerShell
        run: |
          # Check if CMake is already installed
          $cmakeInstalled = Get-Command cmake -ErrorAction SilentlyContinue

          if (-not $cmakeInstalled) {
              Write-Host "CMake not found, installing..." -ForegroundColor Yellow

              # Download CMake installer
              $cmakeVersion = "3.28.1"
              $cmakeUrl = "https://github.com/Kitware/CMake/releases/download/v$cmakeVersion/cmake-$cmakeVersion-windows-x86_64.msi"
              $cmakeInstaller = "cmake-installer.msi"

              Invoke-WebRequest -Uri $cmakeUrl -OutFile $cmakeInstaller -UseBasicParsing

              # Install CMake silently
              Start-Process msiexec.exe -ArgumentList "/i $cmakeInstaller /quiet /norestart" -Wait

              # Add CMake to PATH for this session AND future steps
              $cmakePath = "C:\Program Files\CMake\bin"
              $env:PATH = "$cmakePath;$env:PATH"

              # Persist to GITHUB_PATH for future steps
              echo $cmakePath >> $env:GITHUB_PATH

              # Verify installation
              cmake --version
              if ($LASTEXITCODE -ne 0) {
                  Write-Host "ERROR: CMake installation failed!" -ForegroundColor Red
                  exit 1
              }

              Write-Host "CMake installed successfully and added to PATH!" -ForegroundColor Green
          } else {
              Write-Host "CMake is already installed:" -ForegroundColor Green
              cmake --version
          }

      - name: Install WiX Toolset 5.0.2 (CLI)
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Downloading WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
          $wixUri = "https://github.com/wixtoolset/wix/releases/download/v5.0.2/wix-cli-x64.msi"
          $msiPath = "$env:RUNNER_TEMP\wix-cli-x64.msi"

          Invoke-WebRequest -UseBasicParsing -Uri $wixUri -OutFile $msiPath

          Write-Host "Installing WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
          $p = Start-Process "msiexec.exe" -ArgumentList @("/i", "`"$msiPath`"", "/qn", "/norestart") -PassThru -Wait
          if ($p.ExitCode -ne 0) {
            Write-Host "WiX installer exited with code $($p.ExitCode)" -ForegroundColor Red
            exit $p.ExitCode
          }

      - name: Verify WiX installation
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          # WiX CLI MSI does not always add itself to PATH in non-interactive installs,
          # so we locate it explicitly and then update PATH for subsequent steps.
          $wixDirs = @(
            "C:\Program Files\WiX Toolset v5.0\bin",
            "C:\Program Files (x86)\WiX Toolset v5.0\bin"
          )

          $wixExe = $null
          foreach ($dir in $wixDirs) {
            if (Test-Path (Join-Path $dir "wix.exe")) {
              $wixExe = Join-Path $dir "wix.exe"
              break
            }
          }

          if (-not $wixExe) {
            Write-Host "ERROR: wix.exe not found after installation." -ForegroundColor Red
            Get-ChildItem -Recurse "C:\Program Files" -Filter wix.exe -ErrorAction SilentlyContinue | Select-Object -First 20 | Format-List FullName
            exit 1
          }

          $wixDir = Split-Path $wixExe -Parent

          # Persist wix.exe directory to PATH for all subsequent steps
          "$wixDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

          Write-Host "Using WiX from: $wixExe" -ForegroundColor Green
          & $wixExe --version

      - name: Cache FetchContent dependencies
        uses: actions/cache@v5
        with:
          path: build/_deps
          key: fetchcontent-windows-${{ hashFiles('CMakeLists.txt') }}
          restore-keys: |
            fetchcontent-windows-

      - name: Run setup.ps1 to configure environment
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          .\setup.ps1

      - name: Build installers
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Building C++ server, Tauri app, web app, and MSI installers..." -ForegroundColor Cyan

          # Single build command: wix_installers depends on all C++ targets,
          # tauri-app, and web-app via CMake dependency graph
          cmake --build --preset windows --target wix_installers
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          # Verify outputs
          $failures = @()
          @(
              "build\Release\lemond.exe",
              "build\Release\lemonade-server.exe",
              "build\Release\LemonadeServer.exe",
              "build\app\lemonade-app.exe",
              "build\resources\web-app\index.html",
              "lemonade-server-minimal.msi",
              "lemonade.msi"
          ) | ForEach-Object {
              if (-not (Test-Path $_)) {
                  Write-Host "ERROR: $_ not found!" -ForegroundColor Red
                  $failures += $_
              }
          }

          if ($failures.Count -gt 0) { exit 1 }
          Write-Host "Build and packaging successful!" -ForegroundColor Green

      - name: Upload Lemonade Server Installers
        id: upload-unsigned-msi
        uses: actions/upload-artifact@v7
        with:
          name: Lemonade_Server_MSI
          path: |
            lemonade-server-minimal.msi
            lemonade.msi
          retention-days: 7

  sign-msi-installers:
    name: Sign MSI Installers with SignPath
    runs-on: windows-latest
    needs: build-lemonade-server-installer
    # Sign on tag pushes (releases) or when manually enabled via workflow_dispatch
    if: startsWith(github.ref, 'refs/tags/v') || inputs.enable_signing == true
    steps:
      - name: Sign MSI Installers with SignPath
        id: sign-msi
        uses: signpath/github-action-submit-signing-request@v2
        with:
          api-token: '${{ secrets.SIGNPATH_API_TOKEN }}'
          organization-id: '8103545b-7814-4edc-86d6-a91dc2a2291b'
          project-slug: 'lemonade'
          signing-policy-slug: 'release-signing'
          github-artifact-id: '${{ needs.build-lemonade-server-installer.outputs.unsigned-artifact-id }}'
          wait-for-completion: true
          wait-for-completion-timeout-in-seconds: 3600
          output-artifact-directory: 'signed-msi'
          parameters: |
            version: "${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'test' }}"

      - name: Verify Signed MSI Files
        shell: PowerShell
        run: |
          Write-Host "Verifying signed MSI files..." -ForegroundColor Cyan

          if (-not (Test-Path "signed-msi\lemonade-server-minimal.msi")) {
              Write-Host "ERROR: Signed lemonade-server-minimal.msi not found!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "signed-msi\lemonade.msi")) {
              Write-Host "ERROR: Signed lemonade.msi not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Signed MSI files verified!" -ForegroundColor Green
          Get-ChildItem -Path "signed-msi" -Recurse | Format-Table Name, Length

      - name: Upload Signed MSI Installers
        uses: actions/upload-artifact@v7
        with:
          name: Lemonade_Server_MSI_Signed
          path: |
            signed-msi/lemonade-server-minimal.msi
            signed-msi/lemonade.msi
          retention-days: 7

  build-lemonade-deb:
    name: Build Lemonade .deb Package
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Prepare Debian build
        id: get_version
        uses: ./.github/actions/prepare-debian-build
        with:
          release: '24.04'
          codename: 'noble'

      - name: Build Debian package
        id: build_deb
        uses: ./.github/actions/build-debian-package
        with:
          package-type: binary
          deb-version: ${{ steps.get_version.outputs.version }}

      - name: Upload .deb package
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-deb
          path: ${{ steps.build_deb.outputs.output-dir }}/*.deb
          retention-days: 7

  build-lemonade-rpm:
    name: Build Lemonade .rpm Package
    runs-on: ubuntu-latest
    container:
      image: fedora:latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Install RPM packaging tools
        shell: bash
        run: |
          set -e
          dnf install -y rpm-build

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Build Linux .rpm package
        shell: bash
        run: |
          set -e

          echo "Running setup.sh to configure build environment..."
          bash setup.sh

          echo "Building lemond and lemonade-server for Fedora..."
          cmake --build --preset default

          RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

          cd build

          echo "Creating .rpm package with CPack..."
          cpack -G RPM -V

          if [ ! -f "$RPM_FILE" ]; then
              echo "ERROR: .rpm package not created!"
              echo "Contents of build directory:"
              ls -lR .
              exit 1
          fi

          echo "Package information:"
          rpm -qip "$RPM_FILE"

      - name: Upload .rpm package
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-rpm
          path: build/lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
          retention-days: 7

  build-lemonade-macos-dmg:
    name: Build Lemonade macOS .dmg (with Tauri App)
    runs-on: macos-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
      has_signing: ${{ steps.check_signing.outputs.has_signing }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Check signing secrets
        id: check_signing
        shell: bash
        env:
          APP_CONNECT_KEY: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
        run: |
          if [ -n "$APP_CONNECT_KEY" ]; then
            echo "has_signing=true" >> $GITHUB_OUTPUT
            echo "Signing secrets available - will build signed .pkg"
          else
            echo "has_signing=false" >> $GITHUB_OUTPUT
            echo "No signing secrets - will build and test locally"
          fi

      - name: Setup macOS Keychain
        if: steps.check_signing.outputs.has_signing == 'true'
        uses: ./.github/actions/setup-macos-keychain
        with:
          dev-signing-key: ${{ secrets.MACOS_DEV_SIGNING_IDENTITY_KEY }}
          inst-signing-key: ${{ secrets.MACOS_INST_SIGNING_IDENTITY_KEY }}
          certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
          api-key: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
          api-key-id: '3WFZZ8F948'
          api-issuer-id: '2e545619-8206-4d14-9ba9-ef23eff841b2'

      - name: Build macOS .dmg
        uses: ./.github/actions/build-macos-dmg
        with:
          include-tauri: 'true'
          skip-packaging: ${{ steps.check_signing.outputs.has_signing != 'true' }}

      - name: Upload .pkg package
        if: steps.check_signing.outputs.has_signing == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: build/*.pkg
          retention-days: 7

      # ---- Unsigned local install + test path ----
      - name: Install binaries locally (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        run: |
          set -e
          echo "Installing binaries locally (no signing)..."

          # Copy binaries to /usr/local/bin
          sudo cp build/lemond /usr/local/bin/
          sudo cp build/lemonade-server /usr/local/bin/
          sudo cp build/lemonade /usr/local/bin/
          sudo chmod 755 /usr/local/bin/lemond
          sudo chmod 755 /usr/local/bin/lemonade-server
          sudo chmod 755 /usr/local/bin/lemonade

          # Copy resources
          sudo mkdir -p "/Library/Application Support/Lemonade/resources"
          if [ -d "build/resources" ]; then
            sudo cp -R build/resources/* "/Library/Application Support/Lemonade/resources/" 2>/dev/null || true
          fi

          # Run the postinst script for directory setup
          echo "Running post-install script..."
          sudo bash src/cpp/postinst-full-mac "" "/"

          # Verify installation
          echo "Verifying installation..."
          /usr/local/bin/lemonade-server --version
          /usr/local/bin/lemond --version
          /usr/local/bin/lemonade --version

          echo "Local installation complete!"

      - name: Verify server is running (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        run: |
          echo "Checking server health..."
          for i in $(seq 1 15); do
            if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
              echo "Server is running and healthy"
              exit 0
            fi
            sleep 2
          done

          # Launchd may not start reliably in CI. Fall back to manual start.
          echo "Server not reachable after 30s — starting manually..."
          /usr/local/bin/lemonade-server serve --no-tray > /tmp/lemonade-server.log 2>&1 &

          for i in $(seq 1 15); do
            if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
              echo "Server is running and healthy (started manually)"
              exit 0
            fi
            echo "Waiting for server... ($i/15)"
            sleep 2
          done
          echo "ERROR: Server did not start within 60 seconds"
          cat /tmp/lemonade-server.log 2>/dev/null || true
          exit 1

      - name: Setup Python and virtual environment (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run CLI tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
          GGML_METAL_NO_RESIDENCY: "1"
        run: |
          set -e
          echo "Running CLI tests..."
          .venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server
          .venv/bin/python test/server_cli2.py --server-binary /usr/local/bin/lemonade-server
          echo "CLI tests PASSED!"

      - name: Run endpoint tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
          GGML_METAL_NO_RESIDENCY: "1"
        run: |
          set -e
          echo "Running endpoint tests..."
          .venv/bin/python test/server_endpoints.py --server-binary /usr/local/bin/lemonade-server
          echo "Endpoint tests PASSED!"

      - name: Run Ollama API tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
          GGML_METAL_NO_RESIDENCY: "1"
        run: |
          set -e
          echo "Running Ollama API tests..."
          .venv/bin/python test/test_ollama.py --server-binary /usr/local/bin/lemonade-server
          echo "Ollama API tests PASSED!"

      - name: Run streaming error tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
          GGML_METAL_NO_RESIDENCY: "1"
        run: |
          set -e
          echo "Running streaming error termination tests..."
          .venv/bin/python test/server_streaming_errors.py --server-binary /usr/local/bin/lemonade-server
          echo "Streaming error tests PASSED!"

      - name: Run environment variable tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
        run: |
          set -e
          echo "Running environment variable tests..."
          .venv/bin/python test/server_env_vars.py --lemond-binary /usr/local/bin/lemond
          echo "Environment variable tests PASSED!"

      - name: Cleanup keychain
        if: always() && steps.check_signing.outputs.has_signing == 'true'
        shell: bash
        run: |
          if [ -n "$SIGNING_KEYCHAIN_PATH" ] && [ -f "$SIGNING_KEYCHAIN_PATH" ]; then
            echo "Cleaning up temporary keychain..."
            security delete-keychain "$SIGNING_KEYCHAIN_PATH" 2>/dev/null || true
          fi

  build-lemonade-appimage:
    name: Build Lemonade AppImage
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Run setup.sh to configure environment
        shell: bash
        run: |
          set -e
          echo "Running setup.sh to configure build environment..."
          bash setup.sh
          echo "Build environment configured successfully!"

      - name: Build AppImage
        shell: bash
        run: |
          set -e

          echo "Building Lemonade AppImage..."
          cmake --build --preset default --target appimage

          # Verify AppImage was created
          APPIMAGE_FILE="build/app-appimage/lemonade-app-${LEMONADE_VERSION}-x86_64.AppImage"
          if [ ! -f "$APPIMAGE_FILE" ]; then
              echo "ERROR: AppImage not created!"
              echo "Contents of build/app-appimage directory:"
              ls -lh build/app-appimage/ || echo "Directory does not exist"
              exit 1
          fi

          echo "AppImage created successfully!"
          ls -lh "$APPIMAGE_FILE"

          # Verify it's executable
          chmod +x "$APPIMAGE_FILE"
          "$APPIMAGE_FILE" --appimage-help | head -5

      - name: Upload AppImage artifact
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-appimage
          path: build/app-appimage/lemonade-app-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
          retention-days: 7

  build-lemonade-embeddable-linux:
    name: Build Embeddable Lemonade (Linux)
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Install minimal build dependencies
        run: |
          set -e
          sudo apt-get update
          sudo apt-get install -y cmake ninja-build g++ pkg-config libssl-dev libdrm-dev

      - name: Build embeddable archive
        shell: bash
        run: |
          set -e

          # Do not use setup.sh — we intentionally avoid installing system
          # libraries so that FetchContent statically links all deps, making
          # the embeddable binary portable across Ubuntu versions.
          cmake --preset default -DBUILD_WEB_APP=OFF
          cmake --build --preset default --target embeddable

          ARCHIVE="build/lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz"
          test -f "$ARCHIVE"
          echo "Archive contents:"
          tar tzf "$ARCHIVE"
          ls -lh "$ARCHIVE"

      - name: Upload embeddable archive
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-embeddable-linux
          path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-x64.tar.gz
          retention-days: 7

  build-lemonade-embeddable-windows:
    name: Build Embeddable Lemonade (Windows)
    runs-on: windows-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        shell: PowerShell
        run: |
          $cmakeInstalled = Get-Command cmake -ErrorAction SilentlyContinue

          if (-not $cmakeInstalled) {
              Write-Host "CMake not found, installing..." -ForegroundColor Yellow

              $cmakeVersion = "3.28.1"
              $cmakeUrl = "https://github.com/Kitware/CMake/releases/download/v$cmakeVersion/cmake-$cmakeVersion-windows-x86_64.msi"
              $cmakeInstaller = "cmake-installer.msi"

              Invoke-WebRequest -Uri $cmakeUrl -OutFile $cmakeInstaller -UseBasicParsing

              Start-Process msiexec.exe -ArgumentList "/i $cmakeInstaller /quiet /norestart" -Wait

              $cmakePath = "C:\Program Files\CMake\bin"
              $env:PATH = "$cmakePath;$env:PATH"
              echo $cmakePath >> $env:GITHUB_PATH

              cmake --version
              if ($LASTEXITCODE -ne 0) {
                  Write-Host "ERROR: CMake installation failed!" -ForegroundColor Red
                  exit 1
              }

              Write-Host "CMake installed successfully!" -ForegroundColor Green
          } else {
              Write-Host "CMake is already installed:" -ForegroundColor Green
              cmake --version
          }

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Cache FetchContent dependencies
        uses: actions/cache@v4
        with:
          path: build/_deps
          key: fetchcontent-windows-embeddable-${{ hashFiles('CMakeLists.txt') }}
          restore-keys: fetchcontent-windows-embeddable-

      - name: Build embeddable archive
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          cmake --preset windows -DBUILD_WEB_APP=OFF
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          cmake --build --preset windows --target embeddable
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          $archive = "build\lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64.zip"
          if (-not (Test-Path $archive)) { throw "Archive not found: $archive" }
          Write-Host "Archive created:" -ForegroundColor Green
          Get-ChildItem $archive

      - name: Upload embeddable archive
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
          retention-days: 7


  # ========================================================================
  # TEST JOBS - Inference tests on self-hosted runners
  # ========================================================================

  test-exe-inference:
    name: Test .exe - ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: build-lemonade-server-installer
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
            runner: [Windows, vulkan, rocm]
          - name: ryzenai
            script: server_llm.py
            extra_args: "--wrapped-server ryzenai"
            backends: "cpu hybrid npu"
            runner: [Windows, xdna2]
          - name: flm
            script: server_llm.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Windows, xdna2]
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu npu"
            runner: [Windows, xdna2]
          - name: flm-whisper
            script: server_whisper.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Windows, xdna2]
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu rocm"
            runner: [Windows, rocm]
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
            runner: [self-hosted, Windows]
    env:
      LEMONADE_CI_MODE: "True"
      LEMONADE_CACHE_DIR: ".\\ci-cache"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - uses: actions/checkout@v5

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-windows

      - name: Set environment variables
        shell: PowerShell
        run: |
          $cwd = (Get-Item .\).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install and Verify Lemonade Server
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Install FLM backend for FLM wrapped-server tests
        if: ${{ runner.os == 'Windows' && contains(matrix.extra_args, '--wrapped-server flm') }}
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $lemonadeExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade.exe"
          Write-Host "Installing FLM backend for CI inference tests..." -ForegroundColor Cyan
          & $lemonadeExe backends install flm:npu
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - name: Run tests
        shell: PowerShell
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          $ErrorActionPreference = "Stop"
          $venvPython = ".\.venv\Scripts\python.exe"
          $serverExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade-server.exe"
          $extraArgs = "${{ matrix.extra_args }}" -split " " | Where-Object { $_ }
          $backends = "${{ matrix.backends }}" -split " " | Where-Object { $_ }

          if ($backends.Count -eq 0) {
            Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }}" -ForegroundColor Cyan
            & $venvPython test/${{ matrix.script }} @extraArgs --server-binary $serverExe
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          } else {
            foreach ($backend in $backends) {
              Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend" -ForegroundColor Cyan
              & $venvPython test/${{ matrix.script }} @extraArgs --backend $backend --server-binary $serverExe
              if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
            }
          }

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-exe-${{ matrix.name }}

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-windows

  test-deb-inference:
    name: Test .deb - ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: build-lemonade-deb
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm]
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu rocm"
            runner: [Linux, rocm]
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu vulkan"
            runner: [Linux, vulkan]
          - name: flm
            script: server_llm.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Linux, xdna2]
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
            runner: [self-hosted, Linux]
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-linux

      - name: Set HF_HOME environment variable
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Install FLM backend for FLM wrapped-server tests
        if: ${{ contains(matrix.extra_args, '--wrapped-server flm') }}
        run: |
          set -e
          FLM_VERSION=$(jq -r '.flm.npu' src/cpp/resources/backend_versions.json)
          FLM_VERSION_NUM=$(echo $FLM_VERSION | sed 's/^v//')
          echo "Installing FLM ${FLM_VERSION} for CI inference tests..."
          curl -L -o /tmp/fastflowlm.deb "https://github.com/FastFlowLM/FastFlowLM/releases/download/${FLM_VERSION}/fastflowlm_${FLM_VERSION_NUM}_ubuntu24.04_amd64.deb"
          mkdir -p /tmp/flm-extract
          dpkg-deb -x /tmp/fastflowlm.deb /tmp/flm-extract
          # Add extracted FLM binary to PATH (Linux uses system PATH for FLM discovery)
          FLM_BIN=$(find /tmp/flm-extract -name flm -type f | head -1)
          chmod +x "$FLM_BIN"
          echo "$(dirname "$FLM_BIN")" >> $GITHUB_PATH
          export PATH="$(dirname "$FLM_BIN"):$PATH"
          rm /tmp/fastflowlm.deb
          flm version

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run tests
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          if [ -z "${{ matrix.backends }}" ]; then
            echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }}"
            .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --server-binary lemonade-server
          else
            for backend in ${{ matrix.backends }}; do
              echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend"
              .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend --server-binary lemonade-server
            done
          fi

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-deb-${{ matrix.name }}

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-linux

  test-rpm-package:
    name: Test .rpm - Fedora
    runs-on: ubuntu-latest
    needs: build-lemonade-rpm
    container:
      image: fedora:latest
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
    steps:
      - name: Download Lemonade .rpm Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-rpm
          path: .

      - name: Install and verify Lemonade (.rpm)
        shell: bash
        run: |
          set -e

          RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

          if [ ! -f "$RPM_FILE" ]; then
              echo "ERROR: .rpm file not found: $RPM_FILE"
              ls -la *.rpm 2>/dev/null || echo "No .rpm files found in current directory"
              exit 1
          fi

          dnf install -y shadow-utils "$RPM_FILE"

          echo "Installed package information:"
          rpm -qi lemonade-server

          echo "Installed file list:"
          rpm -ql lemonade-server | sort

          test -f /opt/bin/lemonade-server
          test -f /opt/bin/lemond

          /opt/bin/lemonade-server --version
          /opt/bin/lemond --version

  test-embeddable-linux:
    name: Test Embeddable (Linux)
    runs-on: ubuntu-latest
    needs: build-lemonade-embeddable-linux
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-linux.outputs.version }}
    steps:
      - name: Download embeddable archive
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-linux
          path: .

      - name: Validate archive structure
        shell: bash
        run: |
          set -e

          ARCHIVE="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz"
          test -f "$ARCHIVE"
          tar xzf "$ARCHIVE"

          DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"

          # Verify expected files exist
          test -f "$DIR/lemond"
          test -f "$DIR/lemonade"
          test -f "$DIR/LICENSE"
          test -f "$DIR/resources/server_models.json"
          test -f "$DIR/resources/backend_versions.json"
          test -f "$DIR/resources/defaults.json"
          echo "All expected files present"

          # Verify NO forbidden files
          FORBIDDEN=0
          if ls "$DIR"/lemonade-server* 2>/dev/null; then
            echo "ERROR: Found lemonade-server (legacy CLI) in archive"; FORBIDDEN=1
          fi
          if [ -d "$DIR/resources/web-app" ]; then
            echo "ERROR: Found web-app directory in archive"; FORBIDDEN=1
          fi
          if ls "$DIR"/lemonade-app* 2>/dev/null; then
            echo "ERROR: Found desktop app in archive"; FORBIDDEN=1
          fi
          if ls "$DIR"/lemonade-tray* 2>/dev/null; then
            echo "ERROR: Found tray app in archive"; FORBIDDEN=1
          fi
          if ls "$DIR"/LemonadeServer* 2>/dev/null; then
            echo "ERROR: Found LemonadeServer in archive"; FORBIDDEN=1
          fi
          if [ "$FORBIDDEN" -ne 0 ]; then exit 1; fi
          echo "No forbidden files found"

      - name: Test version commands
        shell: bash
        run: |
          set -e
          DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"
          "$DIR/lemond" --version
          "$DIR/lemonade" --version

      - name: Test lemond startup and health check
        shell: bash
        run: |
          set -e
          DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"
          cd "$DIR"

          ./lemond ./ &
          LEMOND_PID=$!

          # Wait for server to become healthy
          for i in $(seq 1 15); do
            if curl -sf http://localhost:13305/api/v1/health > /dev/null 2>&1; then
              echo "Server is healthy!"
              break
            fi
            if [ $i -eq 15 ]; then
              echo "ERROR: Server failed to start within 30 seconds"
              kill $LEMOND_PID 2>/dev/null || true
              exit 1
            fi
            sleep 2
          done

          # Verify health endpoint responds with valid JSON
          HEALTH=$(curl -sf http://localhost:13305/api/v1/health)
          echo "Health response: $HEALTH"

          # Clean shutdown
          kill $LEMOND_PID 2>/dev/null || true
          wait $LEMOND_PID 2>/dev/null || true
          echo "Embeddable Linux smoke test PASSED!"

  test-embeddable-windows:
    name: Test Embeddable (Windows)
    runs-on: windows-latest
    needs: build-lemonade-embeddable-windows
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-windows.outputs.version }}
    steps:
      - name: Download embeddable archive
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: .

      - name: Validate archive structure
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
          Expand-Archive -Path "$archiveDir.zip" -DestinationPath .

          # Verify expected files
          $expected = @(
              "$archiveDir\lemond.exe",
              "$archiveDir\lemonade.exe",
              "$archiveDir\LICENSE",
              "$archiveDir\resources\server_models.json",
              "$archiveDir\resources\backend_versions.json",
              "$archiveDir\resources\defaults.json"
          )
          $failures = @()
          foreach ($f in $expected) {
              if (-not (Test-Path $f)) {
                  Write-Host "ERROR: $f not found!" -ForegroundColor Red
                  $failures += $f
              }
          }
          if ($failures.Count -gt 0) { exit 1 }
          Write-Host "All expected files present" -ForegroundColor Green

          # Verify NO forbidden files
          $forbidden = @(
              "$archiveDir\lemonade-server.exe",
              "$archiveDir\LemonadeServer.exe",
              "$archiveDir\lemonade-app.exe",
              "$archiveDir\lemonade-tray.exe",
              "$archiveDir\resources\web-app"
          )
          foreach ($f in $forbidden) {
              if (Test-Path $f) {
                  Write-Host "ERROR: Forbidden file found: $f" -ForegroundColor Red
                  exit 1
              }
          }
          Write-Host "No forbidden files found" -ForegroundColor Green

      - name: Test version commands
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
          & "$archiveDir\lemond.exe" --version
          & "$archiveDir\lemonade.exe" --version

      - name: Test lemond startup and health check
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"

          Push-Location $archiveDir
          $proc = Start-Process -FilePath ".\lemond.exe" -ArgumentList ".\" -PassThru -NoNewWindow

          # Wait for server to become healthy
          $healthy = $false
          for ($i = 0; $i -lt 15; $i++) {
            try {
              $response = Invoke-WebRequest -Uri "http://localhost:13305/api/v1/health" -UseBasicParsing -TimeoutSec 2
              if ($response.StatusCode -eq 200) {
                Write-Host "Server is healthy!" -ForegroundColor Green
                Write-Host "Health response: $($response.Content)"
                $healthy = $true
                break
              }
            } catch {}
            Start-Sleep -Seconds 2
          }

          if (-not $healthy) {
            Write-Host "ERROR: Server failed to start within 30 seconds" -ForegroundColor Red
            Stop-Process $proc -Force -ErrorAction SilentlyContinue
            Pop-Location
            exit 1
          }

          Stop-Process $proc -Force -ErrorAction SilentlyContinue
          Pop-Location
          Write-Host "Embeddable Windows smoke test PASSED!" -ForegroundColor Green

  test-dmg-inference:
    name: Test .dmg - llamacpp (metal)
    runs-on: macos-latest
    needs: build-lemonade-macos-dmg
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    # Also skip when no signing secrets (tests already ran inline in build job)
    if: ${{ needs.build-lemonade-macos-dmg.outputs.has_signing == 'true' && !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
      GGML_METAL_NO_RESIDENCY: "1"
    steps:
      - uses: actions/checkout@v5

      - name: Set HF_HOME environment variable
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Install Lemonade Server (.pkg)
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Test llamacpp (metal)
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          .venv/bin/python test/server_llm.py --wrapped-server llamacpp --backend metal --server-binary /usr/local/bin/lemonade-server

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-dmg-llamacpp

  # ========================================================================
  # CLI AND ENDPOINTS TESTS - Run on GitHub-hosted runners (no GPU needed)
  # ========================================================================

  test-cli-endpoints-linux:
    name: Test ${{ matrix.test_type }} (ubuntu-latest)
    runs-on: ubuntu-latest
    needs: build-lemonade-deb
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        test_type: [cli, endpoints, ollama, llamacpp-system, streaming-errors, env-vars]
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Install pip and test dependencies
        shell: bash
        run: |
          set -e
          # Install pip if not already available
          if ! command -v pip3 &> /dev/null; then
            apt-get update
            apt-get install -y python3-pip python3-venv
          fi
          # Create venv and install test dependencies
          python3 -m venv .venv
          .venv/bin/pip install --upgrade pip
          .venv/bin/pip install -r test/requirements.txt

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Set environment
        shell: bash
        run: |
          echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
          echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
          echo "SERVER_BINARY=lemonade-server" >> $GITHUB_ENV

      - name: Run tests
        shell: bash
        run: |
          set -e
          VENV_PYTHON=.venv/bin/python
          SERVER_BINARY=lemonade-server

          if [ "${{ matrix.test_type }}" = "cli" ]; then
              echo "Running CLI tests..."
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --ephemeral
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --listen-all
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --api-key
              $VENV_PYTHON test/server_cli2.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
              echo "Running endpoint tests..."
              $VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY"
              $VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY" --server-per-test
          elif [ "${{ matrix.test_type }}" = "ollama" ]; then
              echo "Running Ollama API tests..."
              $VENV_PYTHON test/test_ollama.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "llamacpp-system" ]; then
              echo "Running LlamaCpp System Backend tests..."
              $VENV_PYTHON test/test_llamacpp_system_backend.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "streaming-errors" ]; then
              echo "Running streaming error termination tests..."
              $VENV_PYTHON test/server_streaming_errors.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "env-vars" ]; then
              echo "Running environment variable tests..."
              $VENV_PYTHON test/server_env_vars.py --lemond-binary ./deb-extract/usr/bin/lemond
          fi

          echo "${{ matrix.test_type }} tests PASSED!"

  test-cli-endpoints:
    name: Test ${{ matrix.test_type }} (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    needs:
      - build-lemonade-server-installer
      - build-lemonade-macos-dmg
    strategy:
      fail-fast: false
      matrix:
        os: [windows-latest, macos-latest]
        test_type: [cli, endpoints, ollama, llamacpp-system, streaming-errors]
        include:
          - os: macos-latest
            test_type: env-vars
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GGML_METAL_NO_RESIDENCY: "1"
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      # ---- Windows Setup ----
      - name: Setup (Windows)
        if: runner.os == 'Windows'
        shell: powershell
        run: |
          $cwd = (Get-Item .).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install Lemonade Server (Windows)
        if: runner.os == 'Windows'
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        if: runner.os == 'Windows'
        shell: powershell
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

      # ---- macOS Setup (only when signed .pkg is available) ----
      - name: Download .pkg package
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        uses: actions/download-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Install Lemonade Server (.pkg)
        id: install-pkg
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}
          download-artifact: 'false'

      - name: Set environment (macOS)
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        shell: bash
        run: |
          echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
          echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
          echo "SERVER_BINARY=${{ steps.install-pkg.outputs.bin-path }}/lemonade-server" >> $GITHUB_ENV

      # ---- Common Setup ----
      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      # ---- Run Tests ----
      - name: Run ${{ matrix.test_type }} tests
        if: ${{ !(runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing != 'true') }}
        shell: bash
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e  # Exit on error

          if [ "${{ matrix.test_type }}" = "cli" ]; then
              echo "Running CLI tests..."
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"
              $VENV_PYTHON test/server_cli2.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
              echo "Running endpoint tests..."
              $VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "ollama" ]; then
              echo "Running Ollama API tests..."
              $VENV_PYTHON test/test_ollama.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "llamacpp-system" ]; then
              echo "Running LlamaCpp System Backend tests..."
              $VENV_PYTHON test/test_llamacpp_system_backend.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "streaming-errors" ]; then
              echo "Running streaming error termination tests..."
              $VENV_PYTHON test/server_streaming_errors.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "env-vars" ]; then
              echo "Running environment variable tests..."
              LEMOND_BINARY="$(dirname "$SERVER_BINARY")/lemond"
              $VENV_PYTHON test/server_env_vars.py --lemond-binary "$LEMOND_BINARY"
          fi

          echo "${{ matrix.test_type }} tests PASSED!"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-${{ matrix.os }}-${{ matrix.test_type }}

  # ========================================================================
  # API KEY TESTS - Separate job with LEMONADE_API_KEY env var
  # ========================================================================

  test-cli-apikey-linux:
    name: Test API Key (ubuntu-latest)
    runs-on: ubuntu-latest
    needs: build-lemonade-deb
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    env:
      LEMONADE_CI_MODE: "True"
      LEMONADE_API_KEY: "test-api-key-12345"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Install pip and test dependencies
        shell: bash
        run: |
          set -e
          if ! command -v pip3 &> /dev/null; then
            apt-get update
            apt-get install -y python3-pip python3-venv
          fi
          python3 -m venv .venv
          .venv/bin/pip install --upgrade pip
          .venv/bin/pip install -r test/requirements.txt

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Verify API key enforcement
        shell: bash
        run: |
          set -e
          PASS=0
          FAIL=0

          # 1. No API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: No API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: No API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 2. Wrong API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: Wrong API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 3. Correct API key → must get 200
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "200" ]; then
            echo "PASS: Correct API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
            FAIL=$((FAIL+1))
          fi

          echo ""
          echo "Results: $PASS passed, $FAIL failed"
          if [ "$FAIL" -gt 0 ]; then
            echo "ERROR: API key enforcement tests failed"
            exit 1
          fi

      - name: Run CLI tests with API key
        shell: bash
        run: |
          set -e
          # CLI commands should work because LEMONADE_API_KEY is in the env
          # and the CLI client reads it automatically
          .venv/bin/python test/server_cli.py --server-binary lemonade-server

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-apikey-ubuntu-latest

  test-cli-apikey:
    name: Test API Key (windows-latest)
    runs-on: windows-latest
    needs:
      - build-lemonade-server-installer
    env:
      LEMONADE_CI_MODE: "True"
      LEMONADE_API_KEY: "test-api-key-12345"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - uses: actions/checkout@v5

      - name: Setup (Windows)
        shell: powershell
        run: |
          $cwd = (Get-Item .).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install Lemonade Server (Windows)
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        shell: powershell
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Verify API key enforcement
        shell: bash
        run: |
          set -e
          PASS=0
          FAIL=0

          # 1. No API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: No API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: No API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 2. Wrong API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: Wrong API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 3. Correct API key → must get 200
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "200" ]; then
            echo "PASS: Correct API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
            FAIL=$((FAIL+1))
          fi

          echo ""
          echo "Results: $PASS passed, $FAIL failed"
          if [ "$FAIL" -gt 0 ]; then
            echo "ERROR: API key enforcement tests failed"
            exit 1
          fi

      - name: Run CLI tests with API key
        shell: bash
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-apikey-windows-latest

  # ========================================================================
  # TEST ROCM STABLE CHANNEL - Verify stable channel on Windows hosted runner
  # ========================================================================

  test-rocm-stable-channel:
    name: Test ROCm Stable Channel (Windows)
    runs-on: windows-latest
    needs:
      - build-lemonade-server-installer
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup (Windows)
        shell: powershell
        run: |
          $cwd = (Get-Item .).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install Lemonade Server (Windows)
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        shell: powershell
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Set ROCm channel to stable
        shell: bash
        run: |
          set -e
          echo "Setting rocm_channel to stable in config.json"
          "$VENV_PYTHON" -c "
          import json
          import os
          from pathlib import Path

          cache_dir = Path.home() / '.cache' / 'lemonade'
          config_file = cache_dir / 'config.json'

          # Create cache dir if needed
          cache_dir.mkdir(parents=True, exist_ok=True)

          # Load or create config
          if config_file.exists():
              with open(config_file, 'r') as f:
                  config = json.load(f)
          else:
              config = {}

          # Set rocm_channel to stable
          config['rocm_channel'] = 'stable'

          # Save config
          with open(config_file, 'w') as f:
              json.dump(config, f, indent=2)

          print(f'Config updated: rocm_channel = stable')
          "

      - name: Verify ROCm stable channel configuration
        shell: bash
        run: |
          set -e
          echo "Verifying rocm_channel setting..."
          "$VENV_PYTHON" -c "
          import json
          from pathlib import Path

          config_file = Path.home() / '.cache' / 'lemonade' / 'config.json'

          if not config_file.exists():
              print('ERROR: config.json not found')
              exit(1)

          with open(config_file, 'r') as f:
              config = json.load(f)

          channel = config.get('rocm_channel', 'NOT_SET')
          print(f'rocm_channel = {channel}')

          if channel != 'stable':
              print(f'ERROR: Expected rocm_channel=stable, got {channel}')
              exit(1)

          print('SUCCESS: rocm_channel is set to stable')
          "

      - name: Test channel switching with lemonade CLI
        shell: bash
        run: |
          set -e
          echo "Setting rocm_channel to stable via lemonade CLI..."

          # Use lemonade config set to change the channel
          "$SERVER_BINARY" config set rocm_channel=stable

          echo "Verifying config was updated..."
          "$VENV_PYTHON" -c "
          import json
          from pathlib import Path

          config_file = Path.home() / '.cache' / 'lemonade' / 'config.json'

          if not config_file.exists():
              print('ERROR: config.json not found')
              exit(1)

          with open(config_file, 'r') as f:
              config = json.load(f)

          channel = config.get('rocm_channel', 'NOT_SET')
          print(f'rocm_channel = {channel}')

          if channel != 'stable':
              print(f'ERROR: Expected rocm_channel=stable, got {channel}')
              exit(1)

          print('SUCCESS: rocm_channel set to stable via CLI')
          "

      - name: Test recipes command shows rocm backend
        shell: bash
        run: |
          set -e
          echo "Testing recipes command..."
          "$SERVER_BINARY" recipes | tee recipes_output.txt

          # Check that rocm backend is listed (not rocm-stable or rocm-preview)
          if grep -q "rocm-stable" recipes_output.txt || grep -q "rocm-preview" recipes_output.txt; then
              echo "ERROR: Found rocm-stable or rocm-preview in recipes output (should only show 'rocm')"
              cat recipes_output.txt
              exit 1
          fi

          echo "SUCCESS: Recipes output looks correct"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-rocm-stable-channel

  # ========================================================================
  # RELEASE JOB - Add artifacts to GitHub release
  # ========================================================================

  release:
    name: Create GitHub Release
    runs-on: ubuntu-latest
    needs:
      - sign-msi-installers
      - build-lemonade-rpm
      - build-lemonade-macos-dmg
      - build-lemonade-appimage
      - build-lemonade-embeddable-linux
      - build-lemonade-embeddable-windows
      - test-cli-endpoints
      - test-rpm-package
      - test-embeddable-linux
      - test-embeddable-windows
    if: startsWith(github.ref, 'refs/tags/v')
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
    steps:
      - name: Checkout for release notes action
        uses: actions/checkout@v5
        with:
          sparse-checkout: .github

      - name: Download Signed Lemonade Server Installer (Windows)
        uses: actions/download-artifact@v7
        with:
          name: Lemonade_Server_MSI_Signed
          path: .

      - name: Download Lemonade .rpm Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-rpm
          path: .

      - name: Download Lemonade macOS .pkg Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Download Lemonade AppImage (Linux)
        uses: actions/download-artifact@v7
        with:
          name: lemonade-appimage
          path: .

      - name: Download Lemonade Embeddable (Linux)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-linux
          path: .

      - name: Download Lemonade Embeddable (Windows)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: .

      - name: Verify release artifacts
        run: |
          echo "Release artifacts:"
          ls -lh lemonade-server-minimal.msi
          ls -lh lemonade.msi
          ls -lh lemonade-server-${LEMONADE_VERSION}.x86_64.rpm
          ls -lh *.pkg
          ls -lh lemonade-app-${LEMONADE_VERSION}-x86_64.AppImage
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-windows-x64.zip

      - name: Generate release notes
        id: release-notes
        uses: ./.github/actions/generate-release-notes
        with:
          version: ${{ env.LEMONADE_VERSION }}
          repo: ${{ github.repository }}
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Release
        uses: softprops/action-gh-release@v2
        with:
          name: ${{ github.ref_name }}
          body_path: ${{ steps.release-notes.outputs.release_notes_file }}
          files: |
            lemonade-server-minimal.msi
            lemonade.msi
            lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
            *.pkg
            lemonade-app-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-x64.tar.gz
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
          fail_on_unmatched_files: true
