Nightly Eval Smoke #19
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Eval Smoke | |
| # Runs the coded UI eval flows (evals/flows) against a real Electron app | |
| # under Xvfb every night, and on demand. Flows that need cloud credentials | |
| # declare requiredEnv and are skipped automatically when secrets are absent. | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| eval-smoke: | |
| runs-on: blacksmith-4vcpu-ubuntu-2204 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Setup Node | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 11.4.0 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: 1.3.9 | |
| - name: Install OpenCode CLI | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| OPENCODE_GITHUB_REPO: ${{ vars.OPENCODE_GITHUB_REPO || 'anomalyco/opencode' }} | |
| run: | | |
| set -euo pipefail | |
| repo="${OPENCODE_GITHUB_REPO:-anomalyco/opencode}" | |
| version="$(node -e "const fs=require('fs'); const parsed=JSON.parse(fs.readFileSync('constants.json','utf8')); process.stdout.write(String(parsed.opencodeVersion||'').trim().replace(/^v/,''));")" | |
| if [ -z "$version" ]; then | |
| echo "Unable to resolve OpenCode version from constants.json." >&2 | |
| exit 1 | |
| fi | |
| url="https://github.com/${repo}/releases/download/v${version}/opencode-linux-x64-baseline.tar.gz" | |
| tmp_dir="$RUNNER_TEMP/opencode" | |
| mkdir -p "$tmp_dir/extracted" | |
| curl -fsSL --retry 5 --retry-all-errors --retry-delay 2 -H "Authorization: Bearer ${GITHUB_TOKEN}" -o "$tmp_dir/opencode.tar.gz" "$url" | |
| tar -xzf "$tmp_dir/opencode.tar.gz" -C "$tmp_dir/extracted" | |
| install_dir="$HOME/.opencode/bin" | |
| mkdir -p "$install_dir" | |
| cp "$tmp_dir/extracted/opencode" "$install_dir/opencode" | |
| chmod 755 "$install_dir/opencode" | |
| echo "$install_dir" >> "$GITHUB_PATH" | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Install Xvfb | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq xvfb | |
| - name: Start Electron app under Xvfb | |
| shell: bash | |
| env: | |
| OPENWORK_DEV_MODE: "1" | |
| ELECTRON_DISABLE_SANDBOX: "1" | |
| OPENWORK_ELECTRON_REMOTE_DEBUG_PORT: "9823" | |
| DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS: "--no-sandbox --disable-gpu --disable-dev-shm-usage --enable-unsafe-swiftshader" | |
| run: | | |
| set -euo pipefail | |
| Xvfb :99 -screen 0 1600x1000x24 & | |
| export DISPLAY=:99 | |
| nohup pnpm dev > /tmp/openwork-dev.log 2>&1 & | |
| echo "Waiting for Electron CDP on :9823..." | |
| for i in $(seq 1 36); do | |
| if curl -sf http://127.0.0.1:9823/json/list > /dev/null 2>&1; then | |
| echo "CDP ready after ~$((i*5))s" | |
| exit 0 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Electron CDP did not come up." >&2 | |
| tail -100 /tmp/openwork-dev.log >&2 | |
| exit 1 | |
| - name: Run eval flows | |
| shell: bash | |
| env: | |
| OPENWORK_EVAL_CDP_URL: "http://127.0.0.1:9823" | |
| # Optional cloud coverage: flows skip cleanly when unset. | |
| OPENWORK_EVAL_DEN_API_URL: ${{ secrets.OPENWORK_EVAL_DEN_API_URL }} | |
| OPENWORK_EVAL_DEN_TOKEN: ${{ secrets.OPENWORK_EVAL_DEN_TOKEN }} | |
| run: pnpm evals --all | |
| - name: Upload eval reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-results | |
| path: | | |
| evals/results/ | |
| /tmp/openwork-dev.log | |
| retention-days: 14 | |
| if-no-files-found: warn |