bench.yml

name: Bench

on:
  pull_request:
  push:
    branches:
      - main

env:
  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
  TURBO_TEAM: hashintel
  TURBO_CACHE: remote:rw

  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  optimize-ci: # Do not run benches in the middle of the stack
    runs-on: ubuntu-24.04
    name: Optimize CI
    outputs:
      skip: ${{ steps.check_skip.outputs.skip }}
    steps:
      - name: Optimize CI
        id: check_skip
        uses: withgraphite/graphite-ci-action@ee395f3a78254c006d11339669c6cabddf196f72
        with:
          graphite_token: ${{ secrets.GRAPHITE_CI_OPTIMIZER_TOKEN }}

  setup:
    needs: [optimize-ci]
    if: needs.optimize-ci.outputs.skip == 'false'
    runs-on: ubuntu-24.04
    permissions:
      id-token: write
    outputs:
      unit: ${{ steps.packages.outputs.unit }}
      integration: ${{ steps.packages.outputs.integration }}
    steps:
      - name: Checkout source code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 2

      - name: Install tools
        uses: ./.github/actions/install-tools
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          vault_address: ${{ secrets.VAULT_ADDR }}
          rust: false

      - name: Determine changed packages
        id: packages
        run: |
          UNIT_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:unit"}}) { items { name path } } }'
          INTEGRATION_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:integration"}}) { items { name path } } }'

          UNIT_BENCH_PACKAGES=$(turbo query "$UNIT_BENCH_QUERY" \
            | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }')
          INTEGRATION_BENCH_PACKAGES=$(turbo query "$INTEGRATION_BENCH_QUERY" \
            | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }')

          echo "unit=$UNIT_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT
          echo "integration=$INTEGRATION_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT

  unit-benches:
    name: Unit
    needs: [setup]
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.setup.outputs.unit) }}
      fail-fast: false
    if: needs.setup.outputs.unit != '{"name":[],"include":[]}'
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout base branch
        if: github.event_name == 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.base.sha }}

      - name: Clean up disk
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/clean-up-disk

      - name: Install tools
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/install-tools
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          vault_address: ${{ secrets.VAULT_ADDR }}

      - name: Prune repository
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.name }}
            @rust/hash-repo-chores

      - name: Warm up repository
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/warm-up-repo

      - name: Run base benches
        if: github.event_name == 'pull_request'
        run: turbo run bench:unit --filter "${{ matrix.name }}"

      - name: Checkout head branch
        if: github.event_name == 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          clean: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Checkout
        if: github.event_name != 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Clean up disk
        if: github.event_name != 'pull_request'
        uses: ./.github/actions/clean-up-disk

      - name: Install tools
        uses: ./.github/actions/install-tools
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          vault_address: ${{ secrets.VAULT_ADDR }}

      - name: Prune repository
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.name }}
            @rust/hash-repo-chores

      - name: Warm up repository
        uses: ./.github/actions/warm-up-repo

      - name: Run head benches
        run: turbo run bench:unit --filter "${{ matrix.name }}"

      - name: Analyze benchmarks
        if: github.event_name == 'pull_request'
        run: |
          TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g')
          echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
          echo '<details><summary><h2>${{ matrix.name }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph
          cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"

      - name: Authenticate Vault
        id: secrets
        if: github.event.pull_request.head.repo.full_name == github.repository
        uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0
        with:
          exportToken: true
          url: ${{ env.VAULT_ADDR }}
          method: jwt
          role: dev
          secrets: |
            aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
            aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
            aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;

      - name: Upload benchmark results
        if: github.event.pull_request.head.repo.full_name == github.repository
        run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph

      - name: Upload benchmark summary
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
          path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md

  integration-benches:
    name: Integration
    needs: [setup]
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.setup.outputs.integration) }}
      fail-fast: false
    if: needs.setup.outputs.integration != '{"name":[],"include":[]}'
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout base branch
        if: github.event_name == 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.base.sha }}

      - name: Clean up disk
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/clean-up-disk

      - name: Install tools
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/install-tools
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          vault_address: ${{ secrets.VAULT_ADDR }}

      - name: Prune repository
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.name }}
            @rust/hash-repo-chores

      - name: Warm up repository
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/warm-up-repo

      - name: Create temp files and folders
        run: mkdir -p var/logs

      - name: Launch external services
        if: github.event_name == 'pull_request'
        working-directory: infra/compose
        run: docker compose --profile dev up -d

      - name: Start background tasks
        if: github.event_name == 'pull_request'
        id: base-background-tasks
        env:
          # Set terminal logging to WARN level and enable verbose file logging for CI
          HASH_GRAPH_LOG_CONSOLE_LEVEL: warn
          HASH_GRAPH_LOG_FILE_ENABLED: true
          HASH_GRAPH_LOG_FILE_LEVEL: info
          HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs
        run: |
          # Optimistically compile background tasks.
          turbo run compile:release --env-mode=loose
          mkdir -p logs
          turbo run start --env-mode=loose --log-order stream >var/logs/base-background-task.log 2>&1 &
          PID=$!
          echo "pid=$PID" | tee -a $GITHUB_OUTPUT

          turbo run start:healthcheck --env-mode=loose

      - name: Run base benches
        if: github.event_name == 'pull_request'
        run: turbo run bench:integration --filter "${{ matrix.name }}"

      - name: Show background tasks logs
        if: always() && github.event_name == 'pull_request'
        run: |
          kill -15 ${{ steps.base-background-tasks.outputs.pid }} || true
          cat var/logs/base-background-task.log

      - name: Tear down external services
        if: github.event_name == 'pull_request'
        working-directory: infra/compose
        run: docker compose --profile dev down

      - name: Checkout head branch
        if: github.event_name == 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          clean: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Checkout
        if: github.event_name != 'pull_request'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Clean up disk
        if: github.event_name != 'pull_request'
        uses: ./.github/actions/clean-up-disk

      - name: Install tools
        uses: ./.github/actions/install-tools
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          vault_address: ${{ secrets.VAULT_ADDR }}
          sccache: false # sccache is already running in the background

      - name: Prune repository
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.name }}
            @rust/hash-repo-chores

      - name: Warm up repository
        uses: ./.github/actions/warm-up-repo

      - name: Create temp files and folders
        run: mkdir -p var/logs

      - name: Launch external services
        working-directory: infra/compose
        run: docker compose --profile dev up -d

      - name: Start background tasks
        id: head-background-tasks
        env:
          # Set terminal logging to WARN level and enable verbose file logging for CI
          HASH_GRAPH_LOG_CONSOLE_LEVEL: warn
          HASH_GRAPH_LOG_FILE_ENABLED: true
          HASH_GRAPH_LOG_FILE_LEVEL: info
          HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs
        run: |
          # Optimistically compile background tasks.
          turbo run compile:release --env-mode=loose
          mkdir -p logs
          turbo run start --env-mode=loose --log-order stream >var/logs/head-background-task.log 2>&1 &
          PID=$!
          echo "pid=$PID" | tee -a $GITHUB_OUTPUT

          turbo run start:healthcheck --env-mode=loose

      - name: Run head benches
        run: turbo run bench:integration --filter "${{ matrix.name }}"

      - name: Show background tasks logs
        if: always()
        run: |
          kill -15 ${{ steps.head-background-tasks.outputs.pid }} || true
          cat var/logs/head-background-task.log

      - name: Analyze benchmarks
        if: github.event_name == 'pull_request'
        run: |
          TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g')
          echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
          echo '<details><summary><h2>${{ matrix.name }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph
          cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"

      - name: Authenticate Vault
        id: secrets
        if: github.event.pull_request.head.repo.full_name == github.repository
        uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0
        with:
          exportToken: true
          url: ${{ env.VAULT_ADDR }}
          method: jwt
          role: dev
          secrets: |
            aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
            aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
            aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;

      - name: Upload benchmark results
        if: github.event.pull_request.head.repo.full_name == github.repository
        run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph

      - name: Upload benchmark summary
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
          path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md

      - name: Show container logs
        if: ${{ success() || failure() }}
        working-directory: infra/compose
        run: docker compose --profile dev logs --timestamps

      - name: Upload background tasks logs
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        if: success() || failure()
        with:
          name: ${{ env.TRIMMED_PACKAGE_NAME }}_logs
          path: |
            var/logs

  passed:
    name: Benches passed
    needs: [setup, unit-benches, integration-benches, optimize-ci]
    if: always() && needs.optimize-ci.outputs.skip == 'false'
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Check setup script
        run: |
          [[ ${{ needs.setup.result }} = success ]]
      - name: Check unit benches
        run: |
          [[ ${{ needs.unit-benches.result }} =~ success|skipped ]]
      - name: Check integration benches
        run: |
          [[ ${{ needs.integration-benches.result }} =~ success|skipped ]]

      - name: Notify Slack on failure
        uses: rtCamp/action-slack-notify@c58b60ee33df2229ed2d2eed86eeaf7e6c527c5a
        if: ${{ failure() && github.event_name == 'merge_group' }}
        env:
          SLACK_LINK_NAMES: true
          SLACK_MESSAGE: "At least one bench job for a Pull Request in the Merge Queue failed <!subteam^S09KH99698T>" # Notifies @devops
          SLACK_TITLE: Tests failed
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_USERNAME: GitHub
          VAULT_ADDR: ""
          VAULT_TOKEN: ""

      - name: Checkout
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Download reports' artifacts
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          path: downloads

      - name: Create comment
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        run: |
          echo "## Benchmark results" > /tmp/summary.md
          cat downloads/*/*.md >> /tmp/summary.md
          cat /tmp/summary.md

      - name: Post comment
        uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository
        with:
          message-path: /tmp/summary.md
          message-id: benchmark-results
          refresh-message-position: true