opik/.github/workflows/python_backend_e2e_tests.yml at main · comet-ml/opik · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
name: Python BE E2E Tests
run-name: "Python BE E2E Tests ${{ github.ref_name }} by @${{ github.actor }}"
permissions:
  contents: read
  checks: write
  pull-requests: write
on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'apps/opik-python-backend/**'
      - '.github/workflows/python_backend_e2e_tests.yml'
  push:
    branches:
      - 'main'
    paths:
      - 'apps/opik-python-backend/**'
      - '.github/workflows/python_backend_e2e_tests.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  OPIK_ENABLE_LITELLM_MODELS_MONITORING: False
  OPIK_SENTRY_ENABLE: False
  OPIK_URL_OVERRIDE: http://localhost:8080
  # The studio job processor streams subprocess logs to the stack's real Redis
  # (password-protected, port-mapped to localhost by `opik.sh --port-mapping`).
  # Not required for pass/fail (log flushing is best-effort), but without it the
  # collector hits localhost:6379 unauthenticated and the log path goes dead.
  REDIS_URL: redis://:opik@localhost:6379/0
  # The Anthropic key is stored in the workspace by the tests (model-storage
  # principle); the optimizer reaches the provider only through the backend
  # gateway, never via this directly.
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
  # Keep the optimization short — we only need it to start, iterate, and finish.
  OPTIMIZER_MAX_TRIALS: "1"
  OPTSTUDIO_DATASET_SAMPLES: "4"

jobs:
  run-python-be-e2e:
    name: Python BE E2E
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Setup Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Set up uv
        uses: astral-sh/setup-uv@v8.3.2

      - name: Run Opik server
        env:
          OPIK_USAGE_REPORT_ENABLED: false
          COMPOSE_BAKE: false
        run: |
          cd ${{ github.workspace }}
          ./opik.sh --backend --port-mapping --build

      - name: Check Opik server availability
        shell: bash
        run: |
          chmod +x ${{ github.workspace }}/tests_end_to_end/installer_utils/*.sh
          cd ${{ github.workspace }}/deployment/docker-compose
          ${{ github.workspace }}/tests_end_to_end/installer_utils/check_docker_compose_pods.sh
          ${{ github.workspace }}/tests_end_to_end/installer_utils/check_backend.sh

      - name: Install opik-python-backend
        run: |
          cd ${{ github.workspace }}/apps/opik-python-backend
          uv pip install --system -r requirements.txt -r tests/test_requirements.txt
          uv pip install --system .

      - name: Run Python BE E2E tests
        run: |
          cd ${{ github.workspace }}/apps/opik-python-backend
          # Retry transient LLM/network failures so a flaky provider call
          # doesn't fail the run; the assertions themselves are deterministic.
          # --log-cli-level streams logs live (incl. the subprocess error +
          # traceback surfaced by process_optimizer_job) so failures are fully
          # visible even though the optimization runs in a child process.
          # --durations=0 logs how long each test took.
          pytest tests/e2e -vv \
            --reruns 2 --reruns-delay 5 \
            --log-cli-level=INFO \
            --durations=0 \
            --junitxml=${{ github.workspace }}/python_be_e2e_results.xml

      - name: Publish Test Report
        uses: EnricoMi/publish-unit-test-result-action/linux@v2
        if: always()
        with:
          action_fail: true
          comment_mode: failures
          check_name: Python BE E2E Tests Results
          files: ${{ github.workspace }}/python_be_e2e_results.xml

      - name: Keep BE logs on failure
        if: failure()
        run: |
          cd ${{ github.workspace }}/deployment/docker-compose
          docker compose logs backend > ${{ github.workspace }}/backend.log 2>&1 || true
          docker compose logs python-backend > ${{ github.workspace }}/python-backend.log 2>&1 || true

      - name: Upload BE logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: python-be-e2e-be-logs
          path: |
            ${{ github.workspace }}/backend.log
            ${{ github.workspace }}/python-backend.log
          if-no-files-found: ignore