feat: reusable AI agent steps with rigid linking, edit/fork, and evals #22833
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend only integration tests | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| paths: | |
| - "backend/**" | |
| - ".github/workflows/backend-test.yml" | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - "backend/**" | |
| - ".github/workflows/backend-test.yml" | |
| defaults: | |
| run: | |
| working-directory: ./backend | |
| jobs: | |
| cargo_test: | |
| runs-on: ubicloud-standard-16 | |
| services: | |
| postgres: | |
| image: postgres | |
| ports: | |
| - 5432:5432 | |
| env: | |
| POSTGRES_DB: windmill | |
| POSTGRES_PASSWORD: changeme | |
| POSTGRES_INITDB_ARGS: "-c max_connections=500" | |
| options: >- | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --shm-size=256mb | |
| mysql: | |
| image: mysql:8.0 | |
| ports: | |
| - 3306:3306 | |
| env: | |
| MYSQL_ROOT_PASSWORD: changeme | |
| MYSQL_DATABASE: windmill_test | |
| options: >- | |
| --health-cmd "mysqladmin ping -h localhost" --health-interval 10s | |
| --health-timeout 5s --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: "9.0.x" | |
| - uses: denoland/setup-deno@v2 | |
| with: | |
| # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI | |
| # tests what production runs, instead of floating on the latest v2.x. | |
| deno-version: 2.2.1 | |
| - uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.21.5 | |
| - uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: 1.3.10 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - uses: astral-sh/setup-uv@v6.2.1 | |
| with: | |
| version: "0.11.24" | |
| - uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: "8.3" | |
| tools: composer | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: "3.3" | |
| bundler-cache: false | |
| - name: Install windmill CLI from source | |
| run: | | |
| cd $GITHUB_WORKSPACE/cli | |
| bash gen_wm_client.sh | |
| bun install | |
| mkdir -p "$HOME/.local/bin" | |
| printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" | |
| chmod +x "$HOME/.local/bin/wmill" | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| working-directory: / | |
| - name: Install PowerShell, mold and clang | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev | |
| working-directory: / | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-workspaces: backend | |
| toolchain: 1.97.0 | |
| - name: Fix stale v8 build cache | |
| working-directory: ./backend | |
| run: | | |
| # Cargo cache may preserve v8 build fingerprints without the actual | |
| # librusty_v8.a library. Since fingerprints look valid, cargo skips | |
| # build.rs re-run, causing "could not find native static library rusty_v8". | |
| for profile in debug release; do | |
| if [ -d "target/$profile/.fingerprint" ] && [ ! -f "target/$profile/gn_out/obj/librusty_v8.a" ]; then | |
| echo "Cleaning stale v8 build artifacts in target/$profile" | |
| rm -rf "target/$profile/build/v8-"* "target/$profile/.fingerprint/v8-"* | |
| fi | |
| done | |
| - name: Read EE repo commit hash | |
| run: | | |
| echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: windmill-labs/windmill-ee-private | |
| path: ./windmill-ee-private | |
| ref: ${{ env.ee_repo_ref }} | |
| token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} | |
| fetch-depth: 0 | |
| - name: Substitute EE code (EE logic is behind feature flag) | |
| run: | | |
| ./substitute_ee_code.sh --copy --dir ./windmill-ee-private | |
| - name: Setup private npm registry with test package | |
| working-directory: /tmp | |
| run: | | |
| set -e | |
| # Install Verdaccio globally | |
| npm install -g verdaccio | |
| # Create Verdaccio config that requires authentication for @windmill-test packages | |
| mkdir -p /tmp/verdaccio/storage | |
| cat > /tmp/verdaccio/config.yaml << 'VERDACCIO_CONFIG' | |
| storage: /tmp/verdaccio/storage | |
| auth: | |
| htpasswd: | |
| file: /tmp/verdaccio/htpasswd | |
| max_users: 100 | |
| uplinks: | |
| npmjs: | |
| url: https://registry.npmjs.org/ | |
| packages: | |
| '@windmill-test/*': | |
| access: $authenticated | |
| publish: $authenticated | |
| '@*/*': | |
| access: $all | |
| publish: $authenticated | |
| proxy: npmjs | |
| '**': | |
| access: $all | |
| publish: $authenticated | |
| proxy: npmjs | |
| server: | |
| keepAliveTimeout: 60 | |
| middlewares: | |
| audit: | |
| enabled: true | |
| log: { type: stdout, format: pretty, level: warn } | |
| VERDACCIO_CONFIG | |
| # Create empty htpasswd file (users will be created via API) | |
| touch /tmp/verdaccio/htpasswd | |
| # Start Verdaccio in background | |
| verdaccio --config /tmp/verdaccio/config.yaml & | |
| VERDACCIO_PID=$! | |
| # Wait for Verdaccio to be ready | |
| echo "Waiting for Verdaccio to start..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:4873/-/ping > /dev/null 2>&1; then | |
| echo "Verdaccio is ready" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| # Login to get a token | |
| echo "Getting auth token..." | |
| RESPONSE=$(curl -s -X PUT \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"name":"testuser","password":"testpass123"}' \ | |
| http://localhost:4873/-/user/org.couchdb.user:testuser) | |
| echo "Auth response: $RESPONSE" | |
| NPM_TOKEN=$(echo "$RESPONSE" | jq -r '.token') | |
| if [ -z "$NPM_TOKEN" ] || [ "$NPM_TOKEN" = "null" ]; then | |
| echo "Failed to get NPM token from response" | |
| exit 1 | |
| fi | |
| echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV | |
| { | |
| echo "TEST_NPMRC<<NPMRC_EOF" | |
| echo "@windmill-test:registry=http://localhost:4873/" | |
| echo "//localhost:4873/:_authToken=${NPM_TOKEN}" | |
| echo "NPMRC_EOF" | |
| } >> $GITHUB_ENV | |
| echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..." | |
| # Configure npm globally with the auth token | |
| echo "//localhost:4873/:_authToken=${NPM_TOKEN}" > ~/.npmrc | |
| echo "Configured ~/.npmrc with auth token" | |
| # Create a simple test package | |
| mkdir -p /tmp/windmill-test-private-pkg | |
| cat > /tmp/windmill-test-private-pkg/package.json << 'PKG_JSON' | |
| { | |
| "name": "@windmill-test/private-pkg", | |
| "version": "1.0.0", | |
| "main": "index.js" | |
| } | |
| PKG_JSON | |
| cat > /tmp/windmill-test-private-pkg/index.js << 'PKG_JS' | |
| module.exports.greet = (name) => `Hello from private package, ${name}!`; | |
| PKG_JS | |
| # Publish to Verdaccio with auth | |
| cd /tmp/windmill-test-private-pkg | |
| echo "Publishing package..." | |
| npm publish --registry http://localhost:4873 | |
| echo "Package published successfully" | |
| # Verify the package requires auth by trying anonymous access (should fail) | |
| rm -f ~/.npmrc | |
| echo "Testing anonymous access (should fail)..." | |
| if npm view @windmill-test/private-pkg --registry http://localhost:4873 2>/dev/null; then | |
| echo "ERROR: Package should require authentication but anonymous access worked" | |
| exit 1 | |
| fi | |
| echo "Verified: Package requires authentication for @windmill-test/private-pkg" | |
| - name: Cache DuckDB FFI module build | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./backend/windmill-duckdb-ffi-internal/target | |
| key: ${{ runner.os }}-duckdb-ffi-${{ hashFiles('./backend/windmill-duckdb-ffi-internal/src/**/*.rs', './backend/windmill-duckdb-ffi-internal/Cargo.toml', './backend/windmill-duckdb-ffi-internal/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-duckdb-ffi- | |
| - name: cargo test | |
| timeout-minutes: 30 | |
| env: | |
| # setup-rust-toolchain exports RUSTFLAGS=-D warnings, and the RUSTFLAGS env | |
| # var fully REPLACES (never merges with) target.*.rustflags in | |
| # backend/.cargo/config.toml. That silently drops the config's | |
| # `-C link-arg=-fuse-ld=mold`, so CI links the many large integration-test | |
| # binaries (v8 + duckdb + every language runtime, statically linked) with the | |
| # default bfd linker. Its peak memory across ~12 parallel links OOM-kills the | |
| # runner mid-link (SIGTERM => exit 143, before any test runs). Re-add the mold | |
| # link arg here so CI links with mold like local dev, keeping -D warnings. | |
| # (config.toml's `linker = "clang"` still applies; env only overrides rustflags.) | |
| RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=mold" | |
| SQLX_OFFLINE: true | |
| DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill | |
| DISABLE_EMBEDDING: true | |
| RUST_LOG: "off" | |
| RUST_LOG_STYLE: never | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| # Cap parallel rustc/link jobs below the 16 available cores. The tail of | |
| # the build links ~128 full-graph test binaries (one per tests/*.rs file | |
| # across the workspace); at high parallelism enough heavy codegen+link | |
| # units (rustc ~2.6GB, mold ~1GB each) overlap to exhaust the 64GB | |
| # runner. Matches backend-test-windows.yml, which already uses 8. | |
| CARGO_BUILD_JOBS: 8 | |
| # Incremental compilation is per-run dead weight in CI: rust-cache | |
| # (cache-workspaces above) restores compiled dependency artifacts but | |
| # never persists target/**/incremental, so there is no prior state to | |
| # reuse in a one-shot `cargo test`. It only adds per-crate memory | |
| # overhead and extra disk. Off here (kept on for local dev via | |
| # .cargo/config.toml). Matches backend-test-windows.yml. | |
| CARGO_INCREMENTAL: "0" | |
| # backend/Cargo.toml leaves profile.dev at the default debug = 2 for | |
| # the (large) windmill workspace crates; that debug info is emitted | |
| # into every object file and embedded in each test binary. Across the | |
| # full --all --features build it is the dominant memory/disk consumer | |
| # when mold links the windmill-api-integration-tests binary, tipping | |
| # the runner over (lost runner reported as a canceled step). CI needs | |
| # no debug info, so drop it entirely for the dev/test profiles here. | |
| # (test profile inherits dev, but the workspace crates link in as | |
| # dev-profile deps, so both must be set.) CI-only; local dev builds | |
| # are unaffected. | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| # Tests' poll-time stack frames (deep nested async fn chains in | |
| # debug builds) reach ~1.8MB, leaving very thin headroom on the | |
| # default 2MB thread stack. 4MB gives ~2x buffer against flaky | |
| # overflows under parallel-test contention. | |
| RUST_MIN_STACK: 4194304 | |
| WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 | |
| WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 | |
| WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 | |
| TEST_NPM_REGISTRY: "http://localhost:4873/:_authToken=${{ env.NPM_TOKEN }}" | |
| run: | | |
| deno --version && bun -v && node --version && go version && python3 --version && php --version && ruby --version && pwsh --version && dotnet --version | |
| cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. | |
| DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline --all -- --nocapture --test-threads=10 |