From 9f2f63594afbad90ee22303e10966dd2e58a093b Mon Sep 17 00:00:00 2001 From: Yoav Katz Date: Sun, 4 Oct 2026 16:43:22 +0300 Subject: [PATCH] CI: Don't fail fork and Dependabot runs on unavailable secrets GitHub withholds repository secrets from workflow runs triggered by forks and by Dependabot. Two steps treated those secrets as mandatory and failed hard when they resolved to an empty string, which is why PR #1986 showed 11 red jobs that never executed a single test. The internal-dependency step wrote secrets.LLMEVALKIT_SSH_KEY to ~/.ssh/id_ed25519 and pip-installed from git+ssh://github.ibm.com. With an empty key this aborts the whole job: Load key "/home/runner/.ssh/id_ed25519": error in libcrypto git@github.ibm.com: Permission denied (publickey). That killed unittests, eager, and all ten preparation shards before the test step. The host is IBM-internal, so it is unreachable from an external fork regardless of the secret. Skip the install when no key is present. The llmevalkit imports in metrics.py are already lazy and guarded by _requirements_list, so the affected metrics raise a clear "install with..." message if something actually instantiates them; everything else is unaffected. The Hugging Face login step interpolated an empty token into `huggingface-cli login --token`, looped five times, and left the run unauthenticated, which then hit the shared-runner rate limit: urllib.error.HTTPError: HTTP Error 429: Too Many Requests Skip the login when no token is present and say so in the log, instead of retrying a call that cannot succeed. Unauthenticated runs remain exposed to Hub rate limits; this reports the cause rather than fixing it. Both guards are no-ops on trusted runs. Prefer a guard inside the step over continue-on-error at the call sites, so a real install failure still fails the job when the secret IS available. Also pass both secrets through env: rather than interpolating them into the script body, and write the key with printf instead of echo, which mangles values containing backslashes or a leading dash. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Yoav Katz --- .github/actions/install-internal-pip/action.yml | 17 ++++++++++++----- .github/workflows/examples_tests.yml | 8 +++++++- .github/workflows/inference_tests.yml | 8 +++++++- .github/workflows/performance.yml | 8 +++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/actions/install-internal-pip/action.yml b/.github/actions/install-internal-pip/action.yml index e33107e585..b605bf54c0 100644 --- a/.github/actions/install-internal-pip/action.yml +++ b/.github/actions/install-internal-pip/action.yml @@ -2,8 +2,10 @@ name: Install internal repo via pip (SSH) description: pip install git+ssh from GHES inputs: ssh-private-key: - required: true - description: OpenSSH private key (with BEGIN/END) + required: false + description: >- + OpenSSH private key (with BEGIN/END). Empty on runs that cannot read repo + secrets (forks, Dependabot), in which case the install is skipped. host: required: false default: github.ibm.com @@ -20,9 +22,15 @@ runs: using: composite steps: - shell: bash + env: + SSH_PRIVATE_KEY: ${{ inputs.ssh-private-key }} run: | + if [ -z "$SSH_PRIVATE_KEY" ]; then + echo "::notice title=Internal install skipped::No SSH key available (fork or Dependabot run); skipping ${{ inputs.repo }}. Metrics requiring it will raise a clear install error if used." + exit 0 + fi mkdir -p ~/.ssh - echo "${{ inputs.ssh-private-key }}" > ~/.ssh/id_ed25519 + printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 ssh-keyscan "${{ inputs.host }}" >> ~/.ssh/known_hosts if [ -n "${{ inputs.ref }}" ]; then @@ -33,5 +41,4 @@ runs: echo "Installing from URL: $URL" pip install --no-cache-dir --force-reinstall "$URL" ${{ inputs.pip-extra-args }} echo "Downgrading pydantic as required LLM eval kit library" - pip install pydantic==2.11.3 - + pip install pydantic==2.11.3 diff --git a/.github/workflows/examples_tests.yml b/.github/workflows/examples_tests.yml index e10d5f38b8..b98d7edfcb 100644 --- a/.github/workflows/examples_tests.yml +++ b/.github/workflows/examples_tests.yml @@ -48,9 +48,15 @@ jobs: ssh-private-key: ${{ secrets.LLMEVALKIT_SSH_KEY }} - name: Hugging Face Login + env: + HF_TOKEN_FOR_TESTS: ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} run: | + if [ -z "$HF_TOKEN_FOR_TESTS" ]; then + echo "::notice title=Hugging Face login skipped::No token available (fork or Dependabot run); continuing unauthenticated, which may hit Hub rate limits." + exit 0 + fi for i in {1..5}; do - huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} && break || sleep $((2 ** i)); + huggingface-cli login --token "$HF_TOKEN_FOR_TESTS" && break || sleep $((2 ** i)); done - name: Run Tests diff --git a/.github/workflows/inference_tests.yml b/.github/workflows/inference_tests.yml index ddaba0baba..ed2b73662d 100644 --- a/.github/workflows/inference_tests.yml +++ b/.github/workflows/inference_tests.yml @@ -45,9 +45,15 @@ jobs: - run: curl -LsSf https://astral.sh/uv/install.sh | sh - run: uv pip install --system ".[tests,watsonx,inference-tests]" - name: Hugging Face Login + env: + HF_TOKEN_FOR_TESTS: ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} run: | + if [ -z "$HF_TOKEN_FOR_TESTS" ]; then + echo "::notice title=Hugging Face login skipped::No token available (fork or Dependabot run); continuing unauthenticated, which may hit Hub rate limits." + exit 0 + fi for i in {1..5}; do - huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} && break || sleep $((2 ** i)); + huggingface-cli login --token "$HF_TOKEN_FOR_TESTS" && break || sleep $((2 ** i)); done - name: Install Ollama run: curl -fsSL https://ollama.com/install.sh | sh diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index a58f02d757..a1adda46aa 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -40,9 +40,15 @@ jobs: uv pip install --system diskcache - name: Hugging Face Login + env: + HF_TOKEN_FOR_TESTS: ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} run: | + if [ -z "$HF_TOKEN_FOR_TESTS" ]; then + echo "::notice title=Hugging Face login skipped::No token available (fork or Dependabot run); continuing unauthenticated, which may hit Hub rate limits." + exit 0 + fi for i in {1..5}; do - huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} && break || sleep $((2 ** i)); + huggingface-cli login --token "$HF_TOKEN_FOR_TESTS" && break || sleep $((2 ** i)); done - name: Prepare the dirs for performance evaluation in main run: |