Test llm validation trigger #12
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LLM Metadata Validation | |
| on: | |
| pull_request: | |
| paths: | |
| - "data/**" | |
| - "scripts/validate_llm_metadata.py" | |
| - "tests/test_validate_llm_metadata.py" | |
| - ".github/workflows/llm-metadata-validation.yml" | |
| workflow_dispatch: | |
| jobs: | |
| validate-llm-metadata: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| env: | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Detect newly added entry IDs | |
| id: changed-ids | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| BASE_SHA="${{ github.event.pull_request.base.sha }}" | |
| HEAD_SHA="${{ github.event.pull_request.head.sha }}" | |
| ADDED_IDS="$(git diff --unified=0 "${BASE_SHA}" "${HEAD_SHA}" -- data/models.yaml data/datasets.yaml data/tools.yaml \ | |
| | grep -oE '^\+\s*-?\s*id:\s*.*' \ | |
| | sed -E 's/^\+\s*-?\s*id:\s*//' \ | |
| | tr -d '"' \ | |
| | tr '\n' ',' \ | |
| | sed 's/,$//')" | |
| echo "entry_ids=${ADDED_IDS}" >> "$GITHUB_OUTPUT" | |
| - name: Skip when no new metadata entries | |
| if: github.event_name == 'pull_request' && steps.changed-ids.outputs.entry_ids == '' | |
| run: | | |
| mkdir -p reports | |
| cat <<EOF > reports/llm_validation_report.json | |
| {"status":"skipped","skipped_reason":"No newly added metadata entries in this PR.","counts":{"pass":0,"warning":0,"fail":0},"results":[]} | |
| EOF | |
| cat <<EOF > reports/llm_validation_summary.md | |
| # LLM Metadata Validation | |
| - Status: \`skipped\` | |
| - Pass: \`0\` | |
| - Warning: \`0\` | |
| - Fail: \`0\` | |
| - Skipped reason: No newly added metadata entries in this PR. | |
| EOF | |
| - name: Run LLM metadata validation | |
| if: github.event_name != 'pull_request' || steps.changed-ids.outputs.entry_ids != '' | |
| run: > | |
| python scripts/validate_llm_metadata.py | |
| --entry-ids "${{ steps.changed-ids.outputs.entry_ids }}" | |
| --output reports/llm_validation_report.json | |
| --summary-output reports/llm_validation_summary.md | |
| - name: Comment validation result on PR | |
| if: github.event_name == 'pull_request' && always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const marker = "<!-- llm-metadata-validation-comment -->"; | |
| const summary = fs.readFileSync("reports/llm_validation_summary.md", "utf8"); | |
| const body = `${marker}\n${summary}`; | |
| const { owner, repo } = context.repo; | |
| const issue_number = context.issue.number; | |
| const comments = await github.paginate( | |
| github.rest.issues.listComments, | |
| { owner, repo, issue_number, per_page: 100 } | |
| ); | |
| const existing = comments.find((comment) => | |
| comment.user?.type === "Bot" && comment.body?.includes(marker) | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body | |
| }); | |
| core.info(`Updated validation comment: ${existing.id}`); | |
| } else { | |
| const created = await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body | |
| }); | |
| core.info(`Created validation comment: ${created.data.id}`); | |
| } | |