evals(openshift): Add evals tasks for node_debug_exec tool

Author: arkadeepsen

.github/workflows/mcpchecker.yaml

@@ -13,7 +13,7 @@ on:
   workflow_dispatch:
     inputs:
       suite:
-        description: 'Which task suite to run (core, helm, kubevirt, kiali, tekton, or all)'
+        description: 'Which task suite to run (core, helm, kubevirt, kiali, tekton, openshift, or all)'
         required: false
         default: 'core'
         type: choice
@@ -23,6 +23,7 @@ on:
           - kubevirt
           - kiali
           - tekton
+          - openshift
           - all
       task-filter:
         description: 'Regular expression to filter tasks (optional)'
@@ -111,7 +112,7 @@ jobs:
             COMMENT_BODY="${COMMENT_BODY//[[:space:]]/ }"
             read -r _ FIRST_WORD _ <<< "$COMMENT_BODY"
             case "${FIRST_WORD,,}" in
-              core|helm|kubevirt|kiali|tekton|all)
+              core|helm|kubevirt|kiali|tekton|openshift|all)
                 SUITE="${FIRST_WORD,,}"
                 ;;
             esac
@@ -136,9 +137,13 @@ jobs:
               echo "label-selector=suite=tekton" >> $GITHUB_OUTPUT
               echo "toolsets=core,tekton" >> $GITHUB_OUTPUT
               ;;
+            openshift)
+              echo "label-selector=suite=openshift" >> $GITHUB_OUTPUT
+              echo "toolsets=core,config,openshift" >> $GITHUB_OUTPUT
+              ;;
             all)
               echo "label-selector=" >> $GITHUB_OUTPUT  # No filter: run all taskSets
-              echo "toolsets=core,config,helm,kiali,kubevirt,tekton" >> $GITHUB_OUTPUT
+              echo "toolsets=core,config,helm,kiali,kubevirt,tekton,openshift" >> $GITHUB_OUTPUT
               ;;
             *)
               # Default to core suite (matches 'core' or any unrecognized suite)

evals/claude-code/eval.yaml

@@ -92,3 +92,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 15
+    # OpenShift tasks
+    - glob: ../tasks/openshift/*/*.yaml
+      labelSelector:
+        suite: openshift
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/gemini-agent/eval.yaml

@@ -26,3 +26,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 20
+    # OpenShift tasks
+    - glob: ../tasks/openshift/*/*.yaml
+      labelSelector:
+        suite: openshift
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/openai-agent/eval.yaml

@@ -92,3 +92,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 15
+    # OpenShift tasks
+    - glob: ../tasks/openshift/*/*.yaml
+      labelSelector:
+        suite: openshift
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/tasks/openshift/README.md

@@ -0,0 +1,59 @@
+# OpenShift Task Stack
+
+OpenShift-focused MCP tasks live here. Each folder under this directory is a self-contained scenario that exercises the OpenShift toolset—primarily `nodes_debug_exec` for privileged node-level debugging.
+
+## Prerequisites
+
+- Kubernetes cluster with at least one control-plane node (Kind is used in CI as `mcp-eval-cluster`)
+- MCP server running with the `openshift` toolset enabled (`TOOLSETS=core,config,openshift`)
+- `read_only = false` and `disable_destructive = false` (the debug tool is destructive and hidden otherwise)
+- ServiceAccount permissions: create/delete pods, exec into pods, schedule on nodes, run privileged pods
+
+## Prompt Design
+
+Prompts describe **what to find**, not **how to find it**. They do not name MCP tools, container images, or implementation details such as `/host` mounts or `chroot`. The agent must discover and use the appropriate OpenShift tooling on its own.
+
+Task-level `assertions.toolsUsed` in each `task.yaml` still verify that `nodes_debug_exec` was called for scenarios that require node debug exec.
+
+## Cleanup and Verification
+
+Each task ships with:
+
+- **`cleanup.sh`** — deletes ephemeral debug pods labeled `app.kubernetes.io/component=node-debug`
+- **`verify.sh`** — asserts no `node-debug-*` pods remain after the agent completes
+
+Response checks use `verify.contains` in the task YAML (e.g. `"Linux"` or `"hosts"` in the agent reply).
+
+## Registration and CI
+
+Tasks are discovered automatically when `metadata.labels.suite` is `openshift`. They are registered in:
+
+- `evals/openai-agent/eval.yaml`
+- `evals/claude-code/eval.yaml`
+- `evals/gemini-agent/eval.yaml`
+
+CI runs the suite via the mcpchecker workflow (`openshift` option or `/run-mcpchecker openshift` on PRs) with `toolsets=core,config,openshift`.
+
+## Running Locally
+
+```bash
+make run-server TOOLSETS=core,config,openshift
+make run-evals EVAL_CONFIG=evals/openai-agent/eval.yaml EVAL_LABEL_SELECTOR=suite=openshift
+
+# Single task:
+mcpchecker check evals/openai-agent/eval.yaml --run "check-node-kernel"
+```
+
+## Adding a New Task
+
+1. Create a new subdirectory (e.g., `my-scenario/`) with `task.yaml` and optional `setup.sh`, `verify.sh`, and `cleanup.sh`.
+2. Set `metadata.labels.suite: openshift` so the task is picked up by the openshift taskSet.
+3. Write a goal-only prompt; pin `nodes_debug_exec` in task-level `assertions.toolsUsed` when the scenario requires it.
+4. Provide `cleanup.sh` (and preferably `verify.sh`) to ensure ephemeral debug pods are removed.
+
+## Tasks
+
+| Task | Difficulty | Prompt | Verification |
+|------|------------|--------|--------------|
+| **check-node-kernel** | easy | What is the kernel version running on the control-plane node? | Response contains `Linux`; no leftover debug pods |
+| **list-host-etc** | medium | On the control-plane node, list the contents of `/etc` and report what you find. | Response contains `hosts`; no leftover debug pods |

evals/tasks/openshift/check-node-kernel/cleanup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl delete pods -A -l app.kubernetes.io/component=node-debug --ignore-not-found --wait=false

evals/tasks/openshift/check-node-kernel/task.yaml

@@ -0,0 +1,18 @@
+kind: Task
+metadata:
+  labels:
+    suite: openshift
+  name: check-node-kernel
+  difficulty: easy
+steps:
+  prompt:
+    inline: What is the kernel version running on the control-plane node?
+  verify:
+    contains: "Linux"
+    file: verify.sh
+  cleanup:
+    file: cleanup.sh
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "nodes_debug_exec"

evals/tasks/openshift/check-node-kernel/verify.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+remaining=$(kubectl get pods -A -l app.kubernetes.io/component=node-debug --no-headers 2>/dev/null | wc -l | tr -d ' ')
+if [ "${remaining}" -gt 0 ]; then
+  echo "Found leftover node-debug pods:"
+  kubectl get pods -A -l app.kubernetes.io/component=node-debug
+  exit 1
+fi

evals/tasks/openshift/list-host-etc/cleanup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl delete pods -A -l app.kubernetes.io/component=node-debug --ignore-not-found --wait=false

evals/tasks/openshift/list-host-etc/task.yaml

@@ -0,0 +1,18 @@
+kind: Task
+metadata:
+  labels:
+    suite: openshift
+  name: list-host-etc
+  difficulty: medium
+steps:
+  prompt:
+    inline: "On the control-plane node, list the contents of /etc and report what you find."
+  verify:
+    contains: "hosts"
+    file: verify.sh
+  cleanup:
+    file: cleanup.sh
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "nodes_debug_exec"