From 252503497ee53513f9a09a8c078cc283aa876633 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 13:29:09 +0000 Subject: [PATCH 1/6] ci: skip CI on the generated eval results commit The eval-refresh workflow pushes "chore: refresh eval results" back onto the PR head branch with GITHUB_TOKEN. That push does not fire `push` workflows, but it does fire `pull_request: synchronize`, so Refresh eval results and Code Quality re-run as github-actions[bot] and sit in "approval required" for a maintainer (see supabase/evals#281). Because the refresh is label-gated, the re-run would also refresh again. Append the skip-ci marker to the direct-push results commit so GitHub creates no runs for it. The scheduled/dispatch path that opens a fresh results PR through create-pull-request is unchanged and still gets normal CI. main has no required status checks, so a skipped head commit does not block merging. Refs: AI-1254 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ECba2eYo5iJPLnVPT8JvkV --- .github/workflows/eval-refresh.yml | 8 +++++++- CONTRIBUTING.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index e7520f4c..9a3fff86 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -533,7 +533,13 @@ jobs: exit 0 fi - git commit -m "chore: refresh eval results" + # A GITHUB_TOKEN push skips `push` workflows but still fires + # `pull_request: synchronize` on the branch's PR, re-running this + # workflow and Code Quality as github-actions[bot]. Those runs need a + # maintainer's approval and re-running here would loop the refresh. + # [skip ci] on the HEAD commit stops GitHub creating the runs at all: + # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/skipping-workflow-runs + git commit -m "chore: refresh eval results [skip ci]" git push - name: Create results pull request diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d8739fe..a62e47d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ When you create a PR, use GitHub Actions to refresh the results in CI so we can You have a few options to run evals in CI: -- Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch. +- Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch. The results commit carries `[skip ci]` so it doesn't re-trigger CI on your PR. - Add the `run-evals` label to run every benchmark eval across the `benchmark` and `no-skills` experiment suites. Use this when a change can affect results broadly, such as framework changes. - Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. It can commit results directly to the selected branch or open a separate results PR. From f5f68c43d8a0945626957fe40252b490b7499ca2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 13:43:19 +0000 Subject: [PATCH 2/6] test: temporary change to resolve-reliability-001-unhealthy-project-recovery to exercise the refresh commit in CI Adds a behaviour-neutral comment to the eval's scorer so the run-evals-changed path resolves to a non-empty pairs list and the workflow's direct-push results commit executes. This commit will be reverted before merge. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ECba2eYo5iJPLnVPT8JvkV --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index 9db50ef3..932b6979 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -6,6 +6,7 @@ import { } from '@supabase-evals/core'; import { stripIndent } from 'common-tags'; +// Judge-only scorer: the transcript is graded against the rubric below. const scorer: ToolScorer = async (ctx) => { const verdict = await judge({ input: serializeTranscript(ctx.transcript), From 48b346e33eaf88df01ddbce5455d096d3eb32de5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:47:15 +0000 Subject: [PATCH 3/6] chore: refresh eval results [skip ci] --- .../web/src/data/regression-eval-results.json | 509 +++++++++--------- 1 file changed, 253 insertions(+), 256 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 663d9ac7..3905b5f3 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -3970,7 +3970,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer rejects pause/restore as an initial fix, distinguishes restore from restart, and gives a concrete, conditional restart step with support escalation." + "judgeNotes": "Recommends restarting as the first recovery step, rejects pause/restore as a reset, distinguishes it from backup restoration, and advises checking platform status and contacting support if needed." } ], "skills": { @@ -3987,19 +3987,21 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore Supabase project recovery\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard database restart support\", limit: 8) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart pause restore project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pause restore project Supabase dashboard pause project restore\", limit: 8) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project unhealthy dashboard restart project pause restore Supabase project\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" @@ -4009,25 +4011,40 @@ "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" } ], - "resultChars": 24720 + "resultChars": 118970 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project health issue dashboard support Supabase\", limit: 5) { nodes { title href content } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging unhealthy project restart dashboard Supabase", "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"unhealthy project database restart health check Supabase monitoring debugging\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4035,43 +4052,38 @@ "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", - "title": "Shared Responsibility Model" + "url": "https://supabase.com/docs/guides/platform/billing-faq", + "title": "Billing FAQ" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" } ], - "resultChars": 43659 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart unhealthy project Supabase", - "pages": [] + "resultChars": 79140 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 89631, - "cacheReadInputTokens": 36861, - "cacheWriteInputTokens": 35471, - "outputTokens": 1977 + "inputTokens": 85009, + "cacheReadInputTokens": 36934, + "cacheWriteInputTokens": 41282, + "outputTokens": 1554 } ], "stepCount": 4, - "toolCallCount": 9, - "agentRunDurationMs": 31826, + "toolCallCount": 7, + "agentRunDurationMs": 27909, "sandboxUsage": { "memory": 8192 }, @@ -4104,7 +4116,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database first, clearly distinguishes restart from backup restore and resuming a paused project, notes resource overload, and directs the user to troubleshooting guidance and support." + "judgeNotes": "Recommends restarting rather than pausing/restoring, distinguishes resuming a paused project from recovering an unhealthy one, and cites relevant Supabase guidance. It could add log and resource checks and advise contacting support if the issue persists." } ], "skills": { @@ -4119,203 +4131,65 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11960 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging project health unhealthy database restart Supabase dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { a: searchDocs(query: \"restart Supabase project unhealthy dashboard restart project\", limit: 5) { nodes { title href content } } b: searchDocs(query: \"pause and restore Supabase project restore project paused project data\", limit: 5) { nodes { title href content } } c: searchDocs(query: \"Supabase monitoring and debugging project unhealthy dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", - "title": "Shared Responsibility Model" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "title": "Supabase dashboard not loading/ Project not loading on dashboard" }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 18601 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - } - ], - "resultChars": 32742 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore paused project unpause project Supabase\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" - } - ], - "resultChars": 16762 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and debugging Supabase project diagnose issues logs health check advisors restart project\", limit: 8) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { - "url": "https://supabase.com/docs/guides/observability/automate-with-agents", - "title": "Hire an agent" - }, - { - "url": "https://supabase.com/docs/guides/observability/automate-with-agents/security", - "title": "Security monitor" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/functions/error-handling", - "title": "Error Handling" + "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", + "title": "Why do I see Auth & API requests in the dashboard? My app has no users" }, { "url": "https://supabase.com/docs/guides/database/postgres/timeouts", "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/observability/advisors", - "title": "Advisors" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 32486 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Restart project Supabase dashboard restart database\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - } - ], - "resultChars": 114211 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase backups restore project data recovery downtime point in time recovery overwrites\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" } ], - "resultChars": 40294 + "resultChars": 68712 }, { "source": "web_search", - "query": "site:supabase.com/docs project restart dashboard unhealthy pause restore project", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] - }, - { - "source": "web_search", - "query": "Supabase status incidents September 24 2026 dashboard database unhealthy", + "query": "site:supabase.com/docs pause restore Supabase project restore project pause resume database", "pages": [] } ] @@ -4323,15 +4197,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 114869, - "cacheReadInputTokens": 45019, - "cacheWriteInputTokens": 49754, - "outputTokens": 2677 + "inputTokens": 64655, + "cacheReadInputTokens": 31315, + "cacheWriteInputTokens": 19668, + "outputTokens": 1588 } ], - "stepCount": 4, - "toolCallCount": 12, - "agentRunDurationMs": 54629, + "stepCount": 3, + "toolCallCount": 4, + "agentRunDurationMs": 23844, "sandboxUsage": { "memory": 8192 }, @@ -4359,12 +4233,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restart as the first recovery step, rejects pause/restore, notes overload as a possible underlying cause, and advises contacting support if controls are unavailable." + "passed": false, + "judgeNotes": "It correctly recommends restarting rather than pausing, but it describes “restore” only as resuming a paused project, without distinguishing backup/data restoration from restart. It also omits concrete follow-up steps if the project remains unhealthy." } ], "skills": { @@ -4373,26 +4247,36 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore Supabase project dashboard unhealthy\", limit: 5) { nodes { ... on Guide { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging dashboard project unhealthy database restart support incident\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5007 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart pause restore project recovery\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"unhealthy project restart project pause restore project recovery troubleshooting\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { "url": "https://supabase.com/docs/guides/platform/upgrading", @@ -4403,20 +4287,65 @@ "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" } ], - "resultChars": 26907 + "resultChars": 24720 }, { - "source": "web_search", - "query": "site:supabase.com/docs restart project Supabase dashboard restart unhealthy project", + "source": "search_docs", + "query": "query { searchDocs(query: \"pause resume restore paused project data preserved Supabase\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Dashboard unavailable status incident support\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase troubleshoot dashboard project unhealthy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", + "title": "Why do I see Auth & API requests in the dashboard? My app has no users" + } + ], + "resultChars": 17113 + }, { "source": "web_search", - "query": "site:supabase.com/docs \"Restart project\" Management API Supabase", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services Supabase restart unhealthy", "pages": [] } ] @@ -4424,15 +4353,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 64345, - "cacheReadInputTokens": 16868, - "cacheWriteInputTokens": 32387, - "outputTokens": 2537 + "inputTokens": 92899, + "cacheReadInputTokens": 38101, + "cacheWriteInputTokens": 36996, + "outputTokens": 1981 } ], - "stepCount": 3, - "toolCallCount": 7, - "agentRunDurationMs": 35754, + "stepCount": 4, + "toolCallCount": 10, + "agentRunDurationMs": 32922, "sandboxUsage": { "memory": 8192 }, @@ -9189,7 +9118,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting unhealthy services rather than pausing and restoring, and gives a concrete first step. It could more clearly distinguish backup restoration from a restart and mention logs and support if the project remains unhealthy." + "judgeNotes": "Recommends restarting rather than pausing/restoring, gives a concrete first step, and links to Supabase’s unhealthy-services guidance. It could more clearly distinguish backup restoration from restart and mention support if the issue persists." } ], "skills": { @@ -9200,28 +9129,43 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", + "query": "site:supabase.com/docs restart project pause restore unhealthy project Supabase", "pages": [] }, { "source": "web_search", - "query": "Supabase status page incidents September 24 2026 dashboard database", - "pages": [] + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/http-api-issues", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/http-api-issues" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 36547, + "inputTokens": 31236, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11846, - "outputTokens": 1173 + "cacheWriteInputTokens": 11848, + "outputTokens": 1084 } ], "stepCount": 1, "toolCallCount": 3, - "agentRunDurationMs": 17238, + "agentRunDurationMs": 17900, "sandboxUsage": { "memory": 8192 }, @@ -9249,12 +9193,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends a database restart as the first recovery step, distinguishes it from restoring a paused project, advises against pause-and-restore, and gives concrete checks and support escalation." + "passed": false, + "judgeNotes": "Recommends restarting first and rejects pause-and-restore as a fix, but incorrectly describes restore as the way to resume a paused project rather than distinguishing backup recovery from restart. It also omits logs, resource checks, and escalation to Supabase support." } ], "skills": { @@ -9265,12 +9209,65 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + } + ], + "resultChars": 18222 + }, { "source": "web_search", - "query": "Supabase status page current incident September 24 2026 Supabase dashboard database health", + "query": "site:supabase.com/docs/reference/api restart project Supabase Management API restart", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Restart a project Management API endpoint POST /projects restart\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-restart-a-project", + "title": "Restarts the given project" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-restore-a-project", + "title": "Restores the given project" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-shutdown-realtime", + "title": "Shutdowns realtime connections for a project" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-list-all-projects", + "title": "List all projects" + } + ], + "resultChars": 15227 + }, + { + "source": "web_search", + "query": "Supabase status dashboard incident September 24 2026 dashboard unavailable", "pages": [] } ] @@ -9278,15 +9275,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 26636, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11846, - "outputTokens": 872 + "inputTokens": 86506, + "cacheReadInputTokens": 35109, + "cacheWriteInputTokens": 29936, + "outputTokens": 1754 } ], - "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 16735, + "stepCount": 3, + "toolCallCount": 6, + "agentRunDurationMs": 37292, "sandboxUsage": { "memory": 8192 }, @@ -9319,7 +9316,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database as a concrete first step and explicitly advises against pausing and restoring/resuming an unhealthy project. Notes that a restart may only temporarily help if the database is overloaded and cites Supabase troubleshooting guidance." + "judgeNotes": "Recommends restart as the first recovery step, distinguishes it from pause/restore, and advises checking for incidents and contacting support if the problem persists." } ], "skills": { @@ -9330,33 +9327,33 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project Supabase dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", "pages": [] }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] + "query": "site:supabase.com/docs/reference/api restart project management api POST projects restart", + "pages": [] + }, + { + "source": "web_search", + "query": "\"Restart a project\" \"v1/projects/{ref}/restart\" Supabase", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 29553, + "inputTokens": 30885, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 1005 + "outputTokens": 1258 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 14276, + "toolCallCount": 3, + "agentRunDurationMs": 23020, "sandboxUsage": { "memory": 8192 }, From ff3c936243b9d007e1a68c7831b7d7c11144885c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 13:52:14 +0000 Subject: [PATCH 4/6] revert: chore: refresh eval results Reverts 48b346e33eaf88df01ddbce5455d096d3eb32de5, the results commit the workflow pushed while exercising the run-evals-changed path on this PR. The results were only generated to prove the skip-ci behaviour; they are not part of this change. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ECba2eYo5iJPLnVPT8JvkV --- .../web/src/data/regression-eval-results.json | 509 +++++++++--------- 1 file changed, 256 insertions(+), 253 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 3905b5f3..663d9ac7 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -3970,7 +3970,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting as the first recovery step, rejects pause/restore as a reset, distinguishes it from backup restoration, and advises checking platform status and contacting support if needed." + "judgeNotes": "The answer rejects pause/restore as an initial fix, distinguishes restore from restart, and gives a concrete, conditional restart step with support escalation." } ], "skills": { @@ -3987,64 +3987,47 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard database restart support\", limit: 8) { nodes { title href content } }", + "query": "query { searchDocs(query: \"restart project pause restore Supabase project recovery\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project Supabase dashboard pause project restore\", limit: 8) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy dashboard restart project pause restore Supabase project\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart pause restore project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", - "title": "Build a User Management App with Svelte" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" } ], - "resultChars": 118970 + "resultChars": 24720 }, { - "source": "web_search", - "query": "site:supabase.com/docs/guides/monitoring-and-debugging unhealthy project restart dashboard Supabase", + "source": "search_docs", + "query": "query { searchDocs(query: \"pause restore project health issue dashboard support Supabase\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"unhealthy project database restart health check Supabase monitoring debugging\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4052,38 +4035,43 @@ "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/billing-faq", - "title": "Billing FAQ" + "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", + "title": "Shared Responsibility Model" }, { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", + "title": "Connect to your database" } ], - "resultChars": 79140 + "resultChars": 43659 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart unhealthy project Supabase", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 85009, - "cacheReadInputTokens": 36934, - "cacheWriteInputTokens": 41282, - "outputTokens": 1554 + "inputTokens": 89631, + "cacheReadInputTokens": 36861, + "cacheWriteInputTokens": 35471, + "outputTokens": 1977 } ], "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 27909, + "toolCallCount": 9, + "agentRunDurationMs": 31826, "sandboxUsage": { "memory": 8192 }, @@ -4116,7 +4104,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting rather than pausing/restoring, distinguishes resuming a paused project from recovering an unhealthy one, and cites relevant Supabase guidance. It could add log and resource checks and advise contacting support if the issue persists." + "judgeNotes": "Recommends restarting the database first, clearly distinguishes restart from backup restore and resuming a paused project, notes resource overload, and directs the user to troubleshooting guidance and support." } ], "skills": { @@ -4131,65 +4119,203 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11960 + }, { "source": "search_docs", - "query": "query { a: searchDocs(query: \"restart Supabase project unhealthy dashboard restart project\", limit: 5) { nodes { title href content } } b: searchDocs(query: \"pause and restore Supabase project restore project paused project data\", limit: 5) { nodes { title href content } } c: searchDocs(query: \"Supabase monitoring and debugging project unhealthy dashboard\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging project health unhealthy database restart Supabase dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" + "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", + "title": "Shared Responsibility Model" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "title": "Supabase dashboard not loading/ Project not loading on dashboard" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" + } + ], + "resultChars": 18601 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + } + ], + "resultChars": 32742 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restore paused project unpause project Supabase\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" + } + ], + "resultChars": 16762 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and debugging Supabase project diagnose issues logs health check advisors restart project\", limit: 8) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" + "url": "https://supabase.com/docs/guides/observability/automate-with-agents", + "title": "Hire an agent" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/security", + "title": "Security monitor" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", - "title": "Why do I see Auth & API requests in the dashboard? My app has no users" + "url": "https://supabase.com/docs/guides/functions/error-handling", + "title": "Error Handling" }, { "url": "https://supabase.com/docs/guides/database/postgres/timeouts", "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/database/connection-management", - "title": "Connection management" + "url": "https://supabase.com/docs/guides/observability/advisors", + "title": "Advisors" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + } + ], + "resultChars": 32486 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Restart project Supabase dashboard restart database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + } + ], + "resultChars": 114211 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase backups restore project data recovery downtime point in time recovery overwrites\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" } ], - "resultChars": 68712 + "resultChars": 40294 }, { "source": "web_search", - "query": "site:supabase.com/docs pause restore Supabase project restore project pause resume database", + "query": "site:supabase.com/docs project restart dashboard unhealthy pause restore project", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] + }, + { + "source": "web_search", + "query": "Supabase status incidents September 24 2026 dashboard database unhealthy", "pages": [] } ] @@ -4197,15 +4323,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 64655, - "cacheReadInputTokens": 31315, - "cacheWriteInputTokens": 19668, - "outputTokens": 1588 + "inputTokens": 114869, + "cacheReadInputTokens": 45019, + "cacheWriteInputTokens": 49754, + "outputTokens": 2677 } ], - "stepCount": 3, - "toolCallCount": 4, - "agentRunDurationMs": 23844, + "stepCount": 4, + "toolCallCount": 12, + "agentRunDurationMs": 54629, "sandboxUsage": { "memory": 8192 }, @@ -4233,12 +4359,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "It correctly recommends restarting rather than pausing, but it describes “restore” only as resuming a paused project, without distinguishing backup/data restoration from restart. It also omits concrete follow-up steps if the project remains unhealthy." + "passed": true, + "judgeNotes": "Recommends restart as the first recovery step, rejects pause/restore, notes overload as a possible underlying cause, and advises contacting support if controls are unavailable." } ], "skills": { @@ -4247,36 +4373,26 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging dashboard project unhealthy database restart support incident\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"restart project pause restore Supabase project dashboard unhealthy\", limit: 5) { nodes { ... on Guide { title href content } } }", "hasContent": true, "pages": [] }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5007 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"unhealthy project restart project pause restore project recovery troubleshooting\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart pause restore project recovery\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { "url": "https://supabase.com/docs/guides/platform/upgrading", @@ -4287,65 +4403,20 @@ "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" } ], - "resultChars": 24720 + "resultChars": 26907 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"pause resume restore paused project data preserved Supabase\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Dashboard unavailable status incident support\", limit: 5) { nodes { title href content } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs restart project Supabase dashboard restart unhealthy project", "pages": [] }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase troubleshoot dashboard project unhealthy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", - "title": "Why do I see Auth & API requests in the dashboard? My app has no users" - } - ], - "resultChars": 17113 - }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services Supabase restart unhealthy", + "query": "site:supabase.com/docs \"Restart project\" Management API Supabase", "pages": [] } ] @@ -4353,15 +4424,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 92899, - "cacheReadInputTokens": 38101, - "cacheWriteInputTokens": 36996, - "outputTokens": 1981 + "inputTokens": 64345, + "cacheReadInputTokens": 16868, + "cacheWriteInputTokens": 32387, + "outputTokens": 2537 } ], - "stepCount": 4, - "toolCallCount": 10, - "agentRunDurationMs": 32922, + "stepCount": 3, + "toolCallCount": 7, + "agentRunDurationMs": 35754, "sandboxUsage": { "memory": 8192 }, @@ -9118,7 +9189,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting rather than pausing/restoring, gives a concrete first step, and links to Supabase’s unhealthy-services guidance. It could more clearly distinguish backup restoration from restart and mention support if the issue persists." + "judgeNotes": "Recommends restarting unhealthy services rather than pausing and restoring, and gives a concrete first step. It could more clearly distinguish backup restoration from a restart and mention logs and support if the project remains unhealthy." } ], "skills": { @@ -9129,43 +9200,28 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs restart project pause restore unhealthy project Supabase", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", "pages": [] }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/http-api-issues", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/http-api-issues" - } - ] + "query": "Supabase status page incidents September 24 2026 dashboard database", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 31236, + "inputTokens": 36547, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11848, - "outputTokens": 1084 + "cacheWriteInputTokens": 11846, + "outputTokens": 1173 } ], "stepCount": 1, "toolCallCount": 3, - "agentRunDurationMs": 17900, + "agentRunDurationMs": 17238, "sandboxUsage": { "memory": 8192 }, @@ -9193,12 +9249,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "Recommends restarting first and rejects pause-and-restore as a fix, but incorrectly describes restore as the way to resume a paused project rather than distinguishing backup recovery from restart. It also omits logs, resource checks, and escalation to Supabase support." + "passed": true, + "judgeNotes": "Recommends a database restart as the first recovery step, distinguishes it from restoring a paused project, advises against pause-and-restore, and gives concrete checks and support escalation." } ], "skills": { @@ -9209,65 +9265,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - } - ], - "resultChars": 18222 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api restart project Supabase Management API restart", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", "pages": [] }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Restart a project Management API endpoint POST /projects restart\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/api/v1-restart-a-project", - "title": "Restarts the given project" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-restore-a-project", - "title": "Restores the given project" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-shutdown-realtime", - "title": "Shutdowns realtime connections for a project" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-list-all-projects", - "title": "List all projects" - } - ], - "resultChars": 15227 - }, { "source": "web_search", - "query": "Supabase status dashboard incident September 24 2026 dashboard unavailable", + "query": "Supabase status page current incident September 24 2026 Supabase dashboard database health", "pages": [] } ] @@ -9275,15 +9278,15 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 86506, - "cacheReadInputTokens": 35109, - "cacheWriteInputTokens": 29936, - "outputTokens": 1754 + "inputTokens": 26636, + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 11846, + "outputTokens": 872 } ], - "stepCount": 3, - "toolCallCount": 6, - "agentRunDurationMs": 37292, + "stepCount": 1, + "toolCallCount": 2, + "agentRunDurationMs": 16735, "sandboxUsage": { "memory": 8192 }, @@ -9316,7 +9319,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restart as the first recovery step, distinguishes it from pause/restore, and advises checking for incidents and contacting support if the problem persists." + "judgeNotes": "Recommends restarting the database as a concrete first step and explicitly advises against pausing and restoring/resuming an unhealthy project. Notes that a restart may only temporarily help if the database is overloaded and cites Supabase troubleshooting guidance." } ], "skills": { @@ -9327,33 +9330,33 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api restart project management api POST projects restart", + "query": "site:supabase.com/docs project unhealthy restart pause restore project Supabase dashboard", "pages": [] }, { "source": "web_search", - "query": "\"Restart a project\" \"v1/projects/{ref}/restart\" Supabase", - "pages": [] + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 30885, + "inputTokens": 29553, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 1258 + "outputTokens": 1005 } ], "stepCount": 1, - "toolCallCount": 3, - "agentRunDurationMs": 23020, + "toolCallCount": 2, + "agentRunDurationMs": 14276, "sandboxUsage": { "memory": 8192 }, From c78e114437e32e9c4edcc9125e31ec71cd08965a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 13:52:14 +0000 Subject: [PATCH 5/6] revert: test: temporary change to resolve-reliability-001-unhealthy-project-recovery Reverts f5f68c43d8a0945626957fe40252b490b7499ca2 now that the refresh commit has been exercised in CI. The PR's diff is back to the workflow and CONTRIBUTING changes only. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ECba2eYo5iJPLnVPT8JvkV --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index 932b6979..9db50ef3 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -6,7 +6,6 @@ import { } from '@supabase-evals/core'; import { stripIndent } from 'common-tags'; -// Judge-only scorer: the transcript is graded against the rubric below. const scorer: ToolScorer = async (ctx) => { const verdict = await judge({ input: serializeTranscript(ctx.transcript), From ceebd9cb02d1c3bb9a6f1b316f5a3ffc4a873b06 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 14:06:30 +0000 Subject: [PATCH 6/6] ci: trim skip-ci comment and drop CONTRIBUTING note Replace the six-line comment above the results commit with a one-line explainer plus the GitHub docs link, and restore CONTRIBUTING.md to main's version, per review feedback on #332. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ECba2eYo5iJPLnVPT8JvkV --- .github/workflows/eval-refresh.yml | 6 +----- CONTRIBUTING.md | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 9a3fff86..5164e10a 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -533,11 +533,7 @@ jobs: exit 0 fi - # A GITHUB_TOKEN push skips `push` workflows but still fires - # `pull_request: synchronize` on the branch's PR, re-running this - # workflow and Code Quality as github-actions[bot]. Those runs need a - # maintainer's approval and re-running here would loop the refresh. - # [skip ci] on the HEAD commit stops GitHub creating the runs at all: + # [skip ci] stops the bot push from re-triggering PR workflows (they'd need maintainer approval and loop the refresh): # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/skipping-workflow-runs git commit -m "chore: refresh eval results [skip ci]" git push diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a62e47d3..3d8739fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ When you create a PR, use GitHub Actions to refresh the results in CI so we can You have a few options to run evals in CI: -- Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch. The results commit carries `[skip ci]` so it doesn't re-trigger CI on your PR. +- Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch. - Add the `run-evals` label to run every benchmark eval across the `benchmark` and `no-skills` experiment suites. Use this when a change can affect results broadly, such as framework changes. - Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. It can commit results directly to the selected branch or open a separate results PR.