From 9574d16da706a3dec89c7c141ec18ed371736657 Mon Sep 17 00:00:00 2001 From: kriptoburak Date: Sun, 14 Jun 2026 10:32:30 +0200 Subject: [PATCH] feat: add xquik eval coverage --- eval/tasks/xquik.yaml | 27 +++++++++++++++++++++++++++ eval/vendors.yaml | 10 +++++++++- src/discovery/crawler.ts | 12 ++++++++++-- tests/discovery/crawler.test.ts | 5 ++++- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 eval/tasks/xquik.yaml diff --git a/eval/tasks/xquik.yaml b/eval/tasks/xquik.yaml new file mode 100644 index 0000000..c501e1b --- /dev/null +++ b/eval/tasks/xquik.yaml @@ -0,0 +1,27 @@ +vendor: xquik +tasks: + - id: search-tweets + goal: "Search recent X posts matching a query." + expected_ops: + - method: GET + path: "/api/v1/x/tweets/search" + - id: get-user-posts + goal: "Fetch posts for an X user." + expected_ops: + - method: GET + path: "/api/v1/x/users/{id}/tweets" + - id: create-post + goal: "Create a new X post." + expected_ops: + - method: POST + path: "/api/v1/x/tweets" + - id: start-extraction + goal: "Start a bulk extraction job." + expected_ops: + - method: POST + path: "/api/v1/extractions" + - id: register-webhook + goal: "Register a webhook endpoint for activity events." + expected_ops: + - method: POST + path: "/api/v1/webhooks" diff --git a/eval/vendors.yaml b/eval/vendors.yaml index 10d8d85..0fc8110 100644 --- a/eval/vendors.yaml +++ b/eval/vendors.yaml @@ -1,6 +1,6 @@ # Evaluation harness: five vendors picked for surface-mix diversity. # Each vendor's actual .skillship/ project lives under eval/projects//. -# That directory is *not* created automatically — seed it by running +# That directory is *not* created automatically - seed it by running # `skillship init --domain ` (or handcraft one for air-gapped runs). vendors: - id: supabase @@ -77,3 +77,11 @@ vendors: surfaces: [rest, docs, llms_txt, mcp] ops_min: 20 baseline_path: null + + - id: xquik + domain: xquik.com + github_org: Xquik-dev + expected: + surfaces: [rest, mcp] + ops_min: 90 + baseline_path: null diff --git a/src/discovery/crawler.ts b/src/discovery/crawler.ts index c2dc0ce..7d428d5 100644 --- a/src/discovery/crawler.ts +++ b/src/discovery/crawler.ts @@ -70,8 +70,16 @@ export function buildProbeTargets(base: URL): ProbeTarget[] { targets.push({ surface: "rest", url }); } } - const mcpUrl = `${base.protocol}//mcp.${base.hostname}/.well-known/oauth-protected-resource/mcp`; - targets.push({ surface: "mcp", url: mcpUrl }); + const mcpProtectedResourcePath = + "/.well-known/oauth-protected-resource/mcp"; + targets.push({ + surface: "mcp", + url: new URL(mcpProtectedResourcePath, base).toString(), + }); + targets.push({ + surface: "mcp", + url: `${base.protocol}//mcp.${base.hostname}${mcpProtectedResourcePath}`, + }); } return targets; } diff --git a/tests/discovery/crawler.test.ts b/tests/discovery/crawler.test.ts index 7a1667f..2d6a302 100644 --- a/tests/discovery/crawler.test.ts +++ b/tests/discovery/crawler.test.ts @@ -24,7 +24,7 @@ describe("normalizeBase", () => { }); describe("buildProbeTargets", () => { - it("emits llms.txt, sitemap, OpenAPI guesses, + mcp subdomain for real hosts", () => { + it("emits llms.txt, sitemap, OpenAPI guesses, and MCP probes for real hosts", () => { const targets = buildProbeTargets(normalizeBase("supabase.com")); const urls = targets.map((t) => t.url); expect(urls).toContain("https://supabase.com/llms.txt"); @@ -37,6 +37,9 @@ describe("buildProbeTargets", () => { expect(urls).toContain( "https://mcp.supabase.com/.well-known/oauth-protected-resource/mcp", ); + expect(urls).toContain( + "https://supabase.com/.well-known/oauth-protected-resource/mcp", + ); }); it("includes runtime-generated OpenAPI probe paths", () => {