From cdcf21bcf7fe333001ac40ce209fe284f5d8610c Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Tue, 15 Sep 2026 11:37:34 -0700 Subject: [PATCH 1/6] feat(evals): eval the custom claims and RBAC guide in supabase.com/docs Every piece of SQL on the guide is inert until the access token hook is switched on, and the guide shows that step as a dashboard click and links out for the local equivalent. The prompt asks for moderators who can delete any post and names no mechanism, so the checks say whether the page carries the hook along with the schema. Eight checks. Two read the workspace and the stack, five drive PostgREST and Auth as a member, a second member and a moderator, and one asserts the page was retrieved with content. The moderator delete check is the only one the missing hook reds: a member deleting their own post never touches the role, so the forum looks like it works while moderators have no powers. projectRunning is false, because `supabase start` renders config.toml into the Auth container's environment at creation time. Closes DOCS-1307 --- .../build-docs-009-custom-claims-rbac/EVAL.ts | 114 ++++++ .../PROMPT.md | 42 +++ .../README.md | 120 ++++++ .../local/supabase/config.toml | 165 +++++++++ .../migrations/20240101000000_init.sql | 35 ++ .../probes.ts | 348 ++++++++++++++++++ .../stack.ts | 91 +++++ 7 files changed, 915 insertions(+) create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/README.md create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/local/supabase/config.toml create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/local/supabase/migrations/20240101000000_init.sql create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/probes.ts create mode 100644 evals/docs/build-docs-009-custom-claims-rbac/stack.ts diff --git a/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts b/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts new file mode 100644 index 00000000..3bade007 --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts @@ -0,0 +1,114 @@ +import { + buildDocsResult, + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +import { + checkMemberCannotDeleteAnothersPost, + checkMemberCannotSelfPromote, + checkMemberDeletesOwnPost, + checkModeratorDeletesAnyPost, + checkRoleReachesTheToken, + type Probes, + setupProbes, +} from './probes.js'; +import { + checkHookIsEnabled, + checkStackIsRunning, + readStackState, +} from './stack.js'; + +const GUIDE_PATH = 'custom-claims-and-role-based-access-control-rbac'; + +const scorer: LocalStackScorer = async (ctx) => { + try { + const stack = await readStackState(ctx); + const hook = await checkHookIsEnabled(ctx); + + const setup = stack.running + ? await setupProbes(ctx) + : { failure: 'the agent left the local stack down' }; + const probes = 'probes' in setup ? setup.probes : undefined; + const blocked = 'failure' in setup ? setup.failure : undefined; + + const checks: CheckResult[] = [ + checkStackIsRunning(stack), + hook, + await gated( + probes, + blocked, + "the moderator's role travels in their token and a member's does not", + async (p) => checkRoleReachesTheToken(p) + ), + await gated(probes, blocked, 'a member can delete their own post', (p) => + checkMemberDeletesOwnPost(ctx, p) + ), + await gated( + probes, + blocked, + "a member cannot delete another member's post", + (p) => checkMemberCannotDeleteAnothersPost(ctx, p) + ), + await gated( + probes, + blocked, + "a moderator can delete another member's post", + (p) => checkModeratorDeletesAnyPost(ctx, p) + ), + await gated( + probes, + blocked, + 'a member cannot make themselves a moderator', + (p) => checkMemberCannotSelfPromote(ctx, p) + ), + checkGuideWasRead(ctx), + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated who may delete a post', + passed: false, + notes: message, + }, + ], + }; + } +}; + +export default scorer; + +async function gated( + probes: Probes | undefined, + blocked: string | undefined, + name: string, + probe: (probes: Probes) => Promise +): Promise { + if (!probes) { + return { name, passed: false, notes: `not run: ${blocked ?? 'no probes'}` }; + } + return probe(probes); +} + +function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { + const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => + call.pages?.some((page) => page.url.includes(GUIDE_PATH)) + ); + const withContent = calls.filter((call) => call.hasContent); + return { + name: 'the agent read the Custom Claims and RBAC guide the prompt referenced', + passed: withContent.length > 0, + notes: + withContent.length > 0 + ? withContent.map((call) => call.source).join(', ') + : calls.length > 0 + ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` + : 'no docs call reached the guide', + }; +} diff --git a/evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md b/evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md new file mode 100644 index 00000000..f2e6acf7 --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md @@ -0,0 +1,42 @@ +--- +stage: build +interface: cli +product: + - auth + - database +topic: + - rls + - security +services: + - gotrue + - kong + - postgrest +projectRunning: false +motivation: >- + the Custom Claims and RBAC guide is the page agents are pointed at to build + role-based permissions, and its own worked example only works once the access + token hook is switched on. The page shows that step as a dashboard click and + links out for the local equivalent, so it never states the setting a local + project needs. Someone who followed it believed their function was broken + when the function was fine and the hook had never been turned on, and asked + for the missing step to be documented. The same report arrives as "the claim + does not appear in the token" from people who copied the page verbatim. The + failure is silent: every table, function and policy is correct, and every + permission check returns false, so moderators quietly have no powers. One + customer gave up on the product over how hard the local version of this is to + test. Nothing in the Supabase agent skill mentions auth hooks or custom + claims, so the page is the only carrier. The prompt deliberately omits the + vocabulary the page teaches, so read README.md before editing it. +--- + +My forum has regular members and moderators. Moderators can delete any post, +and members can only delete their own. Set that up. + +Then get the local stack running so I can sign in as each of them and see it +work. + +Read the guide below before you start and rely on it for how to set this up, +rather than on what you already know. + +REFERENCE +https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md diff --git a/evals/docs/build-docs-009-custom-claims-rbac/README.md b/evals/docs/build-docs-009-custom-claims-rbac/README.md new file mode 100644 index 00000000..83bf6ef7 --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/README.md @@ -0,0 +1,120 @@ +# build-docs-009-custom-claims-rbac + +## What this eval measures + +The subject is +[Custom Claims & Role-based Access Control](https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac), +not the agent. The prompt is a product request plus the page's url, and the checks say whether an agent that read the +page produced working code. A gap in the page counts as a failure here. + +One claim: a moderator can delete a post they did not write, and a member cannot. + +Every piece of SQL on the page is inert until the access token hook is switched on. The page shows that step as a +dashboard click and links out for the local equivalent, so it never states the four lines a local project needs. The +checks are arranged so that the page's own worked example passes with the hook on and fails with it off, and the two +runs differ by nothing else. + +## Do not reintroduce the vocabulary + +The prompt names the feature and the two roles. It never names the mechanism, because whether the page transmits the +mechanism is the measurement. Keep all of these out of `PROMPT.md`: + +claim, custom claim, JWT, token, access token, hook, auth hook, `custom_access_token_hook`, `auth.jwt()`, +`config.toml`, `supabase_auth_admin`, RLS, row level security, policy, grant, revoke, enum, permission, `authorize`, +security definer, `user_role`. + +The seed may carry product vocabulary the user would already have in front of them. The line is the prompt. + +## The seed carries the contract + +`local/supabase/migrations/20240101000000_init.sql` is the forum's existing schema. Three things in it are +load-bearing. + +- **`member_roles` already exists and the comment says the admin tooling fills it.** The scorer has to be able to + make somebody a moderator, and it cannot do that without knowing where roles live. Seeding the table buys a + probeable address at the cost of the design question, and that trade is deliberate: the page's gap is the hook, not + the table shape. +- **The delete grant on `posts`.** New tables in `public` get no DML grants for `authenticated` on current CLI + versions, so without `grant delete` no policy the agent writes can ever take effect and every behavioral check + reds on a grant this page never mentions. Default grants are the subject of + `evals/regression/resolve-dataapi-002-secure-default-grants`. +- **The read and insert policies on `posts`.** Owner-scoped table policies are the subject of + `build-docs-002-rls-guide`, so a mistake there cannot fail this eval for the wrong reason. Delete is left open, + because delete is the claim. + +`member_roles` has row level security on and no grants to `authenticated`, which is the state a fresh project is in. + +`projectRunning: false` is load-bearing. `supabase start` renders `config.toml` into the Auth container's +environment at creation time, so a hook enabled after the stack is up is invisible to it. Letting the agent own the +lifecycle is what makes the claim measurable without the scorer restarting anything. + +## How the scorer is ordered + +`the local stack is running` gates everything. Without it there is no Auth to sign into and no database to read, so +every probe reds with that note rather than throwing. + +The moderator's role is written to `member_roles` **after** sign-up and the moderator then signs in again. Sign-up +issues a token before the role exists, so a scorer that reused it would read a stale token and red a correct +solution. Whatever puts the role in a token has to run on the second sign-in. + +Each probe deletes its own post row, so no probe can change what another one reads. + +## Do not drop the positive controls + +`a member cannot delete another member's post` passes for an agent that wrote no delete policy at all, because with +row level security on and nothing granting delete, nobody can delete anything. What makes it mean something is +`a member can delete their own post` and `a moderator can delete another member's post`. + +**The two member checks both pass when the hook is off.** That is the point. A member deleting their own post never +touches the role, so the forum looks like it works while moderators quietly have no powers. Only +`a moderator can delete another member's post` separates the two, and it is the check to read first. + +`the moderator's role travels in their token and a member's does not` is the diagnostic that says why. It scans the +moderator's token for the role value at any claim name and any depth, minus the standard claims, and requires the +member's token not to carry it. Asserting on `user_role` by name would constrain the page to one spelling. + +`the access token hook is switched on for the local project` is a source-level claim, and named as one. Its +behavioral counterparts are the two checks above. It accepts any `uri`, because the function's name and schema are +the agent's to choose. + +The refusal checks read the outcome, not the error. `a member cannot make themselves a moderator` counts rows as the +superuser rather than trusting the API's message, so a request the probe itself malformed cannot be scored as a +refusal. + +## The guide has to actually be read + +The last check resolves the guide through the harness's own docs result, because a `search_docs` hit carries the url +in its result rather than its request. It requires retrieved content, not just a url that was reached. + +Read `docs.calls` before reading the score. Docs evals run on one experiment and it is a no-skills one, so there is +no second arm to rule out prior knowledge. An empty `docs.calls` means the run measured nothing about the page, +whatever the checks say. + +## What this eval does not score + +- **The shape of the role storage.** The page models roles and permissions as two tables with two enums, and a + single column on a profile row works as well. The seed fixes where roles live so the scorer can write one, and no + check reads the agent's own tables. +- **Whether a permission layer exists.** The page's `authorize` function and its `role_permissions` table are one + design. A policy that checks the role claim directly satisfies every check here. +- **Reading the claim in application code.** The page closes by decoding the token client-side. No client is seeded, + so there is nothing to read it in. +- **Stale claims.** A role granted after a token is issued does not appear until the token is refreshed, and the + page's closing line that the setup "automatically propagates" reads past that. The scorer signs in again precisely + to avoid measuring it. +- **Admin roles and channel deletion.** The page's worked example carries a second role with a second permission. + One claim, so the prompt asks for one. + +## A risk worth knowing + +**Missing grants break sign-up, not just the claim.** Without `grant execute` on the hook function to +`supabase_auth_admin` and access to the role table, Auth answers `Error running hook URI` and nobody can sign up at +all. That reds five checks with a setup failure rather than a policy verdict, so read the notes before reading the +score. The page does carry those grants. + +**`a member cannot make themselves a moderator` is saturated.** The seed grants `authenticated` nothing on +`member_roles`, so only an agent that adds a grant can red it. It is here because a writable role table is a +privilege escalation, and the page's own revoke line is what prevents it. + +**The delete grant in the seed is doing a lot of work.** Remove it and every behavioral check reds on every run, for +a reason this page says nothing about. Anyone narrowing the seed should check that first. diff --git a/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/config.toml b/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/config.toml new file mode 100644 index 00000000..09e52bbb --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-custom-claims-rbac" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/migrations/20240101000000_init.sql b/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/migrations/20240101000000_init.sql new file mode 100644 index 00000000..1a9c14b5 --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/local/supabase/migrations/20240101000000_init.sql @@ -0,0 +1,35 @@ +-- Fernwood, a hobby forum. Members write posts about their projects. A few of +-- them are moderators, and the admin tooling already records that in +-- member_roles, so treat that table as the list of who is one. + +create type public.forum_role as enum ('moderator'); + +create table posts ( + id bigint primary key generated always as identity, + author_id uuid not null references auth.users on delete cascade, + body text not null, + created_at timestamptz not null default now() +); + +create table member_roles ( + member_id uuid not null references auth.users on delete cascade, + role forum_role not null, + primary key (member_id, role) +); + +-- Reading posts and writing your own are settled, and the app depends on both +-- as they are. The delete grant is here so the forum can offer the button at +-- all. Nothing below decides who may use it. + +alter table posts enable row level security; +alter table member_roles enable row level security; + +grant select, insert, delete on table posts to authenticated; + +create policy "members read every post" +on posts for select to authenticated +using (true); + +create policy "members write their own posts" +on posts for insert to authenticated +with check ((select auth.uid()) = author_id); diff --git a/evals/docs/build-docs-009-custom-claims-rbac/probes.ts b/evals/docs/build-docs-009-custom-claims-rbac/probes.ts new file mode 100644 index 00000000..10608caf --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/probes.ts @@ -0,0 +1,348 @@ +import { randomUUID } from 'node:crypto'; +import type { + CheckResult, + LocalStackEvalContext, + SupabaseClient, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +const PASSWORD = 'secret123'; +const ROLE = 'moderator'; + +const STANDARD_CLAIMS = new Set([ + 'iss', + 'sub', + 'aud', + 'exp', + 'iat', + 'jti', + 'nbf', + 'email', + 'phone', + 'role', + 'aal', + 'amr', + 'session_id', + 'is_anonymous', +]); + +export type Probes = { + memberClient: SupabaseClient; + strangerClient: SupabaseClient; + moderatorClient: SupabaseClient; + memberId: string; + strangerId: string; + moderatorId: string; + memberToken: string; + moderatorToken: string; + memberOwnPost: string; + strangerPost: string; + moderatorTargetPost: string; + marker: string; +}; + +export type Setup = { probes: Probes } | { failure: string }; + +export async function setupProbes(ctx: LocalStackEvalContext): Promise { + const run = randomUUID().slice(0, 8); + const marker = `fernwood-${run}`; + + const memberClient = await ctx.getClient(); + const strangerClient = await ctx.getClient(); + const signUpClient = await ctx.getClient(); + + const member = await signUp(memberClient, `member-${run}`); + const stranger = await signUp(strangerClient, `stranger-${run}`); + const moderator = await signUp(signUpClient, `moderator-${run}`); + for (const outcome of [member, stranger, moderator]) { + if ('failure' in outcome) return outcome; + } + const memberUser = member as SignedUp; + const strangerUser = stranger as SignedUp; + const moderatorUser = moderator as SignedUp; + + const granted = await execSql( + ctx, + `INSERT INTO member_roles (member_id, role) VALUES ('${moderatorUser.id}', '${ROLE}') ON CONFLICT DO NOTHING;` + ); + if (!granted.ok) { + return { failure: `could not record the moderator: ${granted.message}` }; + } + + const roleRows = await ctx.query( + `SELECT count(*)::int AS count FROM member_roles WHERE member_id = '${moderatorUser.id}';` + ); + if (Number(roleRows.rows[0]?.count ?? 0) === 0) { + return { + failure: `member_roles has no ${ROLE} row for ${moderatorUser.id}`, + }; + } + + const moderatorClient = await ctx.getClient(); + const { data: reauth, error: reauthError } = + await moderatorClient.auth.signInWithPassword({ + email: moderatorUser.email, + password: PASSWORD, + }); + if (reauthError || !reauth.session) { + return { + failure: `the moderator could not sign in again: ${reauthError?.message ?? 'no session'}`, + }; + } + + const memberSession = await memberClient.auth.getSession(); + const memberToken = memberSession.data.session?.access_token; + if (!memberToken) { + return { failure: 'the member has no access token' }; + } + + const posts = await seedPosts(ctx, { + member: memberUser.id, + stranger: strangerUser.id, + marker, + }); + if ('failure' in posts) return posts; + + return { + probes: { + memberClient, + strangerClient, + moderatorClient, + memberId: memberUser.id, + strangerId: strangerUser.id, + moderatorId: moderatorUser.id, + memberToken, + moderatorToken: reauth.session.access_token, + marker, + ...posts.ids, + }, + }; +} + +export function checkRoleReachesTheToken(probes: Probes): CheckResult { + const moderator = findRole(decode(probes.moderatorToken)); + const member = findRole(decode(probes.memberToken)); + + return { + name: "the moderator's role travels in their token and a member's does not", + passed: moderator !== undefined && member === undefined, + notes: + moderator === undefined + ? `no claim in the moderator's token carries "${ROLE}"` + : member === undefined + ? `the moderator's token carries it at ${moderator}` + : `both tokens carry it, the member's at ${member}`, + }; +} + +export async function checkMemberDeletesOwnPost( + ctx: LocalStackEvalContext, + probes: Probes +): Promise { + const { error } = await probes.memberClient + .from('posts') + .delete() + .eq('id', probes.memberOwnPost); + const gone = await postIsGone(ctx, probes.memberOwnPost); + + return { + name: 'a member can delete their own post', + passed: gone, + notes: gone ? undefined : (error?.message ?? 'the post is still there'), + }; +} + +export async function checkMemberCannotDeleteAnothersPost( + ctx: LocalStackEvalContext, + probes: Probes +): Promise { + const { error } = await probes.memberClient + .from('posts') + .delete() + .eq('id', probes.strangerPost); + const gone = await postIsGone(ctx, probes.strangerPost); + + return { + name: "a member cannot delete another member's post", + passed: !gone, + notes: gone + ? 'a member deleted a post they did not write' + : (error?.message ?? 'the post is still there'), + }; +} + +export async function checkModeratorDeletesAnyPost( + ctx: LocalStackEvalContext, + probes: Probes +): Promise { + const { error } = await probes.moderatorClient + .from('posts') + .delete() + .eq('id', probes.moderatorTargetPost); + const gone = await postIsGone(ctx, probes.moderatorTargetPost); + + return { + name: "a moderator can delete another member's post", + passed: gone, + notes: gone ? undefined : (error?.message ?? 'the post is still there'), + }; +} + +export async function checkMemberCannotSelfPromote( + ctx: LocalStackEvalContext, + probes: Probes +): Promise { + await probes.memberClient + .from('member_roles') + .insert({ member_id: probes.memberId, role: ROLE }); + const { rows } = await ctx.query( + `SELECT count(*)::int AS count FROM member_roles WHERE member_id = '${probes.memberId}';` + ); + const promoted = Number(rows[0]?.count ?? 0) > 0; + + return { + name: 'a member cannot make themselves a moderator', + passed: !promoted, + notes: promoted + ? 'a member wrote their own row into member_roles' + : 'member_roles rejected the write', + }; +} + +type SignedUp = { client: SupabaseClient; id: string; email: string }; + +async function signUp( + client: SupabaseClient, + tag: string +): Promise { + const email = `${tag}@example.com`; + const { data, error } = await client.auth.signUp({ + email, + password: PASSWORD, + }); + if (error || !data.user?.id || !data.session) { + return { + failure: `${tag} could not sign up: ${error?.message ?? 'no session'}`, + }; + } + return { client, id: data.user.id, email }; +} + +async function seedPosts( + ctx: LocalStackEvalContext, + authors: { member: string; stranger: string; marker: string } +): Promise< + | { + ids: { + memberOwnPost: string; + strangerPost: string; + moderatorTargetPost: string; + }; + } + | { failure: string } +> { + const insert = await execSql( + ctx, + stripIndent` + INSERT INTO posts (author_id, body) VALUES + ('${authors.member}', '${authors.marker}-own'), + ('${authors.stranger}', '${authors.marker}-stranger'), + ('${authors.stranger}', '${authors.marker}-target') + RETURNING id; + `, + { quiet: true } + ); + const ids = insert.stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+$/.test(line)); + + if (ids.length !== 3) { + return { + failure: `seeded ${ids.length} posts instead of 3: ${insert.message}`, + }; + } + return { + ids: { + memberOwnPost: ids[0], + strangerPost: ids[1], + moderatorTargetPost: ids[2], + }, + }; +} + +async function postIsGone( + ctx: LocalStackEvalContext, + id: string +): Promise { + const { rows } = await ctx.query( + `SELECT count(*)::int AS count FROM posts WHERE id = ${id};` + ); + return Number(rows[0]?.count ?? 0) === 0; +} + +function decode(token: string): Record { + try { + const payload = token.split('.')[1]; + return JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')); + } catch { + return {}; + } +} + +function findRole( + claims: Record, + path: string[] = [] +): string | undefined { + for (const [key, value] of Object.entries(claims)) { + if (path.length === 0 && STANDARD_CLAIMS.has(key)) continue; + const here = [...path, key]; + if (typeof value === 'string') { + if (value.toLowerCase() === ROLE) return here.join('.'); + continue; + } + if (Array.isArray(value)) { + if (value.some((item) => String(item).toLowerCase() === ROLE)) { + return here.join('.'); + } + continue; + } + if (value && typeof value === 'object') { + const nested = findRole(value as Record, here); + if (nested) return nested; + } + } + return undefined; +} + +async function execSql( + ctx: LocalStackEvalContext, + sql: string, + options: { quiet?: boolean } = {} +): Promise<{ ok: boolean; stdout: string; message: string }> { + const encoded = Buffer.from(sql, 'utf8').toString('base64'); + const flags = options.quiet ? '-q -A -t' : '-q'; + const result = await ctx.exec( + stripIndent` + DB_URL=$(supabase status -o json 2>/dev/null | node -e 'let input = ""; process.stdin.on("data", data => input += data); process.stdin.on("end", () => console.log(JSON.parse(input).DB_URL));') + echo ${encoded} | base64 -d | psql "$DB_URL" ${flags} -v ON_ERROR_STOP=1 + `, + { timeoutMs: 120_000 } + ); + return { + ok: result.exitCode === 0, + stdout: result.stdout ?? '', + message: firstError(result), + }; +} + +function firstError(result: { stdout?: string; stderr?: string }): string { + const lines = `${result.stderr ?? ''}\n${result.stdout ?? ''}` + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const named = lines.find((line) => + /^(ERROR|FATAL|DETAIL|HINT)\b/i.test(line) + ); + return (named ?? lines[0] ?? 'no output').slice(0, 300); +} diff --git a/evals/docs/build-docs-009-custom-claims-rbac/stack.ts b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts new file mode 100644 index 00000000..013154da --- /dev/null +++ b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts @@ -0,0 +1,91 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; + +const HOOK_SECTION = /^\s*\[auth\.hook\.custom_access_token\]\s*$/; +const ENABLED = /^\s*enabled\s*=\s*(true|"true")\s*$/i; +const URI = /^\s*uri\s*=\s*"(.+)"\s*$/; + +export type StackState = { + running: boolean; + statusError?: string; +}; + +export async function readStackState( + ctx: LocalStackEvalContext +): Promise { + const status = await ctx.exec('supabase status -o env 2>/dev/null', { + timeoutMs: 120_000, + }); + const running = /(^|\n)(PUBLISHABLE_KEY|ANON_KEY)=/.test(status.stdout ?? ''); + return { + running, + statusError: running ? undefined : firstError(status), + }; +} + +export function checkStackIsRunning(state: StackState): CheckResult { + return { + name: 'the local stack is running', + passed: state.running, + notes: state.running + ? undefined + : `supabase status reported no api keys: ${state.statusError ?? 'no output'}`, + }; +} + +export async function checkHookIsEnabled( + ctx: LocalStackEvalContext +): Promise { + const name = 'the access token hook is switched on for the local project'; + let config: string; + try { + config = await ctx.readFile('supabase/config.toml'); + } catch (error) { + return { + name, + passed: false, + notes: `could not read supabase/config.toml: ${error instanceof Error ? error.message : String(error)}`, + }; + } + + const lines = config.split('\n'); + const start = lines.findIndex((line) => HOOK_SECTION.test(line)); + if (start === -1) { + return { + name, + passed: false, + notes: + 'supabase/config.toml has no [auth.hook.custom_access_token] section', + }; + } + + let enabled = false; + let uri: string | undefined; + for (const line of lines.slice(start + 1)) { + if (/^\s*\[/.test(line)) break; + if (ENABLED.test(line)) enabled = true; + const match = URI.exec(line); + if (match) uri = match[1]; + } + + return { + name, + passed: enabled && uri !== undefined, + notes: enabled + ? uri + ? `enabled, pointing at ${uri}` + : 'enabled with no uri, so Auth has no function to call' + : 'the section is present but not enabled', + }; +} + +function firstError(result: { + stdout?: string; + stderr?: string; +}): string { + const lines = `${result.stderr ?? ''}\n${result.stdout ?? ''}` + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const named = lines.find((line) => /^(ERROR|FATAL|failed)\b/i.test(line)); + return (named ?? lines[0] ?? 'no output').slice(0, 300); +} From 95d839a2fcf453e254d5db79e9ffae59dd2863d2 Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Mon, 21 Sep 2026 15:01:01 -0700 Subject: [PATCH 2/6] fix(evals): parse config.toml and scope the self-promote count to moderator `the access token hook is switched on for the local project` matched lines. A trailing comment on `enabled`, a trailing comment on the section header, a single-quoted uri, an inline table and a dotted key are all TOML-legal spellings of a working hook, and all five red. The section-header case also reported the section as absent while it was present and working. Parse the file with smol-toml instead. `a member cannot make themselves a moderator` counted every `member_roles` row the member held. The seed enum has one value, so the count is right today, but an agent that adds a baseline `member` value turns a correct solution into a false escalation report. Count the moderator rows. --- .../README.md | 8 ++++ .../probes.ts | 6 +-- .../stack.ts | 47 +++++++++++++------ package.json | 3 +- pnpm-lock.yaml | 10 ++++ 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/evals/docs/build-docs-009-custom-claims-rbac/README.md b/evals/docs/build-docs-009-custom-claims-rbac/README.md index 83bf6ef7..33ae45b5 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/README.md +++ b/evals/docs/build-docs-009-custom-claims-rbac/README.md @@ -53,6 +53,10 @@ lifecycle is what makes the claim measurable without the scorer restarting anyth `the local stack is running` gates everything. Without it there is no Auth to sign into and no database to read, so every probe reds with that note rather than throwing. +`the access token hook is switched on for the local project` parses `config.toml` rather than matching lines in it. +A trailing comment, a single-quoted uri, an inline table and a dotted key are all TOML-legal spellings of a working +hook, and a line matcher reds every one of them. + The moderator's role is written to `member_roles` **after** sign-up and the moderator then signs in again. Sign-up issues a token before the role exists, so a scorer that reused it would read a stale token and red a correct solution. Whatever puts the role in a token has to run on the second sign-in. @@ -116,5 +120,9 @@ score. The page does carry those grants. `member_roles`, so only an agent that adds a grant can red it. It is here because a writable role table is a privilege escalation, and the page's own revoke line is what prevents it. +The check counts the member's `moderator` rows, not every row they hold. The seed enum has one value, so today the +two are the same. An agent that adds a baseline `member` value would make them differ, and counting every row would +red a correct solution and blame it on an escalation that never happened. + **The delete grant in the seed is doing a lot of work.** Remove it and every behavioral check reds on every run, for a reason this page says nothing about. Anyone narrowing the seed should check that first. diff --git a/evals/docs/build-docs-009-custom-claims-rbac/probes.ts b/evals/docs/build-docs-009-custom-claims-rbac/probes.ts index 10608caf..84a0835d 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/probes.ts +++ b/evals/docs/build-docs-009-custom-claims-rbac/probes.ts @@ -196,7 +196,7 @@ export async function checkMemberCannotSelfPromote( .from('member_roles') .insert({ member_id: probes.memberId, role: ROLE }); const { rows } = await ctx.query( - `SELECT count(*)::int AS count FROM member_roles WHERE member_id = '${probes.memberId}';` + `SELECT count(*)::int AS count FROM member_roles WHERE member_id = '${probes.memberId}' AND role = '${ROLE}';` ); const promoted = Number(rows[0]?.count ?? 0) > 0; @@ -204,8 +204,8 @@ export async function checkMemberCannotSelfPromote( name: 'a member cannot make themselves a moderator', passed: !promoted, notes: promoted - ? 'a member wrote their own row into member_roles' - : 'member_roles rejected the write', + ? `a member wrote themselves a ${ROLE} row into member_roles` + : `member_roles rejected the ${ROLE} write`, }; } diff --git a/evals/docs/build-docs-009-custom-claims-rbac/stack.ts b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts index 013154da..055c21d0 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/stack.ts +++ b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts @@ -1,8 +1,5 @@ import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; - -const HOOK_SECTION = /^\s*\[auth\.hook\.custom_access_token\]\s*$/; -const ENABLED = /^\s*enabled\s*=\s*(true|"true")\s*$/i; -const URI = /^\s*uri\s*=\s*"(.+)"\s*$/; +import { parse as parseToml } from 'smol-toml'; export type StackState = { running: boolean; @@ -47,9 +44,22 @@ export async function checkHookIsEnabled( }; } - const lines = config.split('\n'); - const start = lines.findIndex((line) => HOOK_SECTION.test(line)); - if (start === -1) { + let parsed: unknown; + try { + parsed = parseToml(config); + } catch (error) { + return { + name, + passed: false, + notes: `supabase/config.toml is not valid TOML: ${error instanceof Error ? error.message : String(error)}`, + }; + } + + const hook = readTable( + readTable(readTable(parsed, 'auth'), 'hook'), + 'custom_access_token' + ); + if (!hook) { return { name, passed: false, @@ -58,14 +68,9 @@ export async function checkHookIsEnabled( }; } - let enabled = false; - let uri: string | undefined; - for (const line of lines.slice(start + 1)) { - if (/^\s*\[/.test(line)) break; - if (ENABLED.test(line)) enabled = true; - const match = URI.exec(line); - if (match) uri = match[1]; - } + const enabled = hook.enabled === true || hook.enabled === 'true'; + const uri = + typeof hook.uri === 'string' && hook.uri.length > 0 ? hook.uri : undefined; return { name, @@ -78,6 +83,18 @@ export async function checkHookIsEnabled( }; } +function readTable( + value: unknown, + key: string +): Record | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const child = (value as Record)[key]; + if (typeof child !== 'object' || child === null || Array.isArray(child)) { + return undefined; + } + return child as Record; +} + function firstError(result: { stdout?: string; stderr?: string; diff --git a/package.json b/package.json index 71637fe5..f0d1b5a1 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "dependencies": { "@ai-sdk/anthropic": "catalog:", "@ai-sdk/openai": "catalog:", - "common-tags": "^1.8.2" + "common-tags": "^1.8.2", + "smol-toml": "^1.8.0" }, "devDependencies": { "@biomejs/biome": "1.9.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e9bf544..2ed1eab8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: common-tags: specifier: ^1.8.2 version: 1.8.2 + smol-toml: + specifier: ^1.8.0 + version: 1.8.0 devDependencies: '@biomejs/biome': specifier: 1.9.4 @@ -2888,6 +2891,7 @@ packages: eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -4171,6 +4175,10 @@ packages: sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + smol-toml@1.8.0: + resolution: {integrity: sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==} + engines: {node: '>= 18'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -8285,6 +8293,8 @@ snapshots: sisteransi@1.0.5: {} + smol-toml@1.8.0: {} + source-map-js@1.2.1: {} source-map@0.6.1: {} From 35cd4f2045aa1774a4e88d61a844a5b123df3a5d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:07:43 +0000 Subject: [PATCH 3/6] chore: refresh eval results --- apps/web/src/data/docs-eval-results.json | 354 +++++++++++++++++++++++ 1 file changed, 354 insertions(+) diff --git a/apps/web/src/data/docs-eval-results.json b/apps/web/src/data/docs-eval-results.json index 9d3684f3..e3b2a27b 100644 --- a/apps/web/src/data/docs-eval-results.json +++ b/apps/web/src/data/docs-eval-results.json @@ -1728,6 +1728,360 @@ "run": 3, "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-008-storage-access-control/run-3/result.json" }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at user_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search, shell_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -L --fail --silent --show-error https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md | sed -n '1,260p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ], + "resultChars": 7977 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -L --fail --silent --show-error https://supabase.com/docs/guides/auth/auth-hooks | rg -n -C 3 'local|custom_access|pg-functions|hook' | head -160\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks" + } + ], + "resultChars": 295232 + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 1307751, + "cacheReadInputTokens": 1244422, + "cacheWriteInputTokens": 63221, + "outputTokens": 10439 + } + ], + "stepCount": 36, + "toolCallCount": 20, + "agentRunDurationMs": 177824, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-009-custom-claims-rbac/run-1/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at user_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/api custom claims role based access control RBAC Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs local development custom access token hook pg-functions uri config.toml", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 1118988, + "cacheReadInputTokens": 1046708, + "cacheWriteInputTokens": 72202, + "outputTokens": 6623 + } + ], + "stepCount": 26, + "toolCallCount": 15, + "agentRunDurationMs": 118960, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-009-custom-claims-rbac/run-2/result.json" + }, + { + "experiment": "codex-gpt-5.6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at user_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search, web_search" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs local development auth hooks config.toml custom access token hook pg-functions", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-5.6-luna", + "inputTokens": 2217168, + "cacheReadInputTokens": 2146259, + "cacheWriteInputTokens": 70765, + "outputTokens": 8775 + } + ], + "stepCount": 48, + "toolCallCount": 20, + "agentRunDurationMs": 222318, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-009-custom-claims-rbac/run-3/result.json" + }, { "experiment": "codex-gpt-5.6-luna-no-skills", "experimentSuite": "docs", From 3b37047a4902797df658e4d7a4deee0c4de33f8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:26:50 +0000 Subject: [PATCH 4/6] chore: refresh eval results --- apps/web/src/data/docs-eval-results.json | 104 ++++++++--------------- 1 file changed, 36 insertions(+), 68 deletions(-) diff --git a/apps/web/src/data/docs-eval-results.json b/apps/web/src/data/docs-eval-results.json index e3b2a27b..374faa33 100644 --- a/apps/web/src/data/docs-eval-results.json +++ b/apps/web/src/data/docs-eval-results.json @@ -1786,7 +1786,7 @@ { "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", "passed": true, - "notes": "web_search, shell_fetch" + "notes": "web_search" } ], "skills": { @@ -1807,50 +1807,23 @@ }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -L --fail --silent --show-error https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md | sed -n '1,260p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" - } - ], - "resultChars": 7977 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -L --fail --silent --show-error https://supabase.com/docs/guides/auth/auth-hooks | rg -n -C 3 'local|custom_access|pg-functions|hook' | head -160\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks" - } - ], - "resultChars": 295232 + "query": "site:supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac Supabase custom claims role based access control", + "pages": [] } ] }, "usage": [ { "model": "gpt-5.6-luna", - "inputTokens": 1307751, - "cacheReadInputTokens": 1244422, - "cacheWriteInputTokens": 63221, - "outputTokens": 10439 + "inputTokens": 966897, + "cacheReadInputTokens": 906138, + "cacheWriteInputTokens": 60684, + "outputTokens": 6961 } ], - "stepCount": 36, - "toolCallCount": 20, - "agentRunDurationMs": 177824, + "stepCount": 25, + "toolCallCount": 15, + "agentRunDurationMs": 134689, "sandboxUsage": { "memory": 8192 }, @@ -1917,7 +1890,7 @@ { "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", "passed": true, - "notes": "web_search" + "notes": "web_search, web_search" } ], "skills": { @@ -1938,28 +1911,28 @@ }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/api custom claims role based access control RBAC Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs local development custom access token hook pg-functions uri config.toml", - "pages": [] + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac" + } + ] } ] }, "usage": [ { "model": "gpt-5.6-luna", - "inputTokens": 1118988, - "cacheReadInputTokens": 1046708, - "cacheWriteInputTokens": 72202, - "outputTokens": 6623 + "inputTokens": 932738, + "cacheReadInputTokens": 880596, + "cacheWriteInputTokens": 52046, + "outputTokens": 7557 } ], - "stepCount": 26, - "toolCallCount": 15, - "agentRunDurationMs": 118960, + "stepCount": 32, + "toolCallCount": 17, + "agentRunDurationMs": 142080, "sandboxUsage": { "memory": 8192 }, @@ -2026,7 +1999,7 @@ { "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", "passed": true, - "notes": "web_search, web_search" + "notes": "web_search" } ], "skills": { @@ -2047,17 +2020,12 @@ }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac" - } - ] + "query": "site:supabase.com/docs/guides/api custom claims role based access control RBAC Supabase", + "pages": [] }, { "source": "web_search", - "query": "site:supabase.com/docs local development auth hooks config.toml custom access token hook pg-functions", + "query": "site:supabase.com/docs local development auth hooks config.toml custom_access_token_hook pg-functions", "pages": [] } ] @@ -2065,15 +2033,15 @@ "usage": [ { "model": "gpt-5.6-luna", - "inputTokens": 2217168, - "cacheReadInputTokens": 2146259, - "cacheWriteInputTokens": 70765, - "outputTokens": 8775 + "inputTokens": 1088165, + "cacheReadInputTokens": 1029348, + "cacheWriteInputTokens": 58724, + "outputTokens": 7216 } ], - "stepCount": 48, - "toolCallCount": 20, - "agentRunDurationMs": 222318, + "stepCount": 31, + "toolCallCount": 17, + "agentRunDurationMs": 130902, "sandboxUsage": { "memory": 8192 }, From 89cbdfb1a7a784cd581bf3ec281d2239a9352a4d Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Wed, 23 Sep 2026 07:14:38 -0700 Subject: [PATCH 5/6] fix(evals): read the access token hook from the Auth container The check parsed supabase/config.toml, which needed smol-toml in the root package.json. That pulled a docs-owned eval outside /evals/docs/ and put a second CODEOWNERS team on the PR. supabase start renders the hook into the Auth container's environment at creation time, so the container is what Auth obeys. Reading GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED and _URI out of it leaves the TOML reading to the CLI, and reds a hook that was switched on after the stack started, which the file cannot see. The container is found by the project label on whatever container publishes the api port supabase status reports. A name filter alone would match every Supabase project on the daemon. --- .../build-docs-009-custom-claims-rbac/EVAL.ts | 14 ++- .../README.md | 14 ++- .../stack.ts | 104 +++++++++++------- package.json | 3 +- pnpm-lock.yaml | 10 -- 5 files changed, 82 insertions(+), 63 deletions(-) diff --git a/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts b/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts index 3bade007..434188b9 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts +++ b/evals/docs/build-docs-009-custom-claims-rbac/EVAL.ts @@ -25,11 +25,17 @@ const GUIDE_PATH = 'custom-claims-and-role-based-access-control-rbac'; const scorer: LocalStackScorer = async (ctx) => { try { const stack = await readStackState(ctx); - const hook = await checkHookIsEnabled(ctx); + const down = 'the agent left the local stack down'; - const setup = stack.running - ? await setupProbes(ctx) - : { failure: 'the agent left the local stack down' }; + const hook = stack.running + ? await checkHookIsEnabled(ctx, stack) + : { + name: 'the access token hook is switched on for the local project', + passed: false, + notes: `not run: ${down}`, + }; + + const setup = stack.running ? await setupProbes(ctx) : { failure: down }; const probes = 'probes' in setup ? setup.probes : undefined; const blocked = 'failure' in setup ? setup.failure : undefined; diff --git a/evals/docs/build-docs-009-custom-claims-rbac/README.md b/evals/docs/build-docs-009-custom-claims-rbac/README.md index 33ae45b5..34f38bce 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/README.md +++ b/evals/docs/build-docs-009-custom-claims-rbac/README.md @@ -53,9 +53,11 @@ lifecycle is what makes the claim measurable without the scorer restarting anyth `the local stack is running` gates everything. Without it there is no Auth to sign into and no database to read, so every probe reds with that note rather than throwing. -`the access token hook is switched on for the local project` parses `config.toml` rather than matching lines in it. -A trailing comment, a single-quoted uri, an inline table and a dotted key are all TOML-legal spellings of a working -hook, and a line matcher reds every one of them. +`the access token hook is switched on for the local project` reads `GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED` and +`_URI` out of the running Auth container, found by the project label on whatever container publishes the api port +`supabase status` reports. The container is what Auth obeys, so the CLI does the TOML reading and no spelling of the +hook in `config.toml` can red a working one. It also separates a hook that was switched on from one that was +switched on too late, which the file cannot: the note says which. The moderator's role is written to `member_roles` **after** sign-up and the moderator then signs in again. Sign-up issues a token before the role exists, so a scorer that reused it would read a stale token and red a correct @@ -77,9 +79,9 @@ touches the role, so the forum looks like it works while moderators quietly have moderator's token for the role value at any claim name and any depth, minus the standard claims, and requires the member's token not to carry it. Asserting on `user_role` by name would constrain the page to one spelling. -`the access token hook is switched on for the local project` is a source-level claim, and named as one. Its -behavioral counterparts are the two checks above. It accepts any `uri`, because the function's name and schema are -the agent's to choose. +`the access token hook is switched on for the local project` isolates the hook from the SQL, which is why it is +here alongside its two behavioral counterparts above. It accepts any `uri`, because the function's name and schema +are the agent's to choose. The refusal checks read the outcome, not the error. `a member cannot make themselves a moderator` counts rows as the superuser rather than trusting the API's message, so a request the probe itself malformed cannot be scored as a diff --git a/evals/docs/build-docs-009-custom-claims-rbac/stack.ts b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts index 055c21d0..8fb97a08 100644 --- a/evals/docs/build-docs-009-custom-claims-rbac/stack.ts +++ b/evals/docs/build-docs-009-custom-claims-rbac/stack.ts @@ -1,8 +1,10 @@ import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; -import { parse as parseToml } from 'smol-toml'; + +const HOOK_PREFIX = 'GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN'; export type StackState = { running: boolean; + apiUrl?: string; statusError?: string; }; @@ -12,9 +14,11 @@ export async function readStackState( const status = await ctx.exec('supabase status -o env 2>/dev/null', { timeoutMs: 120_000, }); - const running = /(^|\n)(PUBLISHABLE_KEY|ANON_KEY)=/.test(status.stdout ?? ''); + const env = parseEnv(status.stdout ?? ''); + const running = Boolean(env.PUBLISHABLE_KEY ?? env.ANON_KEY); return { running, + apiUrl: env.API_URL, statusError: running ? undefined : firstError(status), }; } @@ -30,69 +34,87 @@ export function checkStackIsRunning(state: StackState): CheckResult { } export async function checkHookIsEnabled( - ctx: LocalStackEvalContext + ctx: LocalStackEvalContext, + state: StackState ): Promise { const name = 'the access token hook is switched on for the local project'; - let config: string; - try { - config = await ctx.readFile('supabase/config.toml'); - } catch (error) { + const port = state.apiUrl ? /:(\d+)\/?$/.exec(state.apiUrl)?.[1] : undefined; + if (!port) { return { name, passed: false, - notes: `could not read supabase/config.toml: ${error instanceof Error ? error.message : String(error)}`, + notes: `supabase status gave no API_URL to locate the stack by${state.apiUrl ? `, only ${state.apiUrl}` : ''}`, }; } - let parsed: unknown; - try { - parsed = parseToml(config); - } catch (error) { + const read = await ctx.exec(readAuthHookEnv(port), { timeoutMs: 120_000 }); + const container = /^container=(.*)$/m.exec(read.stdout ?? '')?.[1]?.trim(); + if (!container) { return { name, passed: false, - notes: `supabase/config.toml is not valid TOML: ${error instanceof Error ? error.message : String(error)}`, + notes: `the stack is up but no auth container carries this project's label: ${firstError(read)}`, }; } - const hook = readTable( - readTable(readTable(parsed, 'auth'), 'hook'), - 'custom_access_token' - ); - if (!hook) { + const env = parseEnv(read.stdout ?? ''); + const enabled = env[`${HOOK_PREFIX}_ENABLED`]; + const uri = env[`${HOOK_PREFIX}_URI`]; + + if (enabled === undefined && uri === undefined) { return { name, passed: false, - notes: - 'supabase/config.toml has no [auth.hook.custom_access_token] section', + notes: `${container} started without the hook in its environment${await configHint(ctx)}`, }; } + if (enabled?.toLowerCase() !== 'true') { + return { + name, + passed: false, + notes: `${container} carries the hook but not switched on (${HOOK_PREFIX}_ENABLED=${enabled ?? 'unset'})`, + }; + } + if (!uri) { + return { + name, + passed: false, + notes: 'enabled with no uri, so Auth has no function to call', + }; + } + return { name, passed: true, notes: `enabled, pointing at ${uri}` }; +} - const enabled = hook.enabled === true || hook.enabled === 'true'; - const uri = - typeof hook.uri === 'string' && hook.uri.length > 0 ? hook.uri : undefined; +function readAuthHookEnv(port: string): string { + const label = 'com.supabase.cli.project'; + return [ + `PROJECT=$(docker ps --filter publish=${port} --filter label=${label} --format '{{.Label "${label}"}}' 2>/dev/null | head -n 1)`, + '[ -n "$PROJECT" ] || { echo "no container publishes the api port" >&2; exit 1; }', + `AUTH=$(docker ps --filter label=${label}="$PROJECT" --format '{{.Names}}' 2>/dev/null | grep '^supabase_auth_' | head -n 1)`, + '[ -n "$AUTH" ] || { echo "project $PROJECT has no running auth container" >&2; exit 1; }', + 'echo "container=$AUTH"', + `docker inspect "$AUTH" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep "^${HOOK_PREFIX}_" || true`, + ].join('\n'); +} - return { - name, - passed: enabled && uri !== undefined, - notes: enabled - ? uri - ? `enabled, pointing at ${uri}` - : 'enabled with no uri, so Auth has no function to call' - : 'the section is present but not enabled', - }; +async function configHint(ctx: LocalStackEvalContext): Promise { + try { + const config = await ctx.readFile('supabase/config.toml'); + return config.includes('custom_access_token') + ? ', though supabase/config.toml names custom_access_token, so it is either not switched on there or was switched on after the stack started' + : ', and supabase/config.toml never names custom_access_token'; + } catch { + return ''; + } } -function readTable( - value: unknown, - key: string -): Record | undefined { - if (typeof value !== 'object' || value === null) return undefined; - const child = (value as Record)[key]; - if (typeof child !== 'object' || child === null || Array.isArray(child)) { - return undefined; +function parseEnv(stdout: string): Record { + const env: Record = {}; + for (const line of stdout.split('\n')) { + const match = /^([A-Z0-9_]+)=(.*)$/.exec(line.trim()); + if (match) env[match[1]] = match[2].replace(/^"|"$/g, ''); } - return child as Record; + return env; } function firstError(result: { diff --git a/package.json b/package.json index f0d1b5a1..71637fe5 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,7 @@ "dependencies": { "@ai-sdk/anthropic": "catalog:", "@ai-sdk/openai": "catalog:", - "common-tags": "^1.8.2", - "smol-toml": "^1.8.0" + "common-tags": "^1.8.2" }, "devDependencies": { "@biomejs/biome": "1.9.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ed1eab8..5e9bf544 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,9 +71,6 @@ importers: common-tags: specifier: ^1.8.2 version: 1.8.2 - smol-toml: - specifier: ^1.8.0 - version: 1.8.0 devDependencies: '@biomejs/biome': specifier: 1.9.4 @@ -2891,7 +2888,6 @@ packages: eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -4175,10 +4171,6 @@ packages: sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} - smol-toml@1.8.0: - resolution: {integrity: sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==} - engines: {node: '>= 18'} - source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -8293,8 +8285,6 @@ snapshots: sisteransi@1.0.5: {} - smol-toml@1.8.0: {} - source-map-js@1.2.1: {} source-map@0.6.1: {} From c5ae829396dbbbbbd000980ebc46214983b4c594 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:25:43 +0000 Subject: [PATCH 6/6] chore: refresh eval results --- apps/web/src/data/docs-eval-results.json | 646 +++++++++++++++++++++++ 1 file changed, 646 insertions(+) diff --git a/apps/web/src/data/docs-eval-results.json b/apps/web/src/data/docs-eval-results.json index 374faa33..f7da5084 100644 --- a/apps/web/src/data/docs-eval-results.json +++ b/apps/web/src/data/docs-eval-results.json @@ -2961,5 +2961,651 @@ "promptSourcePath": "evals/docs/build-docs-012-cli-getting-started/PROMPT.md", "run": 3, "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-012-cli-getting-started/run-3/result.json" + }, + { + "experiment": "codex-gpt-6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at forum_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search, web_search, search_docs, search_docs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac custom claims RBAC Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"local development configure custom access token auth hook config.toml pg-functions uri seeding auth users\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + } + ], + "resultChars": 60766 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Auth hooks local development config.toml [auth.hook.custom_access_token] pg-functions://postgres\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + } + ], + "resultChars": 60766 + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/auth/auth-hooks", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks" + } + ] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1723091, + "cacheReadInputTokens": 1625825, + "cacheWriteInputTokens": 72400, + "outputTokens": 11895 + } + ], + "stepCount": 38, + "toolCallCount": 24, + "agentRunDurationMs": 296082, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-009-custom-claims-rbac/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at user_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search, search_docs, search_docs, search_docs, search_docs, search_docs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac custom claims role based access control Supabase", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"custom claims role based access control auth hook app_metadata row level security policy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/custom-access-token-hook", + "title": "Custom Access Token Hook" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0015_rls_references_user_metadata", + "title": "Database Advisor: Lint 0015_rls_references_user_metadata" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 59371 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Auth Hooks local development custom access token hook config.toml pg-functions URI supabase_auth_admin grants\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/custom-access-token-hook", + "title": "Custom Access Token Hook" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + } + ], + "resultChars": 77012 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"[auth.hook.custom_access_token] enabled uri pg-functions://postgres function local development config.toml\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + } + ], + "resultChars": 56370 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI config auth hook custom_access_token enable pg-functions uri local config.toml [auth.hook.custom_access_token]\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/custom-access-token-hook", + "title": "Custom Access Token Hook" + } + ], + "resultChars": 63577 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase local development seed.sql db.seed enabled seed.sql auth.users password crypt gen_salt\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", + "title": "GitHub integration" + }, + { + "url": "https://supabase.com/docs/guides/local-development/seeding-your-database", + "title": "Seeding your database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + } + ], + "resultChars": 51222 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Custom Claims Role-based Access Control custom_access_token_hook user_role member_roles custom access token event return RLS policy authorize\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 53884 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Custom Access Token hook config.toml auth.hook custom_access_token enabled uri local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" + } + ], + "resultChars": 91286 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1294840, + "cacheReadInputTokens": 1208870, + "cacheWriteInputTokens": 75396, + "outputTokens": 15668 + } + ], + "stepCount": 30, + "toolCallCount": 32, + "agentRunDurationMs": 307945, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-009-custom-claims-rbac/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-no-skills", + "experimentSuite": "docs", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-009-custom-claims-rbac", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "rls", + "security" + ], + "suite": "docs", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the local stack is running", + "passed": true + }, + { + "name": "the access token hook is switched on for the local project", + "passed": true, + "notes": "enabled, pointing at pg-functions://postgres/public/custom_access_token_hook" + }, + { + "name": "the moderator's role travels in their token and a member's does not", + "passed": true, + "notes": "the moderator's token carries it at user_role" + }, + { + "name": "a member can delete their own post", + "passed": true + }, + { + "name": "a member cannot delete another member's post", + "passed": true, + "notes": "the post is still there" + }, + { + "name": "a moderator can delete another member's post", + "passed": true + }, + { + "name": "a member cannot make themselves a moderator", + "passed": true, + "notes": "member_roles rejected the moderator write" + }, + { + "name": "the agent read the Custom Claims and RBAC guide the prompt referenced", + "passed": true, + "notes": "web_search, shell_fetch, search_docs, search_docs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -L --fail --silent --show-error 'https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md" + } + ], + "resultChars": 7977 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Custom Access Token Auth Hook local development config.toml auth hook\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/local-development/customizing-email-templates", + "title": "Customizing email templates" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + } + ], + "resultChars": 65074 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"config.toml auth hook custom_access_token_hook enabled hook postgres function local development\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/custom-access-token-hook", + "title": "Custom Access Token Hook" + } + ], + "resultChars": 63577 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Local development auth hook config.toml pg-functions custom_access_token_hook uri\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", + "title": "Custom Claims & Role-based Access Control (RBAC)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" + } + ], + "resultChars": 44194 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 2178230, + "cacheReadInputTokens": 2095069, + "cacheWriteInputTokens": 79836, + "outputTokens": 26967 + } + ], + "stepCount": 45, + "toolCallCount": 36, + "agentRunDurationMs": 543546, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "My forum has regular members and moderators. Moderators can delete any post,\nand members can only delete their own. Set that up.\n\nThen get the local stack running so I can sign in as each of them and see it\nwork.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac.md", + "promptSourcePath": "evals/docs/build-docs-009-custom-claims-rbac/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-009-custom-claims-rbac/run-3/result.json" } ]