From f47ca1b261630f4416ac408b78559ab08a858fd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 04:22:04 +0000 Subject: [PATCH] docs(skills): an objectstack-data eval asks for a hook's `refusals` catalogue instead of a thrown English sentence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One eval case in hooks-security-seeds-datasources.json: a save refused over the account's credit limit in an app serving English and Chinese users — expected `refusals: { CREDIT_LIMIT: { message } }` on the hook, a `language: 'js'` body throwing `{ code: 'CREDIT_LIMIT' }`, and the zh-CN sentence under the bundle's `hooks: { HOOK: { refusals } }`; refused: a sentence on the throw (`new Error('`) and a `handler:`. Paid down first: five sentences in cases 1-5 that restated their own `must_not_contain` assertions (453 bytes = 114 units); the case adds 1129 bytes = 283 units; the file is 5616 -> 6292 bytes, 1404 -> 1573 tokens. The ratchet row moves 1404 -> 1573, pinned at the measurement under the standing ruling quoted beside it. Patch changeset for `@objectstack/skills`, whose build copies skills/** into its published dist. Claude-Session: https://claude.ai/code/session_01QSeKYmhU5KEj1nukYhp1Tg Co-authored-by: Claude --- .changeset/26834-hook-refusals-eval.md | 12 +++++++++++ scripts/check-skills-token-ratchet.mjs | 20 ++++++++++++++++++- .../hooks-security-seeds-datasources.json | 20 ++++++++++++++----- 3 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 .changeset/26834-hook-refusals-eval.md diff --git a/.changeset/26834-hook-refusals-eval.md b/.changeset/26834-hook-refusals-eval.md new file mode 100644 index 00000000000..5bd7230751a --- /dev/null +++ b/.changeset/26834-hook-refusals-eval.md @@ -0,0 +1,12 @@ +--- +'@objectstack/skills': patch +--- + +docs(skills): an objectstack-data eval asks for a hook's `refusals` catalogue instead of a sentence thrown from the body + +Clause-②: no +Metadata-protocol: none + +- **Before.** `references/data-hooks.md` teaches a hook's declared `refusals` catalogue (the body throws `{ code }`, the message is translated under `hooks.HOOK.refusals.CODE.message`), but no eval in `evals/hooks-security-seeds-datasources.json` exercised it: an agent asked for a refusal in a multi-language app could still write an English sentence into `throw new Error(...)`, which reaches zh-CN users untranslated, and nothing scored that wrong. +- **Now.** Eval 7 asks for an `order` save refused over the account's credit limit in an app serving English and Chinese users, and expects `refusals: { CREDIT_LIMIT: { message } }` on the hook, a `language: 'js'` body that throws `{ code: 'CREDIT_LIMIT' }`, and the zh-CN sentence under the bundle's `hooks: { HOOK: { refusals: { CREDIT_LIMIT: { message } } } }`; it refuses a sentence on the throw (`new Error('`) and a `handler:` (a catalogue without a `js` body is refused at save). Five sentences in evals 1–5 that restated their own `must_not_contain` assertions are gone; every fact and assertion stays. +- No schema, key or behaviour changes. diff --git a/scripts/check-skills-token-ratchet.mjs b/scripts/check-skills-token-ratchet.mjs index 053eb781e4b..29f2cd5724b 100644 --- a/scripts/check-skills-token-ratchet.mjs +++ b/scripts/check-skills-token-ratchet.mjs @@ -930,7 +930,25 @@ export const CEILINGS = new Map([ // `applyOn: 'create'`. +309 tokens at headroom 0, ceiling +309; nothing // duplicated in the file to cut first. Maintainer ruling 2026-10-27 on the decision card 「skills 修复卡的教学和 eval 都撞 token 天 // 花板,怎么付?」, tap A, verbatim and untranslated: 「A:先削重复,余额按实测提高」. - ['skills/objectstack-data/evals/hooks-security-seeds-datasources.json', 1404], + // 1404 -> 1573 (card #26834, the eval owed by the spec PR that added a + // hook's `refusals` catalogue): one case for a save refused over the + // account's credit limit in an app serving English and Chinese users — + // expected `refusals: { CREDIT_LIMIT: { message } }` on the hook, a + // `language: 'js'` body throwing `{ code: 'CREDIT_LIMIT' }`, the zh-CN + // sentence under the bundle's `hooks: { HOOK: { refusals } }`; NO sentence + // on the throw (`new Error('`), NO `handler:`. Triage 6450886908 (room + // inside the package first, no raise) read together with the standing + // ruling, strictest first — maintainer ruling 2026-10-27 on the decision + // card 「skills 修复卡的教学和 eval 都撞 token 天花板,怎么付?」, tap A, verbatim + // and untranslated: 「A:先削重复,余额按实测提高」. PAID DOWN FIRST: five + // sentences in cases 1–5 that restated their own `must_not_contain` + // assertions (case 4 keeps its one non-asserted clause), 453 bytes = 114 + // units; the case adds 1129 bytes = 283 units. On `origin/main` the file + // was 5616 bytes (1404 tokens, headroom 0); the LANDED file is 6292 bytes + // -> 1573 tokens, ceiling +169, pinned AT the measurement, zero headroom. + // The raising PR's body carries the same arithmetic. + // ceil($(wc -c < skills/objectstack-data/evals/hooks-security-seeds-datasources.json) / 4) == ceil(6292 / 4) + ['skills/objectstack-data/evals/hooks-security-seeds-datasources.json', 1573], // 946 -> 1050 (#26297): the invoice-line case gains the owner opt-in // (`ownerMayEdit` on the controlled_by_parent line) and pins `added_by` as // `type: 'user'`, never a lookup to a `user` object. +104 tokens at diff --git a/skills/objectstack-data/evals/hooks-security-seeds-datasources.json b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json index 3b1381b32e6..7ac2d0eb747 100644 --- a/skills/objectstack-data/evals/hooks-security-seeds-datasources.json +++ b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json @@ -4,7 +4,7 @@ { "id": 1, "prompt": "Support agents want to search tasks by the name of the project they belong to.", - "expected_output": "Applies the one prescription: a stored `text` mirror field `project_name` on `task`, listed in `searchableFields: ['name', 'project_name']`, maintained by a `beforeInsert` / `beforeUpdate` hook on `task` (stamp the parent's name for the incoming `project_id`) and an `afterUpdate` hook on `project` (re-stamp on rename), plus a one-off backfill for rows written outside hooks. Never a dotted path (`project_id.name` is refused, `400 INVALID_FIELD`) and never a `formula` mirror (virtual, nothing to scan).", + "expected_output": "Applies the one prescription: a stored `text` mirror field `project_name` on `task`, listed in `searchableFields: ['name', 'project_name']`, maintained by a `beforeInsert` / `beforeUpdate` hook on `task` (stamp the parent's name for the incoming `project_id`) and an `afterUpdate` hook on `project` (re-stamp on rename), plus a one-off backfill for rows written outside hooks.", "files": [], "assertions": { "must_contain": ["searchableFields", "project_name", "beforeInsert", "afterUpdate"], @@ -14,7 +14,7 @@ { "id": 2, "prompt": "Sales reps may only see their own leads, sales managers see their unit and everything below it, and every signed-in user can read the `product` catalog.", - "expected_output": "Access is authored in permission sets, not on the object (an object-level `permissions` key is rejected). `lead` gets `sharingModel: 'private'`; the rep set's grant on `lead` carries `readScope: 'own'`, the manager set's `readScope: 'unit_and_below'` — an ADR-0057 scope that needs the enterprise `@objectstack/security-enterprise` plugin and `requires: ['hierarchy-security']`, failing closed to `own` without it. `product` is `sharingModel: 'public_read'`.", + "expected_output": "Access is authored in permission sets, not on the object. `lead` gets `sharingModel: 'private'`; the rep set's grant on `lead` carries `readScope: 'own'`, the manager set's `readScope: 'unit_and_below'` — an ADR-0057 scope that needs the enterprise `@objectstack/security-enterprise` plugin and `requires: ['hierarchy-security']`, failing closed to `own` without it. `product` is `sharingModel: 'public_read'`.", "files": [], "assertions": { "must_contain": ["sharingModel: 'private'", "readScope", "unit_and_below", "public_read", "hierarchy-security"], @@ -24,7 +24,7 @@ { "id": 3, "prompt": "Expose the existing `customers` table from our Postgres reporting database as a read-only `legacy_customer` object.", - "expected_output": "Declares a `defineDatasource` entry (credentials by `credentialsRef`, never inline) and points the object at it with `datasource: ''`, mapping the physical table via `remoteName` and column names via `columnMap` — there is no `field.columnName` on an external object. Sets the datasource's `schemaMode` so the platform never alters the remote schema, and keeps the external write gate closed.", + "expected_output": "Declares a `defineDatasource` entry (credentials by `credentialsRef`, never inline) and points the object at it with `datasource: ''`, mapping the physical table via `remoteName` and column names via `columnMap`. Sets the datasource's `schemaMode` so the platform never alters the remote schema, and keeps the external write gate closed.", "files": [], "assertions": { "must_contain": ["defineDatasource", "datasource", "remoteName", "columnMap", "schemaMode"], @@ -34,7 +34,7 @@ { "id": 4, "prompt": "Seed the `country` reference table for every environment, and add demo `opportunity` rows (dev/test only) that close 45 days after install and are owned by whoever installs the package. A nightly job advances each opportunity's `stage`; replaying the seed must not reset it.", - "expected_output": "Uses `defineSeed(Country, { externalId: 'code', mode: 'upsert', records })` for reference data and `defineSeed(Opportunity, { env: ['dev', 'test'], ... })` for demo rows, with lookups referenced by natural key and dynamic values as CEL: `close_date: cel`daysFromNow(45)``, `owner_id: cel`os.user.id``. Never `externalId: 'id'`, never `new Date()` / `Date.now()` (breaks build determinism), never `defineDataset` (the analytics layer), parents before children in the exported array. The job-owned column is declared `columns: { stage: { applyOn: 'create' } }`, so an `upsert` replay leaves it as the row holds it.", + "expected_output": "Uses `defineSeed(Country, { externalId: 'code', mode: 'upsert', records })` for reference data and `defineSeed(Opportunity, { env: ['dev', 'test'], ... })` for demo rows, with lookups referenced by natural key and dynamic values as CEL: `close_date: cel`daysFromNow(45)``, `owner_id: cel`os.user.id``. Parents before children in the exported array. The job-owned column is declared `columns: { stage: { applyOn: 'create' } }`, so an `upsert` replay leaves it as the row holds it.", "files": [], "assertions": { "must_contain": ["defineSeed", "externalId", "upsert", "env:", "daysFromNow", "applyOn: 'create'"], @@ -44,7 +44,7 @@ { "id": 5, "prompt": "Finance may edit a contract's legal terms (`governing_law`, `liability_cap`) while it is a draft, but not once it is approved, signing or active. Legal still edits them.", - "expected_output": "Field permission entries with a row condition on the finance set: `'clm_contract.governing_law': { readable: true, editable: false, when: \"status in ['approved', 'signing', 'active']\" }` and the same for `liability_cap`; the legal set keeps its edit grant. No validation rule and no `readonlyWhen`, which would lock the fields for every user.", + "expected_output": "Field permission entries with a row condition on the finance set: `'clm_contract.governing_law': { readable: true, editable: false, when: \"status in ['approved', 'signing', 'active']\" }` and the same for `liability_cap`; the legal set keeps its edit grant.", "files": [], "assertions": { "must_contain": ["when:", "editable: false", "readable: true", "clm_contract.governing_law"], @@ -60,6 +60,16 @@ "must_contain": ["membershipKeys", "current_user.portal_account_ids", "userField", "valueField", "permissionSets", "external"], "must_not_contain": ["permissions: {"] } + }, + { + "id": 7, + "prompt": "Our app serves English and Chinese users. An `order` whose `amount` exceeds its account's `credit_limit` must not be saved; API clients branch on the code `CREDIT_LIMIT`, and a zh-CN rep reads `超出账户信用额度`.", + "expected_output": "A `beforeInsert` / `beforeUpdate` hook on `order` declares the refusal once, `refusals: { CREDIT_LIMIT: { message: 'Over the account credit limit.' } }`, and its sandboxed `body: { language: 'js', source }` reads the account through `ctx.api` and refuses by code only: `throw Object.assign(new Error(), { code: 'CREDIT_LIMIT' })` — the body's own text is never shown. The zh-CN bundle carries `hooks: { : { refusals: { CREDIT_LIMIT: { message: '超出账户信用额度' } } } }`, served whole in the caller's locale; a catalogue on a hook without a `js` body is refused at save.", + "files": [], + "assertions": { + "must_contain": ["refusals:", "CREDIT_LIMIT: {", "code: 'CREDIT_LIMIT'", "language: 'js'", "hooks: {", "超出账户信用额度"], + "must_not_contain": ["new Error('", "handler:"] + } } ] }