From bf5cde54d211fbc72dae76ce6599b93534aebb32 Mon Sep 17 00:00:00 2001 From: JC Rojas <126191042+sajor2000@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:21:48 -0500 Subject: [PATCH 1/4] fix(plan): clarify research decisions before planning --- .../cases/method-plan-grill-handoff/case.yaml | 49 ++++++++++++++ .../cases/method-plan-grill-handoff/prompt.md | 3 + .../method-plan-grill-headless/case.yaml | 64 +++++++++++++++++++ .../method-plan-grill-headless/prompt.md | 3 + .../cases/method-plan-grill-no-sap/case.yaml | 49 ++++++++++++++ .../cases/method-plan-grill-no-sap/prompt.md | 3 + .../skill-creator/method-plan/evals.json | 1 + .../method-plan/trigger-evals.json | 2 +- .../methodsmith/skills/method-grill/SKILL.md | 17 ++++- .../method-grill/references/question-bank.md | 9 +++ .../methodsmith/skills/method-plan/SKILL.md | 16 +++-- .../references/planning-foundations.md | 16 +++++ .../method-plan/references/sap-template.md | 2 + .../references/universal-planning.md | 2 +- tests/method-grill-contract.test.ts | 15 ++++- 15 files changed, 238 insertions(+), 13 deletions(-) create mode 100644 evals/methodsmith/cases/method-plan-grill-handoff/case.yaml create mode 100644 evals/methodsmith/cases/method-plan-grill-handoff/prompt.md create mode 100644 evals/methodsmith/cases/method-plan-grill-headless/case.yaml create mode 100644 evals/methodsmith/cases/method-plan-grill-headless/prompt.md create mode 100644 evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml create mode 100644 evals/methodsmith/cases/method-plan-grill-no-sap/prompt.md create mode 100644 plugins/methodsmith/skills/method-plan/references/planning-foundations.md diff --git a/evals/methodsmith/cases/method-plan-grill-handoff/case.yaml b/evals/methodsmith/cases/method-plan-grill-handoff/case.yaml new file mode 100644 index 00000000..83f98df3 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-handoff/case.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +id: method-plan-grill-handoff +target: + kind: skill + name: method-plan + source: plugins/methodsmith/skills/method-plan/SKILL.md +mode: forward +prompt_path: evals/methodsmith/cases/method-plan-grill-handoff/prompt.md +fixtures: [] +required_behaviors: + - identifies-timing-risk + - asks-decision +prohibited_behaviors: + - no-bias-guarantee + - no-sap +criteria: + - id: identifies-timing-risk + description: Identifies time-zero or immortal-time bias as a consequential design issue. + type: regex + path: response.md + pattern: '[Ii]mmortal.time|[Tt]ime.zero|[Ss]urviv(?:e|al).{0,80}(?:treatment|medication|drug)' + weight: 4 + hard_gate: true + - id: asks-decision + description: Asks a question before writing the plan. + type: text_contains + path: response.md + value: "?" + weight: 3 + hard_gate: true + - id: no-bias-guarantee + description: Does not promise that a proposed method eliminates bias. + type: regex_not + path: response.md + pattern: '(?:[Ee]liminates|[Rr]emoves|[Aa]voids|[Pp]revents) (?:all |immortal.time )?bias' + weight: 3 + hard_gate: true + - id: no-sap + description: Does not create an SAP before the interview. + type: file_exists + path: workspace/analysis/sap.md + expected: false + weight: 3 + hard_gate: true +threshold: 1 +limitations: + - This evaluates a synthetic first interaction; it does not prove later interview turns or scientific validity. + - Regex gates are approximate; review the complete response for question count, preservation of supplied details, and method assumptions. + - The target digest covers method-plan only; independently record the method-grill directory digest because this case exercises both skills. diff --git a/evals/methodsmith/cases/method-plan-grill-handoff/prompt.md b/evals/methodsmith/cases/method-plan-grill-handoff/prompt.md new file mode 100644 index 00000000..e356b2ea --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-handoff/prompt.md @@ -0,0 +1,3 @@ +Use `method-plan` for a synthetic retrospective cohort study. We want an SAP studying whether starting a medication after ICU admission reduces 90-day mortality using hospital EHR data. The medication, treatment window, and comparator have not been chosen. No dataset is available yet. + +Begin the interactive planning conversation. Save your complete first response to `response.md`; stop at the first question and do not write an SAP or run analyses. The workspace is `workspace/`. diff --git a/evals/methodsmith/cases/method-plan-grill-headless/case.yaml b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml new file mode 100644 index 00000000..e9379b28 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml @@ -0,0 +1,64 @@ +schema_version: "1.0" +id: method-plan-grill-headless +target: + kind: skill + name: method-plan + source: plugins/methodsmith/skills/method-plan/SKILL.md +mode: forward +prompt_path: evals/methodsmith/cases/method-plan-grill-headless/prompt.md +fixtures: [] +required_behaviors: + - draft-status + - open-choices + - data-profile-gap +prohibited_behaviors: + - no-final-status + - no-selected-trial-design + - no-invented-endpoint +criteria: + - id: draft-status + description: The SAP remains in draft. + type: regex + path: workspace/analysis/sap.md + pattern: '^status:\s*draft\b' + weight: 4 + hard_gate: true + - id: open-choices + description: The draft marks unresolved treatment decisions as gaps or provisional. + type: regex + path: workspace/analysis/sap.md + pattern: '[Gg][Aa][Pp]|[Pp]rovisional|[Uu]nresolved' + weight: 4 + hard_gate: true + - id: data-profile-gap + description: The response identifies the missing data profile or data QA prerequisite. + type: regex + path: response.md + pattern: '[Dd]ata[- ](?:profile|QA|quality)|method-data-qa' + weight: 3 + hard_gate: true + - id: no-final-status + description: An incomplete SAP is not finalized. + type: regex_not + path: workspace/analysis/sap.md + pattern: '^status:\s*final\b' + weight: 4 + hard_gate: true + - id: no-selected-trial-design + description: Frontmatter does not label an unchosen target-trial design as settled. + type: regex_not + path: workspace/analysis/sap.md + pattern: '^study_type:\s*target[-_]trial' + weight: 4 + hard_gate: true + - id: no-invented-endpoint + description: The primary outcome is not labeled all-cause without confirmation. + type: regex_not + path: workspace/analysis/sap.md + pattern: '(?:[Pp]rimary outcome|[Pp]rimary endpoint|[Tt]o estimate|[Ee]ndpoint with time point)[^\n]{0,100}all.cause mortality' + weight: 4 + hard_gate: true +threshold: 1 +limitations: + - Regex gates require manual review for fabricated answers, inappropriate outcome qualifiers, and questions that block a headless workflow. + - The synthetic case does not establish clinical validity or safety with real data. diff --git a/evals/methodsmith/cases/method-plan-grill-headless/prompt.md b/evals/methodsmith/cases/method-plan-grill-headless/prompt.md new file mode 100644 index 00000000..20fd1eb9 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-headless/prompt.md @@ -0,0 +1,3 @@ +Use `method-plan` in non-interactive pipeline mode to draft an SAP for a synthetic retrospective ICU cohort studying whether starting a medication after admission changes 90-day mortality in hospital EHR data. The drug, treatment window, comparator, and mortality definition are undecided, and no dataset exists. + +Save the SAP to `workspace/analysis/sap.md` and your user-facing response to `response.md`. Do not run analyses. diff --git a/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml b/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml new file mode 100644 index 00000000..708d3fc1 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +id: method-plan-grill-no-sap +target: + kind: skill + name: method-plan + source: plugins/methodsmith/skills/method-plan/SKILL.md +mode: forward +prompt_path: evals/methodsmith/cases/method-plan-grill-no-sap/prompt.md +fixtures: [] +required_behaviors: + - identifies-timing-risk + - asks-decision +prohibited_behaviors: + - no-bias-guarantee + - no-sap +criteria: + - id: identifies-timing-risk + description: Identifies time-zero or immortal-time bias as a consequential design issue. + type: regex + path: response.md + pattern: '[Ii]mmortal.time|[Tt]ime.zero|[Ss]urviv(?:e|al).{0,80}(?:treatment|medication|drug)' + weight: 4 + hard_gate: true + - id: asks-decision + description: Asks a question before writing the plan. + type: text_contains + path: response.md + value: "?" + weight: 3 + hard_gate: true + - id: no-bias-guarantee + description: Does not promise that a proposed method eliminates bias. + type: regex_not + path: response.md + pattern: '(?:[Ee]liminates|[Rr]emoves|[Aa]voids|[Pp]revents) (?:all |immortal.time )?bias' + weight: 3 + hard_gate: true + - id: no-sap + description: Does not create an SAP before the interview. + type: file_exists + path: workspace/analysis/sap.md + expected: false + weight: 3 + hard_gate: true +threshold: 1 +limitations: + - This evaluates a synthetic first interaction; it does not prove later interview turns or scientific validity. + - Regex gates are approximate; review the complete response for question count, preservation of supplied details, and method assumptions. + - The target digest covers method-plan only; independently record the method-grill directory digest because this case exercises both skills. diff --git a/evals/methodsmith/cases/method-plan-grill-no-sap/prompt.md b/evals/methodsmith/cases/method-plan-grill-no-sap/prompt.md new file mode 100644 index 00000000..93fd4426 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-grill-no-sap/prompt.md @@ -0,0 +1,3 @@ +Use `method-plan` for a synthetic study of whether starting a medication after ICU admission changes 90-day mortality in hospital EHR data. The medication, treatment window, and comparator have not been chosen. No dataset is available yet. + +Begin the interactive planning conversation. Save your complete first response to `response.md`; stop at the first question and do not write a plan or run analyses. The workspace is `workspace/`. diff --git a/evals/methodsmith/skill-creator/method-plan/evals.json b/evals/methodsmith/skill-creator/method-plan/evals.json index 9e4d475a..72743ab9 100644 --- a/evals/methodsmith/skill-creator/method-plan/evals.json +++ b/evals/methodsmith/skill-creator/method-plan/evals.json @@ -11,6 +11,7 @@ "Reads reports/data-qa/shock_v1.md before writing variable, missingness, cohort-size, or model sections", "Writes the SAP to analysis/sap.md (the default path) with sap_version and status frontmatter, keeping status: draft while any estimand, comparator, time-zero, adjustment, or missing-data choice remains unresolved", "Requires the method-sap-workbook companion (analysis/sap-tables/01-overview.csv, 02-outputs.csv, 03-variables.csv) or records an explicit blocker", + "When material research decisions remain unresolved, questions them before writing the SAP in interactive mode or records them as gaps in non-interactive mode", "Does not set status: final without user acceptance and does not run inferential analyses", "Grounds SAP-1 background and SAP-5 methods justification in sources read through the PubMed/Paperclip MCP tools and cited by PMID/DOI with an evidence label, or writes a GAP comment, never a citation from memory" ] diff --git a/evals/methodsmith/skill-creator/method-plan/trigger-evals.json b/evals/methodsmith/skill-creator/method-plan/trigger-evals.json index d4f4efb0..bb3cfa37 100644 --- a/evals/methodsmith/skill-creator/method-plan/trigger-evals.json +++ b/evals/methodsmith/skill-creator/method-plan/trigger-evals.json @@ -7,7 +7,7 @@ {"query": "SAP v2 needs an amendment: add a per-protocol sensitivity analysis and change the follow-up horizon to 90 days. Keep the SAP-N.M IDs stable.", "should_trigger": true}, {"query": "Finalize the SAP in docs/plans/ - QA came back GO, all methodological choices are resolved, run the gap checklist and tell me if anything critical is left.", "should_trigger": true}, {"query": "how should we implement the new Table 1 generator feature in the CLI? make a plan w/ test scenarios, output:html", "should_trigger": true}, - {"query": "Create a plan for the analysis of our case-control study of statin exposure and post-op delirium, including the research question, exposure definition, and outcome ascertainment.", "should_trigger": true}, + {"query": "Plan a study of whether starting a medication after ICU admission changes 90-day mortality; help clarify the treatment decision first.", "should_trigger": true}, {"query": "I have a rough idea for an ML early-warning feature in our app. Plan the work - files, units, tests - I'll execute later with method-work.", "should_trigger": true}, {"query": "Build the claims-to-datasets schema, estimand table, and decision evidence for our complex EHR validation SAP that already exists in docs/plans/.", "should_trigger": false}, {"query": "Generate the tabular SAP companion workbook (Overview, Outputs, Master Variables) from our finalized prose SAP.", "should_trigger": false}, diff --git a/plugins/methodsmith/skills/method-grill/SKILL.md b/plugins/methodsmith/skills/method-grill/SKILL.md index 59e696bd..d6e0b31c 100644 --- a/plugins/methodsmith/skills/method-grill/SKILL.md +++ b/plugins/methodsmith/skills/method-grill/SKILL.md @@ -10,18 +10,19 @@ argument-hint: "[project idea, or path to a brief, research-question.yaml, draft - **Problem it solves:** Studies fail at review because the goal, hypothesis, data, analysis, and journal fit were never challenged together before the SAP was written. - **Use when:** The user says grill me, or wants a study idea, hypothesis, or analysis plan stress-tested before planning or submission. -- **Output:** `docs/brainstorms/--grill.md` with every decision, its recommended and chosen answer, ranked open risks, a reviewer attack list, and the next skill to run. +- **Output:** Standalone: `docs/brainstorms/--grill.md` with decisions and risks. In planning integration: confirmed decisions and unresolved risks returned to `method-plan`. - **Ask only if:** The answer is a decision only the researcher can make; look up every fact yourself first. - **Do not do:** Do not write the SAP, run analyses, open patient-level data, or act on the decisions before the user confirms a shared understanding. - **Interaction:** Check repo/config/chat evidence first. Ask one decision-changing question at a time; use the current harness's blocking question UI when available, otherwise present numbered choices and wait. ## How the grill works -Treat the project as a **decision tree** with six branches. Every decision unlocks the decisions that depend on it; the **frontier** is the set of open decisions whose prerequisites are already settled. +Treat the project as a **decision tree** with seven branches. Every decision unlocks the decisions that depend on it; the **frontier** is the set of open decisions whose prerequisites are already settled. | Branch | The decision it forces | |---|---| | Goal | What decision, practice, or policy changes if the study succeeds, and for whom | +| Task and deliverable | What to produce, for whom, under which constraints, and how success will be judged | | Hypothesis | The falsifiable question: population, exposure or intervention, comparator, outcome, time zero, and expected direction | | Data | Whether this source can answer it: provenance, grain, access and approvals, coverage, missingness, and measurement validity | | Proposed analysis | Design, estimand, confounding or causal assumptions, missing-data plan, multiplicity, sensitivity analyses, and sample size | @@ -38,6 +39,18 @@ Before any literature lookup, read `references/evidence-access.md` (PubMed and P ## Workflow +### Planning integration + +When `method-plan` invokes this skill, use the decision tree and question bank for decisions that affect the requested plan. First establish the user's goal, task and deliverable, question or hypothesis when applicable, data, and proposed methods from existing evidence. Check settled choices for consequential weaknesses, not just missing fields; explain the specific concern before asking a decision-changing question. Do not re-ask settled choices without a reason. For descriptive or exploratory work, clarify the aim without requiring a directional hypothesis. Journal and audience questions are conditional on their relevance to the task. + +Resolve the sensitive-data boundary as in standalone step 2 before data access; use schema-only when unset in non-interactive mode. Look up accessible facts within that boundary; ask the researcher only for decisions or facts unavailable to you. Ask one question at a time with a recommended answer and rationale, and recompute dependent questions after each answer. Stop when the planning-critical decisions are clear or the user stops. Summarize choices and risks, ask the user to confirm the shared understanding, then return confirmed choices and unresolved risks to `method-plan`. If the user stops, carry unconfirmed choices as gaps. Do not create a separate grill record or route back to `method-plan` as a new invocation. The user may request a standalone grill record separately. In non-interactive mode return unresolved choices as gaps without inventing answers. + +When comparing methods, state how each changes the population, time zero, estimand, and assumptions where relevant. Do not treat a landmark design as interchangeable with a baseline target-trial strategy or promise that a method eliminates bias. Aligning eligibility and time zero alone does not resolve treatment assignment over a grace period. Until the drug, comparator, treatment window, and relevant data are known, recommend clarifying those decisions rather than selecting a specific causal estimator. If the facts do not support a responsible recommendation, explain the uncertainty and ask for the information needed rather than inventing one. + +Preserve the outcome as stated: do not add all-cause, in-hospital, or other qualifiers unless supported or confirmed. An active comparator can improve comparability when clinically appropriate, but does not inherently reduce confounding; explain the assumptions and remaining confounding rather than promising an advantage. + +### Standalone interview + 1. **Gather facts before asking.** Read whatever exists: `analysis/research-question.yaml`, `docs/brainstorms/`, any SAP (`analysis/sap.md` or a file with `sap_version` frontmatter), `reports/data-qa/` or `analysis/data-qa-report.md`, and `.methodsmith/config.local.yaml`. Look up journal scope, article types, word limits, and required checklists from the journal's author guidelines with available search tools instead of asking. Look up literature facts (prior studies, event rates, validated definitions, the journal's recent similar papers) with the Evidence access tools above, reading each source before citing it. Use parallel subagents for independent lookups when the platform supports them. Never ask the user for something you can find. 2. **Sensitive-data boundary.** Resolve `phi_environment.mode` in `.methodsmith/config.local.yaml` (worktree, then main checkout; legacy `compliant: true` = `approved`, `compliant: false` = `schema-only`, `sensitive_data: false` = `no-sensitive-data`; any schema-only signal wins; unreadable = `schema-only`). `approved` or `no-sensitive-data`: proceed silently and never mention the boundary again. `schema-only`: headings, types, counts, and aggregates only; never open, print, or paste data values; run data-reading scripts as `method-safe-run -- ` (path in the guard's deny message); save results as aggregate tables (counts under 11 suppressed) in `output/` or a `phi_environment.aggregate_dirs` entry. Unset, interactive: ask once with the blocking question tool (else a numbered list) offering `Approved endpoint: full access`, `Not approved or unsure: schema-only`, and `No sensitive data: public or synthetic`; save the answer as `phi_environment.mode` there once `git check-ignore -q .methodsmith/config.local.yaml` passes (else first add `.methodsmith/*.local.yaml` to `.gitignore`) so the guard hook sees it. Unset, headless: schema-only. In `schema-only`, ask about data by schema, counts, and definitions, never for pasted rows. 3. **Ask the frontier, one question at a time.** Pick the frontier question whose answer would change the most downstream decisions. Ask it with the blocking question UI: diff --git a/plugins/methodsmith/skills/method-grill/references/question-bank.md b/plugins/methodsmith/skills/method-grill/references/question-bank.md index 4291ac73..4f970a49 100644 --- a/plugins/methodsmith/skills/method-grill/references/question-bank.md +++ b/plugins/methodsmith/skills/method-grill/references/question-bank.md @@ -3,6 +3,7 @@ ## Contents - Goal +- Task and deliverable - Hypothesis - Data - Proposed analysis @@ -17,8 +18,16 @@ Each branch lists the probes to draw questions from, what a strong answer looks - **Strong answer:** Names the decision and the actor, and states what a null result would still teach. - **Red flags:** The goal is "to explore" with no decision attached; descriptive, predictive, and causal aims mixed in one question; only a positive result would be publishable. +## Task and deliverable + +- **Probes:** What should be produced, who will use or execute it, what counts as success, and which data access, resource, or timing constraints matter? +- **Strong answer:** Names the deliverable, consumer, acceptance criteria, and material constraints. +- **Red flags:** The requested artifact cannot answer the aim, acceptance criteria are unclear, or access is assumed without confirmation. + ## Hypothesis +Adapt this branch to the aim: for prediction clarify target, prediction horizon, intended use, and performance criteria; for descriptive work clarify the question and quantities to estimate. Do not require a directional causal hypothesis for either. + - **Probes:** Population, exposure or intervention, comparator, and outcome (PICO/PECO)? Time zero and follow-up window? Expected direction and a clinically meaningful difference? Primary versus secondary outcomes? What result would falsify it? - **Strong answer:** One primary hypothesis, a defined time zero, a stated direction, and a minimal clinically important difference. - **Red flags:** Several co-primary outcomes; an outcome defined with information from after time zero; a comparator that is "everyone else"; no falsifying result. diff --git a/plugins/methodsmith/skills/method-plan/SKILL.md b/plugins/methodsmith/skills/method-plan/SKILL.md index e033d174..3fde57c8 100644 --- a/plugins/methodsmith/skills/method-plan/SKILL.md +++ b/plugins/methodsmith/skills/method-plan/SKILL.md @@ -43,7 +43,7 @@ Apply the Skill Value interaction rule to every question below; never silently s **Implementation mode** — all other inputs; uses Phases 0-5 below. -**Detection** (Phase 0, before Phase 1): 2+ keywords → announce "Entering SAP mode"; otherwise "Entering implementation mode". The user may override ("use SAP mode" / "use implementation mode") at any point; confirm if detection seems wrong. If the input also names code files, tests, or a repo feature, ask which mode to use. +**Detection** (Phase 0, before Phase 1): an explicit request to write an SAP or 2+ keywords → announce "Entering SAP mode"; otherwise "Entering implementation mode". The user may override ("use SAP mode" / "use implementation mode") at any point; confirm if detection seems wrong. If the input also names code files, tests, or a repo feature, ask which mode to use. ### SAP Mode Workflow @@ -62,10 +62,8 @@ SAP-mode gates that hold regardless of what the reference says: ## Core Principles -1. **Requirements first; decisions, not code** — build from an existing requirements document. Capture approach, boundaries, files, dependencies, risks, and test scenarios; pseudo-code only as explicitly directional guidance. -2. **Research before structuring; right-size the artifact** — compact for small work, more structure for large work. -3. **Separate planning from execution discovery** — explicitly defer execution-time unknowns. Keep the plan portable (no tool-specific executor instructions) and carry implied test-first or characterization-first posture as a lightweight signal. -4. **Honor user-named resources** — a named CLI, MCP server, URL, file, or artifact is authoritative. Discover it (`command -v`, fetch, read) before assuming it is unavailable; if it fails, say so instead of substituting. +Read `references/planning-foundations.md` for principles 1-4; principles 5-7 follow. + 5. **Data shape before SAP, coding, or modeling** - With data available, inspect actual columns, grain, keys, dates, missingness, duplicates, and validity before SAP finalization or implementation planning (`method-data-qa` for tabular, `method-omics-qc` for omics). Without data, write an explicit data-profile prerequisite and keep SAP variable/model sections provisional. 6. **Every new SAP gets a tabular workbook contract** - Not execution-ready without `analysis/sap-tables/01-overview.csv`, `02-outputs.csv`, `03-variables.csv`, and the rendered `-tabular-sap.xlsx` when `openpyxl` is available. If data QA is missing, keep the SAP in draft and list `method-data-qa` before `method-sap-workbook`. 7. **Causal/observational analysis guardrail** - Make the estimand, causal assumptions, unit/grain, key fields, time zero, and success criteria explicit. Record an analyst-approved assumption only when the choice can be responsibly deferred; unresolved methodological choices block finalization rather than being silently inferred. @@ -89,7 +87,7 @@ If the user references an existing plan or an obvious recent match exists in `do ### 0.1b Classify Task Domain and Detect Mode -Detect the mode (Dual-Mode rule); SAP mode continues to 0.2. Otherwise software or analysis code (repos, APIs, databases, pipelines, notebooks) → 0.2 in implementation mode; genuinely ambiguous ("plan a migration") → ask; anything else → read `references/universal-planning.md`, follow it, and skip all later phases. Named tools or links do not change this routing. +Detect the mode (Dual-Mode rule); SAP mode continues to 0.2. A study, research question, or analysis plan also continues to 0.2, even without SAP keywords or code, so its research decisions reach 0.5b. Otherwise software or analysis code (repos, APIs, databases, pipelines, notebooks) → 0.2 in implementation mode; genuinely ambiguous ("plan a migration") → ask; anything else → read `references/universal-planning.md`, follow it, and skip all later phases. Named tools or links do not change this routing. ### 0.2 Find Upstream Requirements Document @@ -100,6 +98,10 @@ Prefer a matching unified artifact in `docs/plans/` (`artifact_contract: ce-unif - **Relevant origin found:** read `references/origin-document.md` and apply 0.3 (carry everything forward; drop nothing silently) and 0.5 (do not plan over true product blockers). - **No relevant origin, or unclear input:** read `references/planning-bootstrap.md` (0.4); it may recommend `method-brainstorm` or route a symptom without a root cause to the `method-debug` skill. +### 0.5b Clarify the Research Decision + +Read `references/planning-foundations.md` and apply its research-decision interview before structuring a study or analysis plan. + ### 0.6 Assess Plan Depth **Lightweight** (small, bounded, low ambiguity), **Standard** (normal feature or bounded refactor), or **Deep** (cross-cutting, strategic, high-risk, or highly ambiguous). If unclear, ask one targeted question. @@ -148,7 +150,7 @@ Read `references/pre-write-review.md` and fix every failed plan check and origin **REQUIRED: Write the plan file to disk before presenting any options.** -For a current-CE unified requirements artifact (`artifact_contract: ce-unified-plan/v1`, `artifact_readiness: requirements-only`), update that same canonical file in place, set `artifact_readiness: implementation-ready`, and preserve the Product Contract and stable R/A/F/AE identifiers. Do not create a sibling `*-plan` file. Otherwise write `docs/plans/YYYY-MM-DD-NNN---plan.` and confirm `Plan written to docs/plans/[filename]`. **Pipeline mode** (LFG, SLFG, or any `disable-model-invocation` context): make needed choices automatically without questions. +For a current-CE unified requirements artifact (`artifact_contract: ce-unified-plan/v1`, `artifact_readiness: requirements-only`), update that same canonical file in place, set `artifact_readiness: implementation-ready`, and preserve the Product Contract and stable R/A/F/AE identifiers. Do not create a sibling `*-plan` file. Otherwise write `docs/plans/YYYY-MM-DD-NNN---plan.` and confirm `Plan written to docs/plans/[filename]`. **Pipeline mode** (LFG, SLFG, or any `disable-model-invocation` context): make routine choices automatically without questions; leave unresolved research and methodological decisions as gaps. ### 5.3 Confidence Check and Deepening diff --git a/plugins/methodsmith/skills/method-plan/references/planning-foundations.md b/plugins/methodsmith/skills/method-plan/references/planning-foundations.md new file mode 100644 index 00000000..e7951720 --- /dev/null +++ b/plugins/methodsmith/skills/method-plan/references/planning-foundations.md @@ -0,0 +1,16 @@ +# Planning Foundations + +## General planning principles + +1. **Requirements first; decisions, not code** — build from an existing requirements document. Capture approach, boundaries, files, dependencies, risks, and test scenarios; pseudo-code only as explicitly directional guidance. +2. **Research before structuring; right-size the artifact** — compact for small work, more structure for large work. +3. **Separate planning from execution discovery** — explicitly defer execution-time unknowns. Keep the plan portable (no tool-specific executor instructions) and carry implied test-first or characterization-first posture as a lightweight signal. +4. **Honor user-named resources** — a named CLI, MCP server, URL, file, or artifact is authoritative. Discover it (`command -v`, fetch, read) before assuming it is unavailable; if it fails, say so instead of substituting. + +## Research decision interview + +For a study or analysis, check the brief and relevant `docs/brainstorms/*-grill.md` records for the goal, research question or hypothesis (when applicable), data, proposed methods, and intended task and deliverable. Treat unconfirmed choices as open. If a material decision is unclear or an apparent design weakness could change the plan, load `method-grill` in its planning-integration mode before structuring the plan. Carry confirmed decisions and unresolved risks forward; do not invent decisions or force a hypothesis for descriptive or exploratory work. Reassess SAP versus implementation mode after the interview; if the appropriate mode changes, announce the switch and follow its gates. + +If the user stops the interview, continue only if they still want a plan; mark unresolved decisions as gaps and never finalize an SAP over open methodological choices. In pipeline or other non-interactive mode, do not interview or fabricate answers: record the gaps and keep affected sections provisional. + +In interactive and non-interactive modes preserve the outcome exactly as supplied until its definition is confirmed; a proposed option is not a chosen endpoint. Do not assign a specific design, estimator, or `study_type` to an unresolved choice, including in frontmatter, headings, or prose; use the supported broad `observational` or `other` study type when justified until a narrower design is settled: describe target-trial framing only as an unselected option when it has not been chosen. An active comparator does not inherently reduce confounding; describe the conditions and remaining uncertainties instead of asserting a benefit. These rules apply throughout the SAP, including its tables and secondary analyses. diff --git a/plugins/methodsmith/skills/method-plan/references/sap-template.md b/plugins/methodsmith/skills/method-plan/references/sap-template.md index 63bbc80c..b04ac248 100644 --- a/plugins/methodsmith/skills/method-plan/references/sap-template.md +++ b/plugins/methodsmith/skills/method-plan/references/sap-template.md @@ -72,6 +72,8 @@ status: draft | final | amended ### SAP-1.4: Estimand(s) +When the endpoint definition is unresolved, carry the supplied outcome wording into every estimand attribute and mark the missing definition as a gap. Do not add an unconfirmed cause, setting, or other qualifier, even in a provisional estimand. + [For each primary and key secondary objective, state the estimand per ICH E9(R1): (1) population, (2) treatment condition(s) being compared, (3) endpoint/variable with time point, (4) each anticipated intercurrent event (e.g., treatment discontinuation, rescue medication, treatment switching, death before outcome assessment) with its strategy (treatment policy, hypothetical, composite variable, while on treatment, principal stratum), and (5) population-level summary (e.g., risk difference, marginal or conditional odds ratio, hazard ratio, RMST difference). Flag hypothetical and principal-stratum strategies as resting on untestable assumptions. For observational causal questions, state the target-trial causal contrast (intention-to-treat or per-protocol analogue) and time zero. Analysis sets in SAP-4 support the estimand; they do not replace it.] --- diff --git a/plugins/methodsmith/skills/method-plan/references/universal-planning.md b/plugins/methodsmith/skills/method-plan/references/universal-planning.md index bf5b7e03..7c1497e1 100644 --- a/plugins/methodsmith/skills/method-plan/references/universal-planning.md +++ b/plugins/methodsmith/skills/method-plan/references/universal-planning.md @@ -42,7 +42,7 @@ When research is recommended, do it — don't just offer. Stale recommendations **Research decomposition pattern:** 1. Identify 2-5 independent research questions based on the task. Good questions target facts the model is least confident about: current prices, hours, availability, recent changes, seasonal specifics. -2. Dispatch parallel research. Prefer user-named surfaces first per Core Principle 4 in SKILL.md; fall back to web search for questions those surfaces don't cover. +2. Dispatch parallel research. Prefer user-named surfaces first per Core Principle 4 in `references/planning-foundations.md`; fall back to web search for questions those surfaces don't cover. 3. Collate findings into a brief research summary before proceeding to planning. Example for "plan a date night in Seattle this Saturday": diff --git a/tests/method-grill-contract.test.ts b/tests/method-grill-contract.test.ts index adc9b99e..7599e4cb 100644 --- a/tests/method-grill-contract.test.ts +++ b/tests/method-grill-contract.test.ts @@ -9,6 +9,17 @@ async function read(rel: string): Promise { } describe("method-grill research-project interview", () => { + test("routes research planning through the scoped interview and preserves headless gaps", async () => { + const planDir = path.join(process.cwd(), "plugins", "methodsmith", "skills", "method-plan") + const plan = await readFile(path.join(planDir, "SKILL.md"), "utf8") + const foundations = await readFile(path.join(planDir, "references", "planning-foundations.md"), "utf8") + const grill = await read("SKILL.md") + expect(plan).toContain("even without SAP keywords or code") + expect(plan).toContain("references/planning-foundations.md") + expect(foundations).toContain("load `method-grill` in its planning-integration mode") + expect(foundations).toContain("do not interview or fabricate answers") + expect(grill).toContain("### Planning integration") + }) test("asks one question at a time with a recommended answer first", async () => { const content = await read("SKILL.md") @@ -19,11 +30,11 @@ describe("method-grill research-project interview", () => { expect(content).toMatch(/blocking question UI/) }) - test("covers all six branches in the skill and the question bank", async () => { + test("covers all seven branches in the skill and the question bank", async () => { const content = await read("SKILL.md") const bank = await read("references/question-bank.md") - for (const branch of ["Goal", "Hypothesis", "Data", "Proposed analysis", "Target journal", "Audience"]) { + for (const branch of ["Goal", "Task and deliverable", "Hypothesis", "Data", "Proposed analysis", "Target journal", "Audience"]) { expect(content).toContain(`| ${branch} |`) expect(bank).toContain(`## ${branch}`) } From 6ab59ba68fd42e8b7d5538c92742765e71f6e81e Mon Sep 17 00:00:00 2001 From: JC Rojas <126191042+sajor2000@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:35:24 -0500 Subject: [PATCH 2/4] fix(plan): route non-SAP studies to research plans --- .../case.yaml | 40 +++++++++++++++++++ .../prompt.md | 3 ++ .../cases/method-plan-grill-no-sap/case.yaml | 8 ++++ .../methodsmith/skills/method-plan/SKILL.md | 6 ++- .../method-plan/references/sap-template.md | 2 +- .../references/universal-planning.md | 2 + 6 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 evals/methodsmith/cases/method-plan-descriptive-research/case.yaml create mode 100644 evals/methodsmith/cases/method-plan-descriptive-research/prompt.md diff --git a/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml b/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml new file mode 100644 index 00000000..51bf5481 --- /dev/null +++ b/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml @@ -0,0 +1,40 @@ +schema_version: "1.0" +id: method-plan-descriptive-research +target: + kind: skill + name: method-plan + source: plugins/methodsmith/skills/method-plan/SKILL.md +mode: forward +prompt_path: evals/methodsmith/cases/method-plan-descriptive-research/prompt.md +fixtures: [] +required_behaviors: + - research-plan-exists + - open-decisions +prohibited_behaviors: + - no-sap +criteria: + - id: research-plan-exists + description: Produces a research plan in the specified location. + type: file_exists + path: workspace/docs/plans/adherence-plan.md + expected: true + weight: 4 + hard_gate: true + - id: open-decisions + description: Records unresolved research choices rather than inventing them. + type: regex + path: workspace/docs/plans/adherence-plan.md + pattern: '[Uu]nresolved|[Pp]rovisional|[Gg][Aa][Pp]|[Oo]pen decision' + weight: 4 + hard_gate: true + - id: no-sap + description: Does not write an SAP when explicitly declined. + type: file_exists + path: workspace/analysis/sap.md + expected: false + weight: 4 + hard_gate: true +threshold: 1 +limitations: + - Manually review the plan for fabricated definitions, software-only structure, and interactive questions. + - The case is synthetic and does not establish clinical validity. diff --git a/evals/methodsmith/cases/method-plan-descriptive-research/prompt.md b/evals/methodsmith/cases/method-plan-descriptive-research/prompt.md new file mode 100644 index 00000000..3f8883ff --- /dev/null +++ b/evals/methodsmith/cases/method-plan-descriptive-research/prompt.md @@ -0,0 +1,3 @@ +Use `method-plan` in non-interactive pipeline mode to plan a synthetic descriptive study of medication adherence in an outpatient clinic. Do not write an SAP or software implementation plan. The medication, population, adherence definition, and data source have not been chosen. + +Save the research plan to `workspace/docs/plans/adherence-plan.md` and your response to `response.md`. Do not run analyses. diff --git a/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml b/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml index 708d3fc1..50c5b8b1 100644 --- a/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml +++ b/evals/methodsmith/cases/method-plan-grill-no-sap/case.yaml @@ -13,6 +13,7 @@ required_behaviors: prohibited_behaviors: - no-bias-guarantee - no-sap + - no-implementation-plan criteria: - id: identifies-timing-risk description: Identifies time-zero or immortal-time bias as a consequential design issue. @@ -42,6 +43,13 @@ criteria: expected: false weight: 3 hard_gate: true + - id: no-implementation-plan + description: The first interaction does not create a plan directory in the empty workspace. + type: file_exists + path: workspace/docs/plans + expected: false + weight: 3 + hard_gate: true threshold: 1 limitations: - This evaluates a synthetic first interaction; it does not prove later interview turns or scientific validity. diff --git a/plugins/methodsmith/skills/method-plan/SKILL.md b/plugins/methodsmith/skills/method-plan/SKILL.md index 3fde57c8..b7344c23 100644 --- a/plugins/methodsmith/skills/method-plan/SKILL.md +++ b/plugins/methodsmith/skills/method-plan/SKILL.md @@ -43,7 +43,7 @@ Apply the Skill Value interaction rule to every question below; never silently s **Implementation mode** — all other inputs; uses Phases 0-5 below. -**Detection** (Phase 0, before Phase 1): an explicit request to write an SAP or 2+ keywords → announce "Entering SAP mode"; otherwise "Entering implementation mode". The user may override ("use SAP mode" / "use implementation mode") at any point; confirm if detection seems wrong. If the input also names code files, tests, or a repo feature, ask which mode to use. +**Detection** (Phase 0, before Phase 1): an explicit request to write an SAP or 2+ keywords → announce "Entering SAP mode"; otherwise "Entering implementation mode". Explicitly declining an SAP overrides keyword detection. The user may override ("use SAP mode" / "use implementation mode") at any point; confirm if detection seems wrong. If the input also names code files, tests, or a repo feature, ask which mode to use. ### SAP Mode Workflow @@ -87,7 +87,7 @@ If the user references an existing plan or an obvious recent match exists in `do ### 0.1b Classify Task Domain and Detect Mode -Detect the mode (Dual-Mode rule); SAP mode continues to 0.2. A study, research question, or analysis plan also continues to 0.2, even without SAP keywords or code, so its research decisions reach 0.5b. Otherwise software or analysis code (repos, APIs, databases, pipelines, notebooks) → 0.2 in implementation mode; genuinely ambiguous ("plan a migration") → ask; anything else → read `references/universal-planning.md`, follow it, and skip all later phases. Named tools or links do not change this routing. +Detect the mode (Dual-Mode rule); SAP mode continues to 0.2. A study or analysis plan also continues to 0.2, even without SAP keywords or code, so its decisions reach 0.5b. For a non-SAP research plan without software work, set `RESEARCH_PLAN_ONLY`. Software or analysis code → 0.2 in implementation mode; genuinely ambiguous ("plan a migration") → ask; anything else → `references/universal-planning.md`, skipping later phases. Named tools or links do not change routing. ### 0.2 Find Upstream Requirements Document @@ -116,6 +116,8 @@ Phase 1.0 is a gate for any study or analysis work with data: inspect the actual Mark each open question (origin deferrals, research gaps, required decisions) **resolved during planning** or **deferred to implementation** (depends on code changes or runtime behavior). Ask only when the answer materially affects architecture, scope, sequencing, or risk and cannot be inferred. Do not run tests, build, or probe runtime behavior. +**Research-plan route:** If `RESEARCH_PLAN_ONLY` remains set and SAP mode was not selected, follow the research-plan route in `references/universal-planning.md` instead of Phases 3-5. + ## Phase 3: Structure the Plan ### 3.1 Title and File Naming diff --git a/plugins/methodsmith/skills/method-plan/references/sap-template.md b/plugins/methodsmith/skills/method-plan/references/sap-template.md index b04ac248..af6515c9 100644 --- a/plugins/methodsmith/skills/method-plan/references/sap-template.md +++ b/plugins/methodsmith/skills/method-plan/references/sap-template.md @@ -72,7 +72,7 @@ status: draft | final | amended ### SAP-1.4: Estimand(s) -When the endpoint definition is unresolved, carry the supplied outcome wording into every estimand attribute and mark the missing definition as a gap. Do not add an unconfirmed cause, setting, or other qualifier, even in a provisional estimand. +When the endpoint definition is unresolved, carry the supplied outcome wording into the endpoint attribute and mark the missing definition as a gap. Do not add an unconfirmed cause, setting, or other qualifier, even in a provisional estimand. [For each primary and key secondary objective, state the estimand per ICH E9(R1): (1) population, (2) treatment condition(s) being compared, (3) endpoint/variable with time point, (4) each anticipated intercurrent event (e.g., treatment discontinuation, rescue medication, treatment switching, death before outcome assessment) with its strategy (treatment policy, hypothetical, composite variable, while on treatment, principal stratum), and (5) population-level summary (e.g., risk difference, marginal or conditional odds ratio, hazard ratio, RMST difference). Flag hypothetical and principal-stratum strategies as resting on untestable assumptions. For observational causal questions, state the target-trial causal contrast (intention-to-treat or per-protocol analogue) and time zero. Analysis sets in SAP-4 support the estimand; they do not replace it.] diff --git a/plugins/methodsmith/skills/method-plan/references/universal-planning.md b/plugins/methodsmith/skills/method-plan/references/universal-planning.md index 7c1497e1..827e1ef4 100644 --- a/plugins/methodsmith/skills/method-plan/references/universal-planning.md +++ b/plugins/methodsmith/skills/method-plan/references/universal-planning.md @@ -10,6 +10,8 @@ This file is loaded when method-plan detects a non-software task (Phase 0.1b). It replaces the software-specific phases (0.2 through 5.1) with a domain-agnostic planning workflow. +**Research-plan route:** When `RESEARCH_PLAN_ONLY` is set by `method-plan`, the research interview, data preflight, and planning questions have already run. Skip the classification and pipeline stop below and Steps 1-1b. Use Step 2's research-plan structure (questions, methods, sources, deliverables, and unresolved decisions). Keep unconfirmed methodological choices as gaps. In pipeline or non-interactive mode, save the provisional plan under `docs/plans/` using the naming convention in Step 3 and return to the caller without questions or a menu. In interactive mode follow Step 3. Do not create an SAP or require software units, file lists, or test files unless the user asks for them. + ## Before starting: verify classification The detection stub in SKILL.md routes here for anything that isn't clearly software. Verify the classification is correct before proceeding: From dc39ce395b337ab11e108fe934462018b44898a2 Mon Sep 17 00:00:00 2001 From: JC Rojas <126191042+sajor2000@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:59 -0500 Subject: [PATCH 3/4] test(plan): tighten research evaluation gates --- .../case.yaml | 40 +++++++++++++++++-- .../method-plan-grill-headless/case.yaml | 4 +- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml b/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml index 51bf5481..8ae3641e 100644 --- a/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml +++ b/evals/methodsmith/cases/method-plan-descriptive-research/case.yaml @@ -9,9 +9,13 @@ prompt_path: evals/methodsmith/cases/method-plan-descriptive-research/prompt.md fixtures: [] required_behaviors: - research-plan-exists - - open-decisions + - open-medication + - open-population + - open-adherence + - open-data-source prohibited_behaviors: - no-sap + - no-sap-structure criteria: - id: research-plan-exists description: Produces a research plan in the specified location. @@ -20,11 +24,32 @@ criteria: expected: true weight: 4 hard_gate: true - - id: open-decisions - description: Records unresolved research choices rather than inventing them. + - id: open-medication + description: Records medication as unresolved. type: regex path: workspace/docs/plans/adherence-plan.md - pattern: '[Uu]nresolved|[Pp]rovisional|[Gg][Aa][Pp]|[Oo]pen decision' + pattern: '(?:^#{1,6}\s*(?:Unresolved|Open)[^\n]*[\s\S]{0,2000}medication|medication[^\n]{0,150}(?:unresolved|not chosen|undecided|open))' + weight: 4 + hard_gate: true + - id: open-population + description: Records population as unresolved. + type: regex + path: workspace/docs/plans/adherence-plan.md + pattern: '(?:^#{1,6}\s*(?:Unresolved|Open)[^\n]*[\s\S]{0,2000}population|population[^\n]{0,150}(?:unresolved|not chosen|undecided|open))' + weight: 4 + hard_gate: true + - id: open-adherence + description: Records adherence as unresolved. + type: regex + path: workspace/docs/plans/adherence-plan.md + pattern: '(?:^#{1,6}\s*(?:Unresolved|Open)[^\n]*[\s\S]{0,2000}adherence (?:definition|measure)|adherence (?:definition|measure)[^\n]{0,150}(?:unresolved|not chosen|undecided|open))' + weight: 4 + hard_gate: true + - id: open-data-source + description: Records data-source as unresolved. + type: regex + path: workspace/docs/plans/adherence-plan.md + pattern: '(?:^#{1,6}\s*(?:Unresolved|Open)[^\n]*[\s\S]{0,2000}data source|data source[^\n]{0,150}(?:unresolved|not chosen|undecided|open))' weight: 4 hard_gate: true - id: no-sap @@ -34,6 +59,13 @@ criteria: expected: false weight: 4 hard_gate: true + - id: no-sap-structure + description: The research plan is not titled or structured as an SAP. + type: regex_not + path: workspace/docs/plans/adherence-plan.md + pattern: '^#{1,6}\s*(?:Statistical Analysis Plan|SAP-[0-9])' + weight: 4 + hard_gate: true threshold: 1 limitations: - Manually review the plan for fabricated definitions, software-only structure, and interactive questions. diff --git a/evals/methodsmith/cases/method-plan-grill-headless/case.yaml b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml index e9379b28..8b790e1e 100644 --- a/evals/methodsmith/cases/method-plan-grill-headless/case.yaml +++ b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml @@ -31,9 +31,9 @@ criteria: weight: 4 hard_gate: true - id: data-profile-gap - description: The response identifies the missing data profile or data QA prerequisite. + description: The SAP identifies the missing data profile or data QA prerequisite. type: regex - path: response.md + path: workspace/analysis/sap.md pattern: '[Dd]ata[- ](?:profile|QA|quality)|method-data-qa' weight: 3 hard_gate: true From 354a4e4e4034f3360d6f5d9bb177fca19fac1985 Mon Sep 17 00:00:00 2001 From: JC Rojas <126191042+sajor2000@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:07:14 -0500 Subject: [PATCH 4/4] test(plan): require data profile prerequisite --- evals/methodsmith/cases/method-plan-grill-headless/case.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/methodsmith/cases/method-plan-grill-headless/case.yaml b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml index 8b790e1e..d50dfba6 100644 --- a/evals/methodsmith/cases/method-plan-grill-headless/case.yaml +++ b/evals/methodsmith/cases/method-plan-grill-headless/case.yaml @@ -34,7 +34,7 @@ criteria: description: The SAP identifies the missing data profile or data QA prerequisite. type: regex path: workspace/analysis/sap.md - pattern: '[Dd]ata[- ](?:profile|QA|quality)|method-data-qa' + pattern: '(?:gap|prerequisite|required)[^\n]{0,100}(?:data[- ](?:profile|QA|quality)|method-data-qa)|(?:data[- ](?:profile|QA|quality)|method-data-qa)[^\n]{0,100}(?:gap|prerequisite|required)' weight: 3 hard_gate: true - id: no-final-status