index: add a per-doc size hint so agents pre-decide --toc vs full read

Each entry in the index now carries `size: "<N> tokens, <M> lines"` —
a body-size hint scanDocs() synthesizes from the parsed body. Token
count uses the canonical chars/4 estimate (±15% on prose / markdown
for any major LLM tokenizer); line count matches markdown.ts so
--section line numbers and the `size` line count agree.

Why now: the index is exactly the place an agent decides whether to
read the whole body or jump to --toc / --section. Without a size
signal it can't make that decision without a probe read; with one,
it's a glance. Naming: `size` (not `metrics`/`stats`) — the field
IS the doc's size; spread order lets a hand-written frontmatter
`size` win, same principle that keeps us from synthesizing other
fields.

Drive-by: pad the inline injection block with blank lines around the
fenced YAML. CommonMark formatters (oxfmt, prettier) would insert
them anyway, and emitting them ourselves prevents a hook → format →
hook ping-pong on mtime. Locked in with a regression test.

Author: ethan-huo

AGENTS.md

@@ -43,6 +43,7 @@ Four commands — `index`, `read`, `install`, `uninstall`.
 
 ```yaml
 - file: ./skills/claw/SKILL.md
+  size: 1741 tokens, 159 lines
   name: claw
   description: Use when reading or indexing markdown knowledge in a workspace —
     listing what docs exist, navigating a long doc by section, embedding a fresh

skills/claw/SKILL.md

@@ -44,6 +44,11 @@ claw index --dir docs                 # index a specific directory
 claw index --inject AGENTS.md         # embed the index inline in AGENTS.md
 ```
 
+Each entry in the index carries a `size` hint like `"1234 tokens, 56 lines"`
+(token count is a chars/4 estimate, ±15% on prose / markdown). Use it to
+decide before reading: a small doc → read the whole body; a large doc →
+go straight to `claw read … --toc` and drill in with `--section`.
+
 Two delivery modes, one underlying scan:
 
 - **stdout (default)** — call `claw index` whenever you need a fresh

src/wiki.test.ts

@@ -70,46 +70,105 @@ test("scans frontmatter docs, skipping reserved, plain, and ignored files", () =
   expect(docs[1]?.data).toEqual({ type: "Spec" });
 });
 
-test("builds a YAML list: file + frontmatter verbatim per doc", () => {
+test("scanDocs measures body size with chars/4 token estimate and line count", () => {
+  // Body is exactly 8 chars on one line: "abcdefgh" → ceil(8/4) = 2 tokens.
+  const root = fixture({ "a.md": "---\ntype: Note\n---\nabcdefgh" });
+  const [doc] = scanDocs(root);
+  expect(doc?.size).toBe("2 tokens, 1 lines");
+});
+
+test("scanDocs counts lines without inflating on a trailing newline", () => {
+  // 4 visible lines, trailing newline; --section semantics agree the doc has 4 lines.
+  const body = "line 1\nline 2\nline 3\nline 4\n";
+  const root = fixture({ "a.md": `---\ntype: Note\n---\n${body}` });
+  const [doc] = scanDocs(root);
+  expect(doc?.size).toBe(`${Math.ceil(body.length / 4)} tokens, 4 lines`);
+});
+
+test("size hint scales with body, not frontmatter — frontmatter is metadata, not payload", () => {
+  // Same body, very different frontmatter weight: size must be identical.
+  const body = "the body content here";
+  const lean = fixture({ "a.md": `---\ntype: Note\n---\n${body}` });
+  const heavy = fixture({
+    "a.md": `---\ntype: Note\ntitle: A long title\ndescription: ${"x".repeat(500)}\n---\n${body}`,
+  });
+  expect(scanDocs(lean)[0]?.size).toBe(scanDocs(heavy)[0]?.size);
+});
+
+test("builds a YAML list: file + size + frontmatter verbatim per doc", () => {
   const docs: DocRecord[] = [
     {
       path: "docs/spec.md",
+      size: "100 tokens, 10 lines",
       data: { type: "Spec", title: "Spec", description: "the spec", tags: ["x"] },
     },
-    { path: "readme.md", data: { type: "Note", when: "on start" } },
+    { path: "readme.md", size: "5 tokens, 1 lines", data: { type: "Note", when: "on start" } },
   ];
   const out = buildIndex(docs);
   expect(out).toContain("- file: ./docs/spec.md"); // a YAML sequence item
+  expect(out).toContain("size: 100 tokens, 10 lines");
   expect(out).toContain("description: the spec");
   expect(out).toContain("when: on start");
-  // round-trips to structured data
+  // round-trips to structured data — size sits next to file, before frontmatter
   expect(parse(out)).toEqual([
-    { file: "./docs/spec.md", type: "Spec", title: "Spec", description: "the spec", tags: ["x"] },
-    { file: "./readme.md", type: "Note", when: "on start" },
+    {
+      file: "./docs/spec.md",
+      size: "100 tokens, 10 lines",
+      type: "Spec",
+      title: "Spec",
+      description: "the spec",
+      tags: ["x"],
+    },
+    {
+      file: "./readme.md",
+      size: "5 tokens, 1 lines",
+      type: "Note",
+      when: "on start",
+    },
+  ]);
+});
+
+test("an author-written `size` in frontmatter wins over the synthesized hint", () => {
+  // Same principle that keeps us from synthesizing other fields: if the
+  // author cared enough to write it, surface their value, not ours.
+  const out = buildIndex([
+    { path: "a.md", size: "100 tokens, 10 lines", data: { type: "Note", size: "tiny" } },
   ]);
+  expect(parse(out)).toEqual([{ file: "./a.md", size: "tiny", type: "Note" }]);
 });
 
 test("buildIndex emits an empty YAML list for no docs", () => {
   expect(parse(buildIndex([]))).toEqual([]);
 });
 
 test("indexBlock embeds the index in a fenced yaml block", () => {
-  const block = indexBlock([{ path: "a.md", data: { type: "Note", description: "d" } }]);
+  const block = indexBlock([
+    { path: "a.md", size: "1 tokens, 1 lines", data: { type: "Note", description: "d" } },
+  ]);
   expect(block.startsWith(BLOCK_START)).toBe(true);
   expect(block.endsWith(BLOCK_END)).toBe(true);
   expect(block).toContain("```yaml");
   expect(block).toContain("- file: ./a.md");
   expect(block).toContain("description: d");
 });
 
+test("indexBlock pads the fence with blank lines — keeps formatters from rewriting it", () => {
+  // CommonMark formatters (oxfmt, prettier) insert blank lines around fenced
+  // code blocks. If claw doesn't emit them, every hook → format → hook cycle
+  // re-dirties the host file. Lock the layout in.
+  const block = indexBlock([{ path: "a.md", size: "1 tokens, 1 lines", data: { type: "Note" } }]);
+  expect(block).toContain("\n\n```yaml");
+  expect(block).toContain("```\n\n<!-- /claw:index -->");
+});
+
 test("injects then replaces a managed block idempotently", () => {
-  const first = indexBlock([{ path: "a.md", data: { type: "Note" } }]);
+  const first = indexBlock([{ path: "a.md", size: "1 tokens, 1 lines", data: { type: "Note" } }]);
   const once = injectManagedBlock("# Project\n", first);
   expect(once).toContain("# Project");
   expect(once).toContain(BLOCK_START);
   expect(once).toContain("file: ./a.md");
 
-  const newer = indexBlock([{ path: "b.md", data: { type: "Note" } }]);
+  const newer = indexBlock([{ path: "b.md", size: "1 tokens, 1 lines", data: { type: "Note" } }]);
   const twice = injectManagedBlock(once, newer);
   expect(twice.split(BLOCK_START).length - 1).toBe(1); // exactly one block
   expect(twice).toContain("file: ./b.md");
@@ -118,7 +177,7 @@ test("injects then replaces a managed block idempotently", () => {
 
 test("injectManagedBlock refuses to write when a marker is unbalanced", () => {
   // start marker present, end marker missing → corrupted; must not append.
-  const block = indexBlock([{ path: "a.md", data: { type: "Note" } }]);
+  const block = indexBlock([{ path: "a.md", size: "1 tokens, 1 lines", data: { type: "Note" } }]);
   expect(() => injectManagedBlock(`# Project\n${BLOCK_START}\nstray\n`, block)).toThrow(
     /unbalanced/,
   );

src/wiki.ts

@@ -6,11 +6,15 @@ import { stringify } from "yaml";
 import { usageError } from "./errors.ts";
 import { parseFrontmatter, type Frontmatter } from "./frontmatter.ts";
 
-// The index entry an agent navigates by: a path plus the doc's frontmatter,
-// dumped verbatim. We deliberately don't normalize/derive fields — agents read
-// the YAML; fabricated metadata would mislead them.
+// The index entry an agent navigates by: a path, a body-size hint, and the
+// doc's frontmatter dumped verbatim. We deliberately don't normalize/derive
+// frontmatter fields — agents read the YAML; fabricated metadata would mislead
+// them. `size` is the one synthesized field, because no human writes it
+// accurately and an agent reads the index expressly to decide whether to
+// fetch full body or jump straight to --toc/--section.
 export type DocRecord = {
   path: string; // posix, relative to the scan root
+  size: string; // body-size hint, e.g. "1234 tokens, 56 lines"
   data: Frontmatter;
 };
 
@@ -44,13 +48,28 @@ export function scanDocs(root: string): DocRecord[] {
     const parsed = parseFrontmatter(text);
     if (!parsed.hasFrontmatter) continue;
 
-    records.push({ path: rel, data: parsed.data });
+    records.push({ path: rel, size: bodySize(parsed.body), data: parsed.data });
   }
 
   records.sort((a, b) => a.path.localeCompare(b.path));
   return records;
 }
 
+// Body-size hint surfaced in the index so an agent can decide, before reading,
+// whether to grab the whole doc or jump straight to --toc / --section.
+//
+// Token estimation uses the canonical no-tokenizer rule of thumb of ~4 chars
+// per token — accurate to ±15% on English prose / markdown for any major LLM
+// tokenizer (cl100k, BPE-based, etc.). Good enough for "is this huge?"; we
+// trade exactness for zero dependencies and zero per-doc overhead.
+function bodySize(body: string): string {
+  const tokens = Math.ceil(body.length / 4);
+  // Match the line-counting convention in markdown.ts: a trailing newline
+  // doesn't add a "line", so --section and `size` agree on what line N means.
+  const lines = body.length === 0 ? 0 : body.split("\n").length - (body.endsWith("\n") ? 1 : 0);
+  return `${tokens} tokens, ${lines} lines`;
+}
+
 // Enumerate candidate markdown as posix paths relative to root. In a git repo
 // this is the authoritative `.gitignore`-respecting set (tracked + untracked,
 // minus ignored); the daemon's coarse watch ignore never has to be exact. Falls
@@ -84,23 +103,32 @@ function listMarkdown(root: string): string[] {
   return candidates.filter((rel) => !rel.split("/").some((seg) => seg.startsWith(".")));
 }
 
-// The index is a YAML list — one entry per concept, `file` plus the doc's
-// frontmatter verbatim. Plain data: readable, ordered, trivially parsed.
+// The index is a YAML list — one entry per concept, `file` + `size` plus the
+// doc's frontmatter verbatim. `size` sits next to `file` because it's tool-
+// derived metadata about the file, distinct from author-written frontmatter.
+// Spread order lets a hand-written `data.size` win — same principle that keeps
+// us from synthesizing other fields.
 export function buildIndex(docs: DocRecord[]): string {
-  return stringify(docs.map((doc) => ({ file: `./${doc.path}`, ...doc.data })));
+  return stringify(docs.map((doc) => ({ file: `./${doc.path}`, size: doc.size, ...doc.data })));
 }
 
 // The injected block: the full index, fenced so a markdown formatter leaves
 // the embedded YAML alone. Embedding gives the agent passive awareness through
 // the host's file-change channel — a static reference would force the agent to
 // read a second file, no better than just calling `claw index` on demand.
+//
+// Blank lines around the fenced block are deliberate: oxfmt and most CommonMark
+// formatters insert them anyway, and emitting them ourselves keeps the block
+// idempotent against any hook → formatter → hook ping-pong.
 export function indexBlock(docs: DocRecord[]): string {
   return [
     BLOCK_START,
     "<!-- Generated by `claw index --inject`. Edit the docs, not this block. -->",
+    "",
     "```yaml",
     buildIndex(docs).trimEnd(),
     "```",
+    "",
     BLOCK_END,
   ].join("\n");
 }