Store front: a redesign, and the Concepts as a tree instead of 752 chips

.claude/launch.json

@@ -6,6 +6,12 @@
       "runtimeExecutable": "npm",
       "runtimeArgs": ["--prefix", "web", "run", "dev"],
       "port": 5173
+    },
+    {
+      "name": "store",
+      "runtimeExecutable": "env",
+      "runtimeArgs": ["STRUT_DEV_INDEX=https://index.stakwork.ai", "npm", "--prefix", "web", "run", "dev", "--", "--port", "5183", "--strictPort"],
+      "port": 5183
     }
   ]
 }

specs/API.md

@@ -2289,7 +2289,7 @@ installed it, it can be a mirror. `file:`, `pk:` and `git+` refs resolve
 | POST   | `/index/requests/:id/refuse` | `{ reason }` → the refusal: the request dropped, and a re-send from that key for that name answered with `reason` for 30 days, without a fetch. Full scope; 404 `listing_unknown` |
 | GET    | `/index/catalog[?digest=]` | The INDEX card as served (RFC 8785 bytes, sealed under `strut-index\n` by the index key; `ETag`, `If-None-Match` → 304; `cache-control: public`): `{ v, key, seq, publishers, urls?, sig }`. **Open to anyone.** A reader checks it with `readCard(bytes, { key, kind: "index" })`; 404 before the first row |
 | GET    | `/index/packages?q=&limit=` | `{ packages: IndexHit[] }` — every listed publisher's public packages, each at its newest release not yanked: `{ package, publisher, key, urls, version, digest, description?, license?, requires?, detail? }`; `q` as the sidebar's matcher (every word in the name, description or publisher), at most `limit` (100; ≤ 500). `detail` is the release's archive as the index READ it at the crawl (`src/packages/detail.ts`, below): `{ apps: [{ name, title, description?, icon?, can }], workflows, steps, concepts (counts), secrets, can: { spends, acts, code }, readme }` — absent until the archive has been fetched. **Open to anyone**; derived and unsigned — an install re-verifies against the publisher |
-| GET    | `/index/packages/:publisher/:name` | `IndexPackage`: `{ package, publisher, key, urls, description?, releases (newest first, yanked kept), channels: { "<public channel>": digest }, readAt, detail?, detailError? }`. `detail` is the newest release not yanked read off its archive — `PackageDetail`: `{ digest, version, derivedAt, readme?, apps: [{ name, title, description?, icon?, grants: { workflows, steps }, can, files }], workflows: [{ name, description?, inputs: [{ name, type, required, description?, default? }], steps, grants, can, secrets, peer?, sealed?, claims?, error? }], steps: [{ name, helper?, description?, effect?, secrets, lint: [string], bytes, claims? }], concepts: [{ name, description?, parent?, docs? }], secrets, can, files, bytes }` (names inside the package; `can` = what running it reaches: `spends` an `agent` / `llm` step, `acts` a step declaring `effect: "write"`, `code` an `exec` or one of the package's own steps; the steps' facts are regexes over the source, the lint `src/step-lint.ts`'s findings — a reading, never a check); `detailError` says why there is none (the archive not fetched yet, or unreadable). **Open to anyone**; 404 when no listed publisher's catalog has it |
+| GET    | `/index/packages/:publisher/:name` | `IndexPackage`: `{ package, publisher, key, urls, description?, releases (newest first, yanked kept), channels: { "<public channel>": digest }, readAt, detail?, detailError? }`. `detail` is the newest release not yanked read off its archive — `PackageDetail`: `{ digest, version, derivedAt, readme?, apps: [{ name, title, description?, icon?, grants: { workflows, steps }, can, files }], workflows: [{ name, description?, inputs: [{ name, type, required, description?, default? }], steps, grants, can, secrets, peer?, sealed?, claims?, error? }], steps: [{ name, helper?, description?, effect?, secrets, lint: [string], bytes, claims? }], concepts: [{ name, description?, parent?, docs? }], secrets, can, files, bytes }` (names inside the package; `can` = what running it reaches: `spends` an `agent` / `llm` step, `acts` a step declaring `effect: "write"`, `code` an `exec` or one of the package's own steps; the steps' facts are regexes over the source, the lint `src/step-lint.ts`'s findings — a reading, never a check); `detailError` says why there is none (the archive not fetched yet, or unreadable). `?docs=0` leaves each Concept page's `docs` out (the store's read: a page's text is `/index/concepts/:name`'s). **Open to anyone**; 404 when no listed publisher's catalog has it |
 | GET    | `/index/concepts?q=` | `{ concepts: IndexConcept[] }` — every Concept page across the crawled packages AND the live rows' public trees, one per name: `{ name, description?, parent?, packages: [{ package, version }], live?: [{ publisher, url }] }` (the first package by name gives the description and parent, then a live tree's; a page only a live view holds has `packages: []`, and its docs are read from `<url>/public/concepts/<name>`; `q` over names and descriptions). **Open to anyone** |
 | GET    | `/index/concepts/:name` | `IndexConceptPage`: the entry plus `docs?` (the first package's text), `children: [{ name, description? }]` (the pages naming this one as parent) and `parents` (nearest first). **Open to anyone**; 404 when no listed package carries it |
 | GET    | `/llms.txt` | On an index strut, the store as ONE markdown page for an agent that fetches URLs (llmstxt.org; `llmsTxt`, `src/packages/index-tools.ts`), `text/markdown`: an H1 naming the index, a blockquote saying what it is (derived, unsigned past the card, an install re-verifies against the publisher), how an agent reaches it (the `claude mcp add` line for `/index/mcp`) and that a PERSON installs; then `## Reads` (the five open routes above, each a link), `## Packages` (every listed package at its newest release — `- [publisher/name](…/index/packages/publisher/name): description — version; N workflows, M steps, K apps; needs SECRET; spends (model calls), acts on the world, runs code`), `## Concepts` (every page the packages carry, each a link to `/index/concepts/<name>`, its parent and packages) and `## Optional` (the store page `?store=1` when the index serves the UI), and, when this strut also lets anyone run something, `# Run on this server` with §17.8's run and tool lists. **Open to anyone**. On a strut without the role it is the public view's page (§17.8), and `404` when nothing is public |

src/createStrut.ts

@@ -111,7 +111,7 @@ import { packArchive } from "./packages/archive.js";
 import { buildPackage, type BuildRequest } from "./packages/build.js";
 import { PUBLISHER_FILE, PUBLISHER_NAME, catalogStore, moveChannel, publicCatalogStore, publishRelease, publisherStore } from "./packages/publisher.js";
 import { SUBSCRIPTIONS_FILE, createSubscriptions, subscriptionProblem, type Subscription, type Subscriptions } from "./packages/subscriptions.js";
-import { createPackageIndex, indexDir, indexRowProblem, liveRowProblem, type PackageIndex } from "./packages/index.js";
+import { createPackageIndex, indexDir, indexRowProblem, liveRowProblem, withoutConceptDocs, type PackageIndex } from "./packages/index.js";
 import { INDEXES_FILE, createIndexes, indexSubscriptionProblem, parseIndexesEnv, type IndexSubscription, type PackageIndexes, type PackagesCapability } from "./packages/indexes.js";
 import { PUBLISHER_PREFIX, Z32_KEY, releaseFor, type IndexLiveRow, type IndexRow } from "./packages/card.js";
 import { LISTING_MAX_BYTES, hostPatternProblem, perMinute } from "./packages/listing.js";
@@ -4731,7 +4731,11 @@ export async function createStrut<TServices = unknown>(
     app.get("/index/packages/:publisher/:name", async (c) => {
       const name = `${c.req.param("publisher")}/${c.req.param("name")}`;
       const page = await index.package(name);
-      return page ? c.json(page, 200, { "cache-control": "public, no-cache" }) : c.json({ error: `no listed publisher's catalog has ${name}` }, 404);
+      if (!page) return c.json({ error: `no listed publisher's catalog has ${name}` }, 404);
+      // `?docs=0`: without each Concept page's text. The store draws a
+      // package's tree from names alone and reads one page's docs from
+      // `/index/concepts/:name`; a package of 750 pages is megabytes of them.
+      return c.json(c.req.query("docs") === "0" ? withoutConceptDocs(page) : page, 200, { "cache-control": "public, no-cache" });
     });
     // The Concept pages across every crawled package (plans/marketplace.md
     // §7, the store): the tree, and one page with its docs.

src/packages/index.test.ts

@@ -22,7 +22,7 @@ import { MemoryRunStore } from "../store.js";
 import { WorkspaceManager } from "../workspace.js";
 import { INDEX_DOMAIN, readCard, type IndexCatalog } from "./card.js";
 import { buildIndexTools, indexInstructions, llmsTxt } from "./index-tools.js";
-import { createPackageIndex, type IndexConcept, type IndexConceptPage, type IndexHit, type IndexInfo, type IndexPackage } from "./index.js";
+import { createPackageIndex, withoutConceptDocs, type IndexConcept, type IndexConceptPage, type IndexHit, type IndexInfo, type IndexPackage } from "./index.js";
 import { packArchive } from "./archive.js";
 import { PackageCache } from "./cache.js";
 import { canonicalJson, keyFromSeed, sealCard, type Catalog } from "./card.js";
@@ -232,6 +232,7 @@ describe("the package index", () => {
     assert.equal((await search("linear acme")).length, 1, "every word, in the name or the publisher");
     assert.equal((await fetch(`${indexUrl}/index/packages/acme/notion`)).status, 404);
     const page = (await (await fetch(`${indexUrl}/index/packages/acme/linear`)).json()) as IndexPackage;
+    assert.deepEqual(await (await fetch(`${indexUrl}/index/packages/acme/linear?docs=0`)).json(), page, "docs=0 leaves out only Concept pages' text, and this package has none");
     assert.equal(page.package, "acme/linear");
     assert.deepEqual(page.releases.map((r) => r.version), ["1.0.0"]);
     assert.deepEqual(page.channels, { public: linear.digest });
@@ -564,6 +565,13 @@ describe("the concept tree across the crawled packages", () => {
     assert.deepEqual((await index.concepts("civil")).map((c) => [c.name, c.parents]), [["Civil procedure", ["Law"]]]);
     assert.deepEqual((await index.package("openlaw/civil"))!.detail!.concepts.find((c) => c.name === "Civil procedure")!.parents, ["Law"]);
 
+    // The store's read of a package (`?docs=0`): the tree's names and parents, no page's text.
+    const whole = (await index.package("openlaw/civil"))!;
+    assert.match(whole.detail!.concepts.find((c) => c.name === "Statute of limitations")!.docs!, /How long you have/);
+    const slim = withoutConceptDocs(whole);
+    assert.deepEqual(slim.detail!.concepts.map((c) => [c.name, c.parents, "docs" in c]), whole.detail!.concepts.map((c) => [c.name, c.parents, false]));
+    assert.deepEqual({ ...slim, detail: undefined }, { ...whole, detail: undefined }, "nothing else of the page changes");
+
     // The index's tools (./index-tools.ts) over the same functions: the page a model reads (Concept docs left out), the tree, one page — each linking its store page when a store is served.
     const server = await toolsetServer({ tools: buildIndexTools(index, { store: "https://store.example" }), instructions: indexInstructions({ publicUrl: "https://store.example", store: "https://store.example" }) });
     const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();

src/packages/index.ts

@@ -25,7 +25,7 @@ import { fetchArchive } from "./archive.js";
 import type { PackageCache } from "./cache.js";
 import { CARD_VERSION, PUBLISHER_PREFIX, Z32_KEY, canonicalJson, keyFromSeed, releaseFor, sealCard, type Catalog, type CatalogRelease, type IndexCatalog, type IndexLiveRow, type IndexRow, type UnsignedIndexCatalog } from "./card.js";
 export type { IndexLiveRow, IndexRow } from "./card.js";
-import { derivePackageDetail, summarizeDetail, type DetailSummary, type PackageDetail, type StepFacts } from "./detail.js";
+import { derivePackageDetail, summarizeDetail, type ConceptDetail, type DetailSummary, type PackageDetail, type StepFacts } from "./detail.js";
 export type { AppDetail, ConceptDetail, DetailSummary, PackageCan, PackageDetail, StepDetail, WorkflowDetail } from "./detail.js";
 import { fetchCatalog, releaseUrls } from "./pk.js";
 import { CatalogStore, type CatalogFile } from "./publisher.js";
@@ -204,6 +204,14 @@ export interface IndexPackage {
   detailError?: string;
 }
 
+/** A package's page without its Concept pages' text: what a reader that
+ *  draws the tree from names needs (the store, `?docs=0`; a model's brief
+ *  page). One page's docs are `concept(name)`'s. */
+export function withoutConceptDocs<T extends { detail?: PackageDetail }>(page: T): Omit<T, "detail"> & { detail?: Omit<PackageDetail, "concepts"> & { concepts: Array<Omit<ConceptDetail, "docs">> } } {
+  const { detail, ...rest } = page;
+  return detail ? { ...rest, detail: { ...detail, concepts: detail.concepts.map(({ docs: _docs, ...c }) => c) } } : rest;
+}
+
 /** One Concept page as the index knows it: across every crawled package. */
 export interface IndexConcept {
   name: string;

web/src/api.ts

@@ -2778,7 +2778,9 @@ async function storeJSON<T>(path: string): Promise<T> {
 }
 export const storeCatalog = () => storeJSON<IndexCard>("/index/catalog");
 export const storeSearch = (q = "", limit = 200) => storeJSON<{ packages: StoreHit[] }>(`/index/packages?q=${encodeURIComponent(q)}&limit=${limit}`).then((r) => r.packages);
-export const storePackage = (pkg: string) => storeJSON<StorePackage>(`/index/packages/${pkg.split("/").map(encodeURIComponent).join("/")}`);
+/** A package's page, without its Concept pages' text (`docs=0`): the tree is
+ *  drawn from names, and a page's docs are `storeConcept`'s. */
+export const storePackage = (pkg: string) => storeJSON<StorePackage>(`/index/packages/${pkg.split("/").map(encodeURIComponent).join("/")}?docs=0`);
 export const storeConcepts = (q = "") => storeJSON<{ concepts: StoreConcept[] }>(`/index/concepts?q=${encodeURIComponent(q)}`).then((r) => r.concepts);
 export const storeConcept = (name: string) => storeJSON<StoreConceptPage>(`/index/concepts/${encodeURIComponent(name)}`);
 /** Where this index's card is served, for the link into a strut: what the