Skip to content

Commit 48cfed4

Browse files
tfireubs-uiclaude
andcommitted
feat(signals): add auto-scoring middleware for signal quality assessment (closes bounty #25)
Adds scoreSignal() — a pure TypeScript function that evaluates five quality dimensions at submission time and stores results with each signal row: - sourceQuality (0-30): 1 source=10pts, 2=20pts, 3+=30pts - thesisClarity (0-25): headline word-count sweet-spot + body length bonus - beatRelevance (0-20): tag overlap with beat slug keywords - timeliness (0-15): source URLs containing current/prev year - disclosure (0-10): non-empty + tool/model keyword mention bonus Schema: adds quality_score INTEGER and score_breakdown TEXT (JSON) columns via MIGRATION_SIGNAL_SCORING_SQL (migration version 12). Both are nullable so existing rows remain valid without any backfill. The scorer is pure (no I/O, no DB calls) so it adds zero async overhead to the POST /signals handler and is fully testable in isolation. Also adds src/__tests__/signal-scorer.test.ts with 20 unit tests covering all five dimensions plus edge cases (empty sources, null disclosure, etc). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 359e9f5 commit 48cfed4

5 files changed

Lines changed: 491 additions & 6 deletions

File tree

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
import { describe, it, expect } from "vitest";
2+
import { scoreSignal } from "../lib/signal-scorer";
3+
4+
/**
5+
* Unit tests for the signal quality auto-scorer.
6+
* All tests are pure — no network, no DB, no cloudflare bindings needed.
7+
*/
8+
9+
const MINIMAL_SIGNAL = {
10+
headline: "Bitcoin price rises",
11+
body: null,
12+
sources: [],
13+
tags: [],
14+
beat_slug: "agent-economy",
15+
disclosure: null,
16+
};
17+
18+
describe("scoreSignal — sourceQuality dimension", () => {
19+
it("returns 0 pts for zero sources", () => {
20+
const result = scoreSignal({ ...MINIMAL_SIGNAL, sources: [] });
21+
expect(result.breakdown.sourceQuality).toBe(0);
22+
});
23+
24+
it("returns 10 pts for 1 source", () => {
25+
const result = scoreSignal({
26+
...MINIMAL_SIGNAL,
27+
sources: [{ url: "https://example.com/news/2026/article", title: "Example" }],
28+
});
29+
expect(result.breakdown.sourceQuality).toBe(10);
30+
});
31+
32+
it("returns 20 pts for 2 sources", () => {
33+
const result = scoreSignal({
34+
...MINIMAL_SIGNAL,
35+
sources: [
36+
{ url: "https://example.com/a", title: "A" },
37+
{ url: "https://example2.com/b", title: "B" },
38+
],
39+
});
40+
expect(result.breakdown.sourceQuality).toBe(20);
41+
});
42+
43+
it("returns 30 pts for 3+ sources", () => {
44+
const result = scoreSignal({
45+
...MINIMAL_SIGNAL,
46+
sources: [
47+
{ url: "https://a.com", title: "A" },
48+
{ url: "https://b.com", title: "B" },
49+
{ url: "https://c.com", title: "C" },
50+
],
51+
});
52+
expect(result.breakdown.sourceQuality).toBe(30);
53+
});
54+
});
55+
56+
describe("scoreSignal — thesisClarity dimension", () => {
57+
it("gives 5 pts for a headline that is too short (< 5 words)", () => {
58+
const result = scoreSignal({ ...MINIMAL_SIGNAL, headline: "Bitcoin rises" });
59+
expect(result.breakdown.thesisClarity).toBe(5);
60+
});
61+
62+
it("gives 10 pts for headline with 5–7 words", () => {
63+
const result = scoreSignal({
64+
...MINIMAL_SIGNAL,
65+
headline: "Bitcoin hits new all-time high today",
66+
});
67+
expect(result.breakdown.thesisClarity).toBe(10);
68+
});
69+
70+
it("gives 15 pts for headline with 8–15 words", () => {
71+
const result = scoreSignal({
72+
...MINIMAL_SIGNAL,
73+
headline: "Bitcoin breaks one hundred thousand dollars driven by institutional ETF demand",
74+
});
75+
expect(result.breakdown.thesisClarity).toBe(15);
76+
});
77+
78+
it("adds 10 pts bonus when body is longer than 200 chars, capped at 25", () => {
79+
const longBody = "A".repeat(201);
80+
const result = scoreSignal({
81+
...MINIMAL_SIGNAL,
82+
headline: "Bitcoin breaks one hundred thousand dollars driven by institutional ETF demand",
83+
body: longBody,
84+
});
85+
expect(result.breakdown.thesisClarity).toBe(25);
86+
});
87+
88+
it("does not award body bonus for body <= 200 chars", () => {
89+
const shortBody = "Short body.";
90+
const result = scoreSignal({
91+
...MINIMAL_SIGNAL,
92+
headline: "Bitcoin breaks one hundred thousand dollars driven by institutional ETF demand",
93+
body: shortBody,
94+
});
95+
expect(result.breakdown.thesisClarity).toBe(15);
96+
});
97+
});
98+
99+
describe("scoreSignal — beatRelevance dimension", () => {
100+
it("returns 0 pts when no tags", () => {
101+
const result = scoreSignal({ ...MINIMAL_SIGNAL, tags: [], beat_slug: "agent-economy" });
102+
expect(result.breakdown.beatRelevance).toBe(0);
103+
});
104+
105+
it("returns 10 pts for 1 tag matching beat keyword", () => {
106+
const result = scoreSignal({
107+
...MINIMAL_SIGNAL,
108+
tags: ["agent"],
109+
beat_slug: "agent-economy",
110+
});
111+
expect(result.breakdown.beatRelevance).toBe(10);
112+
});
113+
114+
it("returns 20 pts for 2+ tags matching beat keywords", () => {
115+
const result = scoreSignal({
116+
...MINIMAL_SIGNAL,
117+
tags: ["agent", "economy"],
118+
beat_slug: "agent-economy",
119+
});
120+
expect(result.breakdown.beatRelevance).toBe(20);
121+
});
122+
123+
it("returns 0 pts when tags don't overlap with beat keywords", () => {
124+
const result = scoreSignal({
125+
...MINIMAL_SIGNAL,
126+
tags: ["ordinals", "runes"],
127+
beat_slug: "agent-economy",
128+
});
129+
expect(result.breakdown.beatRelevance).toBe(0);
130+
});
131+
});
132+
133+
describe("scoreSignal — timeliness dimension", () => {
134+
it("returns 0 pts for empty sources", () => {
135+
const result = scoreSignal({ ...MINIMAL_SIGNAL, sources: [] });
136+
expect(result.breakdown.timeliness).toBe(0);
137+
});
138+
139+
it("returns 15 pts when any source URL contains the current year", () => {
140+
const currentYear = new Date().getFullYear();
141+
const result = scoreSignal({
142+
...MINIMAL_SIGNAL,
143+
sources: [{ url: `https://news.example.com/${currentYear}/story`, title: "Story" }],
144+
});
145+
expect(result.breakdown.timeliness).toBe(15);
146+
});
147+
148+
it("returns 15 pts when source URL contains previous year", () => {
149+
const prevYear = new Date().getFullYear() - 1;
150+
const result = scoreSignal({
151+
...MINIMAL_SIGNAL,
152+
sources: [{ url: `https://news.example.com/${prevYear}/story`, title: "Old Story" }],
153+
});
154+
expect(result.breakdown.timeliness).toBe(15);
155+
});
156+
157+
it("returns 8 pts when no year match found in source URLs", () => {
158+
const result = scoreSignal({
159+
...MINIMAL_SIGNAL,
160+
sources: [{ url: "https://news.example.com/bitcoin-macro-story", title: "Story" }],
161+
});
162+
expect(result.breakdown.timeliness).toBe(8);
163+
});
164+
});
165+
166+
describe("scoreSignal — disclosure dimension", () => {
167+
it("returns 0 pts for empty disclosure", () => {
168+
const result = scoreSignal({ ...MINIMAL_SIGNAL, disclosure: "" });
169+
expect(result.breakdown.disclosure).toBe(0);
170+
});
171+
172+
it("returns 0 pts for null disclosure", () => {
173+
const result = scoreSignal({ ...MINIMAL_SIGNAL, disclosure: null });
174+
expect(result.breakdown.disclosure).toBe(0);
175+
});
176+
177+
it("returns 5 pts for non-empty disclosure without tool/model mention", () => {
178+
const result = scoreSignal({
179+
...MINIMAL_SIGNAL,
180+
disclosure: "I researched this manually.",
181+
});
182+
expect(result.breakdown.disclosure).toBe(5);
183+
});
184+
185+
it("returns 10 pts for disclosure mentioning claude", () => {
186+
const result = scoreSignal({
187+
...MINIMAL_SIGNAL,
188+
disclosure: "Researched using Claude with aibtc MCP tools.",
189+
});
190+
expect(result.breakdown.disclosure).toBe(10);
191+
});
192+
193+
it("returns 10 pts for disclosure mentioning LLM", () => {
194+
const result = scoreSignal({
195+
...MINIMAL_SIGNAL,
196+
disclosure: "LLM-assisted research with manual verification.",
197+
});
198+
expect(result.breakdown.disclosure).toBe(10);
199+
});
200+
});
201+
202+
describe("scoreSignal — total and shape", () => {
203+
it("total equals sum of all breakdown dimensions", () => {
204+
const result = scoreSignal({
205+
headline: "Agent economy signals new wave of sBTC payment adoption on Stacks network",
206+
body: "A".repeat(250),
207+
sources: [
208+
{ url: "https://stacks.org/2026/news", title: "Stacks News" },
209+
{ url: "https://aibtc.com/2026/updates", title: "AIBTC Updates" },
210+
{ url: "https://x.com/2026/post", title: "Post" },
211+
],
212+
tags: ["agent", "economy", "sbtc"],
213+
beat_slug: "agent-economy",
214+
disclosure: "Researched using Claude and aibtc MCP skills.",
215+
});
216+
const { sourceQuality, thesisClarity, beatRelevance, timeliness, disclosure } =
217+
result.breakdown;
218+
expect(result.total).toBe(
219+
sourceQuality + thesisClarity + beatRelevance + timeliness + disclosure
220+
);
221+
});
222+
223+
it("returns a score between 0 and 100 inclusive", () => {
224+
const result = scoreSignal(MINIMAL_SIGNAL);
225+
expect(result.total).toBeGreaterThanOrEqual(0);
226+
expect(result.total).toBeLessThanOrEqual(100);
227+
});
228+
229+
it("returns maximum score of 100 for a fully-formed signal", () => {
230+
const result = scoreSignal({
231+
headline: "Agent economy signals new wave of sBTC payment adoption on Stacks network",
232+
body: "A".repeat(250),
233+
sources: [
234+
{ url: "https://stacks.org/2026/news", title: "Stacks News" },
235+
{ url: "https://aibtc.com/2026/updates", title: "AIBTC Updates" },
236+
{ url: "https://x.com/2026/post", title: "Post" },
237+
],
238+
tags: ["agent", "economy"],
239+
beat_slug: "agent-economy",
240+
disclosure: "Researched using Claude and aibtc MCP skills.",
241+
});
242+
expect(result.total).toBe(100);
243+
});
244+
});

0 commit comments

Comments
 (0)