From 36af9bc92ad754f4fe119bd5ed430414435d9ade Mon Sep 17 00:00:00 2001 From: Doug Fritz Date: Tue, 8 Sep 2026 13:40:05 +0000 Subject: [PATCH] fix: respect CommonMark tilde code fences and prevent silent AST truncation - In CommonMark, tildes (~) only open fenced code blocks when a run of 3 or more tildes appears at the start of a line (indented <= 3 spaces). Previously, any unescaped single tilde in prose (e.g. "~1,110 tasks") or strikethrough was treated as an inline code span opener, suppressing inline math expansion across subsequent prose. Disentangle backticks and tildes, add isFenceLineStart and isFenceLineEnd to prevent premature closure on comments, and validate info strings. - In parser.go, assert all tokens are consumed in parse() (p.peek().typ == tokEOF). Previously, dangling unconsumed tokens or unmatched \end constructs caused parse() to succeed with a partial AST, silently dropping the rest of the expression. Failing with an error allows Expand to preserve raw markdown verbatim rather than emitting a truncated fragment. - Add regression tests covering inline tildes, tilde ranges, strikethrough, comments inside fences, and unconsumed token error handling. --- markdown.go | 80 +++++++++++++++++++++++++++++++++++++++++++++--- markdown_test.go | 64 ++++++++++++++++++++++++++++++++++++++ parser.go | 8 ++++- parser_test.go | 17 ++++++++++ 4 files changed, 164 insertions(+), 5 deletions(-) diff --git a/markdown.go b/markdown.go index ac7416f..08bd643 100644 --- a/markdown.go +++ b/markdown.go @@ -16,6 +16,39 @@ func isEscaped(md string, i int) bool { return n%2 == 1 } +// isFenceLineStart reports whether pos is at the start of a line (or indented +// by at most 3 spaces), matching CommonMark fenced code block rules. +func isFenceLineStart(md string, pos int) bool { + spaces := 0 + for k := pos - 1; k >= 0; k-- { + if md[k] == '\n' || md[k] == '\r' { + return true + } + if md[k] != ' ' { + return false + } + spaces++ + if spaces > 3 { + return false + } + } + return true +} + +// isFenceLineEnd reports whether pos is followed only by optional spaces/tabs +// until the end of the line or end of string, matching CommonMark closing fence rules. +func isFenceLineEnd(md string, pos int) bool { + for k := pos; k < len(md); k++ { + if md[k] == '\n' || md[k] == '\r' { + return true + } + if md[k] != ' ' && md[k] != '\t' { + return false + } + } + return true +} + // Expand scans markdown text for math delimiters and replaces // them with termtex-rendered output. Display math ($$...$$) becomes // fenced code blocks that glamour preserves verbatim. Inline math ($...$) @@ -65,6 +98,7 @@ func Expand(md string, style Style) string { i := 0 n := len(md) inCode := false + isFencedCode := false codeOpenerLength := 0 codeOpenerChar := byte(0) htmlCloser := "" @@ -87,7 +121,15 @@ func Expand(md string, style Style) string { j++ } sb.WriteString(md[runStart:j]) - if runLen == codeOpenerLength { + if isFencedCode { + // CommonMark §4.5: closing fence must have at least as many + // characters as the opener, be indented <= 3 spaces, and be + // followed only by spaces until newline or EOF. + if runLen >= codeOpenerLength && isFenceLineStart(md, runStart) && isFenceLineEnd(md, j) { + inCode = false + isFencedCode = false + } + } else if runLen == codeOpenerLength { inCode = false } i = j @@ -131,10 +173,10 @@ func Expand(md string, style Style) string { } i = j - if char == '`' || char == '~' { + if char == '`' { runStart := i runLen := 0 - for i < n && md[i] == char { + for i < n && md[i] == '`' { runLen++ i++ } @@ -144,7 +186,37 @@ func Expand(md string, style Style) string { } inCode = true codeOpenerLength = runLen - codeOpenerChar = char + codeOpenerChar = '`' + isFencedCode = runLen >= 3 && isFenceLineStart(md, runStart) + sb.WriteString(md[runStart:i]) + continue + } + + if char == '~' { + runStart := i + runLen := 0 + for i < n && md[i] == '~' { + runLen++ + i++ + } + // In CommonMark, tildes only open fenced code blocks (>= 3 tildes at line start). + // Single/double tildes in prose (~1,000, ~~strike~~) are never code openers. + if !isEscaped(md, runStart) && runLen >= 3 && isFenceLineStart(md, runStart) { + // CommonMark §4.5: info strings for tilde blocks cannot contain tildes. + hasTildeInInfo := false + for k := i; k < n && md[k] != '\n' && md[k] != '\r'; k++ { + if md[k] == '~' { + hasTildeInInfo = true + break + } + } + if !hasTildeInInfo { + inCode = true + isFencedCode = true + codeOpenerLength = runLen + codeOpenerChar = '~' + } + } sb.WriteString(md[runStart:i]) continue } diff --git a/markdown_test.go b/markdown_test.go index 2e0541b..e545e36 100644 --- a/markdown_test.go +++ b/markdown_test.go @@ -150,3 +150,67 @@ func TestExpandMathInsideCodeBlocksUnchanged(t *testing.T) { }) } } + +func TestExpandTildeInProse(t *testing.T) { + cases := []struct { + name string + in string + wantMath string + }{ + { + name: "single tilde approximation before math", + in: "Completed ~1,110 tasks, speed is $a^2 + b^2$.", + wantMath: "a² + b²", + }, + { + name: "tilde range before math", + in: "Between ~10 and ~20 items: $a^2$.", + wantMath: "a²", + }, + { + name: "strikethrough containing math", + in: "Old value: ~~$a^2$~~, new value: $b^2$.", + wantMath: "a²", + }, + { + name: "tilde fence with comment inside not closing prematurely", + in: "~~~\n~~~ not a closer\nx := $a^2$\n~~~\nAfter $a^2$.", + wantMath: "a²", + }, + { + name: "indented tilde fence up to 3 spaces", + in: " ~~~\n x := $a^2$\n ~~~\nAfter $a^2$.", + wantMath: "a²", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := Expand(tc.in, Style{}) + if !strings.Contains(out, tc.wantMath) { + t.Errorf("expected math %q in output, got:\n%s", tc.wantMath, out) + } + if tc.name == "tilde fence with comment inside not closing prematurely" { + if !strings.Contains(out, "$a^2$") { + t.Errorf("expected $a^2$ inside code block to remain unexpanded, got:\n%s", out) + } + } + if tc.name == "indented tilde fence up to 3 spaces" { + if !strings.Contains(out, "$a^2$") { + t.Errorf("expected $a^2$ inside code block to remain unexpanded, got:\n%s", out) + } + } + }) + } +} + +func TestExpandPreservesUnparseableMathVerbatim(t *testing.T) { + // If parsing fails (for example, due to unconsumed tokens like an unmatched \end), + // Expand must preserve the raw math expression verbatim rather than emitting + // a silently truncated fragment. + in := "Before $$x \\end{aligned} y$$ After" + out := Expand(in, Style{}) + if out != in { + t.Errorf("Expand should preserve unparseable math verbatim:\ngot: %q\nwant: %q", out, in) + } +} diff --git a/parser.go b/parser.go index aed54f8..0fdd66f 100644 --- a/parser.go +++ b/parser.go @@ -345,10 +345,16 @@ func parse(input string) (*node, error) { tp := lex(input) p := &parser{tokens: *tp} node, err := p.parseRows() - releaseTokens(tp) if err != nil { + releaseTokens(tp) return nil, err } + if p.peek().typ != tokEOF { + t := p.peek() + releaseTokens(tp) + return nil, fmt.Errorf("unexpected token %q at position %d", t.val, p.pos) + } + releaseTokens(tp) return node, nil } diff --git a/parser_test.go b/parser_test.go index 7a07900..684b4aa 100644 --- a/parser_test.go +++ b/parser_test.go @@ -137,3 +137,20 @@ func TestPipeAsAbsValue(t *testing.T) { t.Errorf("|x| should render as |x|, got %q", got) } } + +// TestUnconsumedTokensError verifies that incomplete expressions with +// dangling tokens (e.g. unmatched \end or closing delimiters) return an error +// instead of silently succeeding with a truncated AST. +func TestUnconsumedTokensError(t *testing.T) { + inputs := []string{ + `x \end{aligned}`, + `x } y`, + `x ] y`, + } + for _, in := range inputs { + _, err := parse(in) + if err == nil { + t.Errorf("parse(%q) expected error for unconsumed tokens, got nil", in) + } + } +}