|
| 1 | +//go:build integration |
| 2 | + |
| 3 | +package crosssdk_test |
| 4 | + |
| 5 | +// Reconciliation gate for the published Conformance page |
| 6 | +// (site/src/content/docs/conformance.mdx). |
| 7 | +// |
| 8 | +// The page's results matrix carries a vector count per row, and the page |
| 9 | +// claims those counts are "generated, not asserted". scripts/conformance_matrix/count.py |
| 10 | +// is the generator — it reads the frozen vector files and emits the counts. |
| 11 | +// Nothing, though, forced the page's numbers to equal count.py's output: the |
| 12 | +// table cells and the prose figures were hand-typed, so a change to a vector |
| 13 | +// file could silently desync the two. On a page that is cited to a standards |
| 14 | +// body as evidence, a stale number is worse than none. |
| 15 | +// |
| 16 | +// This test closes that gap. It runs count.py, parses the page, and asserts: |
| 17 | +// - every on-page vector set from count.py appears as a matrix row with the |
| 18 | +// exact count count.py computes (and no matrix row exists that count.py does |
| 19 | +// not know about), and |
| 20 | +// - the two exact prose figures in the honesty callout (the canonicalization |
| 21 | +// total and the MUST-reject case count) match count.py. |
| 22 | +// |
| 23 | +// It lives here, in cross-sdk-tests/, because cross-sdk-tests.yml re-runs on |
| 24 | +// every change to the vector corpora (cross-sdk-tests/** and spec/**), to this |
| 25 | +// page, and to count.py (scripts/conformance_matrix/**) — every input that can |
| 26 | +// move or desync these counts. count.py is invoked as the single source of |
| 27 | +// truth rather than re-deriving counts here, so the two cannot drift. |
| 28 | + |
| 29 | +import ( |
| 30 | + "bytes" |
| 31 | + "encoding/json" |
| 32 | + "os" |
| 33 | + "os/exec" |
| 34 | + "regexp" |
| 35 | + "strconv" |
| 36 | + "strings" |
| 37 | + "testing" |
| 38 | +) |
| 39 | + |
| 40 | +const ( |
| 41 | + countScriptPath = "../scripts/conformance_matrix/count.py" |
| 42 | + conformancePage = "../site/src/content/docs/conformance.mdx" |
| 43 | +) |
| 44 | + |
| 45 | +type countSet struct { |
| 46 | + Name string `json:"name"` |
| 47 | + Count int `json:"count"` |
| 48 | + OnPage bool `json:"onPage"` |
| 49 | +} |
| 50 | + |
| 51 | +type countPayload struct { |
| 52 | + Sets []countSet `json:"sets"` |
| 53 | + Total int `json:"total"` |
| 54 | +} |
| 55 | + |
| 56 | +// reconcileInputs bundles the once-loaded count.py output and page text so the |
| 57 | +// subtests share a single python3 invocation and a single page read. |
| 58 | +type reconcileInputs struct { |
| 59 | + payload countPayload |
| 60 | + counts map[string]int // set name -> count, for O(1) lookup |
| 61 | + page string |
| 62 | +} |
| 63 | + |
| 64 | +// loadReconcileInputs runs count.py --format json and reads the page. If python3 |
| 65 | +// is not on PATH (e.g. a bare local `go test` without Python), the test is |
| 66 | +// skipped rather than failed — CI runners have python3, so the gate still fires |
| 67 | +// where it matters. |
| 68 | +func loadReconcileInputs(t *testing.T) reconcileInputs { |
| 69 | + t.Helper() |
| 70 | + python, err := exec.LookPath("python3") |
| 71 | + if err != nil { |
| 72 | + t.Skipf("python3 not available, skipping conformance-page reconciliation: %v", err) |
| 73 | + } |
| 74 | + cmd := exec.Command(python, countScriptPath, "--format", "json") |
| 75 | + var stderr bytes.Buffer |
| 76 | + cmd.Stderr = &stderr |
| 77 | + out, err := cmd.Output() |
| 78 | + if err != nil { |
| 79 | + // count.py prints "error: could not count vectors: <detail>" to stderr; |
| 80 | + // surface it so a broken vector file is diagnosable from the CI log. |
| 81 | + t.Fatalf("run count.py: %v\nstderr: %s", err, strings.TrimSpace(stderr.String())) |
| 82 | + } |
| 83 | + var payload countPayload |
| 84 | + if err := json.Unmarshal(out, &payload); err != nil { |
| 85 | + t.Fatalf("parse count.py JSON: %v", err) |
| 86 | + } |
| 87 | + if len(payload.Sets) == 0 { |
| 88 | + t.Fatal("count.py returned no vector sets") |
| 89 | + } |
| 90 | + counts := make(map[string]int, len(payload.Sets)) |
| 91 | + for _, s := range payload.Sets { |
| 92 | + counts[s.Name] = s.Count |
| 93 | + } |
| 94 | + pageBytes, err := os.ReadFile(conformancePage) |
| 95 | + if err != nil { |
| 96 | + t.Fatalf("read conformance page: %v", err) |
| 97 | + } |
| 98 | + return reconcileInputs{payload: payload, counts: counts, page: string(pageBytes)} |
| 99 | +} |
| 100 | + |
| 101 | +// section returns the slice of the page between the first line starting with |
| 102 | +// startPrefix (exclusive) and the next line starting with endPrefix (exclusive). |
| 103 | +// Scoping the regex scans to the intended block means an unrelated table or a |
| 104 | +// stray prose figure elsewhere on the page cannot be misread as a matrix row or |
| 105 | +// as a pinned count. |
| 106 | +func section(t *testing.T, page, startPrefix, endPrefix string) string { |
| 107 | + t.Helper() |
| 108 | + lines := strings.Split(page, "\n") |
| 109 | + start := -1 |
| 110 | + for i, line := range lines { |
| 111 | + if strings.HasPrefix(line, startPrefix) { |
| 112 | + start = i + 1 |
| 113 | + break |
| 114 | + } |
| 115 | + } |
| 116 | + if start == -1 { |
| 117 | + t.Fatalf("section start %q not found on the page", startPrefix) |
| 118 | + } |
| 119 | + end := len(lines) |
| 120 | + for i := start; i < len(lines); i++ { |
| 121 | + if strings.HasPrefix(lines[i], endPrefix) { |
| 122 | + end = i |
| 123 | + break |
| 124 | + } |
| 125 | + } |
| 126 | + if end <= start { |
| 127 | + t.Fatalf("section %q..%q is empty", startPrefix, endPrefix) |
| 128 | + } |
| 129 | + return strings.Join(lines[start:end], "\n") |
| 130 | +} |
| 131 | + |
| 132 | +// matrixRow matches a results-matrix data row: a table row whose first cell is a |
| 133 | +// markdown link, e.g. `| [canonicalization](https://…) | … | 44 |`. The link |
| 134 | +// text is the vector-set name; the final numeric cell is the count. |
| 135 | +var matrixRowRe = regexp.MustCompile(`^\|\s*\[([^\]]+)\]\([^)]*\)\s*\|.*\|\s*([0-9]+)\s*\|\s*$`) |
| 136 | + |
| 137 | +// parseMatrix extracts {name: count} for every matrix row in the given section. |
| 138 | +func parseMatrix(t *testing.T, src string) map[string]int { |
| 139 | + t.Helper() |
| 140 | + rows := map[string]int{} |
| 141 | + for _, line := range strings.Split(src, "\n") { |
| 142 | + m := matrixRowRe.FindStringSubmatch(line) |
| 143 | + if m == nil { |
| 144 | + continue |
| 145 | + } |
| 146 | + n, err := strconv.Atoi(m[2]) |
| 147 | + if err != nil { |
| 148 | + t.Fatalf("row %q: non-numeric count %q", m[1], m[2]) |
| 149 | + } |
| 150 | + if _, dup := rows[m[1]]; dup { |
| 151 | + t.Fatalf("duplicate matrix row for %q", m[1]) |
| 152 | + } |
| 153 | + rows[m[1]] = n |
| 154 | + } |
| 155 | + if len(rows) == 0 { |
| 156 | + t.Fatal("no matrix rows parsed from the results-matrix section") |
| 157 | + } |
| 158 | + return rows |
| 159 | +} |
| 160 | + |
| 161 | +var ( |
| 162 | + proseCanonRe = regexp.MustCompile(`(\d+) canonicalization/receipt-hash vectors`) |
| 163 | + proseMalformedRe = regexp.MustCompile(`(\d+)-case MUST-reject corpus`) |
| 164 | +) |
| 165 | + |
| 166 | +func TestConformancePageReconciles(t *testing.T) { |
| 167 | + in := loadReconcileInputs(t) |
| 168 | + |
| 169 | + // The published matrix lives under "## Results matrix" and ends at the |
| 170 | + // "Each linked vector set…" sentence right after the table. |
| 171 | + t.Run("matrix", func(t *testing.T) { |
| 172 | + matrixSrc := section(t, in.page, "## Results matrix", "Each linked vector set") |
| 173 | + pageRows := parseMatrix(t, matrixSrc) |
| 174 | + |
| 175 | + expected := map[string]int{} |
| 176 | + for _, s := range in.payload.Sets { |
| 177 | + if !s.OnPage { |
| 178 | + // Off-page sets (reference fixtures) must NOT appear as matrix rows. |
| 179 | + if _, present := pageRows[s.Name]; present { |
| 180 | + t.Errorf("set %q is on_page=False in count.py but appears in the page matrix", s.Name) |
| 181 | + } |
| 182 | + continue |
| 183 | + } |
| 184 | + expected[s.Name] = s.Count |
| 185 | + got, present := pageRows[s.Name] |
| 186 | + if !present { |
| 187 | + t.Errorf("count.py set %q is missing from the page matrix", s.Name) |
| 188 | + continue |
| 189 | + } |
| 190 | + if got != s.Count { |
| 191 | + t.Errorf("count mismatch for %q: page=%d, count.py=%d", s.Name, got, s.Count) |
| 192 | + } |
| 193 | + } |
| 194 | + // No matrix row may exist that count.py does not account for. |
| 195 | + for name := range pageRows { |
| 196 | + if _, ok := expected[name]; !ok { |
| 197 | + t.Errorf("page matrix row %q has no corresponding on-page set in count.py", name) |
| 198 | + } |
| 199 | + } |
| 200 | + }) |
| 201 | + |
| 202 | + // The two exact numeric claims in the honesty callout are pinned to count.py |
| 203 | + // too, scoped to that Aside so an earlier figure elsewhere cannot shadow them. |
| 204 | + t.Run("prose", func(t *testing.T) { |
| 205 | + callout := section(t, in.page, `<Aside type="caution" title="Honest coverage`, "</Aside>") |
| 206 | + checks := []struct { |
| 207 | + label string |
| 208 | + re *regexp.Regexp |
| 209 | + setName string |
| 210 | + }{ |
| 211 | + {"canonicalization total", proseCanonRe, "canonicalization"}, |
| 212 | + {"MUST-reject case count", proseMalformedRe, "malformed (MUST-reject)"}, |
| 213 | + } |
| 214 | + for _, c := range checks { |
| 215 | + m := c.re.FindStringSubmatch(callout) |
| 216 | + if m == nil { |
| 217 | + t.Errorf("%s: prose figure not found in the honesty callout (pattern %q)", c.label, c.re) |
| 218 | + continue |
| 219 | + } |
| 220 | + got, err := strconv.Atoi(m[1]) |
| 221 | + if err != nil { |
| 222 | + t.Fatalf("%s: non-numeric prose figure %q", c.label, m[1]) |
| 223 | + } |
| 224 | + want, ok := in.counts[c.setName] |
| 225 | + if !ok { |
| 226 | + t.Fatalf("count.py has no set named %q", c.setName) |
| 227 | + } |
| 228 | + if got != want { |
| 229 | + t.Errorf("%s: prose says %d, count.py says %d", c.label, got, want) |
| 230 | + } |
| 231 | + } |
| 232 | + }) |
| 233 | +} |
0 commit comments