diff --git a/packages/flint-js/src/core/filter-overflow.ts b/packages/flint-js/src/core/filter-overflow.ts index 6c2bb3bc..9c0c0357 100644 --- a/packages/flint-js/src/core/filter-overflow.ts +++ b/packages/flint-js/src/core/filter-overflow.ts @@ -264,7 +264,8 @@ function defaultOverflowOrder( } if (explicitSort?.kind === 'values') { const ordered = sortOrder === 'descending' ? [...explicitSort.values].reverse() : explicitSort.values; - return ordered.filter(value => uniqueValues.includes(value)); + const present = new Set(uniqueValues); + return ordered.filter(value => present.has(value)); } // Infer sort field and direction diff --git a/packages/flint-js/tests/filter-overflow.test.ts b/packages/flint-js/tests/filter-overflow.test.ts index 86c68f8d..1146035f 100644 --- a/packages/flint-js/tests/filter-overflow.test.ts +++ b/packages/flint-js/tests/filter-overflow.test.ts @@ -133,6 +133,31 @@ describe('overflow category selection', () => { expect(applyCategoryViewports(data, result.viewports, { x: 1 })) .toEqual([data[1], data[2], data[3]]); }); + it('keeps explicit order duplicates and drops missing values before truncation', () => { + const data = ['Delta', 'Alpha', 'Charlie', 'Bravo'].map(Category => ({ Category, Value: 1 })); + const sortBy = JSON.stringify(['Bravo', 'Missing', 'Delta', 'Bravo', 'Alpha', 'Charlie']); + const run = (sortOrder?: 'ascending' | 'descending') => filterOverflow( + { + x: { field: 'Category', type: 'nominal', semanticAnnotation: annotation }, + y: { field: 'Value', type: 'quantitative', semanticAnnotation: { semanticType: 'Quantity' } }, + }, + { axisFlags: { x: { banded: true } } }, + { x: { field: 'Category', sortBy, sortOrder }, y: { field: 'Value' } }, + data, + budgets, + marks, + ); + + const ascending = run(); + expect(ascending.viewports[0].orderedValues).toEqual(['Bravo', 'Delta', 'Bravo', 'Alpha', 'Charlie']); + expect(ascending.truncations[0].keptValues).toEqual(['Bravo', 'Delta', 'Bravo']); + expect(ascending.warnings[0].message).toBe("1 of 4 values in 'Category' were omitted (showing first 3 in sort order)."); + expect(ascending.filteredData).toEqual([data[0], data[3]]); + + const descending = run('descending'); + expect(descending.viewports[0].orderedValues).toEqual(['Charlie', 'Alpha', 'Bravo', 'Delta', 'Bravo']); + expect(descending.truncations[0].keptValues).toEqual(['Charlie', 'Alpha', 'Bravo']); + }); it('windows rows whose numeric categories the rail holds as text', () => { const rows = [1880, 1881, 1882].map((Year) => ({ Year, Anomaly: 0 })); diff --git a/packages/flint-py/flint/core/filter_overflow.py b/packages/flint-py/flint/core/filter_overflow.py index 8ce82461..ee326930 100644 --- a/packages/flint-py/flint/core/filter_overflow.py +++ b/packages/flint-py/flint/core/filter_overflow.py @@ -190,7 +190,16 @@ def _default_overflow_strategy( sorted_list = json.loads(sort_by) if isinstance(sorted_list, list): ordered_values = list(reversed(sorted_list)) if sort_order == "descending" else sorted_list - return [v for v in ordered_values if v in unique_values][:max_to_keep] + present = set(unique_values) + + def is_present(v: Any) -> bool: + try: + return v in present + except TypeError: + # Unhashable JSON values keep list-membership semantics. + return v in unique_values + + return [v for v in ordered_values if is_present(v)][:max_to_keep] except Exception: pass is_descending = sort_order == "descending" diff --git a/packages/flint-py/tests/unit/test_filter_overflow.py b/packages/flint-py/tests/unit/test_filter_overflow.py index 1469d5f4..7cc72564 100644 --- a/packages/flint-py/tests/unit/test_filter_overflow.py +++ b/packages/flint-py/tests/unit/test_filter_overflow.py @@ -72,3 +72,46 @@ def test_uses_selected_value_sort(): {"type": "nominal"}, {"sortBy": "y", "sortOrder": "descending"}, ) == ["Delta", "Charlie", "Bravo"] + + +def _explicit_order_result(data, sort_by, sort_order=None): + encoding = {"field": "Category", "sortBy": sort_by} + if sort_order: + encoding["sortOrder"] = sort_order + return filter_overflow( + { + "x": {"field": "Category", "type": "nominal", "semanticAnnotation": {"semanticType": "Category"}}, + "y": {"field": "Value", "type": "quantitative", "semanticAnnotation": {"semanticType": "Quantity"}}, + }, + {"axisFlags": {"x": {"banded": True}}}, + {"x": encoding, "y": {"field": "Value"}}, + data, + BUDGETS, + MARKS, + ) + + +def test_explicit_order_keeps_duplicates_and_drops_missing_values(): + data = [{"Category": c, "Value": 1} for c in ["Delta", "Alpha", "Charlie", "Bravo"]] + sort_by = '["Bravo", "Missing", "Delta", "Bravo", "Alpha", "Charlie"]' + + ascending = _explicit_order_result(data, sort_by) + # The default strategy is capped at the distinct-value count, so duplicates shorten the tail. + assert ascending["viewports"][0]["orderedValues"] == ["Bravo", "Delta", "Bravo", "Alpha"] + assert ascending["truncations"][0]["keptValues"] == ["Bravo", "Delta", "Bravo"] + assert ascending["warnings"][0]["message"] == ( + "1 of 4 values in 'Category' were omitted (showing first 3 in sort order)." + ) + assert ascending["filteredData"] == [data[0], data[3]] + + descending = _explicit_order_result(data, sort_by, "descending") + assert descending["viewports"][0]["orderedValues"] == ["Charlie", "Alpha", "Bravo", "Delta"] + assert descending["truncations"][0]["keptValues"] == ["Charlie", "Alpha", "Bravo"] + + +def test_explicit_order_uses_python_equality_and_skips_unhashable_entries(): + data = [{"Category": c, "Value": 1} for c in [1, 2, 3, 4]] + + result = _explicit_order_result(data, '[["nested"], true, 2.0, 9, 3, 4]') + assert result["viewports"][0]["orderedValues"] == [True, 2.0, 3, 4] + assert result["truncations"][0]["keptValues"] == [True, 2.0, 3] diff --git a/scripts/bench-overflow-order/README.md b/scripts/bench-overflow-order/README.md new file mode 100644 index 00000000..46be21f4 --- /dev/null +++ b/scripts/bench-overflow-order/README.md @@ -0,0 +1,43 @@ +# Explicit-order overflow benchmark (#161) + +Measures exported `filterOverflow` (JS) and `filter_overflow` (Python) when an +explicit JSON `sortBy` category order is used with `maxValues.x = 25`. + +Fixture (generated in `bench.mjs` / `bench.py`): N distinct categories `cat-000000…`, +one row each; `sortBy` lists them reversed, with one duplicate entry and two +missing categories. Sizes 1k / 5k / 10k. + +## Reproduce + +```bash +npm ci +scripts/bench-overflow-order/run.sh a794557384adcc3293a77043f6b9aff1c33d152c 3 +# quick smoke: SIZES=1000 scripts/bench-overflow-order/run.sh a794557 1 +``` + +`run.sh` extracts the baseline sources with `git archive`, bundles both sides of +`filter-overflow.ts` with esbuild, and alternates baseline/change order every +round. Each case runs in its own process; JS takes 15 timed calls after warm-up and +Python takes 7. Output is written to the git-ignored `experiments/bench-overflow-order/`. + +## Metrics + +- `medianMs`: median wall time of one call (`process.hrtime.bigint` / + `time.perf_counter_ns`), with a forced GC before each call. +- `digest`: SHA-256 of the JSON-serialized result (rows, warnings, truncations, + viewports). Baseline and change must match. +- `medianAllocBytes` (JS): delta of `v8.getHeapStatistics().total_allocated_bytes` + around one call. Node documents this as "total allocated bytes since the Isolate + creation" (Node ≥ 24; V8 14.6 in Node 26). It is a cumulative counter, not + sampling and not a used-heap delta. It is **approximate**: it is updated at + allocation-buffer granularity, excludes ArrayBuffer backing stores, and shows + ~0.3–0.6 MB of offset on a no-op after `gc()` (`results/alloc-probe.txt`). + Compare only the baseline/change difference. +- `tracemallocPeakBytes` (Python): `tracemalloc` peak traced allocation above the + starting point during one call, from a separate untimed call. +- `maxRssKiB`: process peak RSS from `getrusage`; this includes runtime startup. + +## Recorded results (`results/`) + +Node v26.12.1, Python 3.12.3, esbuild 0.28.1, Linux x86_64, 2 vCPU, 3 interleaved +rounds. See `results/summary.txt` and the raw rows in `results/results.jsonl`. diff --git a/scripts/bench-overflow-order/alloc-probe.mjs b/scripts/bench-overflow-order/alloc-probe.mjs new file mode 100644 index 00000000..4dfbb9e4 --- /dev/null +++ b/scripts/bench-overflow-order/alloc-probe.mjs @@ -0,0 +1,11 @@ +import v8 from 'node:v8'; +const tab = () => v8.getHeapStatistics().total_allocated_bytes; +const keep = []; +const probe = (name, fn) => { globalThis.gc(); const u0 = v8.getHeapStatistics().used_heap_size; const a0 = tab(); keep.push(fn()); const a1 = tab(); const u1 = v8.getHeapStatistics().used_heap_size; console.log(name.padEnd(36), 'tab_delta', a1 - a0, 'used_delta', u1 - u0); }; +probe('nothing', () => 0); +probe('1 small object', () => ({ a: 1 })); +probe('1000 small objects', () => Array.from({ length: 1000 }, (_, i) => ({ i }))); +probe('Float64Array 1e6 (8MB off-heap)', () => new Float64Array(1e6)); +probe('Array 1e6 doubles (~8MB)', () => { const a = new Array(1e6); for (let i = 0; i < 1e6; i++) a[i] = i + 0.5; return a; }); +probe('Set of 10k strings', () => new Set(Array.from({ length: 10000 }, (_, i) => 'k' + i))); +console.log('node', process.version, 'v8', process.versions.v8, 'typeof', typeof v8.getHeapStatistics().total_allocated_bytes); diff --git a/scripts/bench-overflow-order/bench.mjs b/scripts/bench-overflow-order/bench.mjs new file mode 100644 index 00000000..7702621c --- /dev/null +++ b/scripts/bench-overflow-order/bench.mjs @@ -0,0 +1,46 @@ +// Usage: node --expose-gc bench.mjs