Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/flint-js/src/core/filter-overflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,8 @@ function defaultOverflowOrder(
}
if (explicitSort?.kind === 'values') {
const ordered = sortOrder === 'descending' ? [...explicitSort.values].reverse() : explicitSort.values;
return ordered.filter(value => uniqueValues.includes(value));
const present = new Set(uniqueValues);
return ordered.filter(value => present.has(value));
}

// Infer sort field and direction
Expand Down
25 changes: 25 additions & 0 deletions packages/flint-js/tests/filter-overflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,31 @@ describe('overflow category selection', () => {
expect(applyCategoryViewports(data, result.viewports, { x: 1 }))
.toEqual([data[1], data[2], data[3]]);
});
it('keeps explicit order duplicates and drops missing values before truncation', () => {
const data = ['Delta', 'Alpha', 'Charlie', 'Bravo'].map(Category => ({ Category, Value: 1 }));
const sortBy = JSON.stringify(['Bravo', 'Missing', 'Delta', 'Bravo', 'Alpha', 'Charlie']);
const run = (sortOrder?: 'ascending' | 'descending') => filterOverflow(
{
x: { field: 'Category', type: 'nominal', semanticAnnotation: annotation },
y: { field: 'Value', type: 'quantitative', semanticAnnotation: { semanticType: 'Quantity' } },
},
{ axisFlags: { x: { banded: true } } },
{ x: { field: 'Category', sortBy, sortOrder }, y: { field: 'Value' } },
data,
budgets,
marks,
);

const ascending = run();
expect(ascending.viewports[0].orderedValues).toEqual(['Bravo', 'Delta', 'Bravo', 'Alpha', 'Charlie']);
expect(ascending.truncations[0].keptValues).toEqual(['Bravo', 'Delta', 'Bravo']);
expect(ascending.warnings[0].message).toBe("1 of 4 values in 'Category' were omitted (showing first 3 in sort order).");
expect(ascending.filteredData).toEqual([data[0], data[3]]);

const descending = run('descending');
expect(descending.viewports[0].orderedValues).toEqual(['Charlie', 'Alpha', 'Bravo', 'Delta', 'Bravo']);
expect(descending.truncations[0].keptValues).toEqual(['Charlie', 'Alpha', 'Bravo']);
});

it('windows rows whose numeric categories the rail holds as text', () => {
const rows = [1880, 1881, 1882].map((Year) => ({ Year, Anomaly: 0 }));
Expand Down
11 changes: 10 additions & 1 deletion packages/flint-py/flint/core/filter_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,16 @@ def _default_overflow_strategy(
sorted_list = json.loads(sort_by)
if isinstance(sorted_list, list):
ordered_values = list(reversed(sorted_list)) if sort_order == "descending" else sorted_list
return [v for v in ordered_values if v in unique_values][:max_to_keep]
present = set(unique_values)

def is_present(v: Any) -> bool:
try:
return v in present
except TypeError:
# Unhashable JSON values keep list-membership semantics.
return v in unique_values

return [v for v in ordered_values if is_present(v)][:max_to_keep]
except Exception:
pass
is_descending = sort_order == "descending"
Expand Down
43 changes: 43 additions & 0 deletions packages/flint-py/tests/unit/test_filter_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,46 @@ def test_uses_selected_value_sort():
{"type": "nominal"},
{"sortBy": "y", "sortOrder": "descending"},
) == ["Delta", "Charlie", "Bravo"]


def _explicit_order_result(data, sort_by, sort_order=None):
encoding = {"field": "Category", "sortBy": sort_by}
if sort_order:
encoding["sortOrder"] = sort_order
return filter_overflow(
{
"x": {"field": "Category", "type": "nominal", "semanticAnnotation": {"semanticType": "Category"}},
"y": {"field": "Value", "type": "quantitative", "semanticAnnotation": {"semanticType": "Quantity"}},
},
{"axisFlags": {"x": {"banded": True}}},
{"x": encoding, "y": {"field": "Value"}},
data,
BUDGETS,
MARKS,
)


def test_explicit_order_keeps_duplicates_and_drops_missing_values():
data = [{"Category": c, "Value": 1} for c in ["Delta", "Alpha", "Charlie", "Bravo"]]
sort_by = '["Bravo", "Missing", "Delta", "Bravo", "Alpha", "Charlie"]'

ascending = _explicit_order_result(data, sort_by)
# The default strategy is capped at the distinct-value count, so duplicates shorten the tail.
assert ascending["viewports"][0]["orderedValues"] == ["Bravo", "Delta", "Bravo", "Alpha"]
assert ascending["truncations"][0]["keptValues"] == ["Bravo", "Delta", "Bravo"]
assert ascending["warnings"][0]["message"] == (
"1 of 4 values in 'Category' were omitted (showing first 3 in sort order)."
)
assert ascending["filteredData"] == [data[0], data[3]]

descending = _explicit_order_result(data, sort_by, "descending")
assert descending["viewports"][0]["orderedValues"] == ["Charlie", "Alpha", "Bravo", "Delta"]
assert descending["truncations"][0]["keptValues"] == ["Charlie", "Alpha", "Bravo"]


def test_explicit_order_uses_python_equality_and_skips_unhashable_entries():
data = [{"Category": c, "Value": 1} for c in [1, 2, 3, 4]]

result = _explicit_order_result(data, '[["nested"], true, 2.0, 9, 3, 4]')
assert result["viewports"][0]["orderedValues"] == [True, 2.0, 3, 4]
assert result["truncations"][0]["keptValues"] == [True, 2.0, 3]
43 changes: 43 additions & 0 deletions scripts/bench-overflow-order/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Explicit-order overflow benchmark (#161)

Measures exported `filterOverflow` (JS) and `filter_overflow` (Python) when an
explicit JSON `sortBy` category order is used with `maxValues.x = 25`.

Fixture (generated in `bench.mjs` / `bench.py`): N distinct categories `cat-000000…`,
one row each; `sortBy` lists them reversed, with one duplicate entry and two
missing categories. Sizes 1k / 5k / 10k.

## Reproduce

```bash
npm ci
scripts/bench-overflow-order/run.sh a794557384adcc3293a77043f6b9aff1c33d152c 3
# quick smoke: SIZES=1000 scripts/bench-overflow-order/run.sh a794557 1
```

`run.sh` extracts the baseline sources with `git archive`, bundles both sides of
`filter-overflow.ts` with esbuild, and alternates baseline/change order every
round. Each case runs in its own process; JS takes 15 timed calls after warm-up and
Python takes 7. Output is written to the git-ignored `experiments/bench-overflow-order/`.

## Metrics

- `medianMs`: median wall time of one call (`process.hrtime.bigint` /
`time.perf_counter_ns`), with a forced GC before each call.
- `digest`: SHA-256 of the JSON-serialized result (rows, warnings, truncations,
viewports). Baseline and change must match.
- `medianAllocBytes` (JS): delta of `v8.getHeapStatistics().total_allocated_bytes`
around one call. Node documents this as "total allocated bytes since the Isolate
creation" (Node ≥ 24; V8 14.6 in Node 26). It is a cumulative counter, not
sampling and not a used-heap delta. It is **approximate**: it is updated at
allocation-buffer granularity, excludes ArrayBuffer backing stores, and shows
~0.3–0.6 MB of offset on a no-op after `gc()` (`results/alloc-probe.txt`).
Compare only the baseline/change difference.
- `tracemallocPeakBytes` (Python): `tracemalloc` peak traced allocation above the
starting point during one call, from a separate untimed call.
- `maxRssKiB`: process peak RSS from `getrusage`; this includes runtime startup.

## Recorded results (`results/`)

Node v26.12.1, Python 3.12.3, esbuild 0.28.1, Linux x86_64, 2 vCPU, 3 interleaved
rounds. See `results/summary.txt` and the raw rows in `results/results.jsonl`.
11 changes: 11 additions & 0 deletions scripts/bench-overflow-order/alloc-probe.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import v8 from 'node:v8';
const tab = () => v8.getHeapStatistics().total_allocated_bytes;
const keep = [];
const probe = (name, fn) => { globalThis.gc(); const u0 = v8.getHeapStatistics().used_heap_size; const a0 = tab(); keep.push(fn()); const a1 = tab(); const u1 = v8.getHeapStatistics().used_heap_size; console.log(name.padEnd(36), 'tab_delta', a1 - a0, 'used_delta', u1 - u0); };
probe('nothing', () => 0);
probe('1 small object', () => ({ a: 1 }));
probe('1000 small objects', () => Array.from({ length: 1000 }, (_, i) => ({ i })));
probe('Float64Array 1e6 (8MB off-heap)', () => new Float64Array(1e6));
probe('Array 1e6 doubles (~8MB)', () => { const a = new Array(1e6); for (let i = 0; i < 1e6; i++) a[i] = i + 0.5; return a; });
probe('Set of 10k strings', () => new Set(Array.from({ length: 10000 }, (_, i) => 'k' + i)));
console.log('node', process.version, 'v8', process.versions.v8, 'typeof', typeof v8.getHeapStatistics().total_allocated_bytes);
46 changes: 46 additions & 0 deletions scripts/bench-overflow-order/bench.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Usage: node --expose-gc bench.mjs <bundle.mjs> <label> <N> <reps>
import { createHash } from 'node:crypto';
import v8 from 'node:v8';
import { resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
const [bundle, label, nArg, repsArg] = process.argv.slice(2);
const { filterOverflow } = await import(pathToFileURL(resolve(bundle)).href);
const N = Number(nArg), reps = Number(repsArg);
const cats = Array.from({ length: N }, (_, i) => `cat-${String(i).padStart(6, '0')}`);
const data = cats.map((Category, i) => ({ Category, Value: (i * 7919) % 1000 }));
// Explicit order: reversed categories, plus missing entries and duplicates.
const order = [...cats].reverse();
order.splice(3, 0, 'missing-a', order[0]);
order.push('missing-b');
const sortBy = JSON.stringify(order);
const call = () => filterOverflow(
{
x: { field: 'Category', type: 'nominal', semanticAnnotation: { semanticType: 'Category' } },
y: { field: 'Value', type: 'quantitative', semanticAnnotation: { semanticType: 'Quantity' } },
},
{ axisFlags: { x: { banded: true } } },
{ x: { field: 'Category', sortBy }, y: { field: 'Value' } },
data,
{ maxValues: { x: 25 } },
new Set(['bar']),
);
const first = call();
const digest = createHash('sha256').update(JSON.stringify(first)).digest('hex');
for (let i = 0; i < 2; i++) call();
const times = [], allocs = [];
for (let i = 0; i < reps; i++) {
globalThis.gc();
const a0 = v8.getHeapStatistics().total_allocated_bytes;
const t0 = process.hrtime.bigint();
call();
const t1 = process.hrtime.bigint();
allocs.push(v8.getHeapStatistics().total_allocated_bytes - a0);
times.push(Number(t1 - t0) / 1e6);
}
const med = a => [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)];
console.log(JSON.stringify({
lang: 'js', label, N, reps, node: process.version, digest,
orderedLen: first.viewports[0]?.orderedValues.length, kept: first.truncations[0]?.keptValues.length,
medianMs: med(times), minMs: Math.min(...times), times,
medianAllocBytes: med(allocs), allocs, maxRssKiB: process.resourceUsage().maxRSS,
}));
38 changes: 38 additions & 0 deletions scripts/bench-overflow-order/bench.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Usage: python3 bench.py <flint-pkg-parent-dir> <label> <N> <reps>
import gc, hashlib, json, platform, resource, sys, time, tracemalloc
root, label, n, reps = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
sys.path.insert(0, root)
from flint.core.filter_overflow import filter_overflow # noqa: E402
cats = [f"cat-{i:06d}" for i in range(n)]
data = [{"Category": c, "Value": (i * 7919) % 1000} for i, c in enumerate(cats)]
order = list(reversed(cats))
order[3:3] = ["missing-a", order[0]]
order.append("missing-b")
sort_by = json.dumps(order)
def call():
return filter_overflow(
{"x": {"field": "Category", "type": "nominal", "semanticAnnotation": {"semanticType": "Category"}},
"y": {"field": "Value", "type": "quantitative", "semanticAnnotation": {"semanticType": "Quantity"}}},
{"axisFlags": {"x": {"banded": True}}},
{"x": {"field": "Category", "sortBy": sort_by}, "y": {"field": "Value"}},
data, {"maxValues": {"x": 25}}, {"bar"})
first = call()
digest = hashlib.sha256(json.dumps(first, sort_keys=True, default=str).encode()).hexdigest()
call()
times = []
for _ in range(reps):
gc.collect()
t0 = time.perf_counter_ns(); call(); times.append((time.perf_counter_ns() - t0) / 1e6)
gc.collect()
tracemalloc.start()
base = tracemalloc.get_traced_memory()[0]
tracemalloc.reset_peak()
call()
peak = tracemalloc.get_traced_memory()[1] - base
tracemalloc.stop()
s = sorted(times)
print(json.dumps({"lang": "py", "label": label, "N": n, "reps": reps, "python": platform.python_version(),
"digest": digest, "orderedLen": len(first["viewports"][0]["orderedValues"]),
"kept": len(first["truncations"][0]["keptValues"]), "medianMs": s[len(s)//2], "minMs": s[0],
"times": times, "tracemallocPeakBytes": peak,
"maxRssKiB": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}))
7 changes: 7 additions & 0 deletions scripts/bench-overflow-order/results/alloc-probe.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
nothing tab_delta 505496 used_delta 22376
1 small object tab_delta 431240 used_delta 22344
1000 small objects tab_delta 630952 used_delta 79920
Float64Array 1e6 (8MB off-heap) tab_delta 597424 used_delta 2440
Array 1e6 doubles (~8MB) tab_delta 17526800 used_delta 16177552
Set of 10k strings tab_delta 2160072 used_delta 1457280
node v26.12.1 v8 14.6.202.34-node.40 typeof number
12 changes: 12 additions & 0 deletions scripts/bench-overflow-order/results/env.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
revision_baseline=a794557384adcc3293a77043f6b9aff1c33d152c
change_source=working tree at a794557 + this commit's source diff (identical filter-overflow sources)
node=v26.12.1 python=Python 3.12.3
esbuild=0.28.1
Linux 6.12.8+ x86_64
2
bb6bacb78b9ad58c2af5005f42fffb7e5f411c2abfe3354b3be3dc8ad949a4bc bench-artifacts/baseline.mjs
a2caef0c29344da36d334728ed794e1e594a712b57cffa4101754df1c8b8656d bench-artifacts/bench.mjs
25d5ddc7a25f31f7256c6fe07a16d37380efca956598b1f1e2b1ad97d8dc05a7 bench-artifacts/change.mjs
fd2ac6f9931a72f6be27e6de558985e8cf666d8bf9072132fcd4b44bc8b33efe bench-artifacts/bench.py
cb4f9eed2935346c3a6ce188dcc85344faefbe90e2997ca16e455b24615236cd packages/flint-py/flint/core/filter_overflow.py
3fb148dff1eaf078124f53cac0e79e2bae19a8c9ced021a826a3ab43b33c702d bench-artifacts/baseline-src/packages/flint-py/flint/core/filter_overflow.py
Loading