diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 176d43bc..3c71f017 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -12,8 +12,11 @@ on: - 'lib/features/workspace/**' - 'lib/core/database/**' - 'lib/core/csv/**' + - 'lib/features/erd/**' + - 'lib/features/results/**' - 'benchmark/**' - 'scripts/ci/bench_compare.py' + - 'scripts/ci/app_bench_report.py' - '.github/workflows/benchmark.yml' workflow_dispatch: @@ -94,3 +97,43 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: gh pr comment "${{ github.event.pull_request.number }}" --body-file "$RUNNER_TEMP/report.md" + + # #1173: ERD, charts / groupings and multi-tab SQL scenarios of the real app + # (benchmark/app_perf_bench.dart, Demo Playground, no database server), + # checked against per-scenario limits. Informational, like the grid bench. + app-bench: + name: App scenarios performance + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 30 + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Install Linux dependencies + timeout-minutes: 8 + run: | + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y clang cmake ninja-build pkg-config libgtk-3-dev libsecret-1-dev libsqlite3-dev xvfb libgl1-mesa-dri + + - uses: subosito/flutter-action@v2 + with: + flutter-version: '3.47.1' + channel: 'stable' + cache: true + + - name: Run the app scenarios + timeout-minutes: 20 + run: | + flutter pub get + xvfb-run -a -s '-screen 0 1920x1080x24' flutter run --profile -d linux \ + -t benchmark/app_perf_bench.dart \ + --dart-define=ONLY=charts_5000,groupings_5000,multi_tab_run,erd_open_200,erd_hover,erd_drag,erd_auto_layout \ + --dart-define=STREAMS= \ + 2>&1 | tee "$RUNNER_TEMP/app.log" || true + + - name: Report + run: | + python3 scripts/ci/app_bench_report.py "$RUNNER_TEMP/app.log" > "$RUNNER_TEMP/app_report.txt" + cat "$RUNNER_TEMP/app_report.txt" + tail -n +2 "$RUNNER_TEMP/app_report.txt" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 022c66ee..40e5f62f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,8 +137,10 @@ jobs: - name: Analyze code run: flutter analyze - - name: Test the benchmark comparison script - run: python3 scripts/ci/test_bench_compare.py + - name: Test the benchmark scripts + run: | + python3 scripts/ci/test_bench_compare.py + python3 scripts/ci/test_app_bench_report.py # packages/ is excluded above: the bridge is its own dependency-free # package and must keep compiling to a single static binary. diff --git a/benchmark/app_perf_bench.dart b/benchmark/app_perf_bench.dart index ee9efdf5..1dc5abab 100644 --- a/benchmark/app_perf_bench.dart +++ b/benchmark/app_perf_bench.dart @@ -30,6 +30,7 @@ import 'package:querya_desktop/core/storage/connection_secrets_store.dart'; import 'package:querya_desktop/core/storage/local_db.dart'; import 'package:querya_desktop/core/theme/theme_controller.dart'; import 'package:querya_desktop/features/connections/connections_panel.dart'; +import 'package:querya_desktop/features/erd/erd_view.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:querya_desktop/features/workspace/result_grid_view.dart'; import 'package:vm_service/vm_service.dart' as vms; @@ -152,6 +153,24 @@ Future _runAll() async { await _scenario('scroll_h', () => _scrollGrid(Axis.horizontal, 18, 3000)); await _scenario('hover_grid', _hoverGrid); + // #1173: Charts and Groupings over the 5 000 rows of the heavy query. + await _scenario('charts_5000', () async { + await _tap(_byText('Charts')); + await _wait(800); + for (final type in const ['Line', 'Pie', 'Bar']) { + final e = _findText(type); + if (e == null) continue; + await _tap(e); + await _wait(500); + } + }); + await _scenario('groupings_5000', () async { + await _tap(_byText('Groupings')); + await _wait(1000); + }); + await _tap(_byText('Grid')); + await _wait(500); + await _scenario('typing', () async { final end = DateTime.now().add(const Duration(seconds: 3)); var i = 0; @@ -209,6 +228,104 @@ Future _runAll() async { await _wait(400); } }); + + // #1173: five tabs holding results; a run in one of them should rebuild + // only that tab's pane. + for (var i = 0; i < 4; i++) { + SqlEditorCommandBridge.instance.invokeNew(); + await _wait(300); + await _runInActiveTab('SELECT * FROM users;'); + } + final ws = _workspace(); + ws.paneBuildCount = 0; + await _scenario('multi_tab_run', () => _runInActiveTab('SELECT * FROM orders;')); + stdout.writeln('BENCH multi_tab_run panes rebuilt=${ws.paneBuildCount}'); + + // #1173: the diagram of a schema of 200 tables joined by foreign keys. + final ddl = StringBuffer('CREATE TABLE erd_t0 (id INTEGER PRIMARY KEY);\n'); + for (var i = 1; i < 200; i++) { + ddl.writeln('CREATE TABLE erd_t$i (id INTEGER PRIMARY KEY, ' + 'parent_id INTEGER REFERENCES erd_t${(i - 1) ~/ 2}(id));'); + } + SqlEditorCommandBridge.instance.invokeNew(); + await _wait(300); + await _runInActiveTab(ddl.toString()); + await _scenario('erd_open_200', () async { + await _tap(_byKey('open_diagram_tab')); + await _waitFor(() => _find((e) => e.widget.key == const ValueKey('erd_table_erd_t199')) != null, + 'diagram of 200 tables'); + await _wait(800); + }); + await _scenario('erd_hover', () => _hoverOver(_find((e) => e.widget is ErdView)!, 3000)); + await _scenario('erd_drag', () => _dragCard('erd_table_erd_t0', 2000)); + await _scenario('erd_auto_layout', () async { + await _tap(_byKey('erd_auto_layout')); + await _wait(1200); + }); +} + +/// The SQL workspace of the open connection. +dynamic _workspace() => + _findState((s) => s.runtimeType.toString() == 'GenericSqlWorkspaceState')!; + +/// Puts [sql] into the active query tab, runs it and waits for the run to end. +Future _runInActiveTab(String sql) async { + final ws = _workspace(); + final session = ws.activeSession; + session.controller.text = sql; + SqlEditorCommandBridge.instance.invokeExecute(); + await _wait(100); + await _waitFor(() => !(ws.activeSession.running as bool), 'query run'); + await _wait(400); +} + +/// Moves the mouse over [root] for [ms], without pressing. +Future _hoverOver(Element root, int ms) async { + final box = root.renderObject! as RenderBox; + final origin = box.localToGlobal(Offset.zero); + final size = box.size; + final end = DateTime.now().add(Duration(milliseconds: ms)); + var t = 0.0; + Offset? last; + while (DateTime.now().isBefore(end)) { + await SchedulerBinding.instance.endOfFrame; + t += 0.03; + final pos = origin + + Offset(size.width * (0.5 + 0.45 * math.sin(t)), + size.height * (0.5 + 0.4 * math.sin(t * 1.3))); + GestureBinding.instance.handlePointerEvent(PointerHoverEvent( + position: pos, + delta: last == null ? Offset.zero : pos - last, + kind: PointerDeviceKind.mouse, + )); + last = pos; + } +} + +/// Presses the card with [key] and drags it in a circle for [ms]. +Future _dragCard(String key, int ms) async { + final box = _byKey(key).renderObject! as RenderBox; + final start = box.localToGlobal(box.size.center(Offset.zero)); + final b = GestureBinding.instance; + const pointer = 77; + b.handlePointerEvent(PointerDownEvent( + position: start, pointer: pointer, kind: PointerDeviceKind.mouse, + buttons: kPrimaryButton)); + final end = DateTime.now().add(Duration(milliseconds: ms)); + var t = 0.0; + var last = start; + while (DateTime.now().isBefore(end)) { + await SchedulerBinding.instance.endOfFrame; + t += 0.05; + final pos = start + Offset(160 * math.sin(t), 90 * (1 - math.cos(t))); + b.handlePointerEvent(PointerMoveEvent( + position: pos, delta: pos - last, pointer: pointer, + kind: PointerDeviceKind.mouse, buttons: kPrimaryButton)); + last = pos; + } + b.handlePointerEvent(PointerUpEvent( + position: last, pointer: pointer, kind: PointerDeviceKind.mouse)); + await _wait(400); } /// Live scroll position with extent under [root] along [axis], or null. diff --git a/scripts/ci/app_bench_report.py b/scripts/ci/app_bench_report.py new file mode 100644 index 00000000..2debe55f --- /dev/null +++ b/scripts/ci/app_bench_report.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Markdown report of app_perf_bench scenarios against per-scenario limits. + +usage: app_bench_report.py APP_BENCH.log + +Reads the `BENCH frames=.. build p50/p90/max=.. raster ..` lines and +checks each scenario of LIMITS: build p90 against its limit (ms), and for +multi_tab_run the number of query panes rebuilt by one run. The first output +line is `OVER=`, the number of scenarios over a limit or missing. +Informational: always exits 0. +""" +import re +import sys + +# build p90 limits in ms. A frame budget at 60 Hz is 16.7 ms; opening a +# diagram or laying it out again is one long frame by design. +LIMITS = { + "charts_5000": 16.7, + "groupings_5000": 16.7, + "multi_tab_run": 16.7, + "erd_open_200": 50.0, + "erd_hover": 8.3, + "erd_drag": 16.7, + "erd_auto_layout": 50.0, +} + +# A run in one tab should rebuild that tab's pane only. +MAX_PANES_REBUILT = 1 + +_LINE = re.compile( + r"BENCH (\S+)\s+frames=\s*(\d+)\s+" + r"build p50/p90/max=([\d.]+)/([\d.]+)/([\d.]+)\s+" + r"raster p50/p90/max=([\d.]+)/([\d.]+)/([\d.]+).*?stutters=(\d+)") +_PANES = re.compile(r"BENCH multi_tab_run panes rebuilt=(\d+)") + + +def parse(text): + scenarios = {} + for m in _LINE.finditer(text): + scenarios[m[1]] = { + "frames": int(m[2]), + "build_p50": float(m[3]), + "build_p90": float(m[4]), + "build_max": float(m[5]), + "raster_p50": float(m[6]), + "stutters": int(m[9]), + } + panes = _PANES.search(text) + return scenarios, (int(panes[1]) if panes else None) + + +def report(text): + scenarios, panes = parse(text) + over = 0 + lines = [ + "### App benchmark: ERD, charts and groupings, multi-tab SQL", + "", + "| scenario | frames | build p50 | build p90 (limit) | build max | raster p50 | stutters | |", + "|---|---|---|---|---|---|---|---|", + ] + for name, limit in LIMITS.items(): + s = scenarios.get(name) + if s is None: + over += 1 + lines.append(f"| {name} | — | — | — ({limit:g}) | — | — | — | ⚠️ missing |") + continue + bad = s["build_p90"] > limit + over += bad + lines.append( + f"| {name} | {s['frames']} | {s['build_p50']:.1f} | " + f"{s['build_p90']:.1f} ({limit:g}) | {s['build_max']:.1f} | " + f"{s['raster_p50']:.1f} | {s['stutters']} | {'⚠️' if bad else 'ok'} |") + if panes is not None: + bad = panes > MAX_PANES_REBUILT + over += bad + lines += ["", f"Query panes rebuilt by one run in one of five tabs: " + f"**{panes}** (limit {MAX_PANES_REBUILT}){' ⚠️' if bad else ''}."] + lines += ["", "Times in ms, profile build under xvfb with software " + "rendering. Informational, never blocks a merge."] + return over, "\n".join(lines) + + +def main(argv): + try: + with open(argv[1], encoding="utf-8", errors="replace") as f: + text = f.read() + except (IndexError, OSError): + text = "" + over, body = report(text) + print(f"OVER={over}") + print(body) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/ci/test_app_bench_report.py b/scripts/ci/test_app_bench_report.py new file mode 100644 index 00000000..5485b84a --- /dev/null +++ b/scripts/ci/test_app_bench_report.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Tests for app_bench_report.py (#1173). Run: python3 scripts/ci/test_app_bench_report.py""" +import io +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout + +sys.path.insert(0, os.path.dirname(__file__)) +import app_bench_report as abr # noqa: E402 + + +def line(name, build=(1.0, 2.0, 5.0), raster=(4.0, 6.0, 9.0), frames=120, stutters=2): + return (f"BENCH {name.ljust(20)} frames={str(frames).rjust(4)} " + f"build p50/p90/max={build[0]:.1f}/{build[1]:.1f}/{build[2]:.1f} " + f"raster p50/p90/max={raster[0]:.1f}/{raster[1]:.1f}/{raster[2]:.1f} " + f"over8.3 build=0 raster=3 stutters={stutters} gap p50=16.67ms") + + +def all_ok(): + return "\n".join(line(n) for n in abr.LIMITS) + "\nBENCH multi_tab_run panes rebuilt=1\n" + + +class ParseTest(unittest.TestCase): + def test_reads_the_app_bench_line(self): + scenarios, panes = abr.parse(line("erd_hover", build=(0.8, 3.2, 7.5))) + s = scenarios["erd_hover"] + self.assertEqual(s["frames"], 120) + self.assertEqual(s["build_p90"], 3.2) + self.assertEqual(s["build_max"], 7.5) + self.assertEqual(s["raster_p50"], 4.0) + self.assertEqual(s["stutters"], 2) + self.assertIsNone(panes) + + def test_reads_the_pane_count(self): + _, panes = abr.parse("BENCH multi_tab_run panes rebuilt=5") + self.assertEqual(panes, 5) + + +class ReportTest(unittest.TestCase): + def test_everything_within_limits(self): + over, body = abr.report(all_ok()) + self.assertEqual(over, 0) + self.assertNotIn("⚠️", body) + for name in abr.LIMITS: + self.assertIn(f"| {name} |", body) + + def test_a_slow_scenario_is_flagged(self): + text = all_ok().replace(line("erd_hover"), line("erd_hover", build=(2.0, 12.0, 20.0))) + over, body = abr.report(text) + self.assertEqual(over, 1) + self.assertIn("12.0 (8.3)", body) + + def test_a_missing_scenario_is_flagged(self): + text = all_ok().replace(line("erd_drag") + "\n", "") + over, body = abr.report(text) + self.assertEqual(over, 1) + self.assertIn("| erd_drag | — |", body) + + def test_rebuilding_every_pane_is_flagged(self): + text = all_ok().replace("panes rebuilt=1", "panes rebuilt=5") + over, body = abr.report(text) + self.assertEqual(over, 1) + self.assertIn("**5**", body) + + def test_main_prints_the_count_first(self): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "app.log") + with open(p, "w", encoding="utf-8") as f: + f.write(all_ok()) + out = io.StringIO() + with redirect_stdout(out): + abr.main(["app_bench_report.py", p]) + self.assertTrue(out.getvalue().startswith("OVER=0\n")) + + def test_no_log_reports_every_scenario_missing(self): + out = io.StringIO() + with redirect_stdout(out): + abr.main(["app_bench_report.py", "/nonexistent.log"]) + self.assertTrue(out.getvalue().startswith(f"OVER={len(abr.LIMITS)}\n")) + + +if __name__ == "__main__": + unittest.main()