From 1ac60f7b93bdb278a9f56bb0eab2bdb28545ef9d Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 23:11:40 -0700 Subject: [PATCH 1/8] test: add shared aggregate conformance suite --- .github/workflows/aggregate-conformance.yml | 64 +++ README.md | 12 + conformance/.credo.exs | 12 + conformance/.formatter.exs | 6 + conformance/.gitignore | 9 + conformance/GAPS.md | 172 ++++++++ conformance/MATRIX.md | 131 ++++++ conformance/README.md | 147 +++++++ conformance/config/config.exs | 24 ++ conformance/lib/adapter.ex | 91 +++++ conformance/lib/catalog.ex | 20 + conformance/lib/database.ex | 76 ++++ conformance/lib/expectations.ex | 255 ++++++++++++ conformance/lib/fixtures.ex | 66 +++ conformance/lib/manual.ex | 55 +++ .../lib/mix/tasks/conformance.matrix.ex | 14 + conformance/lib/report.ex | 114 ++++++ conformance/lib/resource.ex | 91 +++++ conformance/lib/resources.ex | 376 ++++++++++++++++++ conformance/lib/scenario.ex | 70 ++++ conformance/lib/scenarios/bounds.ex | 78 ++++ conformance/lib/scenarios/context.ex | 94 +++++ conformance/lib/scenarios/filters.ex | 114 ++++++ conformance/lib/scenarios/helpers.ex | 46 +++ conformance/lib/scenarios/operations.ex | 260 ++++++++++++ conformance/lib/scenarios/relationships.ex | 122 ++++++ conformance/mix.exs | 38 ++ conformance/mix.lock | 33 ++ conformance/mix.lock.license | 3 + conformance/test/aggregate_test.exs | 24 ++ conformance/test/catalog_test.exs | 64 +++ conformance/test/runner_test.exs | 84 ++++ conformance/test/test_helper.exs | 8 + 33 files changed, 2773 insertions(+) create mode 100644 .github/workflows/aggregate-conformance.yml create mode 100644 conformance/.credo.exs create mode 100644 conformance/.formatter.exs create mode 100644 conformance/.gitignore create mode 100644 conformance/GAPS.md create mode 100644 conformance/MATRIX.md create mode 100644 conformance/README.md create mode 100644 conformance/config/config.exs create mode 100644 conformance/lib/adapter.ex create mode 100644 conformance/lib/catalog.ex create mode 100644 conformance/lib/database.ex create mode 100644 conformance/lib/expectations.ex create mode 100644 conformance/lib/fixtures.ex create mode 100644 conformance/lib/manual.ex create mode 100644 conformance/lib/mix/tasks/conformance.matrix.ex create mode 100644 conformance/lib/report.ex create mode 100644 conformance/lib/resource.ex create mode 100644 conformance/lib/resources.ex create mode 100644 conformance/lib/scenario.ex create mode 100644 conformance/lib/scenarios/bounds.ex create mode 100644 conformance/lib/scenarios/context.ex create mode 100644 conformance/lib/scenarios/filters.ex create mode 100644 conformance/lib/scenarios/helpers.ex create mode 100644 conformance/lib/scenarios/operations.ex create mode 100644 conformance/lib/scenarios/relationships.ex create mode 100644 conformance/mix.exs create mode 100644 conformance/mix.lock create mode 100644 conformance/mix.lock.license create mode 100644 conformance/test/aggregate_test.exs create mode 100644 conformance/test/catalog_test.exs create mode 100644 conformance/test/runner_test.exs create mode 100644 conformance/test/test_helper.exs diff --git a/.github/workflows/aggregate-conformance.yml b/.github/workflows/aggregate-conformance.yml new file mode 100644 index 0000000..a5c8424 --- /dev/null +++ b/.github/workflows/aggregate-conformance.yml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +name: Aggregate conformance +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + aggregates: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + adapter: [sqlite, postgres] + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + defaults: + run: + working-directory: conformance + env: + MIX_ENV: test + CONFORMANCE_ADAPTERS: ${{ matrix.adapter }} + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + steps: + - uses: actions/checkout@v4 + - uses: erlef/setup-beam@v1 + with: + otp-version: '27.3.4.3' + elixir-version: '1.18.4' + - uses: actions/cache@v4 + with: + path: | + conformance/deps + conformance/_build + key: conformance-${{ runner.os }}-otp27-elixir1.18-${{ hashFiles('conformance/mix.lock') }} + - run: mix deps.get --check-locked + - run: mix check + - run: mix dialyzer + if: matrix.adapter == 'postgres' + - uses: actions/upload-artifact@v4 + if: always() + with: + name: aggregate-contracts-${{ matrix.adapter }} + path: conformance/results/ diff --git a/README.md b/README.md index a15c719..31dca86 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,15 @@ dispatch. Both strategies preserve the source binding and attachment path, so aggregates referenced through joined relationships attach to the related row. Alias reuse is scoped to that path and follows the existing filter/sort identity rules. Public aggregate names, including strings, are retained in results. + +## Aggregate conformance tests + +The repository's `conformance/` Mix project runs shared public Ash aggregate +scenarios against SQLite and PostgreSQL. It records supported behavior, +unsupported features, known defects and unresolved semantics separately. +Adapter dependencies belong to that project and are not part of AshSQL's +production dependencies or Hex package. + +See the [conformance guide](https://github.com/ash-project/ash_sql/blob/main/conformance/README.md) +and [expectation matrix](https://github.com/ash-project/ash_sql/blob/main/conformance/MATRIX.md) +for setup, coverage and extension. diff --git a/conformance/.credo.exs b/conformance/.credo.exs new file mode 100644 index 0000000..917f64c --- /dev/null +++ b/conformance/.credo.exs @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT +%{ + configs: [ + %{ + name: "default", + files: %{included: ["lib/", "test/"]}, + checks: %{disabled: [{Credo.Check.Readability.Specs, []}]} + } + ] +} diff --git a/conformance/.formatter.exs b/conformance/.formatter.exs new file mode 100644 index 0000000..8e6dcaa --- /dev/null +++ b/conformance/.formatter.exs @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 0000000..1f0256c --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT +/_build/ +/deps/ +/tmp/ +/results/ +/cover/ +erl_crash.dump diff --git a/conformance/GAPS.md b/conformance/GAPS.md new file mode 100644 index 0000000..3f49064 --- /dev/null +++ b/conformance/GAPS.md @@ -0,0 +1,172 @@ + + + +# Aggregate implementation and decision tasks + +These are local follow-up tasks for the pinned baseline. No GitHub issue is +implied. The matrix links each nonconforming expectation to an entry below. +Unless marked as a decision, the shared scenario contains the intended result. +An observed Postgres result never replaces that result automatically. + +## Root kinds + +Add root SQLite custom and list aggregates. Reuse bounded root inputs, +custom expressions, windowed JSON lists, result types and defaults. Promote +`root.custom` and `root.list` independently, adding bounded and empty inputs +before claiming complete support. + +## Root relationship + +Implement root aggregation over relationship paths, preserving root scope and +endpoint fields. SQLite rejects this explicitly. The Postgres comparator also +fails the tested public `Ash.aggregate` call by resolving `value` against the +parent instead of the child. Do not assume this gap is SQLite-only. + +## Root first + +Postgres root `first` without an explicit sort dereferences a missing first +relationship. The empty-input scenario should return nil. Explicitly sorted +first and empty-input controls already pass. + +## Many-to-many paths + +Generalize grouped input construction for intermediate many-to-many hops and +first/list/custom at the end of a multi-hop path. Carry attachment identity +through each prepared through and destination input. Scalar final-hop, +single-hop list/first, empty-parent and shared-destination controls pass. + +## Parent correlation + +Support parent values in grouped relationship, aggregate, join and unrelated +filters. Prototype correlated scalar queries or parent-inclusive grouped input. +Keep relationship scope before bounds and aggregate predicates after them. +The public query options must hydrate parent references against the parent; +prehydrating a child query with no parent context is not equivalent. + +## Manual + +Support SQL-capable manual relationships using existing adapter join/subquery +callbacks where possible. The shared fixture is a normal foreign-key manual +relationship. Non-equality callbacks and nested parent aliases need further +scenarios before broader support is advertised. + +## No attributes + +Support grouped attribute-free relationships, separating independent inputs +from parent-dependent ones. Postgres's independent ad hoc count currently +errors on a missing selected field, while the parent-dependent count passes. +The direct relationship-load control returns all five children for each parent. + +## Filter dependencies + +Attach aggregate dependencies to the endpoint input before compiling its filter. +The initial scenario counts children with more than one rating. Add alias, +nested dependency, sibling and cycle cases as the implementation expands. + +## Filter fanout + +Use membership EXISTS or row-identity deduplication so filter joins do not +multiply records being aggregated. Two children with equal value 2 and three +matching ratings must sum to 4, not 6, and count as two records. The average +scenario adds a different value to expose weighting errors. Deduplicating +values is not a valid substitute for deduplicating filter matches. + +SQLite rejects the affected aggregate shapes; Postgres currently returns the +multiplied sum/count/list/custom/average. The Postgres direct read control +returns distinct child records. SQLite's direct read control itself returns a +duplicate, which is recorded as a broader read defect. Composite-key count +coverage also exposes Postgres's filter multiplication. + +## Record identity + +Support distinct record counts with all components of a composite key. Grouped +counts currently reject them explicitly. Include attachment keys in the +deduplication subquery. Loading aggregates on keyless source resources is also +rejected by grouped; determine which operations can use relationship keys and +which require an explicit row identity. Ordinary keyless destination counts +already work. + +## From many + +Respect the implicit one-row bound of `from_many?`. Both extraction strategies +currently count four children for the first parent. The separately stacked fix +is intentionally absent from this branch; merging it must cause an unexpected +pass until the expectation is promoted. + +## Default sort + +Apply relationship `default_sort` when there is no explicit sort. Both +aggregate paths currently choose 2 instead of 7. Direct relationship loading +returns the expected child with value 7 on both adapters. + +## Unsorted bounds + +Avoid an empty `ORDER BY` in grouped relationship windows. The bounded count +must be one regardless of which child is chosen; the fixture does not depend +on an unspecified ordering. SQLite currently raises a syntax error. + +## Root bounds + +Preserve root ordering when materializing a limited aggregate input. The +Postgres comparator discards the ordering and aggregates value 2 instead of 7. +An offset-only root count also raises instead of returning three. Keep root +ordering distinct from a first/list aggregate's own ordering. + +## Relationship context + +Make relationship context available before read preparation and avoid retaining +an earlier filter prepared without it. Both the aggregate and direct +relationship-load controls return no rows. An explicitly prepared child query +with the same context returns the two intended records, and parent shared +context also works. This needs Ash-level investigation as well as adapter work. + +## Authorization bounds + +Apply destination authorization before choosing limited relationship rows. +The aggregate's own predicate must remain after the bound. Both adapters return +nil for the first parent after selecting its hidden highest-valued child. +Direct authorized relationship loading returns the visible child with value 2. +Ash currently combines policy and aggregate filters, so this may require an +Ash change to preserve their distinct ordering. + +## Prepared query + +Retain a prepared endpoint query's action and arguments. SQLite returns the +correct count; the Postgres comparator crashes in `Enumerable.List.reduce/3`. +Configured-action and intermediate-action controls pass on both adapters. + +## Tenant bypass + +Honor explicit aggregate tenancy bypass on endpoints and through resources, +without changing a scoped sibling. The SQLite cases pass. Postgres returns +scoped values for these bypass cases. Investigate where Ash has already added +tenant predicates before changing adapter behavior. + +## Path multiplicity + +Decision: when a destination is reached through two different relationship +paths, does a fieldless count count path occurrences or distinct destinations? +The repeated many-to-many scenario currently returns three and two in Postgres; +SQLite rejects the path. These are strict observations, not accepted semantics. + +## Keyless identity + +Decision: define distinct-record semantics when the destination has no primary +key. PostgreSQL currently counts rows and SQLite rejects the operation. Do not +choose an arbitrary attribute, concatenate values, or assume a SQL rowid exists +for every resource. + +## Many-to-many bounds API + +Decision: expose relationship limits/offsets for many-to-many aggregates in Ash. +The current many-to-many DSL has no `limit` option and `Ash.Query.aggregate` +rejects a limited target query before either adapter runs. This corrects the +earlier assumption that the grouped guard alone was blocking a public feature. +Add the API and settle per-parent ordering before a conformance result is fixed. + +## Unique list order + +Decision: choose which occurrence supplies the ordering value when duplicate +list values have different sort keys. SQLite rejects this shape; Postgres +raises its DISTINCT/ORDER BY restriction. A representative-row rule would need +to be defined and implemented for both adapters. diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md new file mode 100644 index 0000000..bc13156 --- /dev/null +++ b/conformance/MATRIX.md @@ -0,0 +1,131 @@ + + + +# Aggregate expectation matrix + +Generated by `mix conformance.matrix`. These are declared contracts; `results/` contains execution results. Passing an unsupported, known-defect or unresolved expectation does not establish feature conformance. + +| Scenario | sqlite | postgres | +| --- | --- | --- | +| `bounds.default_sort` | [known_defect](GAPS.md#default-sort) | [known_defect](GAPS.md#default-sort) | +| `bounds.default_sort_control` | supported | supported | +| `bounds.filter_after_limit` | supported | supported | +| `bounds.from_many` | [known_defect](GAPS.md#from-many) | [known_defect](GAPS.md#from-many) | +| `bounds.list_filter_after_limit` | supported | supported | +| `bounds.many_to_many_query_limit` | [unresolved](GAPS.md#many-to-many-bounds-api) | [unresolved](GAPS.md#many-to-many-bounds-api) | +| `bounds.relationship_limit` | supported | supported | +| `bounds.relationship_offset` | supported | supported | +| `bounds.relationship_offset_only` | supported | supported | +| `bounds.root_first_distinct_sort` | supported | [known_defect](GAPS.md#root-bounds) | +| `bounds.root_limit` | supported | supported | +| `bounds.root_offset_only` | supported | [known_defect](GAPS.md#root-bounds) | +| `bounds.root_order_then_limit` | supported | [known_defect](GAPS.md#root-bounds) | +| `bounds.root_zero` | supported | supported | +| `bounds.unsorted_limit` | [known_defect](GAPS.md#unsorted-bounds) | supported | +| `context.actor` | supported | supported | +| `context.arguments` | supported | supported | +| `context.attribute_tenant` | supported | supported | +| `context.authorization` | supported | supported | +| `context.authorization_before_bounds` | [known_defect](GAPS.md#authorization-bounds) | [known_defect](GAPS.md#authorization-bounds) | +| `context.authorization_bounds_control` | supported | supported | +| `context.bypass_sibling` | supported | [known_defect](GAPS.md#tenant-bypass) | +| `context.intermediate_action` | supported | supported | +| `context.intermediate_actor` | supported | supported | +| `context.prepared_context_control` | supported | supported | +| `context.prepared_query_arguments` | supported | [known_defect](GAPS.md#prepared-query) | +| `context.read_action` | supported | supported | +| `context.relationship_context` | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | +| `context.relationship_context_control` | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | +| `context.shared` | supported | supported | +| `context.tenant_bypass` | supported | [known_defect](GAPS.md#tenant-bypass) | +| `context.through_arguments` | supported | supported | +| `context.through_bypass` | supported | [known_defect](GAPS.md#tenant-bypass) | +| `context.through_tenant` | supported | supported | +| `field.aggregate` | supported | supported | +| `field.calculation` | supported | supported | +| `field.root_aggregate` | supported | supported | +| `filter.aggregate_dependency` | [unsupported](GAPS.md#filter-dependencies) | supported | +| `filter.exists` | supported | supported | +| `filter.fanout_avg` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| `filter.fanout_count` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| `filter.fanout_count_records` | supported | [known_defect](GAPS.md#filter-fanout) | +| `filter.fanout_custom` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| `filter.fanout_list` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| `filter.fanout_read_control` | [known_defect](GAPS.md#filter-fanout) | supported | +| `filter.fanout_sum` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| `filter.join` | supported | supported | +| `filter.not_exists` | supported | supported | +| `filter.or_exists` | supported | supported | +| `filter.ordinary` | supported | supported | +| `filter.parent` | [unsupported](GAPS.md#parent-correlation) | supported | +| `filter.parent_join` | [unsupported](GAPS.md#parent-correlation) | supported | +| `filter.parent_relationship` | [unsupported](GAPS.md#parent-correlation) | supported | +| `filter.parent_unrelated` | [unsupported](GAPS.md#parent-correlation) | supported | +| `filter.sibling_independence` | supported | supported | +| `identity.composite_count` | [unsupported](GAPS.md#record-identity) | supported | +| `identity.composite_fanout_count` | [unsupported](GAPS.md#record-identity) | [known_defect](GAPS.md#filter-fanout) | +| `identity.keyless_count` | supported | supported | +| `identity.keyless_distinct` | [unresolved](GAPS.md#keyless-identity) | [unresolved](GAPS.md#keyless-identity) | +| `identity.keyless_source` | [unsupported](GAPS.md#record-identity) | supported | +| `identity.root_composite_count` | [unsupported](GAPS.md#record-identity) | supported | +| `loaded.avg` | supported | supported | +| `loaded.count` | supported | supported | +| `loaded.custom` | supported | supported | +| `loaded.exists` | supported | supported | +| `loaded.first` | supported | supported | +| `loaded.list` | supported | supported | +| `loaded.max` | supported | supported | +| `loaded.min` | supported | supported | +| `loaded.sum` | supported | supported | +| `ordering.asc_nils_first` | supported | supported | +| `ordering.asc_nils_last` | supported | supported | +| `ordering.desc_nils_first` | supported | supported | +| `ordering.desc_nils_last` | supported | supported | +| `ordering.expression_first` | supported | supported | +| `ordering.expression_list` | supported | supported | +| `ordering.ties` | supported | supported | +| `ordering.unique_other_field` | [unresolved](GAPS.md#unique-list-order) | [unresolved](GAPS.md#unique-list-order) | +| `path.final_many_to_many_custom` | [unsupported](GAPS.md#many-to-many-paths) | supported | +| `path.final_many_to_many_first` | [unsupported](GAPS.md#many-to-many-paths) | supported | +| `path.final_many_to_many_list` | [unsupported](GAPS.md#many-to-many-paths) | supported | +| `path.final_many_to_many_scalar` | supported | supported | +| `path.intermediate_many_to_many` | [unsupported](GAPS.md#many-to-many-paths) | supported | +| `path.manual` | [unsupported](GAPS.md#manual) | supported | +| `path.many_to_many` | supported | supported | +| `path.many_to_many_first` | supported | supported | +| `path.many_to_many_list` | supported | supported | +| `path.multi_hop` | supported | supported | +| `path.no_attributes` | [unsupported](GAPS.md#no-attributes) | [known_defect](GAPS.md#no-attributes) | +| `path.no_attributes_control` | supported | supported | +| `path.no_attributes_parent` | [unsupported](GAPS.md#no-attributes) | supported | +| `path.repeated_many_to_many` | [unresolved](GAPS.md#path-multiplicity) | [unresolved](GAPS.md#path-multiplicity) | +| `path.root_relationship` | [unsupported](GAPS.md#root-relationship) | [known_defect](GAPS.md#root-relationship) | +| `path.to_one` | supported | supported | +| `path.unrelated` | supported | supported | +| `root.avg` | supported | supported | +| `root.count` | supported | supported | +| `root.custom` | [unsupported](GAPS.md#root-kinds) | supported | +| `root.exists` | supported | supported | +| `root.first` | supported | supported | +| `root.list` | [unsupported](GAPS.md#root-kinds) | supported | +| `root.max` | supported | supported | +| `root.min` | supported | supported | +| `root.sum` | supported | supported | +| `root.unsorted_first_empty` | supported | [known_defect](GAPS.md#root-first) | +| `use.calculation` | supported | supported | +| `use.filter` | supported | supported | +| `use.pagination` | supported | supported | +| `use.sort` | supported | supported | +| `values.constrained_scalar` | supported | supported | +| `values.distinct_count` | supported | supported | +| `values.distinct_list` | supported | supported | +| `values.field_count` | supported | supported | +| `values.filtered_first_default` | supported | supported | +| `values.include_nil_first` | supported | supported | +| `values.include_nil_list` | supported | supported | +| `values.list_default` | supported | supported | +| `values.root_empty` | supported | supported | +| `values.same_name_distinct_definitions` | supported | supported | +| `values.scalar_default` | supported | supported | +| `values.string_constraints` | supported | supported | +| `values.string_name` | supported | supported | diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..de7573d --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,147 @@ + + + +# Shared aggregate conformance + +Run the same aggregate scenarios through Ash's public API on SQLite and +PostgreSQL. A scenario has one expected answer. PostgreSQL output is not the +definition of that answer. + +This is a separate Mix project so AshSQL does not acquire adapter dependencies +or an adapter dependency cycle. Its `:ash_sql` dependency points at the parent +checkout. The two adapters are pinned in `mix.exs` and the full dependency set +is locked in `mix.lock`. Nothing in this directory is part of AshSQL's Hex +package or production API. + +## Run + +Use the Elixir and Erlang versions in the parent `.tool-versions`: + +```sh +cd conformance +mise install +mise exec -- mix deps.get --check-locked +mise exec -- mix check +mise exec -- mix dialyzer +``` + +PostgreSQL 17 must be running. The defaults are `localhost:5432`, user +`postgres`, password `postgres`. Set `PGHOST`, `PGPORT`, `PGUSER`, and +`PGPASSWORD` to override them. `CONFORMANCE_PG_DATABASE` defaults to +`ash_sql_aggregate_conformance`. Use a dedicated test database. Setup creates +it if absent and runs only this suite's migrations. It does not drop databases +or truncate existing tables. Every scenario seeds its fixtures in a sandbox +transaction and rolls them back. + +SQLite uses `tmp/aggregates.sqlite3`. An adapter is started only when selected: + +```sh +CONFORMANCE_ADAPTERS=sqlite mise exec -- mix test +CONFORMANCE_ADAPTERS=postgres mise exec -- mix test +mise exec -- mix test --only scenario:filter.fanout_sum +``` + +The default selects both. Unknown adapter names fail immediately. All +dependencies still compile when selecting one adapter; selection controls +database startup and scenario execution. SQLite-only runs require no running +PostgreSQL server. + +## Read the results + +[MATRIX.md](MATRIX.md) lists the explicit expectations. The ExUnit formatter +writes `results/sqlite-postgres.json`, or a file named for the selected adapter. +CI uploads these reports even when a test fails. + +| Status | A passing test means | +| --- | --- | +| `supported` | The operation returned the shared expected result. | +| `unsupported` | The operation raised the recorded exception with the recorded reason. | +| `known_defect` | The operation produced exactly the documented wrong result or error. | +| `unresolved` | The operation matches the recorded observation, pending a semantic/API decision. | + +A green suite means these contracts still match. It does **not** mean all +features work. Correct results unexpectedly returned by an unsupported or +known-defect case fail the test until its declaration is promoted. A different +wrong value, unrelated exception, or database setup failure also fails. +Unresolved cases link to a decision and make no correctness claim. There are +no blanket skips and no automatic expected-result updates. + +## Coverage and boundaries + +The catalog covers all nine aggregate kinds, root and loaded execution, +unrelated and relationship queries, to-one/to-many/many-to-many paths, multiple +hops, manual and attribute-free relationships, calculation and aggregate +fields, parent references, dependency and fanout filters, join filters, +ordering, bounds, empty/nil/default/unique results, constrained types, +read actions, arguments, actor, authorization, tenancy, shared context, +filtering, sorting, calculations and pagination. + +The fixtures deliberately contain equal-valued records, duplicate filter +matches, nil values, an empty parent and shared destinations. Named resource +aggregates and ad hoc query aggregates both appear. Direct relationship-read +controls distinguish aggregate defects from broader read behavior. Explicit +null ordering avoids treating database defaults as an Ash parity contract. +Numerical helpers round floating-point and decimal aggregate values to six +decimal places for comparisons. Constrained custom scalar values retain their +struct and constraints; string whitespace and nils remain unchanged. + +This is a representative scenario catalog with deliberate interaction cases, +not every possible combination. Schema-based Postgres tenancy, arbitrary +database types/functions, nested parent-stack depths, every manual callback +shape, other SQLite builds and libSQL need additional adapter-specific coverage. +SQL-shape, query-count and performance tests belong alongside the adapter. +Keep the existing adapter regression suites. + +[GAPS.md](GAPS.md) records the implementation tasks and unresolved semantics. +In particular, limited many-to-many relationships need an Ash API decision: +the current relationship DSL lacks `limit`, and aggregate queries reject it. + +## Extend + +1. Add a scenario to the appropriate module in `lib/scenarios/`. Give it a + stable ID, area, explicit expected value, and a public Ash operation. + Keep fixtures and assertions independent of adapter identity. +2. Add fixtures or resource roles only as needed. `Fixtures` seeds data through + Ash; `Resources` provides the shared SQL resource definitions. The `Adapter` + behaviour keeps setup, sandbox lifecycle, resource mapping, and custom SQL + implementations outside the scenario runner. +3. Declare every adapter's expectation in `Expectations`. New IDs and adapters + have no default status. Link every gap to an entry in `GAPS.md`, specifying + an exact wrong value or an exception class plus a narrow message pattern. + Verify the public API and fixture before classifying a failure. +4. Run both adapters, regenerate `MATRIX.md` with `mix conformance.matrix`, and + run `mix check`. Commit the matrix and expectation change with the scenario. + +To add another data layer, implement `Adapter`, supply the same resource roles, +and list it in `Adapter.all/0`. Non-SQL adapters can provide their own resource +definitions and lifecycle; the scenario runner has no Ecto dependency. Add a +status for every existing scenario. Status helpers in `Expectations` currently +name the two SQL adapters and must be extended explicitly. + +When a feature lands, run its existing scenario first. The unexpected-pass +failure identifies declarations to promote to `:supported`. Preserve the shared +expected result unless a separately documented semantic decision changes it. + +## Baseline + +The first baseline uses AshSQL extraction `0985b9f`, AshSQLite `46a4b86`, +AshPostgres `945073e4`, and Ash 3.33.10, with the full dependency versions in +this project's lockfile. +It intentionally excludes the separately stacked from_many and schema fixes. +The newer locked Ash version can expose behavior different from older adapter +test runs; the matrix describes this exact dependency set. + +There are 122 shared scenarios and 12 runner/catalog tests. The initial run +matches all 244 adapter expectations: + +| Adapter | Supported | Unsupported | Known defect | Unresolved | +| --- | ---: | ---: | ---: | ---: | +| SQLite | 87 | 24 | 7 | 4 | +| PostgreSQL | 96 | 0 | 22 | 4 | + +These counts describe scenario outcomes, not distinct bugs or a percentage of +all Ash features. Several scenarios exercise the same underlying gap. + +Local validation uses PostgreSQL 17.5 and the SQLite library bundled with the +locked Exqlite release. The workflow runs separate SQLite and PostgreSQL jobs +on all pull requests, including PRs whose base is another feature branch. diff --git a/conformance/config/config.exs b/conformance/config/config.exs new file mode 100644 index 0000000..aa9b4cc --- /dev/null +++ b/conformance/config/config.exs @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT +import Config + +config :ash, :validate_domain_resource_inclusion?, false +config :ash, :validate_domain_config_inclusion?, false +config :ash, :default_string_length_count, :codepoints +config :logger, level: :warning + +config :ash_sql_conformance, AshSql.Conformance.SqliteRepo, + database: Path.expand("../tmp/aggregates.sqlite3", __DIR__), + pool: Ecto.Adapters.SQL.Sandbox, + pool_size: 2, + migration_lock: false + +config :ash_sql_conformance, AshSql.Conformance.PostgresRepo, + hostname: System.get_env("PGHOST", "localhost"), + port: String.to_integer(System.get_env("PGPORT", "5432")), + username: System.get_env("PGUSER", "postgres"), + password: System.get_env("PGPASSWORD", "postgres"), + database: System.get_env("CONFORMANCE_PG_DATABASE", "ash_sql_aggregate_conformance"), + pool: Ecto.Adapters.SQL.Sandbox, + pool_size: 2 diff --git a/conformance/lib/adapter.ex b/conformance/lib/adapter.ex new file mode 100644 index 0000000..deda25b --- /dev/null +++ b/conformance/lib/adapter.ex @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Adapter do + @moduledoc """ + Database and resource configuration for the shared aggregate scenarios. + + Scenarios receive an adapter and ask it for a resource by role. They never + branch on the adapter ID. Other data layers can implement this contract and + provide the same fixture domain without depending on Ecto in the runner. + """ + + @callback id() :: atom() + @callback resource(atom()) :: module() + @callback custom_aggregate() :: module() + @callback setup!() :: term() + @callback checkout!() :: term() + @callback checkin!() :: term() + + def all, do: [AshSql.Conformance.Sqlite, AshSql.Conformance.Postgres] + + def selected do + requested = System.get_env("CONFORMANCE_ADAPTERS", "sqlite,postgres") |> String.split(",") + known = Map.new(all(), &{to_string(&1.id()), &1}) + + Enum.map(requested, fn name -> + Map.get(known, String.trim(name)) || + raise ArgumentError, "Unknown adapter: #{inspect(name)}" + end) + |> Enum.uniq() + end +end + +defmodule AshSql.Conformance.SqliteRepo do + @moduledoc false + use AshSqlite.Repo, otp_app: :ash_sql_conformance +end + +defmodule AshSql.Conformance.PostgresRepo do + @moduledoc false + use AshPostgres.Repo, otp_app: :ash_sql_conformance, warn_on_missing_ash_functions?: false + def installed_extensions, do: [] + def min_pg_version, do: %Version{major: 17, minor: 0, patch: 0} +end + +defmodule AshSql.Conformance.Sqlite do + @moduledoc false + @behaviour AshSql.Conformance.Adapter + def id, do: :sqlite + def repo, do: AshSql.Conformance.SqliteRepo + + def resource(role), + do: Module.concat(AshSql.Conformance.Sqlite, Macro.camelize(to_string(role))) + + def custom_aggregate, do: AshSql.Conformance.SqliteSum + def setup!, do: AshSql.Conformance.Database.setup!(repo()) + def checkout!, do: Ecto.Adapters.SQL.Sandbox.checkout(repo()) + def checkin!, do: Ecto.Adapters.SQL.Sandbox.checkin(repo()) +end + +defmodule AshSql.Conformance.Postgres do + @moduledoc false + @behaviour AshSql.Conformance.Adapter + def id, do: :postgres + def repo, do: AshSql.Conformance.PostgresRepo + + def resource(role), + do: Module.concat(AshSql.Conformance.Postgres, Macro.camelize(to_string(role))) + + def custom_aggregate, do: AshSql.Conformance.PostgresSum + def setup!, do: AshSql.Conformance.Database.setup!(repo()) + def checkout!, do: Ecto.Adapters.SQL.Sandbox.checkout(repo()) + def checkin!, do: Ecto.Adapters.SQL.Sandbox.checkin(repo()) +end + +defmodule AshSql.Conformance.SqliteSum do + @moduledoc false + use Ash.Resource.Aggregate.CustomAggregate + use AshSqlite.CustomAggregate + import Ecto.Query + def dynamic(opts, binding), do: dynamic(sum(field(as(^binding), ^opts[:field]))) +end + +defmodule AshSql.Conformance.PostgresSum do + @moduledoc false + use Ash.Resource.Aggregate.CustomAggregate + use AshPostgres.CustomAggregate + import Ecto.Query + def dynamic(opts, binding), do: dynamic(fragment("sum(?)", field(as(^binding), ^opts[:field]))) +end diff --git a/conformance/lib/catalog.ex b/conformance/lib/catalog.ex new file mode 100644 index 0000000..6d3c45f --- /dev/null +++ b/conformance/lib/catalog.ex @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Catalog do + @moduledoc false + alias AshSql.Conformance.Scenarios + + def all do + [ + Scenarios.Operations, + Scenarios.Relationships, + Scenarios.Filters, + Scenarios.Bounds, + Scenarios.Context + ] + |> Enum.flat_map(& &1.all()) + |> Enum.sort_by(& &1.id) + end +end diff --git a/conformance/lib/database.ex b/conformance/lib/database.ex new file mode 100644 index 0000000..bce7005 --- /dev/null +++ b/conformance/lib/database.ex @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Database do + @moduledoc false + + def setup!(repo) do + config = repo.config() + + if database = config[:database], do: File.mkdir_p!(Path.dirname(database)) + + case repo.__adapter__().storage_up(config) do + :ok -> :ok + {:error, :already_up} -> :ok + {:error, error} -> raise "Cannot create conformance database: #{inspect(error)}" + end + + {:ok, _} = repo.start_link() + Ecto.Migrator.up(repo, 1, AshSql.Conformance.Schema, log: false) + Ecto.Adapters.SQL.Sandbox.mode(repo, :manual) + end +end + +defmodule AshSql.Conformance.Schema do + @moduledoc false + use Ecto.Migration + + def change do + create table(:ac_parents, primary_key: false) do + add(:id, :bigint, primary_key: true) + add(:label, :text) + add(:threshold, :bigint) + add(:tenant_id, :text) + end + + create table(:ac_children, primary_key: false) do + add(:id, :bigint, primary_key: true) + add(:parent_id, :bigint) + add(:label, :text) + add(:value, :bigint) + add(:visible, :boolean) + add(:tenant_id, :text) + end + + create(index(:ac_children, [:parent_id])) + + create table(:ac_ratings, primary_key: false) do + add(:id, :bigint, primary_key: true) + add(:child_id, :bigint) + add(:score, :bigint) + end + + create table(:ac_tags, primary_key: false) do + add(:id, :bigint, primary_key: true) + add(:label, :text) + add(:value, :bigint) + end + + create table(:ac_links, primary_key: false) do + add(:parent_id, :bigint, primary_key: true) + add(:tag_id, :bigint, primary_key: true) + add(:tenant_id, :text) + end + + create table(:ac_child_tags, primary_key: false) do + add(:child_id, :bigint, primary_key: true) + add(:tag_id, :bigint, primary_key: true) + end + + create table(:ac_events, primary_key: false) do + add(:parent_id, :bigint) + add(:value, :bigint) + end + end +end diff --git a/conformance/lib/expectations.ex b/conformance/lib/expectations.ex new file mode 100644 index 0000000..2471c95 --- /dev/null +++ b/conformance/lib/expectations.ex @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Expectations do + @moduledoc """ + Explicit baseline for the pinned adapters and the parent AshSQL checkout. + + New scenarios and adapters have no implicit status. A gap has a narrow error + or wrong-result signature and a local implementation task. Updating this file + never changes the shared scenario's expected answer. + """ + + @supported_both ~w( + bounds.default_sort_control bounds.filter_after_limit bounds.list_filter_after_limit + bounds.relationship_limit bounds.relationship_offset bounds.relationship_offset_only + bounds.root_limit bounds.root_zero + context.actor context.arguments context.attribute_tenant context.authorization + context.authorization_bounds_control context.intermediate_action context.intermediate_actor + context.prepared_context_control context.read_action context.shared context.through_arguments + context.through_tenant + field.aggregate field.calculation field.root_aggregate + filter.exists filter.join filter.not_exists filter.or_exists filter.ordinary filter.sibling_independence + identity.keyless_count + loaded.avg loaded.count loaded.custom loaded.exists loaded.first loaded.list loaded.max loaded.min loaded.sum + ordering.asc_nils_first ordering.asc_nils_last ordering.desc_nils_first ordering.desc_nils_last + ordering.expression_first ordering.expression_list ordering.ties + path.final_many_to_many_scalar path.many_to_many path.many_to_many_first path.many_to_many_list + path.multi_hop path.no_attributes_control path.to_one path.unrelated + root.avg root.count root.exists root.first root.max root.min root.sum + use.calculation use.filter use.pagination use.sort + values.constrained_scalar values.distinct_count values.distinct_list values.field_count + values.filtered_first_default values.include_nil_first values.include_nil_list values.list_default + values.root_empty values.same_name_distinct_definitions values.scalar_default + values.string_constraints values.string_name + ) + + def for(id, adapter), do: all() |> Map.fetch!(id) |> Map.fetch!(adapter) + + def all do + Map.merge(Map.new(@supported_both, &{&1, both(:supported)}), gaps()) + end + + defp gaps do + %{ + "root.custom" => sqlite(root_unsupported()), + "root.list" => sqlite(root_unsupported()), + "root.unsorted_first_empty" => + postgres( + defect_error( + ~r/\*\* \(KeyError\) key :sort not found in: nil/, + "root-first" + ) + ), + "path.root_relationship" => %{ + sqlite: + unsupported( + ~r/AshSql grouped query aggregates do not yet support relationship aggregate :result/, + "root-relationship" + ), + postgres: + defect_error( + ~r/no such aggregate field: AshSql\.Conformance\.Postgres\.Parent\.value/, + "root-relationship" + ) + }, + "path.final_many_to_many_first" => sqlite(many_to_many()), + "path.final_many_to_many_list" => sqlite(many_to_many()), + "path.final_many_to_many_custom" => sqlite(many_to_many()), + "path.intermediate_many_to_many" => sqlite(many_to_many()), + "path.repeated_many_to_many" => %{ + sqlite: + unresolved_error( + ~r/AshSql does not support loading aggregates over multi-hop paths that include many_to_many relationships/, + "path-multiplicity" + ), + postgres: unresolved_value(%{1 => 3, 2 => 2, 3 => 0}, "path-multiplicity") + }, + "path.manual" => + sqlite( + unsupported( + ~r/AshSql does not support loading aggregates over manual relationships/, + "manual" + ) + ), + "path.no_attributes" => %{ + sqlite: no_attributes(), + postgres: + defect_error( + ~r/field `result` in `select` does not exist in schema AshSql\.Conformance\.Postgres\.Child/, + "no-attributes" + ) + }, + "path.no_attributes_parent" => sqlite(no_attributes()), + "filter.parent" => sqlite(parent_filter()), + "filter.parent_unrelated" => sqlite(parent_filter()), + "filter.parent_relationship" => + sqlite( + unsupported( + ~r/AshSql does not support loading aggregates over relationships with parent-dependent filters/, + "parent-correlation" + ) + ), + "filter.parent_join" => + sqlite( + unsupported( + ~r/AshSql does not support loading aggregates with parent-dependent join filters/, + "parent-correlation" + ) + ), + "filter.aggregate_dependency" => + sqlite( + unsupported( + ~r/AshSql does not support loading aggregates with aggregate filters that reference other aggregates/, + "filter-dependencies" + ) + ), + "filter.fanout_sum" => fanout(6), + "filter.fanout_avg" => fanout(3.25), + "filter.fanout_count" => fanout(3), + "filter.fanout_list" => fanout([2, 2, 2]), + "filter.fanout_custom" => fanout(6), + "filter.fanout_count_records" => + postgres(defect_value(%{1 => 3, 2 => 0, 3 => 0}, "filter-fanout")), + "filter.fanout_read_control" => sqlite(defect_value([11, 11, 12], "filter-fanout")), + "identity.composite_count" => sqlite(composite_count()), + "identity.composite_fanout_count" => %{ + sqlite: composite_count(), + postgres: defect_value(%{1 => 6, 2 => 1, 3 => 0}, "filter-fanout") + }, + "identity.root_composite_count" => sqlite(composite_count()), + "identity.keyless_source" => + sqlite( + unsupported( + ~r/AshSql cannot load aggregates on resources with no primary key/, + "record-identity" + ) + ), + "identity.keyless_distinct" => %{ + sqlite: + unresolved_error( + ~r/requires a single primary key to count distinct records, but AshSql\.Conformance\.Sqlite\.Event has no primary key/, + "keyless-identity" + ), + postgres: unresolved_value(%{1 => 2, 2 => 0, 3 => 0}, "keyless-identity") + }, + "bounds.from_many" => both(defect_value(%{1 => 4, 2 => 1, 3 => 0}, "from-many")), + "bounds.default_sort" => both(defect_value(%{1 => 2, 2 => 4, 3 => nil}, "default-sort")), + "bounds.unsorted_limit" => + sqlite( + defect_error(~r/\*\* \(Exqlite.Error\) near "\)": syntax error/, "unsorted-bounds") + ), + "bounds.root_order_then_limit" => postgres(defect_value(2, "root-bounds")), + "bounds.root_first_distinct_sort" => postgres(defect_value(2, "root-bounds")), + "bounds.root_offset_only" => + postgres(defect_error(~r/\*\* \(BadMapError\) expected a map, got: nil/, "root-bounds")), + "bounds.many_to_many_query_limit" => + both(unresolved_error(~r/Cannot set limit on aggregate query/, "many-to-many-bounds-api")), + "ordering.unique_other_field" => %{ + sqlite: + unresolved_error( + ~r/AshSql only supports uniq list aggregates when sorting by the list aggregate field/, + "unique-list-order" + ), + postgres: + unresolved_error( + ~r/ERROR 42P10 .*in an aggregate with DISTINCT, ORDER BY expressions must appear in argument list/, + "unique-list-order" + ) + }, + "context.relationship_context" => + both(defect_value(%{1 => 0, 2 => 0, 3 => 0}, "relationship-context")), + "context.relationship_context_control" => + both(defect_value(%{1 => [], 2 => [], 3 => []}, "relationship-context")), + "context.authorization_before_bounds" => + both(defect_value(%{1 => nil, 2 => 4, 3 => nil}, "authorization-bounds")), + "context.prepared_query_arguments" => + postgres( + defect_error( + ~r/no function clause matching in Enumerable.List.reduce\/3/, + "prepared-query" + ) + ), + "context.tenant_bypass" => + postgres(defect_value(%{1 => 3, 2 => 0, 3 => 0}, "tenant-bypass")), + "context.bypass_sibling" => postgres(defect_value({3, 3}, "tenant-bypass")), + "context.through_bypass" => + postgres(defect_value(%{1 => 3, 2 => 3, 3 => nil}, "tenant-bypass")) + } + end + + defp both(status), do: %{sqlite: status, postgres: status} + defp sqlite(status), do: %{sqlite: status, postgres: :supported} + defp postgres(status), do: %{sqlite: :supported, postgres: status} + defp task(id), do: "GAPS.md##{id}" + + defp unsupported(pattern, id), + do: {:unsupported, {:error, Ash.Error.Unknown, pattern}, task(id)} + + defp defect_error(pattern, id), + do: {:known_defect, {:error, Ash.Error.Unknown, pattern}, task(id)} + + defp defect_value(value, id), do: {:known_defect, {:value, value}, task(id)} + + defp unresolved_error(pattern, id), + do: {:unresolved, {:error, Ash.Error.Unknown, pattern}, task(id)} + + defp unresolved_value(value, id), do: {:unresolved, {:value, value}, task(id)} + + defp root_unsupported, + do: + {:unsupported, + {:error, Ash.Error.Invalid, + ~r/Data layer for AshSql\.Conformance\.Sqlite\.Child does not support using query aggregates/}, + task("root-kinds")} + + defp many_to_many, + do: + unsupported( + ~r/AshSql does not support loading aggregates over multi-hop paths that include many_to_many relationships/, + "many-to-many-paths" + ) + + defp no_attributes, + do: + unsupported( + ~r/AshSql does not support loading aggregates over no_attributes\? relationships/, + "no-attributes" + ) + + defp parent_filter, + do: + unsupported( + ~r/AshSql does not support loading aggregates with parent-dependent aggregate filters/, + "parent-correlation" + ) + + defp composite_count, + do: + unsupported( + ~r/requires a single primary key to count distinct records, but AshSql\.Conformance\.Sqlite\.Link has composite primary key/, + "record-identity" + ) + + defp fanout(value) do + %{ + sqlite: + unsupported( + ~r/AshSql does not support loading sum, avg, list, custom, or field-based count aggregates with filters that reference to-many relationships/, + "filter-fanout" + ), + postgres: defect_value(value, "filter-fanout") + } + end +end diff --git a/conformance/lib/fixtures.ex b/conformance/lib/fixtures.ex new file mode 100644 index 0000000..6e8f058 --- /dev/null +++ b/conformance/lib/fixtures.ex @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Fixtures do + @moduledoc "Small asymmetric data sets shared by every adapter." + + def seed!(adapter) do + parents = [ + %{id: 1, label: "alpha", threshold: 3, tenant_id: "a"}, + %{id: 2, label: "beta", threshold: 5, tenant_id: "b"}, + %{id: 3, label: "empty", threshold: 9, tenant_id: "a"} + ] + + children = [ + %{id: 11, parent_id: 1, label: "same", value: 2, visible: true, tenant_id: "a"}, + %{id: 12, parent_id: 1, label: "same", value: 2, visible: true, tenant_id: "a"}, + %{id: 13, parent_id: 1, label: "high", value: 7, visible: false, tenant_id: "b"}, + %{id: 14, parent_id: 1, label: nil, value: nil, visible: true, tenant_id: "a"}, + %{id: 21, parent_id: 2, label: "other", value: 4, visible: true, tenant_id: "b"} + ] + + ratings = [ + %{id: 101, child_id: 11, score: 8}, + %{id: 102, child_id: 11, score: 9}, + %{id: 103, child_id: 12, score: 8}, + %{id: 104, child_id: 13, score: 1} + ] + + tags = [%{id: 201, label: "red", value: 3}, %{id: 202, label: "blue", value: 8}] + + links = [ + %{parent_id: 1, tag_id: 201, tenant_id: "a"}, + %{parent_id: 1, tag_id: 202, tenant_id: "b"}, + %{parent_id: 2, tag_id: 201, tenant_id: "a"} + ] + + child_tags = [%{child_id: 11, tag_id: 201}, %{child_id: 12, tag_id: 202}] + events = [%{parent_id: 1, value: 2}, %{parent_id: 1, value: 3}] + + for {role, rows} <- [ + parent: parents, + child: children, + rating: ratings, + tag: tags, + link: links, + child_tag: child_tags, + event: events + ], + row <- rows do + Ash.Seed.seed!(struct(adapter.resource(role), row)) + end + + %{adapter: adapter, parent: adapter.resource(:parent), child: adapter.resource(:child)} + end +end + +defmodule AshSql.Conformance.SqliteResources do + @moduledoc false + use AshSql.Conformance.Resources, namespace: AshSql.Conformance.Sqlite, adapter: :sqlite +end + +defmodule AshSql.Conformance.PostgresResources do + @moduledoc false + use AshSql.Conformance.Resources, namespace: AshSql.Conformance.Postgres, adapter: :postgres +end diff --git a/conformance/lib/manual.ex b/conformance/lib/manual.ex new file mode 100644 index 0000000..a7ebb4f --- /dev/null +++ b/conformance/lib/manual.ex @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Manual do + @moduledoc false + defmacro __using__(opts) do + prefix = Keyword.fetch!(opts, :prefix) + join_fun = String.to_atom("#{prefix}_join") + subquery_fun = String.to_atom("#{prefix}_subquery") + + quote do + use Ash.Resource.ManualRelationship + import Ecto.Query + + def load(parents, _opts, %{query: query, actor: actor, authorize?: authorize?}) do + ids = Enum.map(parents, & &1.id) + + rows = + query + |> Ash.Query.do_filter(parent_id: [in: ids]) + |> Ash.read!(actor: actor, authorize?: authorize?) + + {:ok, Enum.group_by(rows, & &1.parent_id)} + end + + def unquote(join_fun)(query, _opts, parent_binding, child_binding, type, child_query) do + {:ok, + join(query, type, [], child in ^child_query, + as: ^child_binding, + on: child.parent_id == as(^parent_binding).id + )} + end + + def unquote(subquery_fun)(_opts, parent_binding, child_binding, child_query) do + {:ok, + from(row in child_query, + where: field(as(^child_binding), :parent_id) == field(parent_as(^parent_binding), :id) + )} + end + end + end +end + +defmodule AshSql.Conformance.Sqlite.Manual do + @moduledoc false + use AshSqlite.ManualRelationship + use AshSql.Conformance.Manual, prefix: :ash_sqlite +end + +defmodule AshSql.Conformance.Postgres.Manual do + @moduledoc false + use AshPostgres.ManualRelationship + use AshSql.Conformance.Manual, prefix: :ash_postgres +end diff --git a/conformance/lib/mix/tasks/conformance.matrix.ex b/conformance/lib/mix/tasks/conformance.matrix.ex new file mode 100644 index 0000000..92fa0f0 --- /dev/null +++ b/conformance/lib/mix/tasks/conformance.matrix.ex @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule Mix.Tasks.Conformance.Matrix do + @moduledoc false + use Mix.Task + @shortdoc "Generate the declared aggregate capability matrix" + def run([]) do + Mix.Task.run("compile") + File.write!("MATRIX.md", AshSql.Conformance.Report.matrix()) + Mix.shell().info("Wrote MATRIX.md") + end +end diff --git a/conformance/lib/report.ex b/conformance/lib/report.ex new file mode 100644 index 0000000..868baa9 --- /dev/null +++ b/conformance/lib/report.ex @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Report do + @moduledoc false + alias AshSql.Conformance.{Adapter, Catalog, Expectations} + + def declaration_rows do + for scenario <- Catalog.all(), adapter <- Adapter.all() do + expectation = Expectations.for(scenario.id, adapter.id()) + + %{ + scenario: scenario.id, + area: scenario.area, + adapter: adapter.id(), + status: status(expectation), + task: task(expectation) + } + end + end + + def status(:supported), do: :supported + def status({status, _, _}), do: status + def task(:supported), do: nil + def task({_, _, task}), do: task + + def matrix do + rows = Enum.group_by(declaration_rows(), & &1.scenario) + adapters = Enum.map(Adapter.all(), & &1.id()) + + header = """ + + + + # Aggregate expectation matrix + + Generated by `mix conformance.matrix`. These are declared contracts; `results/` contains execution results. Passing an unsupported, known-defect or unresolved expectation does not establish feature conformance. + + | Scenario | #{Enum.join(adapters, " | ")} | + | --- | #{Enum.map_join(adapters, " | ", fn _ -> "---" end)} | + """ + + body = + Enum.map_join(Catalog.all(), "\n", fn scenario -> + cells = + Enum.map(adapters, fn adapter -> + row = Enum.find(Map.fetch!(rows, scenario.id), &(&1.adapter == adapter)) + if row.task, do: "[#{row.status}](#{row.task})", else: to_string(row.status) + end) + + "| `#{scenario.id}` | #{Enum.join(cells, " | ")} |" + end) + + header <> body <> "\n" + end + + def write_results!(rows) do + File.mkdir_p!("results") + name = Adapter.selected() |> Enum.map_join("-", & &1.id()) + rows = Enum.sort_by(rows, &{&1.scenario, &1.adapter}) + counts = Enum.frequencies_by(rows, &"#{&1.status}/#{&1.execution}") + + report = %{ + counts: counts, + scenarios: rows, + meaning: "A matched gap expectation is not feature conformance." + } + + File.write!("results/#{name}.json", Jason.encode!(report, pretty: true) <> "\n") + IO.puts("\nAggregate contracts: #{inspect(counts)}\nReport: results/#{name}.json") + end +end + +defmodule AshSql.Conformance.Formatter do + @moduledoc false + use GenServer + alias AshSql.Conformance.{Expectations, Report} + + def init(_opts), do: {:ok, []} + + def handle_cast( + {:test_finished, %{tags: %{scenario: id, adapter: adapter} = tags} = test}, + rows + ) do + expectation = Expectations.for(id, adapter) + + execution = + case test.state do + nil -> :matched + {:excluded, _} -> :excluded + {:skipped, _} -> :skipped + _ -> :failed + end + + row = %{ + scenario: id, + adapter: adapter, + area: tags.area, + status: Report.status(expectation), + task: Report.task(expectation), + execution: execution + } + + {:noreply, [row | rows]} + end + + def handle_cast({:suite_finished, _times}, rows) do + Report.write_results!(rows) + {:noreply, rows} + end + + def handle_cast(_event, rows), do: {:noreply, rows} +end diff --git a/conformance/lib/resource.ex b/conformance/lib/resource.ex new file mode 100644 index 0000000..5b41202 --- /dev/null +++ b/conformance/lib/resource.ex @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Domain do + @moduledoc false + use Ash.Domain + + resources do + allow_unregistered?(true) + end +end + +defmodule AshSql.Conformance.Resource do + @moduledoc false + + defmacro __using__(opts) do + adapter = Keyword.fetch!(opts, :adapter) + table = Keyword.fetch!(opts, :table) + authorizers = Keyword.get(opts, :authorizers, []) + + {data_layer, config} = + case adapter do + :sqlite -> + {AshSqlite.DataLayer, + quote do + sqlite do + table(unquote(table)) + repo(AshSql.Conformance.SqliteRepo) + end + end} + + :postgres -> + {AshPostgres.DataLayer, + quote do + postgres do + table(unquote(table)) + repo(AshSql.Conformance.PostgresRepo) + end + end} + end + + quote do + use Ash.Resource, + domain: AshSql.Conformance.Domain, + data_layer: unquote(data_layer), + authorizers: unquote(authorizers) + + unquote(config) + + actions do + defaults([:read]) + end + end + end +end + +defmodule AshSql.Conformance.Scope do + @moduledoc false + use Ash.Resource.Preparation + + @impl true + def prepare(query, opts, context) do + value = + case opts[:from] do + :actor -> context.actor && context.actor.label + :context -> query.context[:visible_label] + :tenant -> context.tenant + end + + Ash.Query.do_filter(query, [{opts[:field] || :label, value || "missing-context"}]) + end +end + +defmodule AshSql.Conformance.Quantity do + @moduledoc false + use Ash.Type + defstruct [:value, :unit] + def constraints, do: [unit: [type: :atom, default: :units]] + def storage_type(_), do: :integer + def cast_input(value, constraints), do: cast_stored(value, constraints) + def cast_stored(nil, _), do: {:ok, nil} + def cast_stored(%__MODULE__{} = value, _), do: {:ok, value} + + def cast_stored(value, constraints) when is_integer(value), + do: {:ok, %__MODULE__{value: value, unit: constraints[:unit]}} + + def cast_stored(_, _), do: :error + def dump_to_native(%__MODULE__{value: value}, _), do: {:ok, value} + def dump_to_native(value, _) when is_integer(value) or is_nil(value), do: {:ok, value} +end diff --git a/conformance/lib/resources.ex b/conformance/lib/resources.ex new file mode 100644 index 0000000..c9a639f --- /dev/null +++ b/conformance/lib/resources.ex @@ -0,0 +1,376 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Resources do + @moduledoc "One resource definition per role, instantiated for each SQL adapter." + + defmacro __using__(opts) do + namespace = opts |> Keyword.fetch!(:namespace) |> Macro.expand(__CALLER__) + adapter = Keyword.fetch!(opts, :adapter) + parent = Module.concat(namespace, Parent) + child = Module.concat(namespace, Child) + rating = Module.concat(namespace, Rating) + tag = Module.concat(namespace, Tag) + link = Module.concat(namespace, Link) + child_tag = Module.concat(namespace, ChildTag) + event = Module.concat(namespace, Event) + tenant_child = Module.concat(namespace, TenantChild) + tenant_link = Module.concat(namespace, TenantLink) + authorized_child = Module.concat(namespace, AuthorizedChild) + manual = Module.concat(namespace, Manual) + + quote context: Elixir do + defmodule unquote(parent) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_parents" + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:label, :string, public?: true) + attribute(:threshold, :integer, public?: true) + attribute(:tenant_id, :string, public?: true) + end + + actions do + read :paged do + pagination(offset?: true, countable: true, required?: false) + end + end + + relationships do + has_many(:children, unquote(child), destination_attribute: :parent_id, public?: true) + + has_many(:top_children, unquote(child), + destination_attribute: :parent_id, + sort: [value: :desc_nils_last, id: :asc], + limit: 2 + ) + + has_many(:middle_children, unquote(child), + destination_attribute: :parent_id, + sort: [value: :desc_nils_last, id: :asc], + limit: 1, + offset: 1 + ) + + has_many(:offset_children, unquote(child), + destination_attribute: :parent_id, + sort: [value: :desc_nils_last, id: :asc], + offset: 1 + ) + + has_many(:unsorted_limited, unquote(child), destination_attribute: :parent_id, limit: 1) + + has_one(:top_child, unquote(child), + destination_attribute: :parent_id, + from_many?: true, + sort: [value: :desc_nils_last, id: :asc] + ) + + has_one(:default_top_child, unquote(child), + destination_attribute: :parent_id, + from_many?: true, + default_sort: [value: :desc_nils_last, id: :asc] + ) + + has_many(:above_threshold, unquote(child), + destination_attribute: :parent_id, + filter: expr(value >= parent(threshold)) + ) + + has_many(:all_children, unquote(child), no_attributes?: true) + + has_many(:matching_children, unquote(child), + no_attributes?: true, + filter: expr(value >= parent(threshold)) + ) + + has_many(:manual_children, unquote(child), manual: unquote(manual)) + + has_many(:visible_children, unquote(child), + destination_attribute: :parent_id, + read_action: :visible + ) + + has_many(:argument_children, unquote(child), + destination_attribute: :parent_id, + read_action: :by_label, + read_action_arguments: %{label: "same"} + ) + + has_many(:context_children, unquote(child), + destination_attribute: :parent_id, + read_action: :from_context, + relationship_context: %{visible_label: "same"} + ) + + has_many(:actor_children, unquote(child), + destination_attribute: :parent_id, + read_action: :for_actor + ) + + has_many(:tenant_children, unquote(tenant_child), destination_attribute: :parent_id) + + has_many(:authorized_children, unquote(authorized_child), + destination_attribute: :parent_id + ) + + has_many(:authorized_top, unquote(authorized_child), + destination_attribute: :parent_id, + sort: [value: :desc_nils_last, id: :asc], + limit: 1 + ) + + has_many(:links, unquote(link), destination_attribute: :parent_id) + + has_many(:scoped_links, unquote(link), + destination_attribute: :parent_id, + read_action: :in_tenant, + read_action_arguments: %{tenant_id: "a"} + ) + + has_many(:tenant_links, unquote(tenant_link), destination_attribute: :parent_id) + has_many(:events, unquote(event), destination_attribute: :parent_id) + + many_to_many :tags, unquote(tag) do + through(unquote(link)) + source_attribute_on_join_resource(:parent_id) + destination_attribute_on_join_resource(:tag_id) + end + + many_to_many :scoped_tags, unquote(tag) do + through(unquote(link)) + join_relationship(:scoped_links) + source_attribute_on_join_resource(:parent_id) + destination_attribute_on_join_resource(:tag_id) + end + + many_to_many :tenant_tags, unquote(tag) do + through(unquote(tenant_link)) + join_relationship(:tenant_links) + source_attribute_on_join_resource(:parent_id) + destination_attribute_on_join_resource(:tag_id) + end + end + + aggregates do + count(:child_count, :children, public?: true) + sum(:child_sum, :children, :value, public?: true) + first(:first_value, :children, :value, sort: [value: :asc], public?: true) + count(:visible_count, :visible_children) + count(:argument_count, :argument_children) + count(:context_count, :context_children) + count(:actor_count, :actor_children) + end + + calculations do + calculate(:double_threshold, :integer, expr(threshold * 2), public?: true) + + calculate( + :sum_plus_threshold, + :integer, + expr(if(is_nil(child_sum), do: 0, else: child_sum) + threshold), + public?: true + ) + end + end + + defmodule unquote(child) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_children" + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:parent_id, :integer, public?: true) + + attribute(:label, :string, + public?: true, + constraints: [trim?: false, allow_empty?: true] + ) + + attribute(:value, :integer, public?: true) + attribute(:visible, :boolean, public?: true) + attribute(:tenant_id, :string, public?: true) + end + + actions do + read :visible do + filter(expr(visible == true)) + end + + read :by_label do + argument(:label, :string, allow_nil?: false) + filter(expr(label == ^arg(:label))) + end + + read :from_context do + prepare({AshSql.Conformance.Scope, from: :context}) + end + + read :for_actor do + prepare({AshSql.Conformance.Scope, from: :actor}) + end + + read :for_tenant do + prepare({AshSql.Conformance.Scope, from: :tenant, field: :tenant_id}) + end + end + + relationships do + belongs_to(:parent, unquote(parent), define_attribute?: false, public?: true) + has_many(:ratings, unquote(rating), destination_attribute: :child_id, public?: true) + + many_to_many :tags, unquote(tag) do + through(unquote(child_tag)) + source_attribute_on_join_resource(:child_id) + destination_attribute_on_join_resource(:tag_id) + end + end + + aggregates do + count(:rating_count, :ratings, public?: true) + end + + calculations do + calculate(:double_value, :integer, expr(value * 2), public?: true) + end + end + + defmodule unquote(rating) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_ratings" + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:child_id, :integer, public?: true) + attribute(:score, :integer, public?: true) + end + + relationships do + belongs_to(:child, unquote(child), define_attribute?: false, public?: true) + end + end + + defmodule unquote(tag) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_tags" + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:label, :string, public?: true) + attribute(:value, :integer, public?: true) + end + + relationships do + many_to_many :parents, unquote(parent) do + through(unquote(link)) + source_attribute_on_join_resource(:tag_id) + destination_attribute_on_join_resource(:parent_id) + end + + many_to_many :children, unquote(child) do + through(unquote(child_tag)) + source_attribute_on_join_resource(:tag_id) + destination_attribute_on_join_resource(:child_id) + end + end + end + + defmodule unquote(link) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_links" + + attributes do + attribute(:parent_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:tag_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:tenant_id, :string, public?: true) + end + + relationships do + belongs_to(:parent, unquote(parent), define_attribute?: false) + belongs_to(:tag, unquote(tag), define_attribute?: false) + end + + actions do + read :in_tenant do + argument(:tenant_id, :string, allow_nil?: false) + filter(expr(tenant_id == ^arg(:tenant_id))) + end + end + end + + defmodule unquote(child_tag) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_child_tags" + + attributes do + attribute(:child_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:tag_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + end + end + + defmodule unquote(event) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_events" + + resource do + require_primary_key?(false) + end + + attributes do + attribute(:parent_id, :integer, public?: true) + attribute(:value, :integer, public?: true) + end + + relationships do + belongs_to(:parent, unquote(parent), define_attribute?: false) + end + end + + defmodule unquote(tenant_child) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_children" + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:parent_id, :integer, public?: true) + attribute(:tenant_id, :string, public?: true) + attribute(:value, :integer, public?: true) + end + + multitenancy do + strategy(:attribute) + attribute(:tenant_id) + end + end + + defmodule unquote(tenant_link) do + use AshSql.Conformance.Resource, adapter: unquote(adapter), table: "ac_links" + + attributes do + attribute(:parent_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:tag_id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:tenant_id, :string, public?: true) + end + + multitenancy do + strategy(:attribute) + attribute(:tenant_id) + end + end + + defmodule unquote(authorized_child) do + use AshSql.Conformance.Resource, + adapter: unquote(adapter), + table: "ac_children", + authorizers: [Ash.Policy.Authorizer] + + attributes do + attribute(:id, :integer, primary_key?: true, allow_nil?: false, public?: true) + attribute(:parent_id, :integer, public?: true) + attribute(:value, :integer, public?: true) + attribute(:visible, :boolean, public?: true) + end + + policies do + policy action_type(:read) do + authorize_if(expr(visible == true)) + end + end + end + end + end +end diff --git a/conformance/lib/scenario.ex b/conformance/lib/scenario.ex new file mode 100644 index 0000000..8dd7bcf --- /dev/null +++ b/conformance/lib/scenario.ex @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenario do + @moduledoc "A public Ash operation with an adapter-independent expected result." + @enforce_keys [:id, :area, :expected, :run] + defstruct [:id, :area, :expected, :run] + + def new(id, area, expected, run), + do: %__MODULE__{id: id, area: area, expected: expected, run: run} +end + +defmodule AshSql.Conformance.Runner do + @moduledoc """ + Checks conformance or an explicitly recorded gap. + + Only the operation is captured. Database setup and fixture construction happen + outside this boundary. Rejections require an exception class and message + pattern; wrong-result defects require the exact observed value. An unexpected + pass is a failure until the adapter's expectation is promoted. + """ + import ExUnit.Assertions + + def run!(scenario, expectation, context) do + outcome = capture(fn -> scenario.run.(context) end) + assert_outcome!(scenario, expectation, outcome) + end + + def assert_outcome!(scenario, :supported, {:ok, actual}) do + assert actual == scenario.expected, + "#{scenario.id}: expected #{format(scenario.expected)}, got #{format(actual)}" + end + + def assert_outcome!(scenario, :supported, {:error, exception, message}) do + flunk( + "#{scenario.id}: expected #{format(scenario.expected)}, raised #{inspect(exception)}: #{message}" + ) + end + + def assert_outcome!(scenario, {status, signature, task}, outcome) + when status in [:unsupported, :known_defect, :unresolved] do + assert is_binary(task) and byte_size(task) > 0, + "A gap must link to an implementation task or decision" + + if status != :unresolved and outcome == {:ok, scenario.expected} do + flunk( + "#{scenario.id}: unexpected pass; promote this #{status} expectation to :supported (#{task})" + ) + end + + assert matches?(signature, outcome), + "#{scenario.id}: #{status} signature changed (#{task})\n" <> + "Expected: #{format(signature)}\nObserved: #{format(outcome)}" + end + + def capture(fun) do + {:ok, fun.()} + rescue + exception -> {:error, exception.__struct__, Exception.message(exception)} + end + + defp matches?({:value, expected}, {:ok, actual}), do: expected == actual + + defp matches?({:error, exception, pattern}, {:error, exception, message}), + do: Regex.match?(pattern, message) + + defp matches?(_, _), do: false + defp format(value), do: inspect(value, charlists: :as_lists) +end diff --git a/conformance/lib/scenarios/bounds.ex b/conformance/lib/scenarios/bounds.ex new file mode 100644 index 0000000..877cbbf --- /dev/null +++ b/conformance/lib/scenarios/bounds.ex @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Bounds do + @moduledoc false + import AshSql.Conformance.Scenario, only: [new: 4] + import AshSql.Conformance.Scenarios.Helpers + + def all do + [ + new("bounds.relationship_limit", :bounds, %{1 => 9, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :top_children, field: :value) + end), + new("bounds.relationship_offset", :bounds, %{1 => 2, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :middle_children, field: :value) + end), + new("bounds.relationship_offset_only", :bounds, %{1 => 4, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :offset_children, field: :value) + end), + new("bounds.filter_after_limit", :bounds, %{1 => 2, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :top_children, field: :value, query: [filter: [visible: true]]) + end), + new("bounds.list_filter_after_limit", :bounds, %{1 => [2], 2 => [4], 3 => []}, fn ctx -> + loaded(ctx, :list, :top_children, + field: :value, + query: [filter: [visible: true], sort: [value: :asc]] + ) + end), + new("bounds.unsorted_limit", :bounds, %{1 => 1, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :unsorted_limited) + end), + new("bounds.from_many", :bounds, %{1 => 1, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :top_child) + end), + new("bounds.default_sort", :bounds, %{1 => 7, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :first, :default_top_child, field: :value) + end), + new("bounds.default_sort_control", :bounds, %{1 => [13], 2 => [21], 3 => []}, fn ctx -> + relationship_ids(ctx, :default_top_child) + end), + new("bounds.root_limit", :bounds, 2, fn ctx -> + root(%{ctx | child: Ash.Query.limit(ctx.child, 2)}, :count) + end), + new("bounds.root_offset_only", :bounds, 3, fn ctx -> + query = ctx.child |> Ash.Query.sort(:id) |> Ash.Query.offset(2) + root(%{ctx | child: query}, :count) + end), + new("bounds.root_order_then_limit", :bounds, 7, fn ctx -> + query = ctx.child |> Ash.Query.sort(value: :desc_nils_last) |> Ash.Query.limit(1) + root(%{ctx | child: query}, :sum, field: :value) + end), + new("bounds.root_first_distinct_sort", :bounds, 7, fn ctx -> + query = ctx.child |> Ash.Query.sort(value: :desc_nils_last) |> Ash.Query.limit(1) + root(%{ctx | child: query}, :first, field: :value, query: [sort: [value: :asc]]) + end), + new("bounds.root_zero", :bounds, %{count: 0, first: nil, exists: false}, fn ctx -> + ctx.child + |> Ash.Query.limit(0) + |> Ash.aggregate!( + [ + {:count, :count}, + {:first, :first, field: :value, query: [sort: [value: :asc]]}, + {:exists, :exists} + ], + authorize?: false + ) + end), + new("bounds.many_to_many_query_limit", :bounds, :unresolved, fn ctx -> + query = ctx.adapter.resource(:tag) |> Ash.Query.sort(value: :desc) |> Ash.Query.limit(1) + loaded(ctx, :sum, :tags, field: :value, query: query) + end), + new("ordering.unique_other_field", :ordering, :unresolved, fn ctx -> + loaded(ctx, :list, :children, field: :label, uniq?: true, query: [sort: [value: :desc]]) + end) + ] + end +end diff --git a/conformance/lib/scenarios/context.ex b/conformance/lib/scenarios/context.ex new file mode 100644 index 0000000..d1e7939 --- /dev/null +++ b/conformance/lib/scenarios/context.ex @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Context do + @moduledoc false + import AshSql.Conformance.Scenario, only: [new: 4] + import AshSql.Conformance.Scenarios.Helpers + + def all do + [ + new("context.read_action", :context, %{1 => 3, 2 => 1, 3 => 0}, fn ctx -> + named(ctx, :visible_count) + end), + new("context.arguments", :context, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + named(ctx, :argument_count) + end), + new("context.relationship_context", :context, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + named(ctx, :context_count) + end), + new( + "context.relationship_context_control", + :context, + %{1 => [11, 12], 2 => [], 3 => []}, + fn ctx -> + relationship_ids(ctx, :context_children) + end + ), + new("context.shared", :context, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + query = Ash.Query.set_context(ctx.parent, %{shared: %{visible_label: "same"}}) + named(%{ctx | parent: query}, :context_count) + end), + new("context.prepared_context_control", :context, [11, 12], fn ctx -> + ctx.child + |> Ash.Query.set_context(%{visible_label: "same"}) + |> Ash.Query.for_read(:from_context) + |> Ash.Query.sort(:id) + |> Ash.read!(authorize?: false) + |> Enum.map(& &1.id) + end), + new("context.actor", :context, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + named(ctx, :actor_count, actor: %{label: "same"}) + end), + new("context.prepared_query_arguments", :context, %{1 => 1, 2 => 0, 3 => 0}, fn ctx -> + query = Ash.Query.for_read(ctx.child, :by_label, %{label: "high"}, authorize?: false) + loaded(ctx, :count, :children, query: query) + end), + new("context.intermediate_action", :context, %{1 => 3, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, [:argument_children, :ratings]) + end), + new("context.intermediate_actor", :context, %{1 => 3, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, [:actor_children, :ratings], [], actor: %{label: "same"}) + end), + new("context.attribute_tenant", :context, %{1 => 3, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, :tenant_children, [], tenant: "a") + end), + new("context.tenant_bypass", :context, %{1 => 4, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :tenant_children, [multitenancy: :bypass], tenant: "a") + end), + new("context.bypass_sibling", :context, {3, 4}, fn ctx -> + row = + selected_parent(ctx) + |> Ash.Query.aggregate(:scoped, :count, :tenant_children) + |> Ash.Query.aggregate(:all, :count, :tenant_children, multitenancy: :bypass) + |> Ash.read_one!(tenant: "a", authorize?: false) + + {row.aggregates.scoped, row.aggregates.all} + end), + new("context.through_arguments", :context, %{1 => 3, 2 => 3, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :scoped_tags, field: :value) + end), + new("context.through_tenant", :context, %{1 => 3, 2 => 3, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :tenant_tags, [field: :value], tenant: "a") + end), + new("context.through_bypass", :context, %{1 => 11, 2 => 3, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :tenant_tags, [field: :value, multitenancy: :bypass], tenant: "a") + end), + new("context.authorization", :context, %{1 => 3, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :authorized_children, [], authorize?: true) + end), + new("context.authorization_before_bounds", :context, %{1 => 2, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :authorized_top, [field: :value], authorize?: true) + end), + new( + "context.authorization_bounds_control", + :context, + %{1 => [11], 2 => [21], 3 => []}, + fn ctx -> + relationship_ids(ctx, :authorized_top, authorize?: true) + end + ) + ] + end +end diff --git a/conformance/lib/scenarios/filters.ex b/conformance/lib/scenarios/filters.ex new file mode 100644 index 0000000..f49c5eb --- /dev/null +++ b/conformance/lib/scenarios/filters.ex @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Filters do + @moduledoc false + import AshSql.Conformance.Scenario, only: [new: 4] + import AshSql.Conformance.Scenarios.Helpers + require Ash.Query + require Ash.Expr + + def all do + [ + new("filter.ordinary", :filters, %{1 => 4, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, field: :value, query: [filter: [visible: true]]) + end), + new("filter.exists", :filters, %{1 => 4, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + query: Ash.Query.filter(ctx.child, exists(ratings, score > 5)) + ) + end), + new("filter.aggregate_dependency", :filters, %{1 => 1, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, :children, query: Ash.Query.filter(ctx.child, rating_count > 1)) + end), + new("filter.parent", :filters, %{1 => 7, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + query: [filter: Ash.Expr.expr(value >= parent(threshold))] + ) + end), + new("filter.parent_relationship", :filters, %{1 => 7, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :above_threshold, field: :value) + end), + new("filter.parent_unrelated", :filters, %{1 => 11, 2 => 7, 3 => nil}, fn ctx -> + loaded(ctx, :sum, ctx.child, + field: :value, + query: [filter: Ash.Expr.expr(value >= parent(threshold))] + ) + end), + new("filter.join", :filters, %{1 => 4, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + join_filters: %{[:children] => Ash.Expr.expr(visible == true)} + ) + end), + new("filter.parent_join", :filters, %{1 => 7, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + join_filters: %{[:children] => Ash.Expr.expr(value >= parent(threshold))} + ) + end), + new("filter.fanout_count_records", :filters, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, :children, query: Ash.Query.filter(ctx.child, ratings.score > 5)) + end), + new("filter.fanout_read_control", :filters, [11, 12], fn ctx -> + ctx.child + |> Ash.Query.filter(ratings.score > 5) + |> Ash.Query.sort(:id) + |> Ash.read!(authorize?: false) + |> Enum.map(& &1.id) + end), + new("filter.sibling_independence", :filters, {4, 11}, fn ctx -> + row = + selected_parent(ctx) + |> Ash.Query.aggregate(:filtered, :sum, :children, + field: :value, + query: [filter: [visible: true]] + ) + |> Ash.Query.aggregate(:all, :sum, :children, field: :value) + |> Ash.read_one!(authorize?: false) + + {row.aggregates.filtered, row.aggregates.all} + end), + new("filter.or_exists", :filters, %{1 => 11, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + query: Ash.Query.filter(ctx.child, exists(ratings, score > 5) or value == 7) + ) + end), + new("filter.not_exists", :filters, %{1 => 7, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + query: Ash.Query.filter(ctx.child, not exists(ratings, score > 5)) + ) + end) + ] ++ fanout() + end + + defp fanout do + for {kind, expected} <- [sum: 4, avg: 3.666667, count: 2, list: [2, 2], custom: 4] do + new("filter.fanout_#{kind}", :filters, expected, fn ctx -> + if kind == :avg do + Ash.Seed.seed!(struct(ctx.adapter.resource(:rating), id: 105, child_id: 13, score: 8)) + end + + query = ctx.child |> Ash.Query.filter(ratings.score > 5) |> Ash.Query.sort(value: :asc) + + opts = + if kind == :custom, + do: [type: :integer, implementation: {ctx.adapter.custom_aggregate(), field: :value}], + else: [field: :value] + + loaded( + %{ctx | parent: selected_parent(ctx)}, + kind, + :children, + Keyword.put(opts, :query, query) + ) + |> Map.fetch!(1) + end) + end + end +end diff --git a/conformance/lib/scenarios/helpers.ex b/conformance/lib/scenarios/helpers.ex new file mode 100644 index 0000000..81ab2ed --- /dev/null +++ b/conformance/lib/scenarios/helpers.ex @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Helpers do + @moduledoc false + require Ash.Query + + def loaded(context, kind, path \\ :children, opts \\ [], read_opts \\ []) do + context.parent + |> Ash.Query.build(aggregate: {:result, kind, path, opts}) + |> Ash.Query.sort(:id) + |> Ash.read!(Keyword.put_new(read_opts, :authorize?, false)) + |> Map.new(&{&1.id, normalize(Map.fetch!(&1.aggregates, :result))}) + end + + def root(context, kind, opts \\ []) do + Ash.aggregate!(context.child, [{:result, kind, opts}], authorize?: false) + |> Map.fetch!(:result) + |> normalize() + end + + def named(context, name, read_opts \\ []) do + context.parent + |> Ash.Query.load(name) + |> Ash.read!(Keyword.put_new(read_opts, :authorize?, false)) + |> Map.new(&{&1.id, Map.fetch!(&1, name)}) + end + + def relationship_ids(context, name, read_opts \\ []) do + context.parent + |> Ash.Query.load(name) + |> Ash.read!(Keyword.put_new(read_opts, :authorize?, false)) + |> Map.new(fn row -> + ids = row |> Map.fetch!(name) |> List.wrap() |> Enum.map(& &1.id) |> Enum.sort() + {row.id, ids} + end) + end + + def selected_parent(context, id \\ 1), do: Ash.Query.filter(context.parent, id == ^id) + + def normalize(%Decimal{} = value), do: value |> Decimal.to_float() |> Float.round(6) + def normalize(value) when is_float(value), do: Float.round(value, 6) + def normalize(value) when is_list(value), do: Enum.map(value, &normalize/1) + def normalize(value), do: value +end diff --git a/conformance/lib/scenarios/operations.ex b/conformance/lib/scenarios/operations.ex new file mode 100644 index 0000000..cbbe648 --- /dev/null +++ b/conformance/lib/scenarios/operations.ex @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Operations do + @moduledoc false + import AshSql.Conformance.Scenario, only: [new: 4] + import AshSql.Conformance.Scenarios.Helpers + require Ash.Query + require Ash.Sort + + def all do + loaded_kinds() ++ root_kinds() ++ semantics() ++ query_uses() + end + + defp loaded_kinds do + for {kind, expected, opts} <- [ + {:count, %{1 => 4, 2 => 1, 3 => 0}, []}, + {:sum, %{1 => 11, 2 => 4, 3 => nil}, [field: :value]}, + {:avg, %{1 => 3.666667, 2 => 4.0, 3 => nil}, [field: :value]}, + {:min, %{1 => 2, 2 => 4, 3 => nil}, [field: :value]}, + {:max, %{1 => 7, 2 => 4, 3 => nil}, [field: :value]}, + {:exists, %{1 => true, 2 => true, 3 => false}, []}, + {:first, %{1 => 2, 2 => 4, 3 => nil}, [field: :value, query: [sort: [value: :asc]]]}, + {:list, %{1 => [2, 2, 7], 2 => [4], 3 => []}, + [field: :value, query: [sort: [value: :asc]]]}, + {:custom, %{1 => 11, 2 => 4, 3 => nil}, []} + ] do + new("loaded.#{kind}", :operations, expected, fn ctx -> + opts = + if kind == :custom, + do: [type: :integer, implementation: {ctx.adapter.custom_aggregate(), field: :value}], + else: opts + + loaded(ctx, kind, :children, opts) + end) + end + end + + defp root_kinds do + for {kind, expected, opts} <- [ + {:count, 5, []}, + {:sum, 15, [field: :value]}, + {:avg, 3.75, [field: :value]}, + {:min, 2, [field: :value]}, + {:max, 7, [field: :value]}, + {:exists, true, []}, + {:first, 7, [field: :value, query: [sort: [value: :desc]]]}, + {:list, [2, 2, 4, 7], [field: :value, query: [sort: [value: :asc]]]}, + {:custom, 15, []} + ] do + new("root.#{kind}", :operations, expected, fn ctx -> + opts = + if kind == :custom, + do: [type: :integer, implementation: {ctx.adapter.custom_aggregate(), field: :value}], + else: opts + + root(ctx, kind, opts) + end) + end + end + + defp semantics do + [ + new("values.field_count", :results, %{1 => 3, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :children, field: :value) + end), + new("values.distinct_count", :results, %{1 => 2, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :children, field: :value, uniq?: true) + end), + new("values.distinct_list", :results, %{1 => [2, 7], 2 => [4], 3 => []}, fn ctx -> + loaded(ctx, :list, :children, field: :value, uniq?: true, query: [sort: [value: :asc]]) + end), + new( + "values.include_nil_list", + :results, + %{1 => [nil, 2, 2, 7], 2 => [4], 3 => []}, + fn ctx -> + loaded(ctx, :list, :children, + field: :value, + include_nil?: true, + query: [sort: [value: :asc_nils_first]] + ) + end + ), + new("values.include_nil_first", :results, %{1 => nil, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :first, :children, + field: :value, + include_nil?: true, + query: [sort: [value: :asc_nils_first]] + ) + end), + new("values.scalar_default", :results, %{1 => 11, 2 => 4, 3 => 0}, fn ctx -> + loaded(ctx, :sum, :children, field: :value, default: 0) + end), + new("values.list_default", :results, %{1 => [2, 2, 7], 2 => [4], 3 => [99]}, fn ctx -> + loaded(ctx, :list, :children, field: :value, default: [99], query: [sort: [value: :asc]]) + end), + new("values.filtered_first_default", :results, %{1 => 99, 2 => 99, 3 => 99}, fn ctx -> + loaded(ctx, :first, :children, + field: :value, + default: 99, + query: [filter: [value: [gt: 100]]] + ) + end), + new( + "values.constrained_scalar", + :results, + %{1 => quantity(11), 2 => quantity(4), 3 => quantity(0)}, + fn ctx -> + loaded(ctx, :sum, :children, + field: :value, + default: 0, + type: AshSql.Conformance.Quantity, + constraints: [unit: :points] + ) + end + ), + new( + "values.string_constraints", + :results, + %{1 => ["", " padded ", "high", "same", "same"], 2 => ["other"], 3 => []}, + fn ctx -> + for {id, label} <- [{15, ""}, {16, " padded "}] do + Ash.Seed.seed!(struct(ctx.child, id: id, parent_id: 1, label: label)) + end + + loaded(ctx, :list, :children, field: :label, query: [sort: [label: :asc]]) + end + ), + new( + "values.root_empty", + :results, + %{count: 0, sum: nil, first: nil, exists: false}, + fn ctx -> + ctx.child + |> Ash.Query.filter(id < 0) + |> Ash.aggregate!( + [ + {:count, :count}, + {:sum, :sum, field: :value}, + {:first, :first, field: :value, query: [sort: [value: :asc]]}, + {:exists, :exists} + ], + authorize?: false + ) + end + ), + new("root.unsorted_first_empty", :operations, nil, fn ctx -> + root(%{ctx | child: Ash.Query.filter(ctx.child, id < 0)}, :first, field: :value) + end), + new("values.same_name_distinct_definitions", :results, {11, 7}, fn ctx -> + query = + selected_parent(ctx) |> Ash.Query.aggregate(:total, :sum, :children, field: :value) + + original = Ash.read_one!(query, authorize?: false) + + replaced = + query + |> Ash.Query.aggregate(:total, :sum, :children, + field: :value, + query: [filter: [value: [gt: 3]]] + ) + |> Ash.read_one!(authorize?: false) + + {original.aggregates.total, replaced.aggregates.total} + end), + new("values.string_name", :results, 11, fn ctx -> + result = + selected_parent(ctx) + |> Ash.Query.aggregate("total", :sum, :children, field: :value) + |> Ash.read_one!(authorize?: false) + + result.aggregates["total"] + end) + ] ++ null_sorts() + end + + defp null_sorts do + for {order, expected} <- [ + asc_nils_first: nil, + asc_nils_last: 2, + desc_nils_first: nil, + desc_nils_last: 7 + ] do + new("ordering.#{order}", :ordering, expected, fn ctx -> + ctx = %{ctx | parent: selected_parent(ctx)} + + loaded(ctx, :first, :children, + field: :value, + include_nil?: true, + query: [sort: [value: order]] + ) + |> Map.fetch!(1) + end) + end + end + + defp query_uses do + [ + new("use.filter", :usage, [1], fn ctx -> + ctx.parent + |> Ash.Query.filter(child_sum > 5) + |> Ash.read!(authorize?: false) + |> Enum.map(& &1.id) + end), + new("use.sort", :usage, [2, 1, 3], fn ctx -> + ctx.parent + |> Ash.Query.sort(child_sum: :asc_nils_last) + |> Ash.read!(authorize?: false) + |> Enum.map(& &1.id) + end), + new("use.calculation", :usage, %{1 => 14, 2 => 9, 3 => 9}, fn ctx -> + ctx.parent + |> Ash.Query.load(:sum_plus_threshold) + |> Ash.read!(authorize?: false) + |> Map.new(&{&1.id, &1.sum_plus_threshold}) + end), + new("use.pagination", :usage, {[2], 3}, fn ctx -> + page = + ctx.parent + |> Ash.Query.for_read(:paged) + |> Ash.Query.load(:child_count) + |> Ash.Query.sort(child_count: :desc) + |> Ash.read!(page: [offset: 1, limit: 1, count: true], authorize?: false) + + {Enum.map(page.results, & &1.id), page.count} + end), + new("field.calculation", :expressions, %{1 => 22, 2 => 8, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, field: :double_value) + end), + new("field.aggregate", :expressions, %{1 => 4, 2 => 0, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :children, field: :rating_count) + end), + new("field.root_aggregate", :expressions, 5, fn ctx -> + Ash.aggregate!(ctx.parent, [{:result, :sum, field: :child_count}], authorize?: false).result + end), + new("ordering.expression_first", :ordering, %{1 => 7, 2 => 4, 3 => nil}, fn ctx -> + loaded(ctx, :first, :children, + field: :value, + query: Ash.Query.sort(ctx.child, [{Ash.Sort.expr_sort(value * -1, :integer), :asc}]) + ) + end), + new("ordering.expression_list", :ordering, %{1 => [7, 2, 2], 2 => [4], 3 => []}, fn ctx -> + loaded(ctx, :list, :children, + field: :value, + query: Ash.Query.sort(ctx.child, [{Ash.Sort.expr_sort(value * -1, :integer), :asc}]) + ) + end), + new("ordering.ties", :ordering, %{1 => [12, 11, 13], 2 => [21], 3 => []}, fn ctx -> + loaded(ctx, :list, :children, + field: :id, + query: [filter: [value: [is_nil: false]], sort: [value: :asc, id: :desc]] + ) + end) + ] + end + + defp quantity(value), do: %AshSql.Conformance.Quantity{value: value, unit: :points} +end diff --git a/conformance/lib/scenarios/relationships.ex b/conformance/lib/scenarios/relationships.ex new file mode 100644 index 0000000..b90d085 --- /dev/null +++ b/conformance/lib/scenarios/relationships.ex @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Scenarios.Relationships do + @moduledoc false + import AshSql.Conformance.Scenario, only: [new: 4] + import AshSql.Conformance.Scenarios.Helpers + require Ash.Query + + def all do + [ + new("path.to_one", :relationships, %{11 => 3, 12 => 3, 13 => 3, 14 => 3, 21 => 5}, fn ctx -> + loaded(%{ctx | parent: ctx.child}, :sum, :parent, field: :threshold) + end), + new("path.multi_hop", :relationships, %{1 => 26, 2 => nil, 3 => nil}, fn ctx -> + loaded(ctx, :sum, [:children, :ratings], field: :score) + end), + new("path.many_to_many", :relationships, %{1 => 11, 2 => 3, 3 => nil}, fn ctx -> + loaded(ctx, :sum, :tags, field: :value) + end), + new("path.many_to_many_list", :relationships, %{1 => [3, 8], 2 => [3], 3 => []}, fn ctx -> + loaded(ctx, :list, :tags, field: :value, query: [sort: [value: :asc]]) + end), + new("path.many_to_many_first", :relationships, %{1 => 8, 2 => 3, 3 => nil}, fn ctx -> + loaded(ctx, :first, :tags, field: :value, query: [sort: [value: :desc]]) + end), + new( + "path.final_many_to_many_scalar", + :relationships, + %{1 => 11, 2 => nil, 3 => nil}, + fn ctx -> + loaded(ctx, :sum, [:children, :tags], field: :value) + end + ), + new( + "path.final_many_to_many_first", + :relationships, + %{1 => 8, 2 => nil, 3 => nil}, + fn ctx -> + loaded(ctx, :first, [:children, :tags], field: :value, query: [sort: [value: :desc]]) + end + ), + new( + "path.final_many_to_many_list", + :relationships, + %{1 => [3, 8], 2 => [], 3 => []}, + fn ctx -> + loaded(ctx, :list, [:children, :tags], field: :value, query: [sort: [value: :asc]]) + end + ), + new( + "path.final_many_to_many_custom", + :relationships, + %{1 => 11, 2 => nil, 3 => nil}, + fn ctx -> + loaded(ctx, :custom, [:children, :tags], + type: :integer, + implementation: {ctx.adapter.custom_aggregate(), field: :value} + ) + end + ), + new("path.intermediate_many_to_many", :relationships, %{1 => 4, 2 => 2, 3 => nil}, fn ctx -> + loaded(ctx, :sum, [:tags, :children], field: :value) + end), + new("path.repeated_many_to_many", :relationships, :unresolved, fn ctx -> + loaded(ctx, :count, [:tags, :parents]) + end), + new("path.unrelated", :relationships, %{1 => 5, 2 => 5, 3 => 5}, fn ctx -> + loaded(ctx, :count, ctx.child) + end), + new("path.no_attributes", :relationships, %{1 => 5, 2 => 5, 3 => 5}, fn ctx -> + loaded(ctx, :count, :all_children) + end), + new( + "path.no_attributes_control", + :relationships, + %{1 => [11, 12, 13, 14, 21], 2 => [11, 12, 13, 14, 21], 3 => [11, 12, 13, 14, 21]}, + fn ctx -> + relationship_ids(ctx, :all_children) + end + ), + new("path.no_attributes_parent", :relationships, %{1 => 2, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :matching_children) + end), + new("path.manual", :relationships, %{1 => 4, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :manual_children) + end), + new("path.root_relationship", :relationships, 15, fn ctx -> + Ash.aggregate!( + ctx.parent, + [ + {:result, :sum, path: [:children], field: :value} + ], + authorize?: false + ).result + end), + new("identity.composite_count", :identity, %{1 => 2, 2 => 1, 3 => 0}, fn ctx -> + loaded(ctx, :count, :links, uniq?: true) + end), + new("identity.keyless_distinct", :identity, :unresolved, fn ctx -> + loaded(ctx, :count, :events, uniq?: true) + end), + new("identity.keyless_count", :identity, %{1 => 2, 2 => 0, 3 => 0}, fn ctx -> + loaded(ctx, :count, :events) + end), + new("identity.root_composite_count", :identity, 3, fn ctx -> + root(%{ctx | child: ctx.adapter.resource(:link)}, :count, uniq?: true) + end), + new("identity.composite_fanout_count", :identity, %{1 => 2, 2 => 1, 3 => 0}, fn ctx -> + query = Ash.Query.filter(ctx.adapter.resource(:link), parent.children.value > 0) + loaded(ctx, :count, :links, query: query) + end), + new("identity.keyless_source", :identity, [3, 3], fn ctx -> + ctx.adapter.resource(:event) + |> Ash.Query.aggregate(:result, :sum, :parent, field: :threshold) + |> Ash.read!(authorize?: false) + |> Enum.map(& &1.aggregates.result) + end) + ] + end +end diff --git a/conformance/mix.exs b/conformance/mix.exs new file mode 100644 index 0000000..dd7e261 --- /dev/null +++ b/conformance/mix.exs @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.MixProject do + use Mix.Project + + def project do + [ + app: :ash_sql_conformance, + version: "0.1.0", + elixir: "~> 1.18", + elixirc_paths: ["lib"], + deps: deps(), + dialyzer: [plt_add_apps: [:ex_unit, :mix]], + aliases: [check: ["format --check-formatted", "credo --strict", "test"]] + ] + end + + def application, do: [extra_applications: [:logger]] + + def cli, do: [preferred_envs: [check: :test]] + + defp deps do + [ + {:ash_sql, path: "..", override: true}, + {:simple_sat, "~> 0.1"}, + {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, + {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, + {:ash_sqlite, + git: "https://github.com/wtsnz/ash_sqlite.git", + ref: "46a4b869450a2a961ef9af44b5b69da2d5aff29c"}, + {:ash_postgres, + git: "https://github.com/ash-project/ash_postgres.git", + ref: "945073e431ec6eb3fbbb831a8ce5b561d8f8cd35"} + ] + end +end diff --git a/conformance/mix.lock b/conformance/mix.lock new file mode 100644 index 0000000..d12a650 --- /dev/null +++ b/conformance/mix.lock @@ -0,0 +1,33 @@ +%{ + "ash": {:hex, :ash, "3.33.10", "bd26ca12d0c10dc32f936cd1546f30f2ce194bde5d88d7bd77839dd442b50e1a", [:mix], [{:crux, ">= 0.1.2 and < 1.0.0-0", [hex: :crux, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14", [hex: :ecto, repo: "hexpm", optional: false]}, {:ets, "~> 0.8", [hex: :ets, repo: "hexpm", optional: false]}, {:igniter, ">= 0.6.29 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:picosat_elixir, "~> 0.2", [hex: :picosat_elixir, repo: "hexpm", optional: true]}, {:plug, ">= 0.0.0", [hex: :plug, repo: "hexpm", optional: true]}, {:reactor, "~> 1.0", [hex: :reactor, repo: "hexpm", optional: false]}, {:simple_sat, ">= 0.1.1 and < 1.0.0-0", [hex: :simple_sat, repo: "hexpm", optional: true]}, {:spark, ">= 2.6.0", [hex: :spark, repo: "hexpm", optional: false]}, {:splode, "~> 0.3", [hex: :splode, repo: "hexpm", optional: false]}, {:stream_data, "~> 1.0", [hex: :stream_data, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b73ad7f1d00ec4d0341ecdb4e64748e11bcfef0f73af5a7b91fdea9d27fa3064"}, + "ash_postgres": {:git, "https://github.com/ash-project/ash_postgres.git", "945073e431ec6eb3fbbb831a8ce5b561d8f8cd35", [ref: "945073e431ec6eb3fbbb831a8ce5b561d8f8cd35"]}, + "ash_sqlite": {:git, "https://github.com/wtsnz/ash_sqlite.git", "46a4b869450a2a961ef9af44b5b69da2d5aff29c", [ref: "46a4b869450a2a961ef9af44b5b69da2d5aff29c"]}, + "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "cc_precompiler": {:hex, :cc_precompiler, "0.1.11", "8c844d0b9fb98a3edea067f94f616b3f6b29b959b6b3bf25fee94ffe34364768", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3427232caf0835f94680e5bcf082408a70b48ad68a5f5c0b02a3bea9f3a075b9"}, + "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, + "crux": {:hex, :crux, "0.1.4", "1fa21f5ca886d3498f83871a3ef19379b80abf8cfcf2ed32007e80279e714f1e", [:mix], [{:picosat_elixir, "~> 0.2", [hex: :picosat_elixir, repo: "hexpm", optional: true]}, {:simple_sat, ">= 0.1.1 and < 1.0.0-0", [hex: :simple_sat, repo: "hexpm", optional: true]}, {:stream_data, "~> 1.0", [hex: :stream_data, repo: "hexpm", optional: true]}], "hexpm", "ff7d880cf732d82360aa81e439b8d4831cd30768061c9233f90c97e10162b9c0"}, + "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, + "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "dialyxir": {:hex, :dialyxir, "1.4.8", "7ef671a8aff9948b091d8c30f09467fbb16e77305cda451bce48109a0f5e021c", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cbd5a851571e5dfeb32aaf2e840bfa98b7864cb3071bf2ef5d95d1276b12e072"}, + "ecto": {:hex, :ecto, "3.14.2", "99db28a864293a789c970651de711e3cae184291e0e7ea1166c54055ac41c1f3", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25d60b8c816a07d19d85b80bdf60978bd8b102209dda198d768cd7c6745339a6"}, + "ecto_sql": {:hex, :ecto_sql, "3.14.0", "06446ab8410d2f85bfbb80857ee224ab3b693700cbb38f6535d507449a627b2e", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.8", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f4d8d36faf294c9417b5a37ec7ac8217ee2abdef5fcf197ba690f361548d3949"}, + "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.25.0", "309898d694b17a8ca8cd88d648d3a7fe8479b04ea6831a18d97d56ed3541031b", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.14", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.22", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "7da65c7af38dccf228320db32f93ae49650b0afdd850a09fd2fb191554b3faf5"}, + "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, + "ets": {:hex, :ets, "0.9.0", "79c6a6c205436780486f72d84230c6cba2f8a9920456750ddd1e47389107d5fd", [:mix], [], "hexpm", "2861fdfb04bcaeff370f1a5904eec864f0a56dcfebe5921ea9aadf2a481c822b"}, + "exqlite": {:hex, :exqlite, "0.41.0", "f7b6d9730d19efd8a2c9d4624172e82d4f98f2094743429c111972994967b4e2", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a7e9b6bed529ab72aa07ed2a925ac109c27e6877a7a8af252361c396a4192855"}, + "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, + "iterex": {:hex, :iterex, "0.1.2", "58f9b9b9a22a55cbfc7b5234a9c9c63eaac26d276b3db80936c0e1c60355a5a6", [:mix], [], "hexpm", "2e103b8bcc81757a9af121f6dc0df312c9a17220f302b1193ef720460d03029d"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "multigraph": {:hex, :multigraph, "0.16.1-mg.5", "5118665ea3e2eede7244104ccc716b7c517bba44e33c26fb84d7297d77872646", [:mix], [], "hexpm", "d7e3511f1d29e96ed7e7cd01b92bb52eea487b6c61da838f1e61a2ec6838799d"}, + "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, + "reactor": {:hex, :reactor, "1.0.7", "804bd237609df9162d7c61f68c6243722ef3ea2f9c48e693a04b7f09cf99e4d6", [:mix], [{:igniter, "~> 0.4", [hex: :igniter, repo: "hexpm", optional: true]}, {:iterex, "~> 0.1", [hex: :iterex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:multigraph, "~> 0.16.1-mg.2", [hex: :multigraph, repo: "hexpm", optional: false]}, {:spark, ">= 2.3.3 and < 3.0.0-0", [hex: :spark, repo: "hexpm", optional: false]}, {:splode, "~> 0.2", [hex: :splode, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.2", [hex: :telemetry, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}, {:ymlr, "~> 5.0", [hex: :ymlr, repo: "hexpm", optional: false]}], "hexpm", "d5eddbdcdf1016d76d713c927741f6db21421271cea015b0d355d6b1097206a4"}, + "simple_sat": {:hex, :simple_sat, "0.1.4", "39baf72cdca14f93c0b6ce2b6418b72bbb67da98fa9ca4384e2f79bbc299899d", [:mix], [], "hexpm", "3569b68e346a5fd7154b8d14173ff8bcc829f2eb7b088c30c3f42a383443930b"}, + "spark": {:hex, :spark, "2.7.3", "006c41968d557f4c82ebad5dad29656d604ff59a0f8e705a75f0cc70b76c8e7c", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: true]}], "hexpm", "ffced2f13ee13695f3a7284e5ae03724adb0014bd4b65b2cf40ee48a878d5d05"}, + "splode": {:hex, :splode, "0.3.2", "7716b6b2260a98a6f018c65cc0393da2cdf17314202eb351a0956e8762190dff", [:mix], [], "hexpm", "08fd658f80da7f1cd254b149164dcff8acd44b22f032001d5416b97223d32bc9"}, + "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.12.2", "9dd1330fb4cd9a36a7b0f502e5b12486eff632792ee4a5f0eba52a4d4ec32c9c", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "e7c1b10122f973e6558462d51c39026ba0e14afbc6745318e990ea82cfe9e159"}, + "ymlr": {:hex, :ymlr, "5.1.6", "247479328fd5ea1e222ee59ecd5571ab2885cf1a6329aea458d4944cd035cfc1", [:mix], [], "hexpm", "b36d7c72384d2cb2094d2a3c8b0c3b12303e0ccd1b7b12333b81d75b1fe0b53a"}, +} diff --git a/conformance/mix.lock.license b/conformance/mix.lock.license new file mode 100644 index 0000000..30df3ae --- /dev/null +++ b/conformance/mix.lock.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 ash_sql contributors + +SPDX-License-Identifier: MIT diff --git a/conformance/test/aggregate_test.exs b/conformance/test/aggregate_test.exs new file mode 100644 index 0000000..f2d28b8 --- /dev/null +++ b/conformance/test/aggregate_test.exs @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.AggregateTest do + use ExUnit.Case, async: false + alias AshSql.Conformance.{Adapter, Catalog, Expectations, Fixtures, Runner} + + for adapter <- Adapter.selected(), scenario <- Catalog.all() do + @tag adapter: adapter.id(), scenario: scenario.id, area: scenario.area + test "#{adapter.id()} #{scenario.id}" do + adapter = unquote(adapter) + scenario = Enum.find(Catalog.all(), &(&1.id == unquote(scenario.id))) + :ok = adapter.checkout!() + + try do + context = Fixtures.seed!(adapter) + Runner.run!(scenario, Expectations.for(scenario.id, adapter.id()), context) + after + adapter.checkin!() + end + end + end +end diff --git a/conformance/test/catalog_test.exs b/conformance/test/catalog_test.exs new file mode 100644 index 0000000..f87c686 --- /dev/null +++ b/conformance/test/catalog_test.exs @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.CatalogTest do + use ExUnit.Case, async: true + alias AshSql.Conformance.{Adapter, Catalog, Expectations, Report} + + test "every unique scenario has exactly one explicit status per adapter" do + ids = Enum.map(Catalog.all(), & &1.id) + assert length(ids) == length(Enum.uniq(ids)) + assert Enum.sort(ids) == Enum.sort(Map.keys(Expectations.all())) + adapters = Adapter.all() |> Enum.map(& &1.id()) |> Enum.sort() + + for scenario <- Catalog.all() do + statuses = Map.fetch!(Expectations.all(), scenario.id) + assert Enum.sort(Map.keys(statuses)) == adapters + + for {_adapter, expectation} <- statuses do + case expectation do + :supported -> + refute scenario.expected == :unresolved + + {:unsupported, {:error, exception, %Regex{}}, task} -> + refute scenario.expected == :unresolved + assert is_atom(exception) and is_binary(task) + + {:known_defect, signature, task} -> + refute scenario.expected == :unresolved + refute signature == {:value, scenario.expected} + assert is_binary(task) + + {:unresolved, _signature, task} -> + assert scenario.expected == :unresolved + assert is_binary(task) + end + end + end + end + + test "missing scenarios and adapters cannot silently default to supported" do + assert_raise KeyError, fn -> Expectations.for("missing", :sqlite) end + assert_raise KeyError, fn -> Expectations.for("loaded.count", :missing) end + end + + test "gap links resolve to a decision or implementation task" do + headings = + File.read!("GAPS.md") + |> String.split("\n") + |> Enum.filter(&String.starts_with?(&1, "## ")) + |> Enum.map(fn heading -> + heading |> String.trim_leading("## ") |> String.downcase() |> String.replace(" ", "-") + end) + + for %{task: task} <- Report.declaration_rows(), not is_nil(task) do + assert "GAPS.md#" <> anchor = task + assert anchor in headings, "Missing task: #{task}" + end + end + + test "the checked-in matrix matches the executable declarations" do + assert File.read!("MATRIX.md") == Report.matrix(), "Run mix conformance.matrix" + end +end diff --git a/conformance/test/runner_test.exs b/conformance/test/runner_test.exs new file mode 100644 index 0000000..535f2a8 --- /dev/null +++ b/conformance/test/runner_test.exs @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.RunnerTest do + use ExUnit.Case, async: true + alias AshSql.Conformance.{Runner, Scenario} + + defp scenario, do: Scenario.new("example", :runner, 7, fn _ -> 7 end) + + test "supported scenarios must return the shared expected value" do + assert Runner.assert_outcome!(scenario(), :supported, {:ok, 7}) + + assert_raise ExUnit.AssertionError, ~r/expected 7, got 8/, fn -> + Runner.assert_outcome!(scenario(), :supported, {:ok, 8}) + end + end + + test "supported exceptions fail instead of becoming skips" do + assert_raise ExUnit.AssertionError, ~r/raised ArgumentError/, fn -> + Runner.assert_outcome!(scenario(), :supported, {:error, ArgumentError, "broken"}) + end + end + + test "unsupported errors require both the exception class and message" do + status = {:unsupported, {:error, ArgumentError, ~r/^feature unavailable$/}, "GAPS.md#example"} + + assert Runner.assert_outcome!( + scenario(), + status, + {:error, ArgumentError, "feature unavailable"} + ) + + for outcome <- [ + {:error, RuntimeError, "feature unavailable"}, + {:error, ArgumentError, "database offline"} + ] do + assert_raise ExUnit.AssertionError, ~r/signature changed/, fn -> + Runner.assert_outcome!(scenario(), status, outcome) + end + end + end + + test "unexpected passes require promotion" do + for status <- [:unsupported, :known_defect] do + assert_raise ExUnit.AssertionError, ~r/unexpected pass/, fn -> + Runner.assert_outcome!(scenario(), {status, {:value, 6}, "GAPS.md#example"}, {:ok, 7}) + end + end + end + + test "known wrong results cannot mask a different wrong result or exception" do + status = {:known_defect, {:value, 6}, "GAPS.md#example"} + assert Runner.assert_outcome!(scenario(), status, {:ok, 6}) + + for outcome <- [{:ok, 5}, {:error, ArgumentError, "broken"}] do + assert_raise ExUnit.AssertionError, ~r/signature changed/, fn -> + Runner.assert_outcome!(scenario(), status, outcome) + end + end + end + + test "unresolved semantics preserve a strict observation and a decision link" do + status = {:unresolved, {:value, [1, 2]}, "GAPS.md#semantics"} + assert Runner.assert_outcome!(%{scenario() | expected: :unresolved}, status, {:ok, [1, 2]}) + + assert_raise ExUnit.AssertionError, ~r/signature changed/, fn -> + Runner.assert_outcome!(%{scenario() | expected: :unresolved}, status, {:ok, [2, 1]}) + end + end + + test "all nonconforming statuses require a task or decision" do + assert_raise ExUnit.AssertionError, ~r/link/, fn -> + Runner.assert_outcome!(scenario(), {:known_defect, {:value, 6}, ""}, {:ok, 6}) + end + end + + test "operation errors are captured without catching exits" do + assert {:error, ArgumentError, "bad input"} = + Runner.capture(fn -> raise ArgumentError, "bad input" end) + + assert catch_exit(Runner.capture(fn -> exit(:database_down) end)) == :database_down + end +end diff --git a/conformance/test/test_helper.exs b/conformance/test/test_helper.exs new file mode 100644 index 0000000..8e62a42 --- /dev/null +++ b/conformance/test/test_helper.exs @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT +ExUnit.start(formatters: [ExUnit.CLIFormatter, AshSql.Conformance.Formatter]) + +for adapter <- AshSql.Conformance.Adapter.selected() do + adapter.setup!() +end From 01998bc479f2b7b41d19157b70c5fdef137cf28d Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 23:19:41 -0700 Subject: [PATCH 2/8] test: make string constraint scenario independent of collation --- conformance/README.md | 4 +++- conformance/lib/scenarios/operations.ex | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conformance/README.md b/conformance/README.md index de7573d..d1a6173 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -83,7 +83,9 @@ controls distinguish aggregate defects from broader read behavior. Explicit null ordering avoids treating database defaults as an Ash parity contract. Numerical helpers round floating-point and decimal aggregate values to six decimal places for comparisons. Constrained custom scalar values retain their -struct and constraints; string whitespace and nils remain unchanged. +struct and constraints; string whitespace and nils remain unchanged. String +preservation uses numeric record ordering so database text collations do not +change its expected result. This is a representative scenario catalog with deliberate interaction cases, not every possible combination. Schema-based Postgres tenancy, arbitrary diff --git a/conformance/lib/scenarios/operations.ex b/conformance/lib/scenarios/operations.ex index cbbe648..8477f59 100644 --- a/conformance/lib/scenarios/operations.ex +++ b/conformance/lib/scenarios/operations.ex @@ -119,13 +119,14 @@ defmodule AshSql.Conformance.Scenarios.Operations do new( "values.string_constraints", :results, - %{1 => ["", " padded ", "high", "same", "same"], 2 => ["other"], 3 => []}, + %{1 => ["same", "same", "high", "", " padded "], 2 => ["other"], 3 => []}, fn ctx -> for {id, label} <- [{15, ""}, {16, " padded "}] do Ash.Seed.seed!(struct(ctx.child, id: id, parent_id: 1, label: label)) end - loaded(ctx, :list, :children, field: :label, query: [sort: [label: :asc]]) + # Check string preservation independently of the database's text collation. + loaded(ctx, :list, :children, field: :label, query: [sort: [id: :asc]]) end ), new( From 7281bb0ab638eba5e09c732bb7f63dc143867724 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 23:42:07 -0700 Subject: [PATCH 3/8] test: show and compare aggregate results in CI --- .github/workflows/aggregate-conformance.yml | 63 ++++- conformance/MATRIX.md | 246 +++++++++--------- conformance/README.md | 51 +++- conformance/lib/formatter.ex | 55 ++++ .../lib/mix/tasks/conformance.compare.ex | 48 ++++ conformance/lib/report.ex | 177 ++++++++++--- conformance/lib/report/comparison.ex | 114 ++++++++ conformance/lib/scenario.ex | 23 +- conformance/test/aggregate_test.exs | 7 +- conformance/test/catalog_test.exs | 17 ++ conformance/test/report_test.exs | 132 ++++++++++ conformance/test/runner_test.exs | 23 ++ 12 files changed, 783 insertions(+), 173 deletions(-) create mode 100644 conformance/lib/formatter.ex create mode 100644 conformance/lib/mix/tasks/conformance.compare.ex create mode 100644 conformance/lib/report/comparison.ex create mode 100644 conformance/test/report_test.exs diff --git a/.github/workflows/aggregate-conformance.yml b/.github/workflows/aggregate-conformance.yml index a5c8424..bcb3302 100644 --- a/.github/workflows/aggregate-conformance.yml +++ b/.github/workflows/aggregate-conformance.yml @@ -43,6 +43,11 @@ jobs: PGPASSWORD: postgres steps: - uses: actions/checkout@v4 + - name: Fetch PR comparison base + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: git fetch --no-tags --depth=1 origin "$BASE_SHA" - uses: erlef/setup-beam@v1 with: otp-version: '27.3.4.3' @@ -52,9 +57,65 @@ jobs: path: | conformance/deps conformance/_build - key: conformance-${{ runner.os }}-otp27-elixir1.18-${{ hashFiles('conformance/mix.lock') }} + key: conformance-${{ runner.os }}-otp27-elixir1.18-${{ hashFiles('conformance/mix.lock') }}-${{ matrix.adapter }} + restore-keys: | + conformance-${{ runner.os }}-otp27-elixir1.18-${{ hashFiles('conformance/mix.lock') }} - run: mix deps.get --check-locked - run: mix check + - name: Run base aggregate suite + id: base + if: always() && github.event_name == 'pull_request' + continue-on-error: true + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_DIR: ${{ runner.temp }}/aggregate-conformance-base + run: | + base_suite=$(git ls-tree --full-tree --name-only "$BASE_SHA" -- conformance/mix.exs) + if [ -z "$base_suite" ]; then + echo "available=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "available=true" >> "$GITHUB_OUTPUT" + git worktree add --detach "$BASE_DIR" "$BASE_SHA" + for directory in deps _build; do + if [ -d "$directory" ]; then + cp -a "$directory" "$BASE_DIR/conformance/" + fi + done + set +e + ( + set -e + cd "$BASE_DIR/conformance" + unset GITHUB_STEP_SUMMARY + export CONFORMANCE_PG_DATABASE=ash_sql_aggregate_conformance_base + mix deps.get --check-locked + mix deps.compile ash_sql --force + mix test + ) + base_exit_code=$? + set -e + echo "exit_code=$base_exit_code" >> "$GITHUB_OUTPUT" + mkdir -p results/base + if [ -d "$BASE_DIR/conformance/results" ]; then + cp -a "$BASE_DIR/conformance/results/." results/base/ + fi + exit "$base_exit_code" + - name: Compare aggregate results + if: always() && github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_AVAILABLE: ${{ steps.base.outputs.available }} + BASE_EXIT_CODE: ${{ steps.base.outputs.exit_code }} + run: | + if [ "$BASE_AVAILABLE" = "false" ]; then + mix conformance.compare --initial --base-ref "$BASE_SHA" + elif [ "$BASE_AVAILABLE" = "true" ]; then + mix conformance.compare --base "results/base/$CONFORMANCE_ADAPTERS.json" \ + --base-ref "$BASE_SHA" --base-exit-code "${BASE_EXIT_CODE:-1}" + else + echo "Base suite availability could not be determined; inspect the base step." + exit 1 + fi - run: mix dialyzer if: matrix.adapter == 'postgres' - uses: actions/upload-artifact@v4 diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md index bc13156..cc663ba 100644 --- a/conformance/MATRIX.md +++ b/conformance/MATRIX.md @@ -5,127 +5,129 @@ Generated by `mix conformance.matrix`. These are declared contracts; `results/` contains execution results. Passing an unsupported, known-defect or unresolved expectation does not establish feature conformance. +CI runs the base and PR suites in the same environment, compares their JSON results and shows the changes in its job summary. Elixir scenarios and expectations define the contract. + | Scenario | sqlite | postgres | | --- | --- | --- | -| `bounds.default_sort` | [known_defect](GAPS.md#default-sort) | [known_defect](GAPS.md#default-sort) | -| `bounds.default_sort_control` | supported | supported | -| `bounds.filter_after_limit` | supported | supported | -| `bounds.from_many` | [known_defect](GAPS.md#from-many) | [known_defect](GAPS.md#from-many) | -| `bounds.list_filter_after_limit` | supported | supported | -| `bounds.many_to_many_query_limit` | [unresolved](GAPS.md#many-to-many-bounds-api) | [unresolved](GAPS.md#many-to-many-bounds-api) | -| `bounds.relationship_limit` | supported | supported | -| `bounds.relationship_offset` | supported | supported | -| `bounds.relationship_offset_only` | supported | supported | -| `bounds.root_first_distinct_sort` | supported | [known_defect](GAPS.md#root-bounds) | -| `bounds.root_limit` | supported | supported | -| `bounds.root_offset_only` | supported | [known_defect](GAPS.md#root-bounds) | -| `bounds.root_order_then_limit` | supported | [known_defect](GAPS.md#root-bounds) | -| `bounds.root_zero` | supported | supported | -| `bounds.unsorted_limit` | [known_defect](GAPS.md#unsorted-bounds) | supported | -| `context.actor` | supported | supported | -| `context.arguments` | supported | supported | -| `context.attribute_tenant` | supported | supported | -| `context.authorization` | supported | supported | -| `context.authorization_before_bounds` | [known_defect](GAPS.md#authorization-bounds) | [known_defect](GAPS.md#authorization-bounds) | -| `context.authorization_bounds_control` | supported | supported | -| `context.bypass_sibling` | supported | [known_defect](GAPS.md#tenant-bypass) | -| `context.intermediate_action` | supported | supported | -| `context.intermediate_actor` | supported | supported | -| `context.prepared_context_control` | supported | supported | -| `context.prepared_query_arguments` | supported | [known_defect](GAPS.md#prepared-query) | -| `context.read_action` | supported | supported | -| `context.relationship_context` | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | -| `context.relationship_context_control` | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | -| `context.shared` | supported | supported | -| `context.tenant_bypass` | supported | [known_defect](GAPS.md#tenant-bypass) | -| `context.through_arguments` | supported | supported | -| `context.through_bypass` | supported | [known_defect](GAPS.md#tenant-bypass) | -| `context.through_tenant` | supported | supported | -| `field.aggregate` | supported | supported | -| `field.calculation` | supported | supported | -| `field.root_aggregate` | supported | supported | -| `filter.aggregate_dependency` | [unsupported](GAPS.md#filter-dependencies) | supported | -| `filter.exists` | supported | supported | -| `filter.fanout_avg` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | -| `filter.fanout_count` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | -| `filter.fanout_count_records` | supported | [known_defect](GAPS.md#filter-fanout) | -| `filter.fanout_custom` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | -| `filter.fanout_list` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | -| `filter.fanout_read_control` | [known_defect](GAPS.md#filter-fanout) | supported | -| `filter.fanout_sum` | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | -| `filter.join` | supported | supported | -| `filter.not_exists` | supported | supported | -| `filter.or_exists` | supported | supported | -| `filter.ordinary` | supported | supported | -| `filter.parent` | [unsupported](GAPS.md#parent-correlation) | supported | -| `filter.parent_join` | [unsupported](GAPS.md#parent-correlation) | supported | -| `filter.parent_relationship` | [unsupported](GAPS.md#parent-correlation) | supported | -| `filter.parent_unrelated` | [unsupported](GAPS.md#parent-correlation) | supported | -| `filter.sibling_independence` | supported | supported | -| `identity.composite_count` | [unsupported](GAPS.md#record-identity) | supported | -| `identity.composite_fanout_count` | [unsupported](GAPS.md#record-identity) | [known_defect](GAPS.md#filter-fanout) | -| `identity.keyless_count` | supported | supported | -| `identity.keyless_distinct` | [unresolved](GAPS.md#keyless-identity) | [unresolved](GAPS.md#keyless-identity) | -| `identity.keyless_source` | [unsupported](GAPS.md#record-identity) | supported | -| `identity.root_composite_count` | [unsupported](GAPS.md#record-identity) | supported | -| `loaded.avg` | supported | supported | -| `loaded.count` | supported | supported | -| `loaded.custom` | supported | supported | -| `loaded.exists` | supported | supported | -| `loaded.first` | supported | supported | -| `loaded.list` | supported | supported | -| `loaded.max` | supported | supported | -| `loaded.min` | supported | supported | -| `loaded.sum` | supported | supported | -| `ordering.asc_nils_first` | supported | supported | -| `ordering.asc_nils_last` | supported | supported | -| `ordering.desc_nils_first` | supported | supported | -| `ordering.desc_nils_last` | supported | supported | -| `ordering.expression_first` | supported | supported | -| `ordering.expression_list` | supported | supported | -| `ordering.ties` | supported | supported | -| `ordering.unique_other_field` | [unresolved](GAPS.md#unique-list-order) | [unresolved](GAPS.md#unique-list-order) | -| `path.final_many_to_many_custom` | [unsupported](GAPS.md#many-to-many-paths) | supported | -| `path.final_many_to_many_first` | [unsupported](GAPS.md#many-to-many-paths) | supported | -| `path.final_many_to_many_list` | [unsupported](GAPS.md#many-to-many-paths) | supported | -| `path.final_many_to_many_scalar` | supported | supported | -| `path.intermediate_many_to_many` | [unsupported](GAPS.md#many-to-many-paths) | supported | -| `path.manual` | [unsupported](GAPS.md#manual) | supported | -| `path.many_to_many` | supported | supported | -| `path.many_to_many_first` | supported | supported | -| `path.many_to_many_list` | supported | supported | -| `path.multi_hop` | supported | supported | -| `path.no_attributes` | [unsupported](GAPS.md#no-attributes) | [known_defect](GAPS.md#no-attributes) | -| `path.no_attributes_control` | supported | supported | -| `path.no_attributes_parent` | [unsupported](GAPS.md#no-attributes) | supported | -| `path.repeated_many_to_many` | [unresolved](GAPS.md#path-multiplicity) | [unresolved](GAPS.md#path-multiplicity) | -| `path.root_relationship` | [unsupported](GAPS.md#root-relationship) | [known_defect](GAPS.md#root-relationship) | -| `path.to_one` | supported | supported | -| `path.unrelated` | supported | supported | -| `root.avg` | supported | supported | -| `root.count` | supported | supported | -| `root.custom` | [unsupported](GAPS.md#root-kinds) | supported | -| `root.exists` | supported | supported | -| `root.first` | supported | supported | -| `root.list` | [unsupported](GAPS.md#root-kinds) | supported | -| `root.max` | supported | supported | -| `root.min` | supported | supported | -| `root.sum` | supported | supported | -| `root.unsorted_first_empty` | supported | [known_defect](GAPS.md#root-first) | -| `use.calculation` | supported | supported | -| `use.filter` | supported | supported | -| `use.pagination` | supported | supported | -| `use.sort` | supported | supported | -| `values.constrained_scalar` | supported | supported | -| `values.distinct_count` | supported | supported | -| `values.distinct_list` | supported | supported | -| `values.field_count` | supported | supported | -| `values.filtered_first_default` | supported | supported | -| `values.include_nil_first` | supported | supported | -| `values.include_nil_list` | supported | supported | -| `values.list_default` | supported | supported | -| `values.root_empty` | supported | supported | -| `values.same_name_distinct_definitions` | supported | supported | -| `values.scalar_default` | supported | supported | -| `values.string_constraints` | supported | supported | -| `values.string_name` | supported | supported | +| [`bounds.default_sort`](lib/scenarios/bounds.ex#L36) | [known_defect](GAPS.md#default-sort) | [known_defect](GAPS.md#default-sort) | +| [`bounds.default_sort_control`](lib/scenarios/bounds.ex#L39) | supported | supported | +| [`bounds.filter_after_limit`](lib/scenarios/bounds.ex#L21) | supported | supported | +| [`bounds.from_many`](lib/scenarios/bounds.ex#L33) | [known_defect](GAPS.md#from-many) | [known_defect](GAPS.md#from-many) | +| [`bounds.list_filter_after_limit`](lib/scenarios/bounds.ex#L24) | supported | supported | +| [`bounds.many_to_many_query_limit`](lib/scenarios/bounds.ex#L69) | [unresolved](GAPS.md#many-to-many-bounds-api) | [unresolved](GAPS.md#many-to-many-bounds-api) | +| [`bounds.relationship_limit`](lib/scenarios/bounds.ex#L12) | supported | supported | +| [`bounds.relationship_offset`](lib/scenarios/bounds.ex#L15) | supported | supported | +| [`bounds.relationship_offset_only`](lib/scenarios/bounds.ex#L18) | supported | supported | +| [`bounds.root_first_distinct_sort`](lib/scenarios/bounds.ex#L53) | supported | [known_defect](GAPS.md#root-bounds) | +| [`bounds.root_limit`](lib/scenarios/bounds.ex#L42) | supported | supported | +| [`bounds.root_offset_only`](lib/scenarios/bounds.ex#L45) | supported | [known_defect](GAPS.md#root-bounds) | +| [`bounds.root_order_then_limit`](lib/scenarios/bounds.ex#L49) | supported | [known_defect](GAPS.md#root-bounds) | +| [`bounds.root_zero`](lib/scenarios/bounds.ex#L57) | supported | supported | +| [`bounds.unsorted_limit`](lib/scenarios/bounds.ex#L30) | [known_defect](GAPS.md#unsorted-bounds) | supported | +| [`context.actor`](lib/scenarios/context.ex#L41) | supported | supported | +| [`context.arguments`](lib/scenarios/context.ex#L15) | supported | supported | +| [`context.attribute_tenant`](lib/scenarios/context.ex#L54) | supported | supported | +| [`context.authorization`](lib/scenarios/context.ex#L78) | supported | supported | +| [`context.authorization_before_bounds`](lib/scenarios/context.ex#L81) | [known_defect](GAPS.md#authorization-bounds) | [known_defect](GAPS.md#authorization-bounds) | +| [`context.authorization_bounds_control`](lib/scenarios/context.ex#L84) | supported | supported | +| [`context.bypass_sibling`](lib/scenarios/context.ex#L60) | supported | [known_defect](GAPS.md#tenant-bypass) | +| [`context.intermediate_action`](lib/scenarios/context.ex#L48) | supported | supported | +| [`context.intermediate_actor`](lib/scenarios/context.ex#L51) | supported | supported | +| [`context.prepared_context_control`](lib/scenarios/context.ex#L33) | supported | supported | +| [`context.prepared_query_arguments`](lib/scenarios/context.ex#L44) | supported | [known_defect](GAPS.md#prepared-query) | +| [`context.read_action`](lib/scenarios/context.ex#L12) | supported | supported | +| [`context.relationship_context`](lib/scenarios/context.ex#L18) | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | +| [`context.relationship_context_control`](lib/scenarios/context.ex#L21) | [known_defect](GAPS.md#relationship-context) | [known_defect](GAPS.md#relationship-context) | +| [`context.shared`](lib/scenarios/context.ex#L29) | supported | supported | +| [`context.tenant_bypass`](lib/scenarios/context.ex#L57) | supported | [known_defect](GAPS.md#tenant-bypass) | +| [`context.through_arguments`](lib/scenarios/context.ex#L69) | supported | supported | +| [`context.through_bypass`](lib/scenarios/context.ex#L75) | supported | [known_defect](GAPS.md#tenant-bypass) | +| [`context.through_tenant`](lib/scenarios/context.ex#L72) | supported | supported | +| [`field.aggregate`](lib/scenarios/operations.ex#L233) | supported | supported | +| [`field.calculation`](lib/scenarios/operations.ex#L230) | supported | supported | +| [`field.root_aggregate`](lib/scenarios/operations.ex#L236) | supported | supported | +| [`filter.aggregate_dependency`](lib/scenarios/filters.ex#L23) | [unsupported](GAPS.md#filter-dependencies) | supported | +| [`filter.exists`](lib/scenarios/filters.ex#L17) | supported | supported | +| [`filter.fanout_avg`](lib/scenarios/filters.ex#L92) | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| [`filter.fanout_count`](lib/scenarios/filters.ex#L92) | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| [`filter.fanout_count_records`](lib/scenarios/filters.ex#L53) | supported | [known_defect](GAPS.md#filter-fanout) | +| [`filter.fanout_custom`](lib/scenarios/filters.ex#L92) | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| [`filter.fanout_list`](lib/scenarios/filters.ex#L92) | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| [`filter.fanout_read_control`](lib/scenarios/filters.ex#L56) | [known_defect](GAPS.md#filter-fanout) | supported | +| [`filter.fanout_sum`](lib/scenarios/filters.ex#L92) | [unsupported](GAPS.md#filter-fanout) | [known_defect](GAPS.md#filter-fanout) | +| [`filter.join`](lib/scenarios/filters.ex#L41) | supported | supported | +| [`filter.not_exists`](lib/scenarios/filters.ex#L81) | supported | supported | +| [`filter.or_exists`](lib/scenarios/filters.ex#L75) | supported | supported | +| [`filter.ordinary`](lib/scenarios/filters.ex#L14) | supported | supported | +| [`filter.parent`](lib/scenarios/filters.ex#L26) | [unsupported](GAPS.md#parent-correlation) | supported | +| [`filter.parent_join`](lib/scenarios/filters.ex#L47) | [unsupported](GAPS.md#parent-correlation) | supported | +| [`filter.parent_relationship`](lib/scenarios/filters.ex#L32) | [unsupported](GAPS.md#parent-correlation) | supported | +| [`filter.parent_unrelated`](lib/scenarios/filters.ex#L35) | [unsupported](GAPS.md#parent-correlation) | supported | +| [`filter.sibling_independence`](lib/scenarios/filters.ex#L63) | supported | supported | +| [`identity.composite_count`](lib/scenarios/relationships.ex#L98) | [unsupported](GAPS.md#record-identity) | supported | +| [`identity.composite_fanout_count`](lib/scenarios/relationships.ex#L110) | [unsupported](GAPS.md#record-identity) | [known_defect](GAPS.md#filter-fanout) | +| [`identity.keyless_count`](lib/scenarios/relationships.ex#L104) | supported | supported | +| [`identity.keyless_distinct`](lib/scenarios/relationships.ex#L101) | [unresolved](GAPS.md#keyless-identity) | [unresolved](GAPS.md#keyless-identity) | +| [`identity.keyless_source`](lib/scenarios/relationships.ex#L114) | [unsupported](GAPS.md#record-identity) | supported | +| [`identity.root_composite_count`](lib/scenarios/relationships.ex#L107) | [unsupported](GAPS.md#record-identity) | supported | +| [`loaded.avg`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.count`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.custom`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.exists`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.first`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.list`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.max`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.min`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`loaded.sum`](lib/scenarios/operations.ex#L29) | supported | supported | +| [`ordering.asc_nils_first`](lib/scenarios/operations.ex#L187) | supported | supported | +| [`ordering.asc_nils_last`](lib/scenarios/operations.ex#L187) | supported | supported | +| [`ordering.desc_nils_first`](lib/scenarios/operations.ex#L187) | supported | supported | +| [`ordering.desc_nils_last`](lib/scenarios/operations.ex#L187) | supported | supported | +| [`ordering.expression_first`](lib/scenarios/operations.ex#L239) | supported | supported | +| [`ordering.expression_list`](lib/scenarios/operations.ex#L245) | supported | supported | +| [`ordering.ties`](lib/scenarios/operations.ex#L251) | supported | supported | +| [`ordering.unique_other_field`](lib/scenarios/bounds.ex#L73) | [unresolved](GAPS.md#unique-list-order) | [unresolved](GAPS.md#unique-list-order) | +| [`path.final_many_to_many_custom`](lib/scenarios/relationships.ex#L52) | [unsupported](GAPS.md#many-to-many-paths) | supported | +| [`path.final_many_to_many_first`](lib/scenarios/relationships.ex#L36) | [unsupported](GAPS.md#many-to-many-paths) | supported | +| [`path.final_many_to_many_list`](lib/scenarios/relationships.ex#L44) | [unsupported](GAPS.md#many-to-many-paths) | supported | +| [`path.final_many_to_many_scalar`](lib/scenarios/relationships.ex#L28) | supported | supported | +| [`path.intermediate_many_to_many`](lib/scenarios/relationships.ex#L63) | [unsupported](GAPS.md#many-to-many-paths) | supported | +| [`path.manual`](lib/scenarios/relationships.ex#L86) | [unsupported](GAPS.md#manual) | supported | +| [`path.many_to_many`](lib/scenarios/relationships.ex#L19) | supported | supported | +| [`path.many_to_many_first`](lib/scenarios/relationships.ex#L25) | supported | supported | +| [`path.many_to_many_list`](lib/scenarios/relationships.ex#L22) | supported | supported | +| [`path.multi_hop`](lib/scenarios/relationships.ex#L16) | supported | supported | +| [`path.no_attributes`](lib/scenarios/relationships.ex#L72) | [unsupported](GAPS.md#no-attributes) | [known_defect](GAPS.md#no-attributes) | +| [`path.no_attributes_control`](lib/scenarios/relationships.ex#L75) | supported | supported | +| [`path.no_attributes_parent`](lib/scenarios/relationships.ex#L83) | [unsupported](GAPS.md#no-attributes) | supported | +| [`path.repeated_many_to_many`](lib/scenarios/relationships.ex#L66) | [unresolved](GAPS.md#path-multiplicity) | [unresolved](GAPS.md#path-multiplicity) | +| [`path.root_relationship`](lib/scenarios/relationships.ex#L89) | [unsupported](GAPS.md#root-relationship) | [known_defect](GAPS.md#root-relationship) | +| [`path.to_one`](lib/scenarios/relationships.ex#L13) | supported | supported | +| [`path.unrelated`](lib/scenarios/relationships.ex#L69) | supported | supported | +| [`root.avg`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.count`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.custom`](lib/scenarios/operations.ex#L52) | [unsupported](GAPS.md#root-kinds) | supported | +| [`root.exists`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.first`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.list`](lib/scenarios/operations.ex#L52) | [unsupported](GAPS.md#root-kinds) | supported | +| [`root.max`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.min`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.sum`](lib/scenarios/operations.ex#L52) | supported | supported | +| [`root.unsorted_first_empty`](lib/scenarios/operations.ex#L150) | supported | [known_defect](GAPS.md#root-first) | +| [`use.calculation`](lib/scenarios/operations.ex#L214) | supported | supported | +| [`use.filter`](lib/scenarios/operations.ex#L202) | supported | supported | +| [`use.pagination`](lib/scenarios/operations.ex#L220) | supported | supported | +| [`use.sort`](lib/scenarios/operations.ex#L208) | supported | supported | +| [`values.constrained_scalar`](lib/scenarios/operations.ex#L106) | supported | supported | +| [`values.distinct_count`](lib/scenarios/operations.ex#L68) | supported | supported | +| [`values.distinct_list`](lib/scenarios/operations.ex#L71) | supported | supported | +| [`values.field_count`](lib/scenarios/operations.ex#L65) | supported | supported | +| [`values.filtered_first_default`](lib/scenarios/operations.ex#L99) | supported | supported | +| [`values.include_nil_first`](lib/scenarios/operations.ex#L86) | supported | supported | +| [`values.include_nil_list`](lib/scenarios/operations.ex#L74) | supported | supported | +| [`values.list_default`](lib/scenarios/operations.ex#L96) | supported | supported | +| [`values.root_empty`](lib/scenarios/operations.ex#L132) | supported | supported | +| [`values.same_name_distinct_definitions`](lib/scenarios/operations.ex#L153) | supported | supported | +| [`values.scalar_default`](lib/scenarios/operations.ex#L93) | supported | supported | +| [`values.string_constraints`](lib/scenarios/operations.ex#L119) | supported | supported | +| [`values.string_name`](lib/scenarios/operations.ex#L169) | supported | supported | diff --git a/conformance/README.md b/conformance/README.md index d1a6173..2b37c30 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -48,9 +48,22 @@ PostgreSQL server. ## Read the results -[MATRIX.md](MATRIX.md) lists the explicit expectations. The ExUnit formatter -writes `results/sqlite-postgres.json`, or a file named for the selected adapter. -CI uploads these reports even when a test fails. +[MATRIX.md](MATRIX.md) lists the explicit expectations. Each scenario ID links +to its declaration line, including declarations that generate several cases. +The generator captures those locations from the Elixir source. + +Test logs print each check's status, intended result and actual result. Known +gaps print their accepted signature as well. Only supported, matching results +say `PASS`; recorded gaps say `GAP MATCHED`, and unresolved cases say +`OBSERVATION MATCHED`. Failed checks retain the actual observation. + +GitHub's job summary shows counts and expandable result tables, with failures +expanded. The formatter writes the same Markdown to `results/sqlite-postgres.md` +and full JSON to `results/sqlite-postgres.json`, or files named for the selected +adapter. CI uploads both, including on test failures. Long errors are shortened +for display; JSON retains full error details. Stack frames are kept in `details` +but excluded from the comparable observation, so source line changes do not +look like changed results. Assertions still check the original complete error. | Status | A passing test means | | --- | --- | @@ -66,6 +79,36 @@ wrong value, unrelated exception, or database setup failure also fails. Unresolved cases link to a decision and make no correctness claim. There are no blanket skips and no automatic expected-result updates. +## Compare a PR + +CI checks out the PR base in a separate worktree and runs its suite in the same +job, with the same runtime and PostgreSQL service. The base uses its own locked +dependencies and AshSQL checkout. Copied dependency/build caches reduce repeat +compilation; the base gets its own dependency directory and test database. + +The comparison joins the two generated JSON reports by scenario ID and adapter. +GitHub shows added/removed checks and changed statuses, executions, intended +values, accepted signatures and actual observations. Full JSON remains an +artifact. Elixir scenarios and expectations are the source of truth; there is +no checked-in JSON snapshot to update. + +The first PR establishes the initial baseline when its base has no suite. A +base test failure does not stop comparison with a passing PR; its exit code and +recorded results are shown. Missing reports are errors, and older reports that +lack actual observations are explicitly labelled. These comparisons describe +aggregate scenarios, while the main test step still gates all test failures. + +No local work is required. To compare two saved reports locally: + +```sh +mise exec -- mix conformance.compare \ + --base results/base/sqlite-postgres.json \ + --current results/sqlite-postgres.json +``` + +Changes are also written to `results/changes-sqlite-postgres.md`. CI compares +each adapter separately and includes the base commit in the summary. + ## Coverage and boundaries The catalog covers all nine aggregate kinds, root and loaded execution, @@ -133,7 +176,7 @@ It intentionally excludes the separately stacked from_many and schema fixes. The newer locked Ash version can expose behavior different from older adapter test runs; the matrix describes this exact dependency set. -There are 122 shared scenarios and 12 runner/catalog tests. The initial run +There are 122 shared scenarios and 27 runner/catalog/report tests. The run matches all 244 adapter expectations: | Adapter | Supported | Unsupported | Known defect | Unresolved | diff --git a/conformance/lib/formatter.ex b/conformance/lib/formatter.ex new file mode 100644 index 0000000..4381da0 --- /dev/null +++ b/conformance/lib/formatter.ex @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Formatter do + @moduledoc false + use GenServer + alias AshSql.Conformance.Report + + def record(id, adapter, outcome) do + GenServer.call(__MODULE__, {:result, {id, adapter}, outcome}) + end + + def init(_opts) do + Process.register(self(), __MODULE__) + declarations = Map.new(Report.declaration_rows(), &{{&1.scenario, &1.adapter}, &1}) + {:ok, %{rows: [], outcomes: %{}, declarations: declarations}} + end + + def handle_call({:result, key, outcome}, _from, state) do + {:reply, :ok, put_in(state, [:outcomes, key], Report.observation(outcome))} + end + + def handle_cast( + {:test_finished, %{tags: %{scenario: id, adapter: adapter}} = test}, + state + ) do + key = {id, adapter} + + execution = + case test.state do + nil -> :matched + {:excluded, _} -> :excluded + {:skipped, _} -> :skipped + _ -> :failed + end + + row = + state.declarations + |> Map.fetch!(key) + |> Map.put(:execution, execution) + |> Map.merge( + Map.get(state.outcomes, key, %{actual: "No operation result recorded", details: nil}) + ) + + {:noreply, %{state | rows: [row | state.rows]}} + end + + def handle_cast({:suite_finished, _times}, state) do + Report.write_results!(state.rows) + {:noreply, state} + end + + def handle_cast(_event, state), do: {:noreply, state} +end diff --git a/conformance/lib/mix/tasks/conformance.compare.ex b/conformance/lib/mix/tasks/conformance.compare.ex new file mode 100644 index 0000000..a526d87 --- /dev/null +++ b/conformance/lib/mix/tasks/conformance.compare.ex @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule Mix.Tasks.Conformance.Compare do + @moduledoc false + use Mix.Task + alias AshSql.Conformance.{Adapter, Report} + alias AshSql.Conformance.Report.Comparison + @shortdoc "Compare actual aggregate JSON results from two runs" + + def run(args) do + {opts, []} = + OptionParser.parse!(args, + strict: [ + base: :string, + current: :string, + initial: :boolean, + base_ref: :string, + base_exit_code: :integer + ] + ) + + Mix.Task.run("compile") + name = Adapter.selected() |> Enum.map_join("-", & &1.id()) + current = read!(Keyword.get(opts, :current, "results/#{name}.json")) + + base = + case {opts[:initial], opts[:base]} do + {true, nil} -> :initial + {value, path} when value in [false, nil] and is_binary(path) -> read!(path) + _ -> Mix.raise("Specify --base or --initial when the base has no suite") + end + + comparison = Comparison.summary(base, current, opts) + File.mkdir_p!("results") + File.write!("results/changes-#{name}.md", comparison) + if path = Report.summary_path(), do: File.write!(path, comparison, [:append]) + Mix.shell().info(comparison) + end + + defp read!(path) do + unless File.regular?(path), + do: Mix.raise("Missing run results: #{path}; inspect the suite logs") + + path |> File.read!() |> Comparison.decode!() + end +end diff --git a/conformance/lib/report.ex b/conformance/lib/report.ex index 868baa9..4723ddd 100644 --- a/conformance/lib/report.ex +++ b/conformance/lib/report.ex @@ -15,7 +15,9 @@ defmodule AshSql.Conformance.Report do area: scenario.area, adapter: adapter.id(), status: status(expectation), - task: task(expectation) + task: task(expectation), + expected: expected(scenario), + accepted: accepted(scenario, expectation) } end end @@ -25,6 +27,30 @@ defmodule AshSql.Conformance.Report do def task(:supported), do: nil def task({_, _, task}), do: task + def outcome({:ok, actual}), do: value(actual) + + def outcome({:error, exception, message}) do + # Stack frames contain source line numbers and function hashes, not result semantics. + reason = message |> String.split(~r/\n\s+\([^)]+\) [^\n]*:\d+: /, parts: 2) |> hd() + "#{inspect(exception)}: #{String.trim(reason)}" + end + + def observation({:ok, _} = result), do: %{actual: outcome(result), details: nil} + + def observation({:error, exception, message} = result), + do: %{actual: outcome(result), details: "#{inspect(exception)}: #{message}"} + + defp expected(%{expected: :unresolved}), do: "Semantics unresolved" + defp expected(scenario), do: value(scenario.expected) + defp accepted(scenario, :supported), do: expected(scenario) + defp accepted(_, {_, {:value, actual}, _}), do: value(actual) + + defp accepted(_, {_, {:error, exception, pattern}, _}), + do: "#{inspect(exception)} matching #{inspect(pattern)}" + + defp value(value), + do: inspect(value, charlists: :as_lists, limit: :infinity, printable_limit: :infinity) + def matrix do rows = Enum.group_by(declaration_rows(), & &1.scenario) adapters = Enum.map(Adapter.all(), & &1.id()) @@ -37,6 +63,8 @@ defmodule AshSql.Conformance.Report do Generated by `mix conformance.matrix`. These are declared contracts; `results/` contains execution results. Passing an unsupported, known-defect or unresolved expectation does not establish feature conformance. + CI runs the base and PR suites in the same environment, compares their JSON results and shows the changes in its job summary. Elixir scenarios and expectations define the contract. + | Scenario | #{Enum.join(adapters, " | ")} | | --- | #{Enum.map_join(adapters, " | ", fn _ -> "---" end)} | """ @@ -49,12 +77,108 @@ defmodule AshSql.Conformance.Report do if row.task, do: "[#{row.status}](#{row.task})", else: to_string(row.status) end) - "| `#{scenario.id}` | #{Enum.join(cells, " | ")} |" + source = "#{scenario.source.file}#L#{scenario.source.line}" + "| [`#{scenario.id}`](#{source}) | #{Enum.join(cells, " | ")} |" end) header <> body <> "\n" end + def console(rows) do + rows + |> Enum.sort_by(&{&1.scenario, &1.adapter}) + |> Enum.map_join("\n", fn row -> + "[#{verdict(row)}] #{row.adapter} #{row.scenario} (#{row.status})\n" <> + " intended: #{compact(row.expected)}\n" <> + " observed: #{compact(row.actual)}" <> + if(row.status == :supported, do: "", else: "\n accepted: #{compact(row.accepted)}") + end) + end + + def summary(rows, comparison) do + counts = Enum.frequencies_by(rows, &{&1.adapter, &1.status, &1.execution}) + + count_rows = + counts + |> Enum.sort() + |> Enum.map_join("\n", fn {{adapter, status, execution}, count} -> + "| #{adapter} | #{status} | #{execution} | #{count} |" + end) + + sections = [ + {"Failed checks", Enum.filter(rows, &(&1.execution == :failed)), " open"}, + {"Supported checks", Enum.filter(rows, &(&1.status == :supported)), ""}, + {"Unsupported checks", Enum.filter(rows, &(&1.status == :unsupported)), ""}, + {"Known defects", Enum.filter(rows, &(&1.status == :known_defect)), ""}, + {"Unresolved semantics", Enum.filter(rows, &(&1.status == :unresolved)), ""} + ] + + details = + Enum.map_join(sections, "\n", fn + {_, [], _} -> + "" + + {title, checks, open} -> + "\n#{title} (#{length(checks)})\n\n" <> + results_table(checks) <> "\n\n" + end) + + """ + ## Aggregate conformance results + + Matching a recorded gap does not mean the feature is supported. Excluded or skipped checks did not run. Actual values below come from this run; the PR comparison uses JSON results from a separate base run in the same environment. + + | Adapter | Declared status | Execution | Count | + | --- | --- | --- | ---: | + #{count_rows} + + #{comparison} + #{details} + """ + end + + defp results_table(rows) do + body = + rows + |> Enum.sort_by(&{&1.scenario, &1.adapter}) + |> Enum.map_join("\n", fn row -> + "| #{code("#{row.adapter} #{row.scenario}")} | #{verdict(row)} | " <> + "#{code(row.expected)} | #{code(row.actual)} | #{code(row.accepted)} |" + end) + + "| Check | Result | Intended | Observed | Accepted by this test |\n" <> + "| --- | --- | --- | --- | --- |\n" <> body <> "\n" + end + + def verdict(%{execution: :matched, status: :supported}), do: "PASS" + def verdict(%{execution: :matched, status: :unresolved}), do: "OBSERVATION MATCHED" + def verdict(%{execution: :matched}), do: "GAP MATCHED" + def verdict(%{execution: execution}), do: execution |> to_string() |> String.upcase() + + def compact(text, limit \\ 800) do + text = + text + |> String.replace(~r/\e\[[0-9;]*m/, "") + |> String.replace("\r", "\\r") + |> String.replace("\n", "\\n") + |> String.replace("\t", "\\t") + + if String.length(text) > limit, do: String.slice(text, 0, limit) <> " [truncated]", else: text + end + + def code(text) do + escaped = + text + |> compact(1500) + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + |> String.replace("|", "|") + |> String.replace("`", "`") + + "#{escaped}" + end + def write_results!(rows) do File.mkdir_p!("results") name = Adapter.selected() |> Enum.map_join("-", & &1.id()) @@ -62,53 +186,26 @@ defmodule AshSql.Conformance.Report do counts = Enum.frequencies_by(rows, &"#{&1.status}/#{&1.execution}") report = %{ + schema_version: 1, counts: counts, scenarios: rows, meaning: "A matched gap expectation is not feature conformance." } + markdown = summary(rows, "") File.write!("results/#{name}.json", Jason.encode!(report, pretty: true) <> "\n") - IO.puts("\nAggregate contracts: #{inspect(counts)}\nReport: results/#{name}.json") - end -end + File.write!("results/#{name}.md", markdown) -defmodule AshSql.Conformance.Formatter do - @moduledoc false - use GenServer - alias AshSql.Conformance.{Expectations, Report} - - def init(_opts), do: {:ok, []} - - def handle_cast( - {:test_finished, %{tags: %{scenario: id, adapter: adapter} = tags} = test}, - rows - ) do - expectation = Expectations.for(id, adapter) - - execution = - case test.state do - nil -> :matched - {:excluded, _} -> :excluded - {:skipped, _} -> :skipped - _ -> :failed - end - - row = %{ - scenario: id, - adapter: adapter, - area: tags.area, - status: Report.status(expectation), - task: Report.task(expectation), - execution: execution - } + if path = summary_path(), do: File.write!(path, markdown, [:append]) - {:noreply, [row | rows]} + IO.puts("\n" <> console(rows)) + IO.puts("\nAggregate contracts: #{inspect(counts)}\nReport: results/#{name}.md") end - def handle_cast({:suite_finished, _times}, rows) do - Report.write_results!(rows) - {:noreply, rows} + def summary_path do + case System.get_env("GITHUB_STEP_SUMMARY") do + path when path in [nil, ""] -> nil + path -> path + end end - - def handle_cast(_event, rows), do: {:noreply, rows} end diff --git a/conformance/lib/report/comparison.ex b/conformance/lib/report/comparison.ex new file mode 100644 index 0000000..402a5a5 --- /dev/null +++ b/conformance/lib/report/comparison.ex @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.Report.Comparison do + @moduledoc false + alias AshSql.Conformance.Report + @fields ~w(status execution expected accepted actual) + + def decode!(text) do + report = Jason.decode!(text) + + unless Map.get(report, "schema_version", 1) == 1 do + raise ArgumentError, "Unsupported result report version" + end + + rows = Map.fetch!(report, "scenarios") + Enum.each(rows, &validate_row!/1) + if map_size(index(rows)) != length(rows), do: raise(ArgumentError, "Duplicate result checks") + rows + end + + defp validate_row!(row) do + for field <- ~w(scenario adapter status execution) do + unless is_binary(row[field]), do: raise(ArgumentError, "Invalid result field: #{field}") + end + end + + defp index(rows), do: Map.new(rows, &{{&1["scenario"], &1["adapter"]}, &1}) + + def changes(base, current) do + base = index(base) + current = index(current) + + (Map.keys(base) ++ Map.keys(current)) + |> Enum.uniq() + |> Enum.sort() + |> Enum.flat_map(fn key -> + before = Map.get(base, key) + after_row = Map.get(current, key) + + cond do + is_nil(before) -> + [%{check: key, kind: :added, before: nil, after: after_row}] + + is_nil(after_row) -> + [%{check: key, kind: :removed, before: before, after: nil}] + + Map.take(before, @fields) != Map.take(after_row, @fields) -> + [%{check: key, kind: :changed, before: before, after: after_row}] + + true -> + [] + end + end) + end + + def summary(:initial, current, opts) do + "## Aggregate result changes\n\nBase: #{Report.code(opts[:base_ref] || "unspecified")}. " <> + "The base has no conformance suite. This PR establishes the initial baseline " <> + "of #{length(current)} adapter checks; no improvements or regressions are inferred.\n" + end + + def summary(base, current, opts) do + missing = Enum.any?(base ++ current, &(not Map.has_key?(&1, "actual"))) + + warning = + if missing, + do: + "Some reports do not record actual values. Missing observations are labelled; equality of those results cannot be established.\n\n", + else: "" + + base_status = + if opts[:base_exit_code] && opts[:base_exit_code] != 0, + do: + "The base suite exited with code #{opts[:base_exit_code]}; its recorded failures are included below.\n\n", + else: "" + + "## Aggregate result changes\n\nBase: #{Report.code(opts[:base_ref] || "supplied JSON report")}. " <> + "Compared actual base and PR run artifacts by scenario ID and adapter. " <> + "GAP MATCHED and matched unresolved observations do not establish feature support.\n\n" <> + base_status <> warning <> render_changes(changes(base, current)) + end + + def render_changes([]), do: "No recorded result changes.\n" + + def render_changes(changes) do + counts = Enum.frequencies_by(changes, & &1.kind) + count = fn kind -> Map.get(counts, kind, 0) end + body = Enum.map_join(changes, "\n", &change_rows/1) + + "#{count.(:added)} added, #{count.(:removed)} removed, #{count.(:changed)} changed checks.\n\n" <> + "| Check | Changed field | Base | PR |\n| --- | --- | --- | --- |\n" <> body <> "\n" + end + + defp change_rows(%{check: {id, adapter}, kind: :changed} = change) do + @fields + |> Enum.filter(&(Map.fetch(change.before, &1) != Map.fetch(change.after, &1))) + |> Enum.map_join("\n", fn field -> + "| #{Report.code("#{adapter} #{id}")} | #{field} | " <> + "#{Report.code(change.before[field] || "Not recorded")} | #{Report.code(change.after[field] || "Not recorded")} |" + end) + end + + defp change_rows(%{check: {id, adapter}, kind: kind} = change) do + "| #{Report.code("#{adapter} #{id}")} | #{kind} | " <> + "#{Report.code(describe(change.before))} | #{Report.code(describe(change.after))} |" + end + + defp describe(nil), do: "No check" + + defp describe(row), + do: "#{row["status"]}/#{row["execution"]}; observed: #{row["actual"] || "Not recorded"}" +end diff --git a/conformance/lib/scenario.ex b/conformance/lib/scenario.ex index 8dd7bcf..fd09f65 100644 --- a/conformance/lib/scenario.ex +++ b/conformance/lib/scenario.ex @@ -5,10 +5,24 @@ defmodule AshSql.Conformance.Scenario do @moduledoc "A public Ash operation with an adapter-independent expected result." @enforce_keys [:id, :area, :expected, :run] - defstruct [:id, :area, :expected, :run] + defstruct [:id, :area, :expected, :run, :source] - def new(id, area, expected, run), - do: %__MODULE__{id: id, area: area, expected: expected, run: run} + defmacro new(id, area, expected, run) do + source = %{ + file: Path.relative_to(__CALLER__.file, Path.expand("..", __DIR__)), + line: __CALLER__.line + } + + quote do + %AshSql.Conformance.Scenario{ + id: unquote(id), + area: unquote(area), + expected: unquote(expected), + run: unquote(run), + source: unquote(Macro.escape(source)) + } + end + end end defmodule AshSql.Conformance.Runner do @@ -22,8 +36,9 @@ defmodule AshSql.Conformance.Runner do """ import ExUnit.Assertions - def run!(scenario, expectation, context) do + def run!(scenario, expectation, context, record_result \\ fn _ -> :ok end) do outcome = capture(fn -> scenario.run.(context) end) + record_result.(outcome) assert_outcome!(scenario, expectation, outcome) end diff --git a/conformance/test/aggregate_test.exs b/conformance/test/aggregate_test.exs index f2d28b8..d648dd6 100644 --- a/conformance/test/aggregate_test.exs +++ b/conformance/test/aggregate_test.exs @@ -4,7 +4,7 @@ defmodule AshSql.Conformance.AggregateTest do use ExUnit.Case, async: false - alias AshSql.Conformance.{Adapter, Catalog, Expectations, Fixtures, Runner} + alias AshSql.Conformance.{Adapter, Catalog, Expectations, Fixtures, Formatter, Runner} for adapter <- Adapter.selected(), scenario <- Catalog.all() do @tag adapter: adapter.id(), scenario: scenario.id, area: scenario.area @@ -15,7 +15,10 @@ defmodule AshSql.Conformance.AggregateTest do try do context = Fixtures.seed!(adapter) - Runner.run!(scenario, Expectations.for(scenario.id, adapter.id()), context) + + Runner.run!(scenario, Expectations.for(scenario.id, adapter.id()), context, fn outcome -> + Formatter.record(scenario.id, adapter.id(), outcome) + end) after adapter.checkin!() end diff --git a/conformance/test/catalog_test.exs b/conformance/test/catalog_test.exs index f87c686..be0492d 100644 --- a/conformance/test/catalog_test.exs +++ b/conformance/test/catalog_test.exs @@ -61,4 +61,21 @@ defmodule AshSql.Conformance.CatalogTest do test "the checked-in matrix matches the executable declarations" do assert File.read!("MATRIX.md") == Report.matrix(), "Run mix conformance.matrix" end + + test "every scenario links to its declaration, including generated cases" do + for scenario <- Catalog.all() do + assert String.starts_with?(scenario.source.file, "lib/scenarios/") + + line = + scenario.source.file + |> File.read!() + |> String.split("\n") + |> Enum.at(scenario.source.line - 1) + + assert line =~ "new(", "#{scenario.id} source link must point to its declaration" + + assert Report.matrix() =~ + "[`#{scenario.id}`](#{scenario.source.file}#L#{scenario.source.line})" + end + end end diff --git a/conformance/test/report_test.exs b/conformance/test/report_test.exs new file mode 100644 index 0000000..92b8501 --- /dev/null +++ b/conformance/test/report_test.exs @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: 2026 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Conformance.ReportTest do + use ExUnit.Case, async: true + alias AshSql.Conformance.Report + alias AshSql.Conformance.Report.Comparison + + defp row(overrides \\ %{}) do + Map.merge( + %{ + scenario: "loaded.example", + area: :results, + adapter: :sqlite, + status: :supported, + expected: "7", + accepted: "7", + task: nil, + execution: :matched, + actual: "7" + }, + overrides + ) + end + + defp results(rows), + do: %{schema_version: 1, scenarios: rows} |> Jason.encode!() |> Comparison.decode!() + + test "console prints intended and observed values and distinguishes matched gaps" do + rows = [ + row(), + row(%{status: :known_defect, actual: "6", accepted: "6"}), + row(%{status: :unresolved}), + row(%{execution: :failed, actual: "8"}) + ] + + text = Report.console(rows) + assert text =~ "[PASS] sqlite loaded.example (supported)" + assert text =~ "[GAP MATCHED] sqlite loaded.example (known_defect)" + assert text =~ "[OBSERVATION MATCHED]" + assert text =~ "[FAILED]" + assert text =~ "intended: 7\n observed: 8" + assert text =~ "observed: 6\n accepted: 6" + end + + test "the summary escapes table and HTML content without changing string whitespace" do + text = Report.summary([row(%{actual: "\" padded \" |