From 3758d09eedfba96e9c97db95ff570204f263e9ee Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:48:00 +0000 Subject: [PATCH 1/3] Support NOT, <>, NOT IN, NOT BETWEEN, ILIKE and NOT ILIKE These parsed but were refused by the translator. Now: - `<>` / `!=` is {:cmp, :!=, ...}. NOT IN, NOT BETWEEN, ILIKE and NOT ILIKE are {:not_in, ...}, {:not_range, ...}, {:ilike, ...} and {:not_ilike, ...}. They're checked on the rows read (Predicate.pushdown/1 is :filter), since no index can serve them. A NULL field matches none of them, as in SQL, so a NULL status is neither = nor <> 'paid'. - NOT is pushed into the condition it negates, where SQL's NULL rules keep each opposite exact: NOT (a < 1) is a >= 1, NOT (a = 1) is a <> 1, NOT IN/LIKE/ILIKE/BETWEEN/IS NULL flip, NOT NOT cancels, and NOT (a OR b) is NOT a AND NOT b. NOT (a AND b) would need OR, so it's refused with OR's message. - Rewrite turns a single-value NOT IN into <>. - A filter-only condition on the primary key '_' now says what '_' supports (=, <, <=, >, >=, BETWEEN, IN) instead of "cannot be combined with this query shape". The README, the TUI help and completion cover the new forms. There are unit tests for the translation (every NOT case), evaluation (NULL semantics, case-insensitive and non-ASCII ILIKE) and the rewrite, plus FoundationDB tests on the seeded users. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 24 +++++++++++-- lib/efsql/complete.ex | 2 +- lib/efsql/parser.ex | 80 ++++++++++++++++++++++++++++++----------- lib/efsql/planner.ex | 4 ++- lib/efsql/predicate.ex | 64 +++++++++++++++++++++------------ lib/efsql/rewrite.ex | 3 +- lib/efsql/tui/help.ex | 3 ++ test/negation_test.exs | 61 +++++++++++++++++++++++++++++++ test/parser_test.exs | 57 +++++++++++++++++++++++++---- test/predicate_test.exs | 47 ++++++++++++++++++++++-- test/rewrite_test.exs | 9 +++++ 11 files changed, 297 insertions(+), 57 deletions(-) create mode 100644 test/negation_test.exs diff --git a/README.md b/README.md index b49ede9..8097340 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,7 @@ so if you forget. Comments (`-- ...` and `/* ... */`) can go anywhere. A syntax error reports the line and column it was found at, and features efsql doesn't support (`OR`, -`NOT`, `<>`, joins, `HAVING`, functions other than aggregates) are rejected by -name. +joins, `HAVING`, functions other than aggregates) are rejected by name. ### Storage IDs @@ -276,6 +275,27 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals). +### Negation and ILIKE + +```sql +select col_a from tenant_id.table_name where status <> 'paid'; -- or != +select col_a from tenant_id.table_name where status not in ('paid', 'refunded'); +select col_a from tenant_id.table_name where total not between 10 and 20; +select col_a from tenant_id.table_name where name not like 'test%'; +select col_a from tenant_id.table_name where name ilike 'al%'; -- ignores case +select col_a from tenant_id.table_name where not (status = 'paid' or total > 100); +``` + +`NOT` works on any condition, and is read as its opposite: `not (a < 1)` is +`a >= 1`, `not (a = 1 or b = 2)` is `a <> 1 and b <> 2`. `not (a and b)` +would need `OR`, which isn't supported. + +As in SQL, a NULL field matches no comparison, negated ones included: a +row whose `status` is NULL matches neither `status = 'paid'` nor +`status <> 'paid'`. Negations and `ILIKE` are checked on the rows read, as +no index can serve them; the rest of the `WHERE` clause still narrows what +is read. + ### Filter by NULL ```sql diff --git a/lib/efsql/complete.ex b/lib/efsql/complete.ex index fe86918..14a12c1 100644 --- a/lib/efsql/complete.ex +++ b/lib/efsql/complete.ex @@ -14,7 +14,7 @@ defmodule Efsql.Complete do @statement_start ~w(select) @post_expr ~w(and group order limit) - @operators ~w(= > >= < <= like in between not is) + @operators ~w(= <> != > >= < <= like ilike in between not is) def complete(input, context) do {word_start, word} = current_word(input) diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 71fe778..ec45301 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -14,7 +14,11 @@ defmodule Efsql.Parser do alias Efsql.SQL.AST alias Efsql.Types - @flipped %{:< => :>, :> => :<, :<= => :>=, :>= => :<=, := => :=} + @flipped %{:< => :>, :> => :<, :<= => :>=, :>= => :<=, := => :=, :<> => :<>} + + # NOT of a comparison, which SQL's NULL rules keep exact: NOT (a < 1) and + # a >= 1 are both false when a is NULL. + @negated %{:= => :<>, :<> => :=, :< => :>=, :>= => :<, :> => :<=, :<= => :>} @max_versionstamp Bitwise.bsl(1, 96) - 1 def sql_to_logical(sql) do @@ -174,14 +178,36 @@ defmodule Efsql.Parser do # -- WHERE -- defp predicates(nil), do: [] - defp predicates(expr), do: expr |> conjuncts([]) |> Enum.map(&predicate/1) + defp predicates(expr), do: expr |> push_not() |> conjuncts([]) |> Enum.map(&predicate/1) + + # NOT is pushed down to the conditions it negates, each of which has an + # exact opposite: NOT (a = 1) is a <> 1, NOT (a IN ...) is a NOT IN ..., + # and NOT (a OR b) is NOT a AND NOT b. NOT (a AND b) is NOT a OR NOT b, + # which needs OR. + defp push_not({:not, expr}), do: negate(expr) + + defp push_not({op, left, right}) when op in [:and, :or], + do: {op, push_not(left), push_not(right)} + + defp push_not(expr), do: expr + + defp negate({:not, expr}), do: push_not(expr) + defp negate({:and, left, right}), do: {:or, negate(left), negate(right)} + defp negate({:or, left, right}), do: {:and, negate(left), negate(right)} + defp negate({:compare, op, left, right}), do: {:compare, Map.fetch!(@negated, op), left, right} + + defp negate({:between, expr, low, high, negated?}), + do: {:between, expr, low, high, not negated?} + + defp negate({kind, expr, values, negated?}) when kind in [:in, :like, :ilike], + do: {kind, expr, values, not negated?} + + defp negate({:is_null, expr, negated?}), do: {:is_null, expr, not negated?} + defp negate(expr), do: {:not, expr} defp conjuncts({:and, left, right}, acc), do: conjuncts(left, conjuncts(right, acc)) defp conjuncts(expr, acc), do: [expr | acc] - defp predicate({:compare, :<>, _left, _right}), - do: raise(Unsupported, "<> and != are not supported") - defp predicate({:compare, op, {:column, _} = column, {:column, _}}), do: raise(Unsupported, "comparing two fields (#{describe(column)} #{op} ...) is not supported") @@ -193,20 +219,34 @@ defmodule Efsql.Parser do defp predicate({:compare, op, value, {:column, field}}), do: {:cmp, logical_op(Map.fetch!(@flipped, op)), field(field), value(value)} - defp predicate({:between, {:column, field}, low, high, false}), - do: {:range, field(field), {:>=, value(low)}, {:<=, value(high)}} + defp predicate({:between, {:column, field}, low, high, negated?}), + do: + {if(negated?, do: :not_range, else: :range), field(field), {:>=, value(low)}, + {:<=, value(high)}} - defp predicate({:in, {:column, field}, values, false}), - do: {:in, field(field), Enum.map(values, &value/1)} + defp predicate({:in, {:column, field}, values, negated?}), + do: {if(negated?, do: :not_in, else: :in), field(field), Enum.map(values, &value/1)} - defp predicate({:like, {:column, field}, pattern, negated?}) do + defp predicate({like, {:column, field}, pattern, negated?}) when like in [:like, :ilike] do pattern = case value(pattern) do - pattern when is_binary(pattern) -> pattern - other -> raise Unsupported, "a LIKE pattern must be a string, got #{inspect(other)}" + pattern when is_binary(pattern) -> + pattern + + other -> + what = if like == :ilike, do: "an ILIKE", else: "a LIKE" + raise Unsupported, "#{what} pattern must be a string, got #{inspect(other)}" end - {if(negated?, do: :not_like, else: :like), field(field), pattern} + kind = + case {like, negated?} do + {:like, false} -> :like + {:like, true} -> :not_like + {:ilike, false} -> :ilike + {:ilike, true} -> :not_ilike + end + + {kind, field(field), pattern} end defp predicate({:is_null, {:column, "_"}, _negated?}), @@ -215,16 +255,13 @@ defmodule Efsql.Parser do defp predicate({:is_null, {:column, field}, negated?}), do: {if(negated?, do: :not_null, else: :is_null), field(field)} - defp predicate({:or, _, _}), do: raise(Unsupported, "OR is not supported") - defp predicate({:not, _}), do: raise(Unsupported, "NOT is not supported") - - defp predicate({:between, _, _, _, true}), - do: raise(Unsupported, "NOT BETWEEN is not supported") + defp predicate({:or, _, _}), + do: raise(Unsupported, "OR is not supported (nor NOT over AND, which means OR)") - defp predicate({:in, _, _, true}), do: raise(Unsupported, "NOT IN is not supported") - defp predicate({:ilike, _, _, _}), do: raise(Unsupported, "ILIKE is not supported") + defp predicate({:not, expr}), + do: raise(Unsupported, "#{describe(expr)} is not a condition; NOT needs a condition") - defp predicate({kind, _, _, _}) when kind in [:in, :like], + defp predicate({kind, _, _, _}) when kind in [:in, :like, :ilike], do: raise(Unsupported, "#{String.upcase(to_string(kind))} needs a field on its left") defp predicate({:between, _, _, _, _}), @@ -239,6 +276,7 @@ defmodule Efsql.Parser do do: raise(Unsupported, "#{describe(expr)} is not a condition; compare it to something") defp logical_op(:=), do: :== + defp logical_op(:<>), do: :!= defp logical_op(op), do: op # Fields are atoms, which can't be longer than 255 characters. diff --git a/lib/efsql/planner.ex b/lib/efsql/planner.ex index 234ea34..32acf12 100644 --- a/lib/efsql/planner.ex +++ b/lib/efsql/planner.ex @@ -368,8 +368,10 @@ defmodule Efsql.Planner do defp ensure_residual_evaluable!(pred) do if Predicate.field(pred) == @pk_field do + # Rows carry the key under its field's name, not `_`, so a condition + # on `_` must be one the key range can serve. raise Unsupported, - "a constraint on the primary key '_' cannot be combined with this query shape" + "on the primary key '_', only =, <, <=, >, >=, BETWEEN and IN are supported" end end diff --git a/lib/efsql/predicate.ex b/lib/efsql/predicate.ex index eb42d73..f2c0751 100644 --- a/lib/efsql/predicate.ex +++ b/lib/efsql/predicate.ex @@ -5,13 +5,18 @@ defmodule Efsql.Predicate do the Ecto expression that pushes it to the adapter. A query's predicates are an implicitly AND-ed list of: - * `{:cmp, op, field, value}` — op in `:== :> :>= :< :<=` + * `{:cmp, op, field, value}` — op in `:== :!= :> :>= :< :<=` * `{:range, field, {lower_op, value}, {upper_op, value}}` — a two-sided - bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=` - * `{:like, field, pattern}` / `{:not_like, field, pattern}` - * `{:in, field, values}` + bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=`; `:not_range` + is its negation (`NOT BETWEEN`) + * `{:like, field, pattern}` / `{:not_like, field, pattern}`, and the + case-insensitive `:ilike` / `:not_ilike` + * `{:in, field, values}` / `{:not_in, field, values}` * `{:is_null, field}` / `{:not_null, field}` + SQL's `NOT` never reaches here: `Efsql.Parser` pushes it into the + condition it negates (`NOT (a < 1)` is `a >= 1`). + The primary key is the pseudo-field `:_`. A new kind of predicate is added here, clause by clause, plus its @@ -23,16 +28,17 @@ defmodule Efsql.Predicate do @type field :: atom() @type t :: - {:cmp, :== | :> | :>= | :< | :<=, field(), term()} - | {:range, field(), {:> | :>=, term()}, {:< | :<=, term()}} - | {:like | :not_like, field(), String.t()} - | {:in, field(), [term()]} + {:cmp, :== | :!= | :> | :>= | :< | :<=, field(), term()} + | {:range | :not_range, field(), {:> | :>=, term()}, {:< | :<=, term()}} + | {:like | :not_like | :ilike | :not_ilike, field(), String.t()} + | {:in | :not_in, field(), [term()]} | {:is_null | :not_null, field()} @pk_field :_ @cmp_results %{ ==: [:eq], + !=: [:lt, :gt], >: [:gt], >=: [:gt, :eq], <: [:lt], @@ -42,19 +48,18 @@ defmodule Efsql.Predicate do @doc "The field a predicate constrains." @spec field(t()) :: field() def field({:cmp, _op, field, _value}), do: field - def field({:range, field, _lower, _upper}), do: field - def field({:like, field, _pattern}), do: field - def field({:not_like, field, _pattern}), do: field - def field({:in, field, _values}), do: field - def field({:is_null, field}), do: field - def field({:not_null, field}), do: field + def field({kind, field, _lower, _upper}) when kind in [:range, :not_range], do: field + def field({kind, field, _arg}) when kind in [:like, :not_like, :ilike, :not_ilike], do: field + def field({kind, field, _values}) when kind in [:in, :not_in], do: field + def field({kind, field}) when kind in [:is_null, :not_null], do: field # -- evaluation -- @doc """ Whether `row` satisfies every one of `predicates`, with SQL NULL semantics: a NULL (nil or absent) field matches no comparison, LIKE or - IN, NOT LIKE included, and IS NULL matches it. Values compare with + IN, negated ones included (a NULL is neither `= 1` nor `<> 1`), and + IS NULL matches it. Values compare with `Efsql.Types.compare/2`. """ @spec matches?(map(), [t()]) :: boolean() @@ -66,18 +71,28 @@ defmodule Efsql.Predicate do defp eval({:range, field, {lower_op, lower}, {upper_op, upper}}, row), do: eval({:cmp, lower_op, field, lower}, row) and eval({:cmp, upper_op, field, upper}, row) - defp eval({:in, field, values}, row), - do: - on_value(row, field, fn value -> Enum.any?(values, &(Types.compare(value, &1) == :eq)) end) + defp eval({:not_range, field, lower, upper}, row), + do: on_value(row, field, fn _ -> not eval({:range, field, lower, upper}, row) end) + + defp eval({:in, field, values}, row), do: on_value(row, field, &member?(&1, values)) + defp eval({:not_in, field, values}, row), do: on_value(row, field, &(not member?(&1, values))) defp eval({:is_null, field}, row), do: Map.get(row, field) == nil defp eval({:not_null, field}, row), do: Map.get(row, field) != nil defp eval({:like, field, pattern}, row), - do: on_value(row, field, &Regex.match?(like_regex(pattern), &1)) + do: on_value(row, field, &Regex.match?(like_regex(pattern, ""), &1)) defp eval({:not_like, field, pattern}, row), - do: on_value(row, field, &(not Regex.match?(like_regex(pattern), &1))) + do: on_value(row, field, &(not Regex.match?(like_regex(pattern, ""), &1))) + + defp eval({:ilike, field, pattern}, row), + do: on_value(row, field, &Regex.match?(like_regex(pattern, "iu"), &1)) + + defp eval({:not_ilike, field, pattern}, row), + do: on_value(row, field, &(not Regex.match?(like_regex(pattern, "iu"), &1))) + + defp member?(value, values), do: Enum.any?(values, &(Types.compare(value, &1) == :eq)) defp on_value(row, field, fun) do case Map.get(row, field) do @@ -86,14 +101,14 @@ defmodule Efsql.Predicate do end end - defp like_regex(pattern) do + defp like_regex(pattern, flags) do source = pattern |> Regex.escape() |> String.replace("%", ".*") |> String.replace("_", ".") - Regex.compile!("\\A" <> source <> "\\z", "s") + Regex.compile!("\\A" <> source <> "\\z", "s" <> flags) end # -- planning -- @@ -105,9 +120,12 @@ defmodule Efsql.Predicate do * `:in` - an `IN` list, read as one lookup per value when an index or the key can serve them * `:index` - an equality or range an index may serve - * `:filter` - only checked on the rows read + * `:filter` - only checked on the rows read: LIKE, IS NULL, and + every negation (`<>`, NOT IN, NOT BETWEEN, NOT LIKE), which an index + can't serve """ @spec pushdown(t()) :: :key | :in | :index | :filter + def pushdown({:cmp, :!=, _field, _value}), do: :filter def pushdown({:in, _field, _values}), do: :in def pushdown({:cmp, _op, @pk_field, _value}), do: :key def pushdown({:range, @pk_field, _lower, _upper}), do: :key diff --git a/lib/efsql/rewrite.ex b/lib/efsql/rewrite.ex index 22bcb13..e098839 100644 --- a/lib/efsql/rewrite.ex +++ b/lib/efsql/rewrite.ex @@ -123,11 +123,12 @@ defmodule Efsql.Rewrite do end end - # `field in (v)` is an equality. + # `field in (v)` is an equality, and `field not in (v)` an inequality. def pass_in_singleton(%Logical.Select{predicates: preds} = logical) do predicates = Enum.map(preds, fn {:in, field, [value]} -> {:cmp, :==, field, value} + {:not_in, field, [value]} -> {:cmp, :!=, field, value} pred -> pred end) diff --git a/lib/efsql/tui/help.ex b/lib/efsql/tui/help.ex index 505479c..4c1583e 100644 --- a/lib/efsql/tui/help.ex +++ b/lib/efsql/tui/help.ex @@ -56,7 +56,10 @@ defmodule Efsql.Tui.Help do {:sql, "where age between 30 and 40 inclusive both ends"}, {:sql, "where name like 'Al%' % and _ wildcards"}, {:sql, "where name not like 'A%'"}, + {:sql, "where name ilike 'al%' ignores case"}, {:sql, "where status in ('paid', 'shipped')"}, + {:sql, "where status <> 'paid' also !=, not in, not between"}, + {:sql, "where not (a = 1 or b = 2)"}, {:sql, "where city = 'Osaka' and age > 40"}, "Literals: 'text', 42, 1.5, true, false, null.", {:note, "or is not supported; conditions combine with and only"}, diff --git a/test/negation_test.exs b/test/negation_test.exs new file mode 100644 index 0000000..702517f --- /dev/null +++ b/test/negation_test.exs @@ -0,0 +1,61 @@ +defmodule EfsqlTest.Integration.Negation do + use EfsqlTest.Case, async: true + + alias Efsql.Exception.Unsupported + alias Efsql.Physical.Plan + + # Alice "Lorem ipsum", Bob "foobar", Charles with no notes. + + defp names(context, condition) do + "select name from #{context[:tenant_id]}.users where #{condition};" + |> Efsql.all() + |> Enum.map(& &1.name) + |> Enum.sort() + end + + test "<> and != and NOT (=)", context do + for condition <- ["name <> 'Bob'", "name != 'Bob'", "not (name = 'Bob')"] do + assert names(context, condition) == ["Alice", "Charles"], condition + end + end + + test "a NULL field is neither equal nor unequal", context do + assert names(context, "notes <> 'foobar'") == ["Alice"] + assert names(context, "notes not in ('foobar')") == ["Alice"] + assert names(context, "notes not like 'foo%'") == ["Alice"] + end + + test "NOT IN and NOT BETWEEN", context do + assert names(context, "name not in ('Alice', 'Bob')") == ["Charles"] + assert names(context, "not name in ('Alice', 'Bob')") == ["Charles"] + assert names(context, "name not between 'B' and 'Bz'") == ["Alice", "Charles"] + end + + test "ILIKE and NOT ILIKE ignore case", context do + assert names(context, "name ilike 'al%'") == ["Alice"] + assert names(context, "name ilike '%E%'") == ["Alice", "Charles"] + assert names(context, "name not ilike 'a%'") == ["Bob", "Charles"] + end + + test "NOT over IS NULL and over OR", context do + assert names(context, "not (notes is null)") == ["Alice", "Bob"] + assert names(context, "not (name = 'Alice' or name = 'Bob')") == ["Charles"] + end + + test "a negation is checked on the rows read, alongside what an index serves", context do + {%Plan{access: access, ops: ops}, rows, _} = + Efsql.qall( + "select name from #{context[:tenant_id]}.users where name >= 'B' and name <> 'Bob';" + ) + + assert {:index_scan, %Ecto.Query{wheres: [_]}, _} = access + assert {:filter, [{:cmp, :!=, :name, "Bob"}]} in ops + assert rows == [%{name: "Charles"}] + end + + test "the primary key takes only what a key range can serve", context do + assert_raise Unsupported, ~r/on the primary key '_', only =, <, <=, >, >=/, fn -> + Efsql.all("select name from #{context[:tenant_id]}.users where _ <> '0001';") + end + end +end diff --git a/test/parser_test.exs b/test/parser_test.exs index 88af595..c232a58 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -457,6 +457,54 @@ defmodule Efsql.ParserTest do ) end + test "<> and != are inequalities, either way round" do + for condition <- ["age <> 40", "age != 40", "40 <> age"] do + assert %Logical.Select{predicates: [{:cmp, :!=, :age, 40}]} = + parse("select id from t.users where #{condition};") + end + end + + test "NOT IN, NOT BETWEEN, ILIKE and NOT ILIKE" do + assert %Logical.Select{ + predicates: [ + {:not_in, :a, [1, 2]}, + {:not_range, :b, {:>=, 1}, {:<=, 5}}, + {:ilike, :c, "al%"}, + {:not_ilike, :d, "x_"} + ] + } = + parse( + "select id from t.users where a not in (1, 2) and b not between 1 and 5 " <> + "and c ilike 'al%' and d not ilike 'x_';" + ) + end + + test "NOT is pushed into the condition it negates" do + for {condition, predicates} <- [ + {"not (a = 1)", [{:cmp, :!=, :a, 1}]}, + {"not a <> 1", [{:cmp, :==, :a, 1}]}, + {"not a < 1", [{:cmp, :>=, :a, 1}]}, + {"not a >= 1", [{:cmp, :<, :a, 1}]}, + {"not a > 1", [{:cmp, :<=, :a, 1}]}, + {"not a <= 1", [{:cmp, :>, :a, 1}]}, + {"not not a = 1", [{:cmp, :==, :a, 1}]}, + {"not a in (1, 2)", [{:not_in, :a, [1, 2]}]}, + {"not a not in (1, 2)", [{:in, :a, [1, 2]}]}, + {"not (a between 1 and 5)", [{:not_range, :a, {:>=, 1}, {:<=, 5}}]}, + {"not a like 'x%'", [{:not_like, :a, "x%"}]}, + {"not a ilike 'x%'", [{:not_ilike, :a, "x%"}]}, + {"not (a is null)", [{:not_null, :a}]}, + {"not a isnull", [{:not_null, :a}]}, + {"not (a = 1 or b like 'x%')", [{:cmp, :!=, :a, 1}, {:not_like, :b, "x%"}]}, + {"c = 2 and not (a = 1 or not b = 3)", + [{:cmp, :==, :c, 2}, {:cmp, :!=, :a, 1}, {:cmp, :==, :b, 3}]} + ] do + assert %Logical.Select{predicates: ^predicates} = + parse("select id from t.users where #{condition};"), + condition + end + end + test "a value on the left is flipped" do assert %Logical.Select{predicates: [{:cmp, :>, :age, 40}, {:cmp, :==, :name, "x"}]} = parse("select id from t.users where 40 < age and 'x' = name;") @@ -465,12 +513,9 @@ defmodule Efsql.ParserTest do test "unsupported conditions say what is unsupported" do for {condition, message} <- [ {"a = 1 or b = 2", "OR is not supported"}, - {"not a = 1", "NOT is not supported"}, - {"a not in (1)", "NOT IN is not supported"}, - {"a not between 1 and 2", "NOT BETWEEN is not supported"}, - {"a ilike 'x'", "ILIKE is not supported"}, - {"a <> 1", "<> and != are not supported"}, - {"a != 1", "<> and != are not supported"}, + {"not (a = 1 and b = 2)", "OR is not supported"}, + {"not a", "the field a is not a condition; NOT needs a condition"}, + {"a ilike 1", "an ILIKE pattern must be a string"}, {"a = b", "comparing two fields"}, {"1 = 1", "needs a field on one side"}, {"'x' in ('x')", "IN needs a field on its left"}, diff --git a/test/predicate_test.exs b/test/predicate_test.exs index 7256f6d..0020475 100644 --- a/test/predicate_test.exs +++ b/test/predicate_test.exs @@ -43,6 +43,33 @@ defmodule Efsql.PredicateTest do assert holds?({:not_null, :name}) end + test "negations and ILIKE" do + assert holds?({:cmp, :!=, :age, 31}) + refute holds?({:cmp, :!=, :age, 30}) + assert holds?({:cmp, :!=, :price, Decimal.new("9.49")}) + refute holds?({:cmp, :!=, :price, Decimal.new("9.500")}) + assert holds?({:not_in, :age, [10, 20]}) + refute holds?({:not_in, :age, [10, 30]}) + assert holds?({:not_range, :age, {:>=, 31}, {:<=, 40}}) + refute holds?({:not_range, :age, {:>=, 30}, {:<=, 40}}) + assert holds?({:ilike, :name, "al%"}) + assert holds?({:ilike, :name, "ALICE"}) + refute holds?({:not_ilike, :name, "a_ice"}) + assert Predicate.matches?(%{name: "Émile"}, [{:ilike, :name, "émile"}]) + end + + test "a NULL field matches no negation either" do + for predicate <- [ + {:cmp, :!=, :notes, "x"}, + {:not_in, :notes, ["x"]}, + {:not_range, :notes, {:>=, "a"}, {:<=, "b"}}, + {:ilike, :notes, "%"}, + {:not_ilike, :notes, "x"} + ] do + refute holds?(predicate), inspect(predicate) + end + end + test "every predicate must hold" do assert Predicate.matches?(@row, []) refute Predicate.matches?(@row, [{:cmp, :==, :age, 30}, {:is_null, :name}]) @@ -56,7 +83,12 @@ defmodule Efsql.PredicateTest do {{:not_like, :d, "x"}, :d}, {{:in, :e, [1]}, :e}, {{:is_null, :f}, :f}, - {{:not_null, :g}, :g} + {{:not_null, :g}, :g}, + {{:cmp, :!=, :h, 1}, :h}, + {{:not_in, :i, [1]}, :i}, + {{:not_range, :j, {:>=, 1}, {:<=, 2}}, :j}, + {{:ilike, :k, "x"}, :k}, + {{:not_ilike, :l, "x"}, :l} ] do assert Predicate.field(predicate) == field end @@ -70,7 +102,18 @@ defmodule Efsql.PredicateTest do assert Predicate.pushdown({:cmp, :>, :age, 1}) == :index assert Predicate.pushdown({:range, :age, {:>, 1}, {:<, 2}}) == :index - for predicate <- [{:like, :a, "x"}, {:not_like, :a, "x"}, {:is_null, :a}, {:not_null, :a}] do + for predicate <- [ + {:like, :a, "x"}, + {:not_like, :a, "x"}, + {:ilike, :a, "x"}, + {:not_ilike, :a, "x"}, + {:is_null, :a}, + {:not_null, :a}, + {:cmp, :!=, :a, 1}, + {:cmp, :!=, :_, "k"}, + {:not_in, :a, [1]}, + {:not_range, :a, {:>=, 1}, {:<=, 2}} + ] do assert Predicate.pushdown(predicate) == :filter end end diff --git a/test/rewrite_test.exs b/test/rewrite_test.exs index 2c008a5..441a486 100644 --- a/test/rewrite_test.exs +++ b/test/rewrite_test.exs @@ -73,6 +73,15 @@ defmodule Efsql.RewriteTest do test "multi-value in is untouched" do assert [{:in, :name, ["a", "b"]}] = normalize([{:in, :name, ["a", "b"]}]) end + + test "single-value not in becomes an inequality" do + assert [{:cmp, :!=, :name, "a"}] = normalize([{:not_in, :name, ["a"]}]) + assert [{:not_in, :name, ["a", "b"]}] = normalize([{:not_in, :name, ["a", "b"]}]) + end + + test "ilike is not a key range, whatever its prefix" do + assert [{:ilike, :name, "Al%"}] = normalize([{:ilike, :name, "Al%"}]) + end end test "Predicate.take_first takes the first match out, keeping the rest in order" do From ef6a5f7b7a0ee660a809c86e4bddee9649e54779 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:38:04 +0000 Subject: [PATCH 2/3] Run operators over rows as they arrive (Efsql.Pipeline) Operators used to take the whole list of rows, and a batched query across tenants concatenated every batch before the first one ran. So memory held every row read, and a LIMIT still read every batch. Efsql.Pipeline runs a plan's operators a chunk at a time (new/1, feed/2, finish/1), and each operator keeps only what it has to: - filter and project keep nothing; - limit counts, and once it has its rows feed/2 says :halt, so the source stops reading; - a sort directly followed by a limit is fused into a running top-n; - a sort alone keeps every row, since it has to; - aggregate keeps a running total per group. Efsql.Aggregate is now new/2, add/2 and finish/1, with run/3 unchanged. The executor feeds every plan through a pipeline. A single read is one chunk. Batches are a stream (Executor.stream_concurrently/3, replacing map_concurrently/3), so each batch is processed as it arrives, and a halt stops the batches not yet read, cancelling any in flight. sum and avg now switch to Decimal at the first Decimal value instead of converting every value when any is one. The results differ only if one field mixes floats with Decimals. A property test checks that for random rows, random planner-shaped operator chains and random chunking, the pipeline's result equals the operators applied to all the rows at once, early halts included. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 5 +- lib/efsql/aggregate.ex | 147 +++++++++++++++++------------ lib/efsql/executor.ex | 76 ++++++--------- lib/efsql/pipeline.ex | 121 ++++++++++++++++++++++++ test/cross_tenant_test.exs | 11 +++ test/executor_concurrency_test.exs | 44 +++++++-- test/pipeline_test.exs | 116 +++++++++++++++++++++++ 7 files changed, 408 insertions(+), 112 deletions(-) create mode 100644 lib/efsql/pipeline.ex create mode 100644 test/pipeline_test.exs diff --git a/README.md b/README.md index 8097340..acb1aa1 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,10 @@ When the tenants don't fit in one transaction, read them in batches: ``` Each batch of 25 tenants is then its own transaction, and there is no -tenant limit. Up to four batches are read at once (the +tenant limit. Batches are processed as they arrive, so a batched query keeps +only what its result needs: a `LIMIT` without `ORDER BY` stops reading once +it has its rows, `ORDER BY ... LIMIT n` keeps the best `n` so far, and +`GROUP BY` keeps a running total per group. Up to four batches are read at once (the `:efsql, :batch_concurrency` setting, or `batch_concurrency:` from Elixir). Grouping, ordering and `LIMIT` still apply to all the rows together, but the result is no longer one snapshot: each batch sees the database at a slightly different moment, and the result says how many diff --git a/lib/efsql/aggregate.ex b/lib/efsql/aggregate.ex index 23b17b4..65afe48 100644 --- a/lib/efsql/aggregate.ex +++ b/lib/efsql/aggregate.ex @@ -1,7 +1,10 @@ defmodule Efsql.Aggregate do @moduledoc """ - `GROUP BY` and aggregates over pulled rows, for `Efsql.Executor`'s - `{:aggregate, group_by, aggregates}` operator. + `GROUP BY` and aggregates, for the `{:aggregate, group_by, aggregates}` + operator. It runs over rows as they arrive (`new/2`, `add/2`, + `finish/1`), keeping one small running total per group rather than the + rows themselves, so a query across many tenants can aggregate each batch + as it is read. Rows are grouped by the values of the `group_by` fields; equal values group together even when their terms differ (`Decimal` 1.0 and 1.00, or @@ -12,9 +15,10 @@ defmodule Efsql.Aggregate do SQL semantics: `count(*)` counts rows, every other aggregate skips NULLs, and `sum`, `min`, `max` and `avg` of no values are NULL. `sum` and `avg` - take numbers and `Decimal`s, and are `Decimal` when any input is. - `min` and `max` order with `Efsql.Types.compare/2`, so they work on - strings and datetimes too. Groups come out ordered by their key. + take numbers and `Decimal`s, and become `Decimal` from the first + `Decimal` on. `min` and `max` order with `Efsql.Types.compare/2`, so + they work on strings and datetimes too. Groups come out ordered by their + key. """ alias Efsql.Exception.Unsupported @@ -22,79 +26,106 @@ defmodule Efsql.Aggregate do @type aggregate :: {name :: atom, :count | :sum | :min | :max | :avg, atom | :star} + defstruct group_by: [], aggregates: [], groups: %{} + + @opaque t :: %__MODULE__{} + + @doc "All at once: `new/2`, `add/2` and `finish/1` over `rows`." @spec run([map], [atom], [aggregate]) :: [map] - def run(rows, group_by, aggregates) do - rows - |> Enum.group_by(fn row -> Enum.map(group_by, &Types.equality_key(Map.get(row, &1))) end) - |> ensure_one_group(group_by) - |> Enum.map(fn {_key, rows} -> - first = List.first(rows, %{}) - group = Map.new(group_by, &{&1, Map.get(first, &1)}) - - Enum.reduce(aggregates, group, fn {name, function, arg}, acc -> - Map.put(acc, name, compute(function, arg, rows)) + def run(rows, group_by, aggregates), + do: group_by |> new(aggregates) |> add(rows) |> finish() + + @spec new([atom], [aggregate]) :: t() + def new(group_by, aggregates), do: %__MODULE__{group_by: group_by, aggregates: aggregates} + + @doc "Adds rows to the running totals." + @spec add(t(), [map]) :: t() + def add(%__MODULE__{group_by: group_by, aggregates: aggregates} = state, rows) do + groups = + Enum.reduce(rows, state.groups, fn row, groups -> + key = Enum.map(group_by, &Types.equality_key(Map.get(row, &1))) + + {values, accs} = + Map.get_lazy(groups, key, fn -> + {Map.new(group_by, &{&1, Map.get(row, &1)}), Enum.map(aggregates, &initial/1)} + end) + + accs = Enum.zip_with(aggregates, accs, &accumulate(&1, &2, row)) + Map.put(groups, key, {values, accs}) + end) + + %__MODULE__{state | groups: groups} + end + + @doc "One row per group, ordered by the group fields." + @spec finish(t()) :: [map] + def finish(%__MODULE__{group_by: group_by, aggregates: aggregates, groups: groups}) do + # A whole-table aggregate answers even when no row matched. + groups = + if group_by == [] and groups == %{}, + do: [{%{}, Enum.map(aggregates, &initial/1)}], + else: Map.values(groups) + + groups + |> Enum.map(fn {values, accs} -> + aggregates + |> Enum.zip(accs) + |> Enum.reduce(values, fn {{name, function, _arg}, acc}, row -> + Map.put(row, name, result(function, acc)) end) end) |> Efsql.Executor.sort(Enum.map(group_by, &{:asc, &1})) end - # A whole-table aggregate answers even when no row matched. - defp ensure_one_group(groups, []) when groups == %{}, do: [{[], []}] - defp ensure_one_group(groups, _group_by), do: Enum.to_list(groups) + # -- running totals -- - # -- aggregates -- + defp initial({_name, :count, _arg}), do: 0 + defp initial({_name, :avg, _arg}), do: {nil, 0} + defp initial({_name, _function, _arg}), do: nil - defp compute(:count, :star, rows), do: length(rows) - defp compute(:count, field, rows), do: length(values(rows, field)) + defp accumulate({_name, :count, :star}, n, _row), do: n + 1 - defp compute(:min, field, rows), do: extreme(values(rows, field), :lt) - defp compute(:max, field, rows), do: extreme(values(rows, field), :gt) - - defp compute(:sum, field, rows) do - case values(rows, field) do - [] -> nil - values -> sum(values, field) + defp accumulate({_name, function, field}, acc, row) do + case Map.get(row, field) do + nil -> acc + value -> step(function, field, acc, value) end end - defp compute(:avg, field, rows) do - case values(rows, field) do - [] -> - nil + defp step(:count, _field, n, _value), do: n + 1 + defp step(:sum, field, total, value), do: add_number(total, value, field) + defp step(:avg, field, {total, n}, value), do: {add_number(total, value, field), n + 1} + defp step(:min, _field, best, value), do: better(best, value, :lt) + defp step(:max, _field, best, value), do: better(best, value, :gt) - values -> - case sum(values, field) do - %Decimal{} = total -> Decimal.div(total, length(values)) - total -> total / length(values) - end - end - end + defp better(nil, value, _keep), do: value - defp values(rows, field) do - rows |> Enum.map(&Map.get(&1, field)) |> Enum.reject(&is_nil/1) - end + defp better(best, value, keep), + do: if(Types.compare(value, best) == keep, do: value, else: best) - defp extreme([], _keep), do: nil + defp add_number(total, value, field) do + unless is_number(value) or is_struct(value, Decimal) do + raise Unsupported, + "sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}" + end - defp extreme([first | rest], keep) do - Enum.reduce(rest, first, fn value, best -> - if Types.compare(value, best) == keep, do: value, else: best - end) - end + cond do + total == nil -> + value - defp sum(values, field) do - Enum.each(values, fn value -> - unless is_number(value) or is_struct(value, Decimal) do - raise Unsupported, - "sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}" - end - end) + is_struct(total, Decimal) or is_struct(value, Decimal) -> + Decimal.add(decimal(total), decimal(value)) - if Enum.any?(values, &is_struct(&1, Decimal)), - do: values |> Enum.map(&decimal/1) |> Enum.reduce(&Decimal.add/2), - else: Enum.sum(values) + true -> + total + value + end end + defp result(:avg, {nil, 0}), do: nil + defp result(:avg, {%Decimal{} = total, n}), do: Decimal.div(total, n) + defp result(:avg, {total, n}), do: total / n + defp result(_function, acc), do: acc + defp decimal(%Decimal{} = d), do: d defp decimal(n) when is_integer(n), do: Decimal.new(n) defp decimal(n) when is_float(n), do: Decimal.from_float(n) diff --git a/lib/efsql/executor.ex b/lib/efsql/executor.ex index 242f67f..52085c0 100644 --- a/lib/efsql/executor.ex +++ b/lib/efsql/executor.ex @@ -1,12 +1,11 @@ defmodule Efsql.Executor do @moduledoc """ - Executes an `Efsql.Physical.Plan`: pulls rows from the plan's access node, - then folds the operator pipeline over them. - - SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is - false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first - descending, matching PostgreSQL's defaults. Values compare with - `Efsql.Types.compare/2`, so datetimes order chronologically. + Executes an `Efsql.Physical.Plan`: reads rows through the plan's access + node and feeds them, as they arrive, through an `Efsql.Pipeline` of its + operators. A batched read across tenants arrives a batch at a time, and + stops early when the pipeline has all it needs (a `LIMIT` met). + Predicates evaluate with `Efsql.Predicate`'s SQL NULL semantics, and + `sort/2` orders NULLs as PostgreSQL does. A query across tenants (`{:fan_out, [{tenant_name, plan}]}`) reads every tenant in one FoundationDB transaction on the database: each tenant's @@ -17,26 +16,30 @@ defmodule Efsql.Executor do alias Efsql.Exception.Unsupported alias Efsql.Physical.Plan - alias Efsql.Predicate + alias Efsql.Pipeline alias Efsql.Types # transaction_too_old (retryable) and transaction_timed_out @too_long [1007, 1031] def run(%Plan{access: access, ops: ops}) do - Enum.reduce(ops, fetch(access), &apply_op/2) + access + |> chunks() + |> Enum.reduce_while(Pipeline.new(ops), &Pipeline.feed(&2, &1)) + |> Pipeline.finish() end @doc """ - Maps `fun` over `items`, up to `max` at a time, each in a process of its - own, and returns the results in order. The first failure stops the rest - and is raised (or thrown, or exited) here as it was there, so a caller's - `rescue` sees the original exception rather than a linked process's - exit. With `max` 1 it all runs in the calling process. + A stream of `fun` over `items`, up to `max` at a time, each in a process + of its own, the results in order. Halting the stream stops the work not + yet done. A failure is raised (or thrown, or exited) in the consumer as + it was in its process, so a caller's `rescue` sees the original + exception rather than a linked process's exit. With `max` 1 it all runs + in the consuming process, one item at a time. """ - def map_concurrently(items, 1, fun), do: Enum.map(items, fun) + def stream_concurrently(items, 1, fun), do: Stream.map(items, fun) - def map_concurrently(items, max, fun) do + def stream_concurrently(items, max, fun) do items |> Task.async_stream( fn item -> @@ -50,21 +53,21 @@ defmodule Efsql.Executor do ordered: true, timeout: :infinity ) - |> Enum.reduce_while([], fn - {:ok, {:ok, result}}, results -> {:cont, [result | results]} - {:ok, failed}, _results -> {:halt, failed} + |> Stream.map(fn + {:ok, {:ok, result}} -> result + {:ok, {:failed, kind, reason, stacktrace}} -> :erlang.raise(kind, reason, stacktrace) end) - |> case do - {:failed, kind, reason, stacktrace} -> :erlang.raise(kind, reason, stacktrace) - results -> Enum.reverse(results) - end end # -- access nodes -- - # One transaction per batch, up to `concurrency` at a time. - defp fetch({:batches, batches, concurrency}), - do: batches |> map_concurrently(concurrency, &fetch/1) |> Enum.concat() + # The rows a plan reads, as the chunks they arrive in: one per batch, a + # transaction each, up to `concurrency` at a time, or else just one. A + # pipeline that halts stops the batches not yet read. + defp chunks({:batches, batches, concurrency}), + do: stream_concurrently(batches, concurrency, &fetch/1) + + defp chunks(access), do: [fetch(access)] defp fetch({:fan_out, []}), do: [] @@ -95,7 +98,7 @@ defmodule Efsql.Executor do rows = plan.ops - |> Enum.reduce(List.flatten(mine), &apply_op/2) + |> Pipeline.run(List.flatten(mine)) |> Enum.map(&Map.put(&1, :_tenant, name)) {rows, rest} @@ -132,25 +135,6 @@ defmodule Efsql.Executor do Efsql.Repo.async_all_from_source(query, options) end - # -- operators -- - - defp apply_op({:filter, predicates}, rows), - do: Enum.filter(rows, &Predicate.matches?(&1, predicates)) - - defp apply_op({:aggregate, group_by, aggregates}, rows) do - Efsql.Aggregate.run(rows, group_by, aggregates) - end - - defp apply_op({:sort, order}, rows), do: sort(rows, order) - - defp apply_op({:limit, n}, rows) do - Enum.take(rows, n) - end - - defp apply_op({:project, fields}, rows) do - Enum.map(rows, &Map.take(&1, fields)) - end - # -- sorting -- @doc """ diff --git a/lib/efsql/pipeline.ex b/lib/efsql/pipeline.ex new file mode 100644 index 0000000..4131551 --- /dev/null +++ b/lib/efsql/pipeline.ex @@ -0,0 +1,121 @@ +defmodule Efsql.Pipeline do + @moduledoc """ + Runs a plan's operators (see `Efsql.Physical`) over rows as they arrive, + a chunk at a time: `new/1`, then `feed/2` for each chunk, then + `finish/1` for the result. Each operator keeps only what it has to: + + * `{:filter, predicates}` and `{:project, fields}` - nothing; rows + pass straight through + * `{:limit, n}` - how many rows are still wanted; once it has them, + `feed/2` says `:halt`, and whatever feeds the pipeline can stop + reading + * `{:sort, order}` directly followed by `{:limit, n}` - only the best + `n` rows so far (a running top-n), not every row + * `{:sort, order}` alone - every row, since the last one read may sort + first + * `{:aggregate, group_by, aggregates}` - one running total per group + (see `Efsql.Aggregate`) + + So a query across many tenants, read in batches, holds a limit's worth + of rows, or a group's worth of totals, rather than every row it read, + and a `LIMIT` without `ORDER BY` stops reading as soon as it is met. + A single read is just one chunk. + """ + + alias Efsql.Aggregate + alias Efsql.Executor + alias Efsql.Predicate + + defstruct stages: [], out: [] + + @opaque t :: %__MODULE__{} + + @spec new([tuple()]) :: t() + def new(ops), do: %__MODULE__{stages: ops |> fuse() |> Enum.map(&{&1, init(&1)})} + + @doc """ + Feeds a chunk of rows through: `{:cont, pipeline}`, or `{:halt, pipeline}` + when a limit has all its rows and no more input could change the result. + Its shape suits `Enum.reduce_while/3`. + """ + @spec feed(t(), [map()]) :: {:cont | :halt, t()} + def feed(%__MODULE__{} = pipeline, rows) do + {stages, out, halt?} = push(pipeline.stages, rows) + pipeline = %__MODULE__{stages: stages, out: [out | pipeline.out]} + {if(halt?, do: :halt, else: :cont), pipeline} + end + + @doc "The result: every row the operators let through, in order." + @spec finish(t()) :: [map()] + def finish(%__MODULE__{} = pipeline) do + # A stage's held rows go on through the stages after it. + rest = + Enum.reduce(pipeline.stages, [], fn {op, state}, pending -> + {state, out, _halt?} = step(op, state, pending) + out ++ flush(op, state) + end) + + pipeline.out |> Enum.reverse() |> Enum.concat() |> Kernel.++(rest) + end + + @doc "All at once: `new/1`, `feed/2` and `finish/1` over `rows`." + @spec run([tuple()], [map()]) :: [map()] + def run(ops, rows) do + {_, pipeline} = ops |> new() |> feed(rows) + finish(pipeline) + end + + # A sort that a limit then cuts is a running top-n. + defp fuse([{:sort, order}, {:limit, n} | rest]), do: [{:top, order, n} | fuse(rest)] + defp fuse([op | rest]), do: [op | fuse(rest)] + defp fuse([]), do: [] + + defp init({:limit, n}), do: n + defp init({:sort, _order}), do: [] + defp init({:top, _order, _n}), do: [] + defp init({:aggregate, group_by, aggregates}), do: Aggregate.new(group_by, aggregates) + defp init(_op), do: nil + + # Each stage takes the rows the one before let through. + defp push(stages, rows) do + {stages, {out, halt?}} = + Enum.map_reduce(stages, {rows, false}, fn {op, state}, {rows, halt?} -> + {state, out, stop?} = step(op, state, rows) + {{op, state}, {out, halt? or stop?}} + end) + + {stages, out, halt?} + end + + # {state, rows passed on, whether no more input is needed} + defp step({:filter, predicates}, nil, rows), + do: {nil, Enum.filter(rows, &Predicate.matches?(&1, predicates)), false} + + defp step({:project, fields}, nil, rows), + do: {nil, Enum.map(rows, &Map.take(&1, fields)), false} + + defp step({:limit, _n}, remaining, rows) do + taken = Enum.take(rows, remaining) + remaining = remaining - length(taken) + {remaining, taken, remaining == 0} + end + + defp step({:sort, _order}, chunks, []), do: {chunks, [], false} + defp step({:sort, _order}, chunks, rows), do: {[rows | chunks], [], false} + + defp step({:top, _order, _n}, best, []), do: {best, [], false} + + defp step({:top, order, n}, best, rows), + do: {(best ++ rows) |> Executor.sort(order) |> Enum.take(n), [], false} + + defp step({:aggregate, _group_by, _aggregates}, totals, rows), + do: {Aggregate.add(totals, rows), [], false} + + # What a stage still holds once the input is done. + defp flush({:sort, order}, chunks), + do: chunks |> Enum.reverse() |> Enum.concat() |> Executor.sort(order) + + defp flush({:top, _order, _n}, best), do: best + defp flush({:aggregate, _group_by, _aggregates}, totals), do: Aggregate.finish(totals) + defp flush(_op, _state), do: [] +end diff --git a/test/cross_tenant_test.exs b/test/cross_tenant_test.exs index 8ac8a8a..a45a3e9 100644 --- a/test/cross_tenant_test.exs +++ b/test/cross_tenant_test.exs @@ -128,6 +128,17 @@ defmodule EfsqlTest.Integration.CrossTenant do end end + test "ORDER BY ... LIMIT and a plain LIMIT across batches", context do + batched = [tenant_batch: 1, max_tenants: 1, batch_concurrency: 2] + + top = "select name from *.users where #{context[:both]} order by name desc limit 2;" + assert {_plan, [%{name: "Eve"}, %{name: "Dora"}], _} = Efsql.qall(top, batched) + + # without ORDER BY any 2 rows will do, and reading stops once it has them + any = "select name from *.users where #{context[:both]} limit 2;" + assert {_plan, [_, _], _} = Efsql.qall(any, batched) + end + test "a read too old for its transaction fails at once instead of retrying", context do sql = "select name from *.users where #{context[:both]};" diff --git a/test/executor_concurrency_test.exs b/test/executor_concurrency_test.exs index c18b414..b65ebb2 100644 --- a/test/executor_concurrency_test.exs +++ b/test/executor_concurrency_test.exs @@ -4,8 +4,11 @@ defmodule Efsql.ExecutorConcurrencyTest do alias Efsql.Exception.Unsupported alias Efsql.Executor + defp map_concurrently(items, max, fun), + do: items |> Executor.stream_concurrently(max, fun) |> Enum.to_list() + test "results come back in order, whatever order the work finishes in" do - assert Executor.map_concurrently([30, 10, 20, 0], 4, fn ms -> + assert map_concurrently([30, 10, 20, 0], 4, fn ms -> Process.sleep(ms) ms end) == [30, 10, 20, 0] @@ -14,7 +17,7 @@ defmodule Efsql.ExecutorConcurrencyTest do test "at most max run at once, and more than one does" do running = :counters.new(2, []) - Executor.map_concurrently(1..12, 3, fn _ -> + map_concurrently(1..12, 3, fn _ -> :counters.add(running, 1, 1) now = :counters.get(running, 1) if now > :counters.get(running, 2), do: :counters.put(running, 2, now) @@ -27,21 +30,21 @@ defmodule Efsql.ExecutorConcurrencyTest do test "a failure reaches the caller as the original exception" do assert_raise Unsupported, "too big", fn -> - Executor.map_concurrently([1, 2, 3], 2, fn + map_concurrently([1, 2, 3], 2, fn 2 -> raise Unsupported, "too big" n -> n end) end - assert catch_throw(Executor.map_concurrently([1], 2, fn _ -> throw(:stop) end)) == :stop - assert catch_exit(Executor.map_concurrently([1], 2, fn _ -> exit(:gone) end)) == :gone + assert catch_throw(map_concurrently([1], 2, fn _ -> throw(:stop) end)) == :stop + assert catch_exit(map_concurrently([1], 2, fn _ -> exit(:gone) end)) == :gone end test "a failure stops the work not yet done" do started = :counters.new(1, []) assert_raise Unsupported, fn -> - Executor.map_concurrently(1..50, 2, fn n -> + map_concurrently(1..50, 2, fn n -> :counters.add(started, 1, 1) if n == 1, do: raise(Unsupported, "stop"), else: Process.sleep(50) end) @@ -52,6 +55,33 @@ defmodule Efsql.ExecutorConcurrencyTest do test "with max 1 everything runs in the calling process" do me = self() - assert Executor.map_concurrently([1, 2], 1, fn _ -> self() end) == [me, me] + assert map_concurrently([1, 2], 1, fn _ -> self() end) == [me, me] + end + + test "halting the stream stops the work not yet started" do + started = :counters.new(1, []) + + first = + 1..100 + |> Executor.stream_concurrently(2, fn n -> + :counters.add(started, 1, 1) + Process.sleep(5) + n + end) + |> Enum.take(3) + + assert first == [1, 2, 3] + assert :counters.get(started, 1) < 10 + + # one at a time, lazily: nothing past what was taken + :counters.put(started, 1, 0) + + count = fn n -> + :counters.add(started, 1, 1) + n + end + + assert Enum.take(Executor.stream_concurrently(1..100, 1, count), 2) == [1, 2] + assert :counters.get(started, 1) == 2 end end diff --git a/test/pipeline_test.exs b/test/pipeline_test.exs new file mode 100644 index 0000000..468234c --- /dev/null +++ b/test/pipeline_test.exs @@ -0,0 +1,116 @@ +defmodule Efsql.PipelineTest do + use ExUnit.Case, async: true + + alias Efsql.Aggregate + alias Efsql.Executor + alias Efsql.Pipeline + alias Efsql.Predicate + + # The operators applied to all the rows at once: what the pipeline must + # match however the rows are chunked. + defp reference(ops, rows) do + Enum.reduce(ops, rows, fn + {:filter, preds}, rows -> Enum.filter(rows, &Predicate.matches?(&1, preds)) + {:sort, order}, rows -> Executor.sort(rows, order) + {:limit, n}, rows -> Enum.take(rows, n) + {:project, fields}, rows -> Enum.map(rows, &Map.take(&1, fields)) + {:aggregate, group_by, aggs}, rows -> Aggregate.run(rows, group_by, aggs) + end) + end + + # Feeds chunk by chunk, stopping when the pipeline halts. + defp streamed(ops, chunks) do + chunks + |> Enum.reduce_while(Pipeline.new(ops), &Pipeline.feed(&2, &1)) + |> Pipeline.finish() + end + + defp random_rows(n) do + for i <- 1..n//1 do + %{ + id: i, + city: Enum.random(["Lima", "Osaka", "Oslo", nil]), + age: Enum.random([nil, 18, 30, 30, 45, 60]), + price: Enum.random([nil, 1, 2.5, Decimal.new("3.25")]) + } + end + end + + defp random_chunks(rows) do + Enum.chunk_every(rows, Enum.random(1..max(length(rows), 1))) + end + + defp random_ops() do + filter = + Enum.random([[], [{:filter, [{:cmp, :>=, :age, 30}]}], [{:filter, [{:not_null, :city}]}]]) + + order = Enum.random([[asc: :age], [desc: :age, asc: :id], [asc: :city, desc: :id]]) + limit = Enum.random([[], [{:limit, 0}], [{:limit, 1}], [{:limit, 3}], [{:limit, 1000}]]) + + case Enum.random(1..3) do + 1 -> + filter ++ + Enum.random([[], [{:sort, order}]]) ++ + limit ++ Enum.random([[], [{:project, [:id, :age]}]]) + + 2 -> + aggs = [{:n, :count, :star}, {:a, :avg, :age}, {:s, :sum, :price}, {:m, :max, :city}] + + filter ++ + [{:aggregate, Enum.random([[], [:city], [:city, :age]]), aggs}] ++ + Enum.random([[], [{:sort, [desc: :n]}]]) ++ limit + + 3 -> + filter ++ [{:sort, order}, {:limit, Enum.random(0..5)}, {:project, [:id]}] + end + end + + test "however the rows are chunked, the result is the operators over all of them" do + for _ <- 1..500 do + rows = random_rows(Enum.random(0..40)) + ops = random_ops() + + assert streamed(ops, random_chunks(rows)) == reference(ops, rows), + "ops: #{inspect(ops)}" + end + end + + test "a limit halts the feeding once it has its rows" do + chunks = [[%{id: 1}, %{id: 2}], [%{id: 3}], [%{id: 4}]] + pipeline = Pipeline.new([{:filter, [{:cmp, :>, :id, 1}]}, {:limit, 2}]) + + {:cont, pipeline} = Pipeline.feed(pipeline, Enum.at(chunks, 0)) + {:halt, pipeline} = Pipeline.feed(pipeline, Enum.at(chunks, 1)) + assert Pipeline.finish(pipeline) == [%{id: 2}, %{id: 3}] + end + + test "a sort a limit cuts keeps only the best rows between chunks" do + pipeline = Pipeline.new([{:sort, [desc: :n]}, {:limit, 2}]) + + pipeline = + Enum.reduce([[%{n: 1}, %{n: 5}], [%{n: 3}, %{n: 9}], [%{n: 4}]], pipeline, fn chunk, p -> + {:cont, p} = Pipeline.feed(p, chunk) + # never more than the limit held + assert [{{:top, _, 2}, best}] = p.stages + assert length(best) <= 2 + p + end) + + assert Pipeline.finish(pipeline) == [%{n: 9}, %{n: 5}] + end + + test "an aggregate adds each chunk to its running totals" do + ops = [{:aggregate, [:g], [{:n, :count, :star}, {:t, :sum, :v}, {:a, :avg, :v}]}] + chunks = [[%{g: 1, v: 2}, %{g: 2, v: 5}], [%{g: 1, v: 4}], [%{g: 1, v: nil}]] + + assert streamed(ops, chunks) == [ + %{g: 1, n: 3, t: 6, a: 3.0}, + %{g: 2, n: 1, t: 5, a: 5.0} + ] + end + + test "no chunks at all" do + assert streamed([{:sort, [asc: :a]}, {:limit, 2}], []) == [] + assert streamed([{:aggregate, [], [{:n, :count, :star}]}], []) == [%{n: 0}] + end +end From abc24a3c5fe37b654ba25d109603274412c25143 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:54:17 +0000 Subject: [PATCH 3/3] Mangle test queries by codepoint, not grapheme SQLGen.mangle/2 inserts invalid UTF-8 on purpose (a lone 0xC3). The next mangling round then split the string with String.graphemes/1, which raises on invalid UTF-8 under OTP 28's unicode_util, so the "mangled queries never crash" property test failed on some seeds. The string is now cut into codepoints, with each invalid byte a unit of its own. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- test/support/efsql_test/sql_gen.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/support/efsql_test/sql_gen.ex b/test/support/efsql_test/sql_gen.ex index 8b119b5..ec0b0a7 100644 --- a/test/support/efsql_test/sql_gen.ex +++ b/test/support/efsql_test/sql_gen.ex @@ -313,7 +313,7 @@ defmodule EfsqlTest.SQLGen do def mangle(sql, 0), do: sql def mangle(sql, n) do - chars = String.graphemes(sql) + chars = units(sql) at = Enum.random(0..length(chars)) junk = Enum.random(@junk) @@ -329,6 +329,13 @@ defmodule EfsqlTest.SQLGen do mangle(Enum.join(chars), n - 1) end + # A mangled query can hold invalid UTF-8 (a junk byte), which + # String.graphemes/1 raises on (in OTP 28's unicode_util), so it is cut + # into codepoints, each invalid byte a unit of its own. + defp units(<>), do: [<> | units(rest)] + defp units(<>), do: [<> | units(rest)] + defp units(<<>>), do: [] + @alphabet String.graphemes("selctfromwhandi*,.;()'\"=<>!:-+/%_ \n\t0123456789eé") ++ ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>]