diff --git a/README.md b/README.md index b49ede9..acb1aa1 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,7 @@ so if you forget. Comments (`-- ...` and `/* ... */`) can go anywhere. A syntax error reports the line and column it was found at, and features efsql doesn't support (`OR`, -`NOT`, `<>`, joins, `HAVING`, functions other than aggregates) are rejected by -name. +joins, `HAVING`, functions other than aggregates) are rejected by name. ### Storage IDs @@ -220,7 +219,10 @@ When the tenants don't fit in one transaction, read them in batches: ``` Each batch of 25 tenants is then its own transaction, and there is no -tenant limit. Up to four batches are read at once (the +tenant limit. Batches are processed as they arrive, so a batched query keeps +only what its result needs: a `LIMIT` without `ORDER BY` stops reading once +it has its rows, `ORDER BY ... LIMIT n` keeps the best `n` so far, and +`GROUP BY` keeps a running total per group. Up to four batches are read at once (the `:efsql, :batch_concurrency` setting, or `batch_concurrency:` from Elixir). Grouping, ordering and `LIMIT` still apply to all the rows together, but the result is no longer one snapshot: each batch sees the database at a slightly different moment, and the result says how many @@ -276,6 +278,27 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals). +### Negation and ILIKE + +```sql +select col_a from tenant_id.table_name where status <> 'paid'; -- or != +select col_a from tenant_id.table_name where status not in ('paid', 'refunded'); +select col_a from tenant_id.table_name where total not between 10 and 20; +select col_a from tenant_id.table_name where name not like 'test%'; +select col_a from tenant_id.table_name where name ilike 'al%'; -- ignores case +select col_a from tenant_id.table_name where not (status = 'paid' or total > 100); +``` + +`NOT` works on any condition, and is read as its opposite: `not (a < 1)` is +`a >= 1`, `not (a = 1 or b = 2)` is `a <> 1 and b <> 2`. `not (a and b)` +would need `OR`, which isn't supported. + +As in SQL, a NULL field matches no comparison, negated ones included: a +row whose `status` is NULL matches neither `status = 'paid'` nor +`status <> 'paid'`. Negations and `ILIKE` are checked on the rows read, as +no index can serve them; the rest of the `WHERE` clause still narrows what +is read. + ### Filter by NULL ```sql diff --git a/lib/efsql/aggregate.ex b/lib/efsql/aggregate.ex index 23b17b4..65afe48 100644 --- a/lib/efsql/aggregate.ex +++ b/lib/efsql/aggregate.ex @@ -1,7 +1,10 @@ defmodule Efsql.Aggregate do @moduledoc """ - `GROUP BY` and aggregates over pulled rows, for `Efsql.Executor`'s - `{:aggregate, group_by, aggregates}` operator. + `GROUP BY` and aggregates, for the `{:aggregate, group_by, aggregates}` + operator. It runs over rows as they arrive (`new/2`, `add/2`, + `finish/1`), keeping one small running total per group rather than the + rows themselves, so a query across many tenants can aggregate each batch + as it is read. Rows are grouped by the values of the `group_by` fields; equal values group together even when their terms differ (`Decimal` 1.0 and 1.00, or @@ -12,9 +15,10 @@ defmodule Efsql.Aggregate do SQL semantics: `count(*)` counts rows, every other aggregate skips NULLs, and `sum`, `min`, `max` and `avg` of no values are NULL. `sum` and `avg` - take numbers and `Decimal`s, and are `Decimal` when any input is. - `min` and `max` order with `Efsql.Types.compare/2`, so they work on - strings and datetimes too. Groups come out ordered by their key. + take numbers and `Decimal`s, and become `Decimal` from the first + `Decimal` on. `min` and `max` order with `Efsql.Types.compare/2`, so + they work on strings and datetimes too. Groups come out ordered by their + key. """ alias Efsql.Exception.Unsupported @@ -22,79 +26,106 @@ defmodule Efsql.Aggregate do @type aggregate :: {name :: atom, :count | :sum | :min | :max | :avg, atom | :star} + defstruct group_by: [], aggregates: [], groups: %{} + + @opaque t :: %__MODULE__{} + + @doc "All at once: `new/2`, `add/2` and `finish/1` over `rows`." @spec run([map], [atom], [aggregate]) :: [map] - def run(rows, group_by, aggregates) do - rows - |> Enum.group_by(fn row -> Enum.map(group_by, &Types.equality_key(Map.get(row, &1))) end) - |> ensure_one_group(group_by) - |> Enum.map(fn {_key, rows} -> - first = List.first(rows, %{}) - group = Map.new(group_by, &{&1, Map.get(first, &1)}) - - Enum.reduce(aggregates, group, fn {name, function, arg}, acc -> - Map.put(acc, name, compute(function, arg, rows)) + def run(rows, group_by, aggregates), + do: group_by |> new(aggregates) |> add(rows) |> finish() + + @spec new([atom], [aggregate]) :: t() + def new(group_by, aggregates), do: %__MODULE__{group_by: group_by, aggregates: aggregates} + + @doc "Adds rows to the running totals." + @spec add(t(), [map]) :: t() + def add(%__MODULE__{group_by: group_by, aggregates: aggregates} = state, rows) do + groups = + Enum.reduce(rows, state.groups, fn row, groups -> + key = Enum.map(group_by, &Types.equality_key(Map.get(row, &1))) + + {values, accs} = + Map.get_lazy(groups, key, fn -> + {Map.new(group_by, &{&1, Map.get(row, &1)}), Enum.map(aggregates, &initial/1)} + end) + + accs = Enum.zip_with(aggregates, accs, &accumulate(&1, &2, row)) + Map.put(groups, key, {values, accs}) + end) + + %__MODULE__{state | groups: groups} + end + + @doc "One row per group, ordered by the group fields." + @spec finish(t()) :: [map] + def finish(%__MODULE__{group_by: group_by, aggregates: aggregates, groups: groups}) do + # A whole-table aggregate answers even when no row matched. + groups = + if group_by == [] and groups == %{}, + do: [{%{}, Enum.map(aggregates, &initial/1)}], + else: Map.values(groups) + + groups + |> Enum.map(fn {values, accs} -> + aggregates + |> Enum.zip(accs) + |> Enum.reduce(values, fn {{name, function, _arg}, acc}, row -> + Map.put(row, name, result(function, acc)) end) end) |> Efsql.Executor.sort(Enum.map(group_by, &{:asc, &1})) end - # A whole-table aggregate answers even when no row matched. - defp ensure_one_group(groups, []) when groups == %{}, do: [{[], []}] - defp ensure_one_group(groups, _group_by), do: Enum.to_list(groups) + # -- running totals -- - # -- aggregates -- + defp initial({_name, :count, _arg}), do: 0 + defp initial({_name, :avg, _arg}), do: {nil, 0} + defp initial({_name, _function, _arg}), do: nil - defp compute(:count, :star, rows), do: length(rows) - defp compute(:count, field, rows), do: length(values(rows, field)) + defp accumulate({_name, :count, :star}, n, _row), do: n + 1 - defp compute(:min, field, rows), do: extreme(values(rows, field), :lt) - defp compute(:max, field, rows), do: extreme(values(rows, field), :gt) - - defp compute(:sum, field, rows) do - case values(rows, field) do - [] -> nil - values -> sum(values, field) + defp accumulate({_name, function, field}, acc, row) do + case Map.get(row, field) do + nil -> acc + value -> step(function, field, acc, value) end end - defp compute(:avg, field, rows) do - case values(rows, field) do - [] -> - nil + defp step(:count, _field, n, _value), do: n + 1 + defp step(:sum, field, total, value), do: add_number(total, value, field) + defp step(:avg, field, {total, n}, value), do: {add_number(total, value, field), n + 1} + defp step(:min, _field, best, value), do: better(best, value, :lt) + defp step(:max, _field, best, value), do: better(best, value, :gt) - values -> - case sum(values, field) do - %Decimal{} = total -> Decimal.div(total, length(values)) - total -> total / length(values) - end - end - end + defp better(nil, value, _keep), do: value - defp values(rows, field) do - rows |> Enum.map(&Map.get(&1, field)) |> Enum.reject(&is_nil/1) - end + defp better(best, value, keep), + do: if(Types.compare(value, best) == keep, do: value, else: best) - defp extreme([], _keep), do: nil + defp add_number(total, value, field) do + unless is_number(value) or is_struct(value, Decimal) do + raise Unsupported, + "sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}" + end - defp extreme([first | rest], keep) do - Enum.reduce(rest, first, fn value, best -> - if Types.compare(value, best) == keep, do: value, else: best - end) - end + cond do + total == nil -> + value - defp sum(values, field) do - Enum.each(values, fn value -> - unless is_number(value) or is_struct(value, Decimal) do - raise Unsupported, - "sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}" - end - end) + is_struct(total, Decimal) or is_struct(value, Decimal) -> + Decimal.add(decimal(total), decimal(value)) - if Enum.any?(values, &is_struct(&1, Decimal)), - do: values |> Enum.map(&decimal/1) |> Enum.reduce(&Decimal.add/2), - else: Enum.sum(values) + true -> + total + value + end end + defp result(:avg, {nil, 0}), do: nil + defp result(:avg, {%Decimal{} = total, n}), do: Decimal.div(total, n) + defp result(:avg, {total, n}), do: total / n + defp result(_function, acc), do: acc + defp decimal(%Decimal{} = d), do: d defp decimal(n) when is_integer(n), do: Decimal.new(n) defp decimal(n) when is_float(n), do: Decimal.from_float(n) diff --git a/lib/efsql/complete.ex b/lib/efsql/complete.ex index fe86918..14a12c1 100644 --- a/lib/efsql/complete.ex +++ b/lib/efsql/complete.ex @@ -14,7 +14,7 @@ defmodule Efsql.Complete do @statement_start ~w(select) @post_expr ~w(and group order limit) - @operators ~w(= > >= < <= like in between not is) + @operators ~w(= <> != > >= < <= like ilike in between not is) def complete(input, context) do {word_start, word} = current_word(input) diff --git a/lib/efsql/executor.ex b/lib/efsql/executor.ex index 242f67f..52085c0 100644 --- a/lib/efsql/executor.ex +++ b/lib/efsql/executor.ex @@ -1,12 +1,11 @@ defmodule Efsql.Executor do @moduledoc """ - Executes an `Efsql.Physical.Plan`: pulls rows from the plan's access node, - then folds the operator pipeline over them. - - SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is - false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first - descending, matching PostgreSQL's defaults. Values compare with - `Efsql.Types.compare/2`, so datetimes order chronologically. + Executes an `Efsql.Physical.Plan`: reads rows through the plan's access + node and feeds them, as they arrive, through an `Efsql.Pipeline` of its + operators. A batched read across tenants arrives a batch at a time, and + stops early when the pipeline has all it needs (a `LIMIT` met). + Predicates evaluate with `Efsql.Predicate`'s SQL NULL semantics, and + `sort/2` orders NULLs as PostgreSQL does. A query across tenants (`{:fan_out, [{tenant_name, plan}]}`) reads every tenant in one FoundationDB transaction on the database: each tenant's @@ -17,26 +16,30 @@ defmodule Efsql.Executor do alias Efsql.Exception.Unsupported alias Efsql.Physical.Plan - alias Efsql.Predicate + alias Efsql.Pipeline alias Efsql.Types # transaction_too_old (retryable) and transaction_timed_out @too_long [1007, 1031] def run(%Plan{access: access, ops: ops}) do - Enum.reduce(ops, fetch(access), &apply_op/2) + access + |> chunks() + |> Enum.reduce_while(Pipeline.new(ops), &Pipeline.feed(&2, &1)) + |> Pipeline.finish() end @doc """ - Maps `fun` over `items`, up to `max` at a time, each in a process of its - own, and returns the results in order. The first failure stops the rest - and is raised (or thrown, or exited) here as it was there, so a caller's - `rescue` sees the original exception rather than a linked process's - exit. With `max` 1 it all runs in the calling process. + A stream of `fun` over `items`, up to `max` at a time, each in a process + of its own, the results in order. Halting the stream stops the work not + yet done. A failure is raised (or thrown, or exited) in the consumer as + it was in its process, so a caller's `rescue` sees the original + exception rather than a linked process's exit. With `max` 1 it all runs + in the consuming process, one item at a time. """ - def map_concurrently(items, 1, fun), do: Enum.map(items, fun) + def stream_concurrently(items, 1, fun), do: Stream.map(items, fun) - def map_concurrently(items, max, fun) do + def stream_concurrently(items, max, fun) do items |> Task.async_stream( fn item -> @@ -50,21 +53,21 @@ defmodule Efsql.Executor do ordered: true, timeout: :infinity ) - |> Enum.reduce_while([], fn - {:ok, {:ok, result}}, results -> {:cont, [result | results]} - {:ok, failed}, _results -> {:halt, failed} + |> Stream.map(fn + {:ok, {:ok, result}} -> result + {:ok, {:failed, kind, reason, stacktrace}} -> :erlang.raise(kind, reason, stacktrace) end) - |> case do - {:failed, kind, reason, stacktrace} -> :erlang.raise(kind, reason, stacktrace) - results -> Enum.reverse(results) - end end # -- access nodes -- - # One transaction per batch, up to `concurrency` at a time. - defp fetch({:batches, batches, concurrency}), - do: batches |> map_concurrently(concurrency, &fetch/1) |> Enum.concat() + # The rows a plan reads, as the chunks they arrive in: one per batch, a + # transaction each, up to `concurrency` at a time, or else just one. A + # pipeline that halts stops the batches not yet read. + defp chunks({:batches, batches, concurrency}), + do: stream_concurrently(batches, concurrency, &fetch/1) + + defp chunks(access), do: [fetch(access)] defp fetch({:fan_out, []}), do: [] @@ -95,7 +98,7 @@ defmodule Efsql.Executor do rows = plan.ops - |> Enum.reduce(List.flatten(mine), &apply_op/2) + |> Pipeline.run(List.flatten(mine)) |> Enum.map(&Map.put(&1, :_tenant, name)) {rows, rest} @@ -132,25 +135,6 @@ defmodule Efsql.Executor do Efsql.Repo.async_all_from_source(query, options) end - # -- operators -- - - defp apply_op({:filter, predicates}, rows), - do: Enum.filter(rows, &Predicate.matches?(&1, predicates)) - - defp apply_op({:aggregate, group_by, aggregates}, rows) do - Efsql.Aggregate.run(rows, group_by, aggregates) - end - - defp apply_op({:sort, order}, rows), do: sort(rows, order) - - defp apply_op({:limit, n}, rows) do - Enum.take(rows, n) - end - - defp apply_op({:project, fields}, rows) do - Enum.map(rows, &Map.take(&1, fields)) - end - # -- sorting -- @doc """ diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 71fe778..ec45301 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -14,7 +14,11 @@ defmodule Efsql.Parser do alias Efsql.SQL.AST alias Efsql.Types - @flipped %{:< => :>, :> => :<, :<= => :>=, :>= => :<=, := => :=} + @flipped %{:< => :>, :> => :<, :<= => :>=, :>= => :<=, := => :=, :<> => :<>} + + # NOT of a comparison, which SQL's NULL rules keep exact: NOT (a < 1) and + # a >= 1 are both false when a is NULL. + @negated %{:= => :<>, :<> => :=, :< => :>=, :>= => :<, :> => :<=, :<= => :>} @max_versionstamp Bitwise.bsl(1, 96) - 1 def sql_to_logical(sql) do @@ -174,14 +178,36 @@ defmodule Efsql.Parser do # -- WHERE -- defp predicates(nil), do: [] - defp predicates(expr), do: expr |> conjuncts([]) |> Enum.map(&predicate/1) + defp predicates(expr), do: expr |> push_not() |> conjuncts([]) |> Enum.map(&predicate/1) + + # NOT is pushed down to the conditions it negates, each of which has an + # exact opposite: NOT (a = 1) is a <> 1, NOT (a IN ...) is a NOT IN ..., + # and NOT (a OR b) is NOT a AND NOT b. NOT (a AND b) is NOT a OR NOT b, + # which needs OR. + defp push_not({:not, expr}), do: negate(expr) + + defp push_not({op, left, right}) when op in [:and, :or], + do: {op, push_not(left), push_not(right)} + + defp push_not(expr), do: expr + + defp negate({:not, expr}), do: push_not(expr) + defp negate({:and, left, right}), do: {:or, negate(left), negate(right)} + defp negate({:or, left, right}), do: {:and, negate(left), negate(right)} + defp negate({:compare, op, left, right}), do: {:compare, Map.fetch!(@negated, op), left, right} + + defp negate({:between, expr, low, high, negated?}), + do: {:between, expr, low, high, not negated?} + + defp negate({kind, expr, values, negated?}) when kind in [:in, :like, :ilike], + do: {kind, expr, values, not negated?} + + defp negate({:is_null, expr, negated?}), do: {:is_null, expr, not negated?} + defp negate(expr), do: {:not, expr} defp conjuncts({:and, left, right}, acc), do: conjuncts(left, conjuncts(right, acc)) defp conjuncts(expr, acc), do: [expr | acc] - defp predicate({:compare, :<>, _left, _right}), - do: raise(Unsupported, "<> and != are not supported") - defp predicate({:compare, op, {:column, _} = column, {:column, _}}), do: raise(Unsupported, "comparing two fields (#{describe(column)} #{op} ...) is not supported") @@ -193,20 +219,34 @@ defmodule Efsql.Parser do defp predicate({:compare, op, value, {:column, field}}), do: {:cmp, logical_op(Map.fetch!(@flipped, op)), field(field), value(value)} - defp predicate({:between, {:column, field}, low, high, false}), - do: {:range, field(field), {:>=, value(low)}, {:<=, value(high)}} + defp predicate({:between, {:column, field}, low, high, negated?}), + do: + {if(negated?, do: :not_range, else: :range), field(field), {:>=, value(low)}, + {:<=, value(high)}} - defp predicate({:in, {:column, field}, values, false}), - do: {:in, field(field), Enum.map(values, &value/1)} + defp predicate({:in, {:column, field}, values, negated?}), + do: {if(negated?, do: :not_in, else: :in), field(field), Enum.map(values, &value/1)} - defp predicate({:like, {:column, field}, pattern, negated?}) do + defp predicate({like, {:column, field}, pattern, negated?}) when like in [:like, :ilike] do pattern = case value(pattern) do - pattern when is_binary(pattern) -> pattern - other -> raise Unsupported, "a LIKE pattern must be a string, got #{inspect(other)}" + pattern when is_binary(pattern) -> + pattern + + other -> + what = if like == :ilike, do: "an ILIKE", else: "a LIKE" + raise Unsupported, "#{what} pattern must be a string, got #{inspect(other)}" end - {if(negated?, do: :not_like, else: :like), field(field), pattern} + kind = + case {like, negated?} do + {:like, false} -> :like + {:like, true} -> :not_like + {:ilike, false} -> :ilike + {:ilike, true} -> :not_ilike + end + + {kind, field(field), pattern} end defp predicate({:is_null, {:column, "_"}, _negated?}), @@ -215,16 +255,13 @@ defmodule Efsql.Parser do defp predicate({:is_null, {:column, field}, negated?}), do: {if(negated?, do: :not_null, else: :is_null), field(field)} - defp predicate({:or, _, _}), do: raise(Unsupported, "OR is not supported") - defp predicate({:not, _}), do: raise(Unsupported, "NOT is not supported") - - defp predicate({:between, _, _, _, true}), - do: raise(Unsupported, "NOT BETWEEN is not supported") + defp predicate({:or, _, _}), + do: raise(Unsupported, "OR is not supported (nor NOT over AND, which means OR)") - defp predicate({:in, _, _, true}), do: raise(Unsupported, "NOT IN is not supported") - defp predicate({:ilike, _, _, _}), do: raise(Unsupported, "ILIKE is not supported") + defp predicate({:not, expr}), + do: raise(Unsupported, "#{describe(expr)} is not a condition; NOT needs a condition") - defp predicate({kind, _, _, _}) when kind in [:in, :like], + defp predicate({kind, _, _, _}) when kind in [:in, :like, :ilike], do: raise(Unsupported, "#{String.upcase(to_string(kind))} needs a field on its left") defp predicate({:between, _, _, _, _}), @@ -239,6 +276,7 @@ defmodule Efsql.Parser do do: raise(Unsupported, "#{describe(expr)} is not a condition; compare it to something") defp logical_op(:=), do: :== + defp logical_op(:<>), do: :!= defp logical_op(op), do: op # Fields are atoms, which can't be longer than 255 characters. diff --git a/lib/efsql/pipeline.ex b/lib/efsql/pipeline.ex new file mode 100644 index 0000000..4131551 --- /dev/null +++ b/lib/efsql/pipeline.ex @@ -0,0 +1,121 @@ +defmodule Efsql.Pipeline do + @moduledoc """ + Runs a plan's operators (see `Efsql.Physical`) over rows as they arrive, + a chunk at a time: `new/1`, then `feed/2` for each chunk, then + `finish/1` for the result. Each operator keeps only what it has to: + + * `{:filter, predicates}` and `{:project, fields}` - nothing; rows + pass straight through + * `{:limit, n}` - how many rows are still wanted; once it has them, + `feed/2` says `:halt`, and whatever feeds the pipeline can stop + reading + * `{:sort, order}` directly followed by `{:limit, n}` - only the best + `n` rows so far (a running top-n), not every row + * `{:sort, order}` alone - every row, since the last one read may sort + first + * `{:aggregate, group_by, aggregates}` - one running total per group + (see `Efsql.Aggregate`) + + So a query across many tenants, read in batches, holds a limit's worth + of rows, or a group's worth of totals, rather than every row it read, + and a `LIMIT` without `ORDER BY` stops reading as soon as it is met. + A single read is just one chunk. + """ + + alias Efsql.Aggregate + alias Efsql.Executor + alias Efsql.Predicate + + defstruct stages: [], out: [] + + @opaque t :: %__MODULE__{} + + @spec new([tuple()]) :: t() + def new(ops), do: %__MODULE__{stages: ops |> fuse() |> Enum.map(&{&1, init(&1)})} + + @doc """ + Feeds a chunk of rows through: `{:cont, pipeline}`, or `{:halt, pipeline}` + when a limit has all its rows and no more input could change the result. + Its shape suits `Enum.reduce_while/3`. + """ + @spec feed(t(), [map()]) :: {:cont | :halt, t()} + def feed(%__MODULE__{} = pipeline, rows) do + {stages, out, halt?} = push(pipeline.stages, rows) + pipeline = %__MODULE__{stages: stages, out: [out | pipeline.out]} + {if(halt?, do: :halt, else: :cont), pipeline} + end + + @doc "The result: every row the operators let through, in order." + @spec finish(t()) :: [map()] + def finish(%__MODULE__{} = pipeline) do + # A stage's held rows go on through the stages after it. + rest = + Enum.reduce(pipeline.stages, [], fn {op, state}, pending -> + {state, out, _halt?} = step(op, state, pending) + out ++ flush(op, state) + end) + + pipeline.out |> Enum.reverse() |> Enum.concat() |> Kernel.++(rest) + end + + @doc "All at once: `new/1`, `feed/2` and `finish/1` over `rows`." + @spec run([tuple()], [map()]) :: [map()] + def run(ops, rows) do + {_, pipeline} = ops |> new() |> feed(rows) + finish(pipeline) + end + + # A sort that a limit then cuts is a running top-n. + defp fuse([{:sort, order}, {:limit, n} | rest]), do: [{:top, order, n} | fuse(rest)] + defp fuse([op | rest]), do: [op | fuse(rest)] + defp fuse([]), do: [] + + defp init({:limit, n}), do: n + defp init({:sort, _order}), do: [] + defp init({:top, _order, _n}), do: [] + defp init({:aggregate, group_by, aggregates}), do: Aggregate.new(group_by, aggregates) + defp init(_op), do: nil + + # Each stage takes the rows the one before let through. + defp push(stages, rows) do + {stages, {out, halt?}} = + Enum.map_reduce(stages, {rows, false}, fn {op, state}, {rows, halt?} -> + {state, out, stop?} = step(op, state, rows) + {{op, state}, {out, halt? or stop?}} + end) + + {stages, out, halt?} + end + + # {state, rows passed on, whether no more input is needed} + defp step({:filter, predicates}, nil, rows), + do: {nil, Enum.filter(rows, &Predicate.matches?(&1, predicates)), false} + + defp step({:project, fields}, nil, rows), + do: {nil, Enum.map(rows, &Map.take(&1, fields)), false} + + defp step({:limit, _n}, remaining, rows) do + taken = Enum.take(rows, remaining) + remaining = remaining - length(taken) + {remaining, taken, remaining == 0} + end + + defp step({:sort, _order}, chunks, []), do: {chunks, [], false} + defp step({:sort, _order}, chunks, rows), do: {[rows | chunks], [], false} + + defp step({:top, _order, _n}, best, []), do: {best, [], false} + + defp step({:top, order, n}, best, rows), + do: {(best ++ rows) |> Executor.sort(order) |> Enum.take(n), [], false} + + defp step({:aggregate, _group_by, _aggregates}, totals, rows), + do: {Aggregate.add(totals, rows), [], false} + + # What a stage still holds once the input is done. + defp flush({:sort, order}, chunks), + do: chunks |> Enum.reverse() |> Enum.concat() |> Executor.sort(order) + + defp flush({:top, _order, _n}, best), do: best + defp flush({:aggregate, _group_by, _aggregates}, totals), do: Aggregate.finish(totals) + defp flush(_op, _state), do: [] +end diff --git a/lib/efsql/planner.ex b/lib/efsql/planner.ex index 234ea34..32acf12 100644 --- a/lib/efsql/planner.ex +++ b/lib/efsql/planner.ex @@ -368,8 +368,10 @@ defmodule Efsql.Planner do defp ensure_residual_evaluable!(pred) do if Predicate.field(pred) == @pk_field do + # Rows carry the key under its field's name, not `_`, so a condition + # on `_` must be one the key range can serve. raise Unsupported, - "a constraint on the primary key '_' cannot be combined with this query shape" + "on the primary key '_', only =, <, <=, >, >=, BETWEEN and IN are supported" end end diff --git a/lib/efsql/predicate.ex b/lib/efsql/predicate.ex index eb42d73..f2c0751 100644 --- a/lib/efsql/predicate.ex +++ b/lib/efsql/predicate.ex @@ -5,13 +5,18 @@ defmodule Efsql.Predicate do the Ecto expression that pushes it to the adapter. A query's predicates are an implicitly AND-ed list of: - * `{:cmp, op, field, value}` — op in `:== :> :>= :< :<=` + * `{:cmp, op, field, value}` — op in `:== :!= :> :>= :< :<=` * `{:range, field, {lower_op, value}, {upper_op, value}}` — a two-sided - bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=` - * `{:like, field, pattern}` / `{:not_like, field, pattern}` - * `{:in, field, values}` + bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=`; `:not_range` + is its negation (`NOT BETWEEN`) + * `{:like, field, pattern}` / `{:not_like, field, pattern}`, and the + case-insensitive `:ilike` / `:not_ilike` + * `{:in, field, values}` / `{:not_in, field, values}` * `{:is_null, field}` / `{:not_null, field}` + SQL's `NOT` never reaches here: `Efsql.Parser` pushes it into the + condition it negates (`NOT (a < 1)` is `a >= 1`). + The primary key is the pseudo-field `:_`. A new kind of predicate is added here, clause by clause, plus its @@ -23,16 +28,17 @@ defmodule Efsql.Predicate do @type field :: atom() @type t :: - {:cmp, :== | :> | :>= | :< | :<=, field(), term()} - | {:range, field(), {:> | :>=, term()}, {:< | :<=, term()}} - | {:like | :not_like, field(), String.t()} - | {:in, field(), [term()]} + {:cmp, :== | :!= | :> | :>= | :< | :<=, field(), term()} + | {:range | :not_range, field(), {:> | :>=, term()}, {:< | :<=, term()}} + | {:like | :not_like | :ilike | :not_ilike, field(), String.t()} + | {:in | :not_in, field(), [term()]} | {:is_null | :not_null, field()} @pk_field :_ @cmp_results %{ ==: [:eq], + !=: [:lt, :gt], >: [:gt], >=: [:gt, :eq], <: [:lt], @@ -42,19 +48,18 @@ defmodule Efsql.Predicate do @doc "The field a predicate constrains." @spec field(t()) :: field() def field({:cmp, _op, field, _value}), do: field - def field({:range, field, _lower, _upper}), do: field - def field({:like, field, _pattern}), do: field - def field({:not_like, field, _pattern}), do: field - def field({:in, field, _values}), do: field - def field({:is_null, field}), do: field - def field({:not_null, field}), do: field + def field({kind, field, _lower, _upper}) when kind in [:range, :not_range], do: field + def field({kind, field, _arg}) when kind in [:like, :not_like, :ilike, :not_ilike], do: field + def field({kind, field, _values}) when kind in [:in, :not_in], do: field + def field({kind, field}) when kind in [:is_null, :not_null], do: field # -- evaluation -- @doc """ Whether `row` satisfies every one of `predicates`, with SQL NULL semantics: a NULL (nil or absent) field matches no comparison, LIKE or - IN, NOT LIKE included, and IS NULL matches it. Values compare with + IN, negated ones included (a NULL is neither `= 1` nor `<> 1`), and + IS NULL matches it. Values compare with `Efsql.Types.compare/2`. """ @spec matches?(map(), [t()]) :: boolean() @@ -66,18 +71,28 @@ defmodule Efsql.Predicate do defp eval({:range, field, {lower_op, lower}, {upper_op, upper}}, row), do: eval({:cmp, lower_op, field, lower}, row) and eval({:cmp, upper_op, field, upper}, row) - defp eval({:in, field, values}, row), - do: - on_value(row, field, fn value -> Enum.any?(values, &(Types.compare(value, &1) == :eq)) end) + defp eval({:not_range, field, lower, upper}, row), + do: on_value(row, field, fn _ -> not eval({:range, field, lower, upper}, row) end) + + defp eval({:in, field, values}, row), do: on_value(row, field, &member?(&1, values)) + defp eval({:not_in, field, values}, row), do: on_value(row, field, &(not member?(&1, values))) defp eval({:is_null, field}, row), do: Map.get(row, field) == nil defp eval({:not_null, field}, row), do: Map.get(row, field) != nil defp eval({:like, field, pattern}, row), - do: on_value(row, field, &Regex.match?(like_regex(pattern), &1)) + do: on_value(row, field, &Regex.match?(like_regex(pattern, ""), &1)) defp eval({:not_like, field, pattern}, row), - do: on_value(row, field, &(not Regex.match?(like_regex(pattern), &1))) + do: on_value(row, field, &(not Regex.match?(like_regex(pattern, ""), &1))) + + defp eval({:ilike, field, pattern}, row), + do: on_value(row, field, &Regex.match?(like_regex(pattern, "iu"), &1)) + + defp eval({:not_ilike, field, pattern}, row), + do: on_value(row, field, &(not Regex.match?(like_regex(pattern, "iu"), &1))) + + defp member?(value, values), do: Enum.any?(values, &(Types.compare(value, &1) == :eq)) defp on_value(row, field, fun) do case Map.get(row, field) do @@ -86,14 +101,14 @@ defmodule Efsql.Predicate do end end - defp like_regex(pattern) do + defp like_regex(pattern, flags) do source = pattern |> Regex.escape() |> String.replace("%", ".*") |> String.replace("_", ".") - Regex.compile!("\\A" <> source <> "\\z", "s") + Regex.compile!("\\A" <> source <> "\\z", "s" <> flags) end # -- planning -- @@ -105,9 +120,12 @@ defmodule Efsql.Predicate do * `:in` - an `IN` list, read as one lookup per value when an index or the key can serve them * `:index` - an equality or range an index may serve - * `:filter` - only checked on the rows read + * `:filter` - only checked on the rows read: LIKE, IS NULL, and + every negation (`<>`, NOT IN, NOT BETWEEN, NOT LIKE), which an index + can't serve """ @spec pushdown(t()) :: :key | :in | :index | :filter + def pushdown({:cmp, :!=, _field, _value}), do: :filter def pushdown({:in, _field, _values}), do: :in def pushdown({:cmp, _op, @pk_field, _value}), do: :key def pushdown({:range, @pk_field, _lower, _upper}), do: :key diff --git a/lib/efsql/rewrite.ex b/lib/efsql/rewrite.ex index 22bcb13..e098839 100644 --- a/lib/efsql/rewrite.ex +++ b/lib/efsql/rewrite.ex @@ -123,11 +123,12 @@ defmodule Efsql.Rewrite do end end - # `field in (v)` is an equality. + # `field in (v)` is an equality, and `field not in (v)` an inequality. def pass_in_singleton(%Logical.Select{predicates: preds} = logical) do predicates = Enum.map(preds, fn {:in, field, [value]} -> {:cmp, :==, field, value} + {:not_in, field, [value]} -> {:cmp, :!=, field, value} pred -> pred end) diff --git a/lib/efsql/tui/help.ex b/lib/efsql/tui/help.ex index 505479c..4c1583e 100644 --- a/lib/efsql/tui/help.ex +++ b/lib/efsql/tui/help.ex @@ -56,7 +56,10 @@ defmodule Efsql.Tui.Help do {:sql, "where age between 30 and 40 inclusive both ends"}, {:sql, "where name like 'Al%' % and _ wildcards"}, {:sql, "where name not like 'A%'"}, + {:sql, "where name ilike 'al%' ignores case"}, {:sql, "where status in ('paid', 'shipped')"}, + {:sql, "where status <> 'paid' also !=, not in, not between"}, + {:sql, "where not (a = 1 or b = 2)"}, {:sql, "where city = 'Osaka' and age > 40"}, "Literals: 'text', 42, 1.5, true, false, null.", {:note, "or is not supported; conditions combine with and only"}, diff --git a/test/cross_tenant_test.exs b/test/cross_tenant_test.exs index 8ac8a8a..a45a3e9 100644 --- a/test/cross_tenant_test.exs +++ b/test/cross_tenant_test.exs @@ -128,6 +128,17 @@ defmodule EfsqlTest.Integration.CrossTenant do end end + test "ORDER BY ... LIMIT and a plain LIMIT across batches", context do + batched = [tenant_batch: 1, max_tenants: 1, batch_concurrency: 2] + + top = "select name from *.users where #{context[:both]} order by name desc limit 2;" + assert {_plan, [%{name: "Eve"}, %{name: "Dora"}], _} = Efsql.qall(top, batched) + + # without ORDER BY any 2 rows will do, and reading stops once it has them + any = "select name from *.users where #{context[:both]} limit 2;" + assert {_plan, [_, _], _} = Efsql.qall(any, batched) + end + test "a read too old for its transaction fails at once instead of retrying", context do sql = "select name from *.users where #{context[:both]};" diff --git a/test/executor_concurrency_test.exs b/test/executor_concurrency_test.exs index c18b414..b65ebb2 100644 --- a/test/executor_concurrency_test.exs +++ b/test/executor_concurrency_test.exs @@ -4,8 +4,11 @@ defmodule Efsql.ExecutorConcurrencyTest do alias Efsql.Exception.Unsupported alias Efsql.Executor + defp map_concurrently(items, max, fun), + do: items |> Executor.stream_concurrently(max, fun) |> Enum.to_list() + test "results come back in order, whatever order the work finishes in" do - assert Executor.map_concurrently([30, 10, 20, 0], 4, fn ms -> + assert map_concurrently([30, 10, 20, 0], 4, fn ms -> Process.sleep(ms) ms end) == [30, 10, 20, 0] @@ -14,7 +17,7 @@ defmodule Efsql.ExecutorConcurrencyTest do test "at most max run at once, and more than one does" do running = :counters.new(2, []) - Executor.map_concurrently(1..12, 3, fn _ -> + map_concurrently(1..12, 3, fn _ -> :counters.add(running, 1, 1) now = :counters.get(running, 1) if now > :counters.get(running, 2), do: :counters.put(running, 2, now) @@ -27,21 +30,21 @@ defmodule Efsql.ExecutorConcurrencyTest do test "a failure reaches the caller as the original exception" do assert_raise Unsupported, "too big", fn -> - Executor.map_concurrently([1, 2, 3], 2, fn + map_concurrently([1, 2, 3], 2, fn 2 -> raise Unsupported, "too big" n -> n end) end - assert catch_throw(Executor.map_concurrently([1], 2, fn _ -> throw(:stop) end)) == :stop - assert catch_exit(Executor.map_concurrently([1], 2, fn _ -> exit(:gone) end)) == :gone + assert catch_throw(map_concurrently([1], 2, fn _ -> throw(:stop) end)) == :stop + assert catch_exit(map_concurrently([1], 2, fn _ -> exit(:gone) end)) == :gone end test "a failure stops the work not yet done" do started = :counters.new(1, []) assert_raise Unsupported, fn -> - Executor.map_concurrently(1..50, 2, fn n -> + map_concurrently(1..50, 2, fn n -> :counters.add(started, 1, 1) if n == 1, do: raise(Unsupported, "stop"), else: Process.sleep(50) end) @@ -52,6 +55,33 @@ defmodule Efsql.ExecutorConcurrencyTest do test "with max 1 everything runs in the calling process" do me = self() - assert Executor.map_concurrently([1, 2], 1, fn _ -> self() end) == [me, me] + assert map_concurrently([1, 2], 1, fn _ -> self() end) == [me, me] + end + + test "halting the stream stops the work not yet started" do + started = :counters.new(1, []) + + first = + 1..100 + |> Executor.stream_concurrently(2, fn n -> + :counters.add(started, 1, 1) + Process.sleep(5) + n + end) + |> Enum.take(3) + + assert first == [1, 2, 3] + assert :counters.get(started, 1) < 10 + + # one at a time, lazily: nothing past what was taken + :counters.put(started, 1, 0) + + count = fn n -> + :counters.add(started, 1, 1) + n + end + + assert Enum.take(Executor.stream_concurrently(1..100, 1, count), 2) == [1, 2] + assert :counters.get(started, 1) == 2 end end diff --git a/test/negation_test.exs b/test/negation_test.exs new file mode 100644 index 0000000..702517f --- /dev/null +++ b/test/negation_test.exs @@ -0,0 +1,61 @@ +defmodule EfsqlTest.Integration.Negation do + use EfsqlTest.Case, async: true + + alias Efsql.Exception.Unsupported + alias Efsql.Physical.Plan + + # Alice "Lorem ipsum", Bob "foobar", Charles with no notes. + + defp names(context, condition) do + "select name from #{context[:tenant_id]}.users where #{condition};" + |> Efsql.all() + |> Enum.map(& &1.name) + |> Enum.sort() + end + + test "<> and != and NOT (=)", context do + for condition <- ["name <> 'Bob'", "name != 'Bob'", "not (name = 'Bob')"] do + assert names(context, condition) == ["Alice", "Charles"], condition + end + end + + test "a NULL field is neither equal nor unequal", context do + assert names(context, "notes <> 'foobar'") == ["Alice"] + assert names(context, "notes not in ('foobar')") == ["Alice"] + assert names(context, "notes not like 'foo%'") == ["Alice"] + end + + test "NOT IN and NOT BETWEEN", context do + assert names(context, "name not in ('Alice', 'Bob')") == ["Charles"] + assert names(context, "not name in ('Alice', 'Bob')") == ["Charles"] + assert names(context, "name not between 'B' and 'Bz'") == ["Alice", "Charles"] + end + + test "ILIKE and NOT ILIKE ignore case", context do + assert names(context, "name ilike 'al%'") == ["Alice"] + assert names(context, "name ilike '%E%'") == ["Alice", "Charles"] + assert names(context, "name not ilike 'a%'") == ["Bob", "Charles"] + end + + test "NOT over IS NULL and over OR", context do + assert names(context, "not (notes is null)") == ["Alice", "Bob"] + assert names(context, "not (name = 'Alice' or name = 'Bob')") == ["Charles"] + end + + test "a negation is checked on the rows read, alongside what an index serves", context do + {%Plan{access: access, ops: ops}, rows, _} = + Efsql.qall( + "select name from #{context[:tenant_id]}.users where name >= 'B' and name <> 'Bob';" + ) + + assert {:index_scan, %Ecto.Query{wheres: [_]}, _} = access + assert {:filter, [{:cmp, :!=, :name, "Bob"}]} in ops + assert rows == [%{name: "Charles"}] + end + + test "the primary key takes only what a key range can serve", context do + assert_raise Unsupported, ~r/on the primary key '_', only =, <, <=, >, >=/, fn -> + Efsql.all("select name from #{context[:tenant_id]}.users where _ <> '0001';") + end + end +end diff --git a/test/parser_test.exs b/test/parser_test.exs index 88af595..c232a58 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -457,6 +457,54 @@ defmodule Efsql.ParserTest do ) end + test "<> and != are inequalities, either way round" do + for condition <- ["age <> 40", "age != 40", "40 <> age"] do + assert %Logical.Select{predicates: [{:cmp, :!=, :age, 40}]} = + parse("select id from t.users where #{condition};") + end + end + + test "NOT IN, NOT BETWEEN, ILIKE and NOT ILIKE" do + assert %Logical.Select{ + predicates: [ + {:not_in, :a, [1, 2]}, + {:not_range, :b, {:>=, 1}, {:<=, 5}}, + {:ilike, :c, "al%"}, + {:not_ilike, :d, "x_"} + ] + } = + parse( + "select id from t.users where a not in (1, 2) and b not between 1 and 5 " <> + "and c ilike 'al%' and d not ilike 'x_';" + ) + end + + test "NOT is pushed into the condition it negates" do + for {condition, predicates} <- [ + {"not (a = 1)", [{:cmp, :!=, :a, 1}]}, + {"not a <> 1", [{:cmp, :==, :a, 1}]}, + {"not a < 1", [{:cmp, :>=, :a, 1}]}, + {"not a >= 1", [{:cmp, :<, :a, 1}]}, + {"not a > 1", [{:cmp, :<=, :a, 1}]}, + {"not a <= 1", [{:cmp, :>, :a, 1}]}, + {"not not a = 1", [{:cmp, :==, :a, 1}]}, + {"not a in (1, 2)", [{:not_in, :a, [1, 2]}]}, + {"not a not in (1, 2)", [{:in, :a, [1, 2]}]}, + {"not (a between 1 and 5)", [{:not_range, :a, {:>=, 1}, {:<=, 5}}]}, + {"not a like 'x%'", [{:not_like, :a, "x%"}]}, + {"not a ilike 'x%'", [{:not_ilike, :a, "x%"}]}, + {"not (a is null)", [{:not_null, :a}]}, + {"not a isnull", [{:not_null, :a}]}, + {"not (a = 1 or b like 'x%')", [{:cmp, :!=, :a, 1}, {:not_like, :b, "x%"}]}, + {"c = 2 and not (a = 1 or not b = 3)", + [{:cmp, :==, :c, 2}, {:cmp, :!=, :a, 1}, {:cmp, :==, :b, 3}]} + ] do + assert %Logical.Select{predicates: ^predicates} = + parse("select id from t.users where #{condition};"), + condition + end + end + test "a value on the left is flipped" do assert %Logical.Select{predicates: [{:cmp, :>, :age, 40}, {:cmp, :==, :name, "x"}]} = parse("select id from t.users where 40 < age and 'x' = name;") @@ -465,12 +513,9 @@ defmodule Efsql.ParserTest do test "unsupported conditions say what is unsupported" do for {condition, message} <- [ {"a = 1 or b = 2", "OR is not supported"}, - {"not a = 1", "NOT is not supported"}, - {"a not in (1)", "NOT IN is not supported"}, - {"a not between 1 and 2", "NOT BETWEEN is not supported"}, - {"a ilike 'x'", "ILIKE is not supported"}, - {"a <> 1", "<> and != are not supported"}, - {"a != 1", "<> and != are not supported"}, + {"not (a = 1 and b = 2)", "OR is not supported"}, + {"not a", "the field a is not a condition; NOT needs a condition"}, + {"a ilike 1", "an ILIKE pattern must be a string"}, {"a = b", "comparing two fields"}, {"1 = 1", "needs a field on one side"}, {"'x' in ('x')", "IN needs a field on its left"}, diff --git a/test/pipeline_test.exs b/test/pipeline_test.exs new file mode 100644 index 0000000..468234c --- /dev/null +++ b/test/pipeline_test.exs @@ -0,0 +1,116 @@ +defmodule Efsql.PipelineTest do + use ExUnit.Case, async: true + + alias Efsql.Aggregate + alias Efsql.Executor + alias Efsql.Pipeline + alias Efsql.Predicate + + # The operators applied to all the rows at once: what the pipeline must + # match however the rows are chunked. + defp reference(ops, rows) do + Enum.reduce(ops, rows, fn + {:filter, preds}, rows -> Enum.filter(rows, &Predicate.matches?(&1, preds)) + {:sort, order}, rows -> Executor.sort(rows, order) + {:limit, n}, rows -> Enum.take(rows, n) + {:project, fields}, rows -> Enum.map(rows, &Map.take(&1, fields)) + {:aggregate, group_by, aggs}, rows -> Aggregate.run(rows, group_by, aggs) + end) + end + + # Feeds chunk by chunk, stopping when the pipeline halts. + defp streamed(ops, chunks) do + chunks + |> Enum.reduce_while(Pipeline.new(ops), &Pipeline.feed(&2, &1)) + |> Pipeline.finish() + end + + defp random_rows(n) do + for i <- 1..n//1 do + %{ + id: i, + city: Enum.random(["Lima", "Osaka", "Oslo", nil]), + age: Enum.random([nil, 18, 30, 30, 45, 60]), + price: Enum.random([nil, 1, 2.5, Decimal.new("3.25")]) + } + end + end + + defp random_chunks(rows) do + Enum.chunk_every(rows, Enum.random(1..max(length(rows), 1))) + end + + defp random_ops() do + filter = + Enum.random([[], [{:filter, [{:cmp, :>=, :age, 30}]}], [{:filter, [{:not_null, :city}]}]]) + + order = Enum.random([[asc: :age], [desc: :age, asc: :id], [asc: :city, desc: :id]]) + limit = Enum.random([[], [{:limit, 0}], [{:limit, 1}], [{:limit, 3}], [{:limit, 1000}]]) + + case Enum.random(1..3) do + 1 -> + filter ++ + Enum.random([[], [{:sort, order}]]) ++ + limit ++ Enum.random([[], [{:project, [:id, :age]}]]) + + 2 -> + aggs = [{:n, :count, :star}, {:a, :avg, :age}, {:s, :sum, :price}, {:m, :max, :city}] + + filter ++ + [{:aggregate, Enum.random([[], [:city], [:city, :age]]), aggs}] ++ + Enum.random([[], [{:sort, [desc: :n]}]]) ++ limit + + 3 -> + filter ++ [{:sort, order}, {:limit, Enum.random(0..5)}, {:project, [:id]}] + end + end + + test "however the rows are chunked, the result is the operators over all of them" do + for _ <- 1..500 do + rows = random_rows(Enum.random(0..40)) + ops = random_ops() + + assert streamed(ops, random_chunks(rows)) == reference(ops, rows), + "ops: #{inspect(ops)}" + end + end + + test "a limit halts the feeding once it has its rows" do + chunks = [[%{id: 1}, %{id: 2}], [%{id: 3}], [%{id: 4}]] + pipeline = Pipeline.new([{:filter, [{:cmp, :>, :id, 1}]}, {:limit, 2}]) + + {:cont, pipeline} = Pipeline.feed(pipeline, Enum.at(chunks, 0)) + {:halt, pipeline} = Pipeline.feed(pipeline, Enum.at(chunks, 1)) + assert Pipeline.finish(pipeline) == [%{id: 2}, %{id: 3}] + end + + test "a sort a limit cuts keeps only the best rows between chunks" do + pipeline = Pipeline.new([{:sort, [desc: :n]}, {:limit, 2}]) + + pipeline = + Enum.reduce([[%{n: 1}, %{n: 5}], [%{n: 3}, %{n: 9}], [%{n: 4}]], pipeline, fn chunk, p -> + {:cont, p} = Pipeline.feed(p, chunk) + # never more than the limit held + assert [{{:top, _, 2}, best}] = p.stages + assert length(best) <= 2 + p + end) + + assert Pipeline.finish(pipeline) == [%{n: 9}, %{n: 5}] + end + + test "an aggregate adds each chunk to its running totals" do + ops = [{:aggregate, [:g], [{:n, :count, :star}, {:t, :sum, :v}, {:a, :avg, :v}]}] + chunks = [[%{g: 1, v: 2}, %{g: 2, v: 5}], [%{g: 1, v: 4}], [%{g: 1, v: nil}]] + + assert streamed(ops, chunks) == [ + %{g: 1, n: 3, t: 6, a: 3.0}, + %{g: 2, n: 1, t: 5, a: 5.0} + ] + end + + test "no chunks at all" do + assert streamed([{:sort, [asc: :a]}, {:limit, 2}], []) == [] + assert streamed([{:aggregate, [], [{:n, :count, :star}]}], []) == [%{n: 0}] + end +end diff --git a/test/predicate_test.exs b/test/predicate_test.exs index 7256f6d..0020475 100644 --- a/test/predicate_test.exs +++ b/test/predicate_test.exs @@ -43,6 +43,33 @@ defmodule Efsql.PredicateTest do assert holds?({:not_null, :name}) end + test "negations and ILIKE" do + assert holds?({:cmp, :!=, :age, 31}) + refute holds?({:cmp, :!=, :age, 30}) + assert holds?({:cmp, :!=, :price, Decimal.new("9.49")}) + refute holds?({:cmp, :!=, :price, Decimal.new("9.500")}) + assert holds?({:not_in, :age, [10, 20]}) + refute holds?({:not_in, :age, [10, 30]}) + assert holds?({:not_range, :age, {:>=, 31}, {:<=, 40}}) + refute holds?({:not_range, :age, {:>=, 30}, {:<=, 40}}) + assert holds?({:ilike, :name, "al%"}) + assert holds?({:ilike, :name, "ALICE"}) + refute holds?({:not_ilike, :name, "a_ice"}) + assert Predicate.matches?(%{name: "Émile"}, [{:ilike, :name, "émile"}]) + end + + test "a NULL field matches no negation either" do + for predicate <- [ + {:cmp, :!=, :notes, "x"}, + {:not_in, :notes, ["x"]}, + {:not_range, :notes, {:>=, "a"}, {:<=, "b"}}, + {:ilike, :notes, "%"}, + {:not_ilike, :notes, "x"} + ] do + refute holds?(predicate), inspect(predicate) + end + end + test "every predicate must hold" do assert Predicate.matches?(@row, []) refute Predicate.matches?(@row, [{:cmp, :==, :age, 30}, {:is_null, :name}]) @@ -56,7 +83,12 @@ defmodule Efsql.PredicateTest do {{:not_like, :d, "x"}, :d}, {{:in, :e, [1]}, :e}, {{:is_null, :f}, :f}, - {{:not_null, :g}, :g} + {{:not_null, :g}, :g}, + {{:cmp, :!=, :h, 1}, :h}, + {{:not_in, :i, [1]}, :i}, + {{:not_range, :j, {:>=, 1}, {:<=, 2}}, :j}, + {{:ilike, :k, "x"}, :k}, + {{:not_ilike, :l, "x"}, :l} ] do assert Predicate.field(predicate) == field end @@ -70,7 +102,18 @@ defmodule Efsql.PredicateTest do assert Predicate.pushdown({:cmp, :>, :age, 1}) == :index assert Predicate.pushdown({:range, :age, {:>, 1}, {:<, 2}}) == :index - for predicate <- [{:like, :a, "x"}, {:not_like, :a, "x"}, {:is_null, :a}, {:not_null, :a}] do + for predicate <- [ + {:like, :a, "x"}, + {:not_like, :a, "x"}, + {:ilike, :a, "x"}, + {:not_ilike, :a, "x"}, + {:is_null, :a}, + {:not_null, :a}, + {:cmp, :!=, :a, 1}, + {:cmp, :!=, :_, "k"}, + {:not_in, :a, [1]}, + {:not_range, :a, {:>=, 1}, {:<=, 2}} + ] do assert Predicate.pushdown(predicate) == :filter end end diff --git a/test/rewrite_test.exs b/test/rewrite_test.exs index 2c008a5..441a486 100644 --- a/test/rewrite_test.exs +++ b/test/rewrite_test.exs @@ -73,6 +73,15 @@ defmodule Efsql.RewriteTest do test "multi-value in is untouched" do assert [{:in, :name, ["a", "b"]}] = normalize([{:in, :name, ["a", "b"]}]) end + + test "single-value not in becomes an inequality" do + assert [{:cmp, :!=, :name, "a"}] = normalize([{:not_in, :name, ["a"]}]) + assert [{:not_in, :name, ["a", "b"]}] = normalize([{:not_in, :name, ["a", "b"]}]) + end + + test "ilike is not a key range, whatever its prefix" do + assert [{:ilike, :name, "Al%"}] = normalize([{:ilike, :name, "Al%"}]) + end end test "Predicate.take_first takes the first match out, keeping the rest in order" do diff --git a/test/support/efsql_test/sql_gen.ex b/test/support/efsql_test/sql_gen.ex index 8b119b5..ec0b0a7 100644 --- a/test/support/efsql_test/sql_gen.ex +++ b/test/support/efsql_test/sql_gen.ex @@ -313,7 +313,7 @@ defmodule EfsqlTest.SQLGen do def mangle(sql, 0), do: sql def mangle(sql, n) do - chars = String.graphemes(sql) + chars = units(sql) at = Enum.random(0..length(chars)) junk = Enum.random(@junk) @@ -329,6 +329,13 @@ defmodule EfsqlTest.SQLGen do mangle(Enum.join(chars), n - 1) end + # A mangled query can hold invalid UTF-8 (a junk byte), which + # String.graphemes/1 raises on (in OTP 28's unicode_util), so it is cut + # into codepoints, each invalid byte a unit of its own. + defp units(<>), do: [<> | units(rest)] + defp units(<>), do: [<> | units(rest)] + defp units(<<>>), do: [] + @alphabet String.graphemes("selctfromwhandi*,.;()'\"=<>!:-+/%_ \n\t0123456789eé") ++ ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>]