Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,7 @@ so if you forget.

Comments (`-- ...` and `/* ... */`) can go anywhere. A syntax error reports the
line and column it was found at, and features efsql doesn't support (`OR`,
`NOT`, `<>`, joins, `HAVING`, functions other than aggregates) are rejected by
name.
joins, `HAVING`, functions other than aggregates) are rejected by name.

### Storage IDs

Expand Down Expand Up @@ -220,7 +219,10 @@ When the tenants don't fit in one transaction, read them in batches:
```

Each batch of 25 tenants is then its own transaction, and there is no
tenant limit. Up to four batches are read at once (the
tenant limit. Batches are processed as they arrive, so a batched query keeps
only what its result needs: a `LIMIT` without `ORDER BY` stops reading once
it has its rows, `ORDER BY ... LIMIT n` keeps the best `n` so far, and
`GROUP BY` keeps a running total per group. Up to four batches are read at once (the
`:efsql, :batch_concurrency` setting, or `batch_concurrency:` from Elixir). Grouping, ordering and `LIMIT` still apply to all the rows
together, but the result is no longer one snapshot: each batch sees the
database at a slightly different moment, and the result says how many
Expand Down Expand Up @@ -276,6 +278,27 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and

Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals).

### Negation and ILIKE

```sql
select col_a from tenant_id.table_name where status <> 'paid'; -- or !=
select col_a from tenant_id.table_name where status not in ('paid', 'refunded');
select col_a from tenant_id.table_name where total not between 10 and 20;
select col_a from tenant_id.table_name where name not like 'test%';
select col_a from tenant_id.table_name where name ilike 'al%'; -- ignores case
select col_a from tenant_id.table_name where not (status = 'paid' or total > 100);
```

`NOT` works on any condition, and is read as its opposite: `not (a < 1)` is
`a >= 1`, `not (a = 1 or b = 2)` is `a <> 1 and b <> 2`. `not (a and b)`
would need `OR`, which isn't supported.

As in SQL, a NULL field matches no comparison, negated ones included: a
row whose `status` is NULL matches neither `status = 'paid'` nor
`status <> 'paid'`. Negations and `ILIKE` are checked on the rows read, as
no index can serve them; the rest of the `WHERE` clause still narrows what
is read.

### Filter by NULL

```sql
Expand Down
147 changes: 89 additions & 58 deletions lib/efsql/aggregate.ex
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
defmodule Efsql.Aggregate do
@moduledoc """
`GROUP BY` and aggregates over pulled rows, for `Efsql.Executor`'s
`{:aggregate, group_by, aggregates}` operator.
`GROUP BY` and aggregates, for the `{:aggregate, group_by, aggregates}`
operator. It runs over rows as they arrive (`new/2`, `add/2`,
`finish/1`), keeping one small running total per group rather than the
rows themselves, so a query across many tenants can aggregate each batch
as it is read.

Rows are grouped by the values of the `group_by` fields; equal values
group together even when their terms differ (`Decimal` 1.0 and 1.00, or
Expand All @@ -12,89 +15,117 @@ defmodule Efsql.Aggregate do

SQL semantics: `count(*)` counts rows, every other aggregate skips NULLs,
and `sum`, `min`, `max` and `avg` of no values are NULL. `sum` and `avg`
take numbers and `Decimal`s, and are `Decimal` when any input is.
`min` and `max` order with `Efsql.Types.compare/2`, so they work on
strings and datetimes too. Groups come out ordered by their key.
take numbers and `Decimal`s, and become `Decimal` from the first
`Decimal` on. `min` and `max` order with `Efsql.Types.compare/2`, so
they work on strings and datetimes too. Groups come out ordered by their
key.
"""

alias Efsql.Exception.Unsupported
alias Efsql.Types

@type aggregate :: {name :: atom, :count | :sum | :min | :max | :avg, atom | :star}

defstruct group_by: [], aggregates: [], groups: %{}

@opaque t :: %__MODULE__{}

@doc "All at once: `new/2`, `add/2` and `finish/1` over `rows`."
@spec run([map], [atom], [aggregate]) :: [map]
def run(rows, group_by, aggregates) do
rows
|> Enum.group_by(fn row -> Enum.map(group_by, &Types.equality_key(Map.get(row, &1))) end)
|> ensure_one_group(group_by)
|> Enum.map(fn {_key, rows} ->
first = List.first(rows, %{})
group = Map.new(group_by, &{&1, Map.get(first, &1)})

Enum.reduce(aggregates, group, fn {name, function, arg}, acc ->
Map.put(acc, name, compute(function, arg, rows))
def run(rows, group_by, aggregates),
do: group_by |> new(aggregates) |> add(rows) |> finish()

@spec new([atom], [aggregate]) :: t()
def new(group_by, aggregates), do: %__MODULE__{group_by: group_by, aggregates: aggregates}

@doc "Adds rows to the running totals."
@spec add(t(), [map]) :: t()
def add(%__MODULE__{group_by: group_by, aggregates: aggregates} = state, rows) do
groups =
Enum.reduce(rows, state.groups, fn row, groups ->
key = Enum.map(group_by, &Types.equality_key(Map.get(row, &1)))

{values, accs} =
Map.get_lazy(groups, key, fn ->
{Map.new(group_by, &{&1, Map.get(row, &1)}), Enum.map(aggregates, &initial/1)}
end)

accs = Enum.zip_with(aggregates, accs, &accumulate(&1, &2, row))
Map.put(groups, key, {values, accs})
end)

%__MODULE__{state | groups: groups}
end

@doc "One row per group, ordered by the group fields."
@spec finish(t()) :: [map]
def finish(%__MODULE__{group_by: group_by, aggregates: aggregates, groups: groups}) do
# A whole-table aggregate answers even when no row matched.
groups =
if group_by == [] and groups == %{},
do: [{%{}, Enum.map(aggregates, &initial/1)}],
else: Map.values(groups)

groups
|> Enum.map(fn {values, accs} ->
aggregates
|> Enum.zip(accs)
|> Enum.reduce(values, fn {{name, function, _arg}, acc}, row ->
Map.put(row, name, result(function, acc))
end)
end)
|> Efsql.Executor.sort(Enum.map(group_by, &{:asc, &1}))
end

# A whole-table aggregate answers even when no row matched.
defp ensure_one_group(groups, []) when groups == %{}, do: [{[], []}]
defp ensure_one_group(groups, _group_by), do: Enum.to_list(groups)
# -- running totals --

# -- aggregates --
defp initial({_name, :count, _arg}), do: 0
defp initial({_name, :avg, _arg}), do: {nil, 0}
defp initial({_name, _function, _arg}), do: nil

defp compute(:count, :star, rows), do: length(rows)
defp compute(:count, field, rows), do: length(values(rows, field))
defp accumulate({_name, :count, :star}, n, _row), do: n + 1

defp compute(:min, field, rows), do: extreme(values(rows, field), :lt)
defp compute(:max, field, rows), do: extreme(values(rows, field), :gt)

defp compute(:sum, field, rows) do
case values(rows, field) do
[] -> nil
values -> sum(values, field)
defp accumulate({_name, function, field}, acc, row) do
case Map.get(row, field) do
nil -> acc
value -> step(function, field, acc, value)
end
end

defp compute(:avg, field, rows) do
case values(rows, field) do
[] ->
nil
defp step(:count, _field, n, _value), do: n + 1
defp step(:sum, field, total, value), do: add_number(total, value, field)
defp step(:avg, field, {total, n}, value), do: {add_number(total, value, field), n + 1}
defp step(:min, _field, best, value), do: better(best, value, :lt)
defp step(:max, _field, best, value), do: better(best, value, :gt)

values ->
case sum(values, field) do
%Decimal{} = total -> Decimal.div(total, length(values))
total -> total / length(values)
end
end
end
defp better(nil, value, _keep), do: value

defp values(rows, field) do
rows |> Enum.map(&Map.get(&1, field)) |> Enum.reject(&is_nil/1)
end
defp better(best, value, keep),
do: if(Types.compare(value, best) == keep, do: value, else: best)

defp extreme([], _keep), do: nil
defp add_number(total, value, field) do
unless is_number(value) or is_struct(value, Decimal) do
raise Unsupported,
"sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}"
end

defp extreme([first | rest], keep) do
Enum.reduce(rest, first, fn value, best ->
if Types.compare(value, best) == keep, do: value, else: best
end)
end
cond do
total == nil ->
value

defp sum(values, field) do
Enum.each(values, fn value ->
unless is_number(value) or is_struct(value, Decimal) do
raise Unsupported,
"sum and avg need numbers, but #{field} holds #{inspect(value, limit: 5)}"
end
end)
is_struct(total, Decimal) or is_struct(value, Decimal) ->
Decimal.add(decimal(total), decimal(value))

if Enum.any?(values, &is_struct(&1, Decimal)),
do: values |> Enum.map(&decimal/1) |> Enum.reduce(&Decimal.add/2),
else: Enum.sum(values)
true ->
total + value
end
end

defp result(:avg, {nil, 0}), do: nil
defp result(:avg, {%Decimal{} = total, n}), do: Decimal.div(total, n)
defp result(:avg, {total, n}), do: total / n
defp result(_function, acc), do: acc

defp decimal(%Decimal{} = d), do: d
defp decimal(n) when is_integer(n), do: Decimal.new(n)
defp decimal(n) when is_float(n), do: Decimal.from_float(n)
Expand Down
2 changes: 1 addition & 1 deletion lib/efsql/complete.ex
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ defmodule Efsql.Complete do

@statement_start ~w(select)
@post_expr ~w(and group order limit)
@operators ~w(= > >= < <= like in between not is)
@operators ~w(= <> != > >= < <= like ilike in between not is)

def complete(input, context) do
{word_start, word} = current_word(input)
Expand Down
76 changes: 30 additions & 46 deletions lib/efsql/executor.ex
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
defmodule Efsql.Executor do
@moduledoc """
Executes an `Efsql.Physical.Plan`: pulls rows from the plan's access node,
then folds the operator pipeline over them.

SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is
false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first
descending, matching PostgreSQL's defaults. Values compare with
`Efsql.Types.compare/2`, so datetimes order chronologically.
Executes an `Efsql.Physical.Plan`: reads rows through the plan's access
node and feeds them, as they arrive, through an `Efsql.Pipeline` of its
operators. A batched read across tenants arrives a batch at a time, and
stops early when the pipeline has all it needs (a `LIMIT` met).
Predicates evaluate with `Efsql.Predicate`'s SQL NULL semantics, and
`sort/2` orders NULLs as PostgreSQL does.

A query across tenants (`{:fan_out, [{tenant_name, plan}]}`) reads every
tenant in one FoundationDB transaction on the database: each tenant's
Expand All @@ -17,26 +16,30 @@ defmodule Efsql.Executor do

alias Efsql.Exception.Unsupported
alias Efsql.Physical.Plan
alias Efsql.Predicate
alias Efsql.Pipeline
alias Efsql.Types

# transaction_too_old (retryable) and transaction_timed_out
@too_long [1007, 1031]

def run(%Plan{access: access, ops: ops}) do
Enum.reduce(ops, fetch(access), &apply_op/2)
access
|> chunks()
|> Enum.reduce_while(Pipeline.new(ops), &Pipeline.feed(&2, &1))
|> Pipeline.finish()
end

@doc """
Maps `fun` over `items`, up to `max` at a time, each in a process of its
own, and returns the results in order. The first failure stops the rest
and is raised (or thrown, or exited) here as it was there, so a caller's
`rescue` sees the original exception rather than a linked process's
exit. With `max` 1 it all runs in the calling process.
A stream of `fun` over `items`, up to `max` at a time, each in a process
of its own, the results in order. Halting the stream stops the work not
yet done. A failure is raised (or thrown, or exited) in the consumer as
it was in its process, so a caller's `rescue` sees the original
exception rather than a linked process's exit. With `max` 1 it all runs
in the consuming process, one item at a time.
"""
def map_concurrently(items, 1, fun), do: Enum.map(items, fun)
def stream_concurrently(items, 1, fun), do: Stream.map(items, fun)

def map_concurrently(items, max, fun) do
def stream_concurrently(items, max, fun) do
items
|> Task.async_stream(
fn item ->
Expand All @@ -50,21 +53,21 @@ defmodule Efsql.Executor do
ordered: true,
timeout: :infinity
)
|> Enum.reduce_while([], fn
{:ok, {:ok, result}}, results -> {:cont, [result | results]}
{:ok, failed}, _results -> {:halt, failed}
|> Stream.map(fn
{:ok, {:ok, result}} -> result
{:ok, {:failed, kind, reason, stacktrace}} -> :erlang.raise(kind, reason, stacktrace)
end)
|> case do
{:failed, kind, reason, stacktrace} -> :erlang.raise(kind, reason, stacktrace)
results -> Enum.reverse(results)
end
end

# -- access nodes --

# One transaction per batch, up to `concurrency` at a time.
defp fetch({:batches, batches, concurrency}),
do: batches |> map_concurrently(concurrency, &fetch/1) |> Enum.concat()
# The rows a plan reads, as the chunks they arrive in: one per batch, a
# transaction each, up to `concurrency` at a time, or else just one. A
# pipeline that halts stops the batches not yet read.
defp chunks({:batches, batches, concurrency}),
do: stream_concurrently(batches, concurrency, &fetch/1)

defp chunks(access), do: [fetch(access)]

defp fetch({:fan_out, []}), do: []

Expand Down Expand Up @@ -95,7 +98,7 @@ defmodule Efsql.Executor do

rows =
plan.ops
|> Enum.reduce(List.flatten(mine), &apply_op/2)
|> Pipeline.run(List.flatten(mine))
|> Enum.map(&Map.put(&1, :_tenant, name))

{rows, rest}
Expand Down Expand Up @@ -132,25 +135,6 @@ defmodule Efsql.Executor do
Efsql.Repo.async_all_from_source(query, options)
end

# -- operators --

defp apply_op({:filter, predicates}, rows),
do: Enum.filter(rows, &Predicate.matches?(&1, predicates))

defp apply_op({:aggregate, group_by, aggregates}, rows) do
Efsql.Aggregate.run(rows, group_by, aggregates)
end

defp apply_op({:sort, order}, rows), do: sort(rows, order)

defp apply_op({:limit, n}, rows) do
Enum.take(rows, n)
end

defp apply_op({:project, fields}, rows) do
Enum.map(rows, &Map.take(&1, fields))
end

# -- sorting --

@doc """
Expand Down
Loading
Loading