Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,7 @@ its case (`"CreatedAt"`). Most SQL keywords work as bare names, so columns like
so if you forget.

Comments (`-- ...` and `/* ... */`) can go anywhere. A syntax error reports the
line and column it was found at, and features efsql doesn't support (`OR`,
joins, `HAVING`, functions other than aggregates) are rejected by name.
line and column it was found at, and features efsql doesn't support (joins, `HAVING`, functions other than aggregates) are rejected by name.

### Storage IDs

Expand Down Expand Up @@ -278,6 +277,26 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and

Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals).

### OR

```sql
select col_a from tenant_id.table_name where status = 'paid' or total > 100;
select col_a from tenant_id.table_name where city = 'Osaka' and (age < 20 or age > 60);
select col_a from tenant_id.table_name where status = 'paid' or status = 'shipped';
```

`AND` binds tighter than `OR`, so `a and b or c` is `(a and b) or c`. An
`OR` is checked on the rows read, while the conditions `AND`-ed with it
still narrow what is read: in the second query, an index on `city` serves
`city = 'Osaka'`. `OR`-ed equalities on one field are an `IN`
(`status in ('paid', 'shipped')`), read as one lookup per value when an
index or the primary key serves the field. As in SQL, a NULL field fails
only its own branch: a row whose `status` is NULL still matches
`status = 'paid' or total > 100` when its total is over 100.

In a query across tenants, an `OR` on `_tenant` alone picks the tenants,
but one can't mix `_tenant` with other fields.

### Negation and ILIKE

```sql
Expand All @@ -290,8 +309,8 @@ select col_a from tenant_id.table_name where not (status = 'paid' or total > 100
```

`NOT` works on any condition, and is read as its opposite: `not (a < 1)` is
`a >= 1`, `not (a = 1 or b = 2)` is `a <> 1 and b <> 2`. `not (a and b)`
would need `OR`, which isn't supported.
`a >= 1`, `not (a = 1 or b = 2)` is `a <> 1 and b <> 2`, and `not (a and b)`
is `not a or not b`.

As in SQL, a NULL field matches no comparison, negated ones included: a
row whose `status` is NULL matches neither `status = 'paid'` nor
Expand Down
2 changes: 1 addition & 1 deletion lib/efsql/complete.ex
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ defmodule Efsql.Complete do
"""

@statement_start ~w(select)
@post_expr ~w(and group order limit)
@post_expr ~w(and or group order limit)
@operators ~w(= <> != > >= < <= like ilike in between not is)

def complete(input, context) do
Expand Down
9 changes: 8 additions & 1 deletion lib/efsql/fanout.ex
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,14 @@ defmodule Efsql.Fanout do
end

{tenant_preds, row_preds} =
Enum.split_with(logical.predicates, &(Predicate.field(&1) == :_tenant))
Enum.split_with(logical.predicates, &(Predicate.fields(&1) == [:_tenant]))

# Each tenant's own reads check row_preds before its rows get _tenant.
for pred <- row_preds, :_tenant in Predicate.fields(pred) do
raise Unsupported,
"an OR can't mix _tenant with other fields; " <>
"put the condition on _tenant on its own (where _tenant = 'a' and (x = 1 or y = 2))"
end

names =
storage_id
Expand Down
20 changes: 14 additions & 6 deletions lib/efsql/parser.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ defmodule Efsql.Parser do
@moduledoc """
SQL text to `Efsql.Logical.Select`: parses with `Efsql.SQL.Parser`, then
translates the syntax tree. Syntax problems raise `Efsql.SQL.SyntaxError`;
valid SQL that efsql can't execute (`OR`, comparing two columns, ...)
valid SQL that efsql can't execute (comparing two columns, ...)
raises `Efsql.Exception.Unsupported`.

Purely a translation — normalization (range merging, LIKE rewriting)
Expand Down Expand Up @@ -53,7 +53,7 @@ defmodule Efsql.Parser do
defp check_tenant_field(logical) do
used =
if(is_list(logical.projection), do: logical.projection, else: []) ++
Enum.map(logical.predicates, &Efsql.Predicate.field/1) ++
Enum.flat_map(logical.predicates, &Efsql.Predicate.fields/1) ++
Enum.map(logical.order, &elem(&1, 1)) ++
(logical.group_by || []) ++ Enum.map(logical.aggregates, &elem(&1, 2))

Expand Down Expand Up @@ -182,8 +182,7 @@ defmodule Efsql.Parser do

# NOT is pushed down to the conditions it negates, each of which has an
# exact opposite: NOT (a = 1) is a <> 1, NOT (a IN ...) is a NOT IN ...,
# and NOT (a OR b) is NOT a AND NOT b. NOT (a AND b) is NOT a OR NOT b,
# which needs OR.
# NOT (a OR b) is NOT a AND NOT b, and NOT (a AND b) is NOT a OR NOT b.
defp push_not({:not, expr}), do: negate(expr)

defp push_not({op, left, right}) when op in [:and, :or],
Expand All @@ -208,6 +207,9 @@ defmodule Efsql.Parser do
defp conjuncts({:and, left, right}, acc), do: conjuncts(left, conjuncts(right, acc))
defp conjuncts(expr, acc), do: [expr | acc]

defp disjuncts({:or, left, right}, acc), do: disjuncts(left, disjuncts(right, acc))
defp disjuncts(expr, acc), do: [expr | acc]

defp predicate({:compare, op, {:column, _} = column, {:column, _}}),
do:
raise(Unsupported, "comparing two fields (#{describe(column)} #{op} ...) is not supported")
Expand Down Expand Up @@ -255,8 +257,14 @@ defmodule Efsql.Parser do
defp predicate({:is_null, {:column, field}, negated?}),
do: {if(negated?, do: :not_null, else: :is_null), field(field)}

defp predicate({:or, _, _}),
do: raise(Unsupported, "OR is not supported (nor NOT over AND, which means OR)")
# `a OR b OR (c AND d)` is one OR of three branches, each an AND-ed list.
defp predicate({:or, _, _} = expr) do
branches =
for branch <- disjuncts(expr, []),
do: branch |> conjuncts([]) |> Enum.map(&predicate/1)

{:or, branches}
end

defp predicate({:not, expr}),
do: raise(Unsupported, "#{describe(expr)} is not a condition; NOT needs a condition")
Expand Down
7 changes: 4 additions & 3 deletions lib/efsql/planner.ex
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ defmodule Efsql.Planner do

defp takes(fields, residual, sort) do
needed =
(Enum.map(residual, &Predicate.field/1) ++ Enum.map(sort, fn {_dir, f} -> f end))
(Enum.flat_map(residual, &Predicate.fields/1) ++ Enum.map(sort, fn {_dir, f} -> f end))
|> Enum.uniq()

case needed -- fields do
Expand All @@ -367,11 +367,12 @@ defmodule Efsql.Planner do
end

defp ensure_residual_evaluable!(pred) do
if Predicate.field(pred) == @pk_field do
if @pk_field in Predicate.fields(pred) do
# Rows carry the key under its field's name, not `_`, so a condition
# on `_` must be one the key range can serve.
raise Unsupported,
"on the primary key '_', only =, <, <=, >, >=, BETWEEN and IN are supported"
"on the primary key '_', only =, <, <=, >, >=, BETWEEN and IN are supported " <>
"(and an OR of = and IN); use the primary key field name for anything else"
end
end

Expand Down
22 changes: 19 additions & 3 deletions lib/efsql/predicate.ex
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ defmodule Efsql.Predicate do
case-insensitive `:ilike` / `:not_ilike`
* `{:in, field, values}` / `{:not_in, field, values}`
* `{:is_null, field}` / `{:not_null, field}`
* `{:or, branches}` — true when any branch is, each branch itself an
AND-ed list of predicates (`a = 1 OR (b = 2 AND c = 3)` is
`{:or, [[a = 1], [b = 2, c = 3]]}`)

SQL's `NOT` never reaches here: `Efsql.Parser` pushes it into the
condition it negates (`NOT (a < 1)` is `a >= 1`).
Expand All @@ -33,6 +36,7 @@ defmodule Efsql.Predicate do
| {:like | :not_like | :ilike | :not_ilike, field(), String.t()}
| {:in | :not_in, field(), [term()]}
| {:is_null | :not_null, field()}
| {:or, [[t()]]}

@pk_field :_

Expand All @@ -45,14 +49,21 @@ defmodule Efsql.Predicate do
<=: [:lt, :eq]
}

@doc "The field a predicate constrains."
@doc "The field a predicate constrains; see `fields/1` for an OR."
@spec field(t()) :: field()
def field({:cmp, _op, field, _value}), do: field
def field({kind, field, _lower, _upper}) when kind in [:range, :not_range], do: field
def field({kind, field, _arg}) when kind in [:like, :not_like, :ilike, :not_ilike], do: field
def field({kind, field, _values}) when kind in [:in, :not_in], do: field
def field({kind, field}) when kind in [:is_null, :not_null], do: field

@doc "Every field a predicate reads, in order, without duplicates."
@spec fields(t()) :: [field()]
def fields({:or, branches}),
do: branches |> Enum.concat() |> Enum.flat_map(&fields/1) |> Enum.uniq()

def fields(predicate), do: [field(predicate)]

# -- evaluation --

@doc """
Expand All @@ -77,6 +88,10 @@ defmodule Efsql.Predicate do
defp eval({:in, field, values}, row), do: on_value(row, field, &member?(&1, values))
defp eval({:not_in, field, values}, row), do: on_value(row, field, &(not member?(&1, values)))

# A NULL makes a branch false, not the whole OR: `a = 1 OR b = 2` holds
# when a is NULL and b is 2.
defp eval({:or, branches}, row), do: Enum.any?(branches, &matches?(row, &1))

defp eval({:is_null, field}, row), do: Map.get(row, field) == nil
defp eval({:not_null, field}, row), do: Map.get(row, field) != nil

Expand Down Expand Up @@ -120,9 +135,10 @@ defmodule Efsql.Predicate do
* `:in` - an `IN` list, read as one lookup per value when an index or
the key can serve them
* `:index` - an equality or range an index may serve
* `:filter` - only checked on the rows read: LIKE, IS NULL, and
* `:filter` - only checked on the rows read: LIKE, IS NULL, OR, and
every negation (`<>`, NOT IN, NOT BETWEEN, NOT LIKE), which an index
can't serve
can't serve (an OR of equalities on one field becomes an IN first;
see `Efsql.Rewrite`)
"""
@spec pushdown(t()) :: :key | :in | :index | :filter
def pushdown({:cmp, :!=, _field, _value}), do: :filter
Expand Down
44 changes: 38 additions & 6 deletions lib/efsql/rewrite.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ defmodule Efsql.Rewrite do
@moduledoc """
Normalization passes over an `Efsql.Logical` query. Each pass is a pure
`logical -> logical` function; `normalize/1` runs them in order. New
rewrite rules (OR to DNF, NOT pushdown, contradiction detection, ...) are
added here as further passes.
rewrite rules (contradiction detection, ...) are added here as further
passes. (NOT is pushed down earlier, by `Efsql.Parser`.)
"""

alias Efsql.Logical
alias Efsql.Predicate
alias Efsql.Types

@lower_ops ~w[> >=]a
@upper_ops ~w[< <=]a
Expand All @@ -16,6 +17,7 @@ defmodule Efsql.Rewrite do
logical
|> pass_merge_ranges()
|> pass_like_prefix()
|> pass_or_in()
|> pass_in_singleton()
end

Expand Down Expand Up @@ -123,13 +125,43 @@ defmodule Efsql.Rewrite do
end
end

# `field in (v)` is an equality, and `field not in (v)` an inequality.
# `a = 1 OR a = 2 OR a IN (3, 4)` is `a IN (1, 2, 3, 4)`, which an index
# or the primary key serves as one lookup per value; any other OR is
# only checked on the rows read.
def pass_or_in(%Logical.Select{predicates: preds} = logical) do
%Logical.Select{logical | predicates: Enum.map(preds, &or_in/1)}
end

defp or_in({:or, branches} = pred) do
values = Enum.map(branches, &in_values/1)

case Enum.uniq_by(values, &elem(&1, 0)) do
[{field, _}] when field != nil -> {:in, field, Enum.flat_map(values, &elem(&1, 1))}
_ -> pred
end
end

defp or_in(pred), do: pred

defp in_values([{:cmp, :==, field, value}]), do: {field, [value]}
defp in_values([{:in, field, values}]), do: {field, values}
defp in_values(_branch), do: {nil, []}

# An IN list holds each value once, since each is read on its own (`a IN
# (1, 1)` would read a = 1 twice). Then `field in (v)` is an equality,
# and `field not in (v)` an inequality.
def pass_in_singleton(%Logical.Select{predicates: preds} = logical) do
predicates =
Enum.map(preds, fn
{:in, field, [value]} -> {:cmp, :==, field, value}
{:not_in, field, [value]} -> {:cmp, :!=, field, value}
pred -> pred
{kind, field, values} when kind in [:in, :not_in] ->
case {kind, Enum.uniq_by(values, &Types.equality_key/1)} do
{:in, [value]} -> {:cmp, :==, field, value}
{:not_in, [value]} -> {:cmp, :!=, field, value}
{kind, values} -> {kind, field, values}
end

pred ->
pred
end)

%Logical.Select{logical | predicates: predicates}
Expand Down
4 changes: 3 additions & 1 deletion lib/efsql/tui/help.ex
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@ defmodule Efsql.Tui.Help do
{:sql, "where status <> 'paid' also !=, not in, not between"},
{:sql, "where not (a = 1 or b = 2)"},
{:sql, "where city = 'Osaka' and age > 40"},
{:sql, "where city = 'Osaka' or (city = 'Kyoto' and age > 40)"},
"Literals: 'text', 42, 1.5, true, false, null.",
{:note, "or is not supported; conditions combine with and only"},
{:note, "and binds tighter than or; an or is checked on the rows read,"},
{:note, "except or-ed equalities on one field, read like in (...)"},
{:note, "a null field matches no comparison, like in SQL"}
]},
{"Ordering and limits",
Expand Down
1 change: 1 addition & 0 deletions test/complete_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ defmodule Efsql.CompleteTest do

{_, candidates} = Complete.complete("select id from users where name is null ", @context)
assert "and" in candidates
assert "or" in candidates
end

test "order by offers fields" do
Expand Down
17 changes: 17 additions & 0 deletions test/cross_tenant_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,23 @@ defmodule EfsqlTest.Integration.CrossTenant do
assert rows == [%{_tenant: context[:other], name: "Dora"}]
end

test "an OR on _tenant chooses the tenants; mixed with other fields it is refused",
context do
either = "_tenant = '#{context[:tenant_id]}' or _tenant = '#{context[:other]}'"

assert {%Plan{access: {:fan_out, [_, _]}}, rows, _} =
Efsql.qall("select name from *.users where #{either} order by name;")

assert Enum.map(rows, & &1.name) == ["Alice", "Bob", "Charles", "Dora", "Eve"]

assert_raise Unsupported, ~r/an OR can't mix _tenant with other fields/, fn ->
Efsql.all(
"select name from *.users where #{context[:both]} and " <>
"(_tenant = '#{context[:other]}' or name = 'Alice');"
)
end
end

test "more tenants than the limit is refused", context do
assert_raise Unsupported, ~r/would read 2 tenants, more than the limit of 1/, fn ->
Efsql.all("select name from *.users where #{context[:both]};", max_tenants: 1)
Expand Down
Loading
Loading