diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c372a9a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +env: + # Keep in step with release.yml. + ELIXIR_VERSION: "1.19" + OTP_VERSION: "28" + FDB_VERSION: "7.3.69" + ERLFDB_COMPILE_API_VERSION: "730" + MIX_ENV: test + +jobs: + test: + name: mix test + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ env.OTP_VERSION }} + elixir-version: ${{ env.ELIXIR_VERSION }} + + # The client links the erlfdb NIF; the server package provides the + # fdbserver binary that EctoFoundationDB.Sandbox runs the tests against. + - name: Install FoundationDB + run: | + BASE="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}" + curl -fsSL -o fdb-clients.deb "$BASE/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" + curl -fsSL -o fdb-server.deb "$BASE/foundationdb-server_${FDB_VERSION}-1_amd64.deb" + sudo dpkg -i fdb-clients.deb fdb-server.deb + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: test-${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-fdb${{ env.FDB_VERSION }}-mix-${{ hashFiles('mix.lock') }} + restore-keys: test-${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-fdb${{ env.FDB_VERSION }}-mix- + + - run: mix deps.get --only test + - run: mix deps.compile + - run: mix compile + - run: mix test diff --git a/README.md b/README.md index d34497f..038b15f 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,75 @@ select col_a, col_b from tenant_id.table_name where index_col >= 'baz' and index select col_a, col_b from tenant_id.table_name where index_col between 'baz' and 'zaz'; ``` -Since efsql doesn't have access to the Ecto schema, type checking is loosened. For example, a `naive_datetime` indexed column must be queried using its string representation. +Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals). + +### Filter by NULL + +```sql +select col_a from tenant_id.table_name where col_b is null; +select col_a from tenant_id.table_name where col_b is not null; +``` + +A field that is `nil` or absent from the stored record is NULL. The +PostgreSQL shorthands `col_b isnull` and `col_b notnull` also work. As in SQL, +`col_b = null` would never be true, so efsql rejects it with a pointer to +`IS NULL`. + +### Typed literals + +A quoted literal is always a string. To compare against an Elixir term that +has no SQL literal, annotate the literal with a type using the PostgreSQL cast +operator `::`, or the standard `CAST(... AS ...)`: + +```sql +-- atoms +select col_a from tenant_id.table_name where status = 'active'::atom; +select col_a from tenant_id.table_name where status = cast('active' as atom); +select col_a from tenant_id.table_name where status in ('active'::atom, 'pending'::atom); + +-- module names are atoms too +select col_a from tenant_id.table_name where kind = 'Elixir.MyApp.Widget'::atom; + +-- NaiveDateTime, for :naive_datetime / :naive_datetime_usec fields +select col_a from tenant_id.table_name where inserted_at >= '2024-03-01 12:00:00'::timestamp; +select col_a from tenant_id.table_name where inserted_at >= '2024-03-01'::timestamp; + +-- DateTime, for :utc_datetime / :utc_datetime_usec fields +select col_a from tenant_id.table_name where seen_at >= '2024-03-01T12:00:00Z'::timestamptz; +select col_a from tenant_id.table_name where seen_at >= '2024-03-01T14:00:00+02:00'::timestamptz; + +-- Date and Time, for :date and :time / :time_usec fields +select col_a from tenant_id.table_name where birthday = '2024-03-01'::date; +select col_a from tenant_id.table_name where opens_at < '09:30:00'::time; +``` + +| Type | Elixir term | Alias | +| --- | --- | --- | +| `atom` | `Atom` | | +| `timestamp` | `NaiveDateTime` | `naive_datetime` | +| `timestamptz` | `DateTime` (UTC) | `utc_datetime` | +| `date` | `Date` | | +| `time` | `Time` | | + +Elixir has two datetime types and Ecto stores whichever the field declares, +so pick the literal type that matches the column: a `NaiveDateTime` never +equals a `DateTime`, and index lookups compare the exact encoding. A +`timestamp` literal with a time zone offset is rejected rather than having the +offset silently dropped; a `timestamptz` literal without one is taken as UTC. +A bare date means midnight. Fractional seconds are optional, and values +compare equal regardless of precision. + +Numbers need no type: a numeric literal compares by value against a +`:decimal` field, so `where price < 100` and `where price = 0.1` work. This +holds for filtering and sorting, but not for an index on a decimal field, +which the adapter keys by the `Decimal`'s term encoding rather than its value. + +Ecto stores `Ecto.Enum` fields as strings, so query those with a plain string +literal, not an atom. + +`BETWEEN` and `LIKE` don't accept typed literals; write a range as +`inserted_at >= '2024-01-01'::timestamp and inserted_at < '2025-01-01'::timestamp` +instead. ### Limit diff --git a/lib/efsql.ex b/lib/efsql.ex index fab1145..ef0e7bc 100644 --- a/lib/efsql.ex +++ b/lib/efsql.ex @@ -14,10 +14,7 @@ defmodule Efsql do import Ecto.Query - def lex_and_parse(sql) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - SQL.Parser.parse(tokens, context) - end + def lex_and_parse(sql), do: Efsql.Parser.parse(sql) def hello() do tenant = EctoFoundationDB.Tenant.open!(Efsql.Repo, "localhost") @@ -48,9 +45,7 @@ defmodule Efsql do end def sql_to_logical(sql, tenants \\ %{}) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - logical = %Efsql.Logical.Select{} = Efsql.Parser.to_logical(parsed) + logical = %Efsql.Logical.Select{} = Efsql.Parser.sql_to_logical(sql) resolve_tenant(logical, tenants) end diff --git a/lib/efsql/complete.ex b/lib/efsql/complete.ex index 647f25d..a17a8a6 100644 --- a/lib/efsql/complete.ex +++ b/lib/efsql/complete.ex @@ -14,7 +14,7 @@ defmodule Efsql.Complete do @statement_start ~w(select) @post_expr ~w(and order limit) - @operators ~w(= > >= < <= like in between not) + @operators ~w(= > >= < <= like in between not is) def complete(input, context) do {word_start, word} = current_word(input) @@ -33,8 +33,17 @@ defmodule Efsql.Complete do defp candidates_for([], _context, _tokens), do: @statement_start - defp candidates_for([last | _], context, tokens) do + defp candidates_for([last | earlier], context, tokens) do cond do + last == "is" -> + ~w(null not) + + last == "not" and List.first(earlier) == "is" -> + ~w(null) + + last == "null" -> + @post_expr + last == "select" or last == "," -> fields(context, tokens) ++ ["*"] diff --git a/lib/efsql/executor.ex b/lib/efsql/executor.ex index a9c5074..1e9f6f0 100644 --- a/lib/efsql/executor.ex +++ b/lib/efsql/executor.ex @@ -4,13 +4,22 @@ defmodule Efsql.Executor do then folds the operator pipeline over them. SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is - false — including NOT LIKE. Sorting places NULLs last ascending and first - descending, matching PostgreSQL's defaults. + false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first + descending, matching PostgreSQL's defaults. Values compare with + `Efsql.Types.compare/2`, so datetimes order chronologically. """ alias Efsql.Physical.Plan + alias Efsql.Types @cmp_ops ~w[== > >= < <=]a + @cmp_results %{ + ==: [:eq], + >: [:gt], + >=: [:gt, :eq], + <: [:lt], + <=: [:lt, :eq] + } def run(%Plan{access: access, ops: ops}) do Enum.reduce(ops, fetch(access), &apply_op/2) @@ -72,7 +81,7 @@ defmodule Efsql.Executor do defp eval({:cmp, op, field, param}, row) when op in @cmp_ops do case Map.get(row, field) do nil -> false - value -> apply(Kernel, op, [value, param]) + value -> Types.compare(value, param) in Map.fetch!(@cmp_results, op) end end @@ -83,10 +92,13 @@ defmodule Efsql.Executor do defp eval({:in, field, values}, row) do case Map.get(row, field) do nil -> false - value -> value in values + value -> Enum.any?(values, &(Types.compare(value, &1) == :eq)) end end + defp eval({:is_null, field}, row), do: Map.get(row, field) == nil + defp eval({:not_null, field}, row), do: Map.get(row, field) != nil + defp eval({:like, field, pattern}, row) do case Map.get(row, field) do nil -> false @@ -117,10 +129,20 @@ defmodule Efsql.Executor do defp compare(a, b, [{dir, field} | rest]) do case {Map.get(a, field), Map.get(b, field)} do - {v, v} -> compare(a, b, rest) - {nil, _} -> if dir == :asc, do: :gt, else: :lt - {_, nil} -> if dir == :asc, do: :lt, else: :gt - {va, vb} -> order(if(va < vb, do: :lt, else: :gt), dir) + {v, v} -> + compare(a, b, rest) + + {nil, _} -> + if dir == :asc, do: :gt, else: :lt + + {_, nil} -> + if dir == :asc, do: :lt, else: :gt + + {va, vb} -> + case Types.compare(va, vb) do + :eq -> compare(a, b, rest) + cmp -> order(cmp, dir) + end end end diff --git a/lib/efsql/logical.ex b/lib/efsql/logical.ex index 7d61767..384d346 100644 --- a/lib/efsql/logical.ex +++ b/lib/efsql/logical.ex @@ -10,6 +10,7 @@ defmodule Efsql.Logical do bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=` * `{:like, field, pattern}` / `{:not_like, field, pattern}` * `{:in, field, values}` + * `{:is_null, field}` / `{:not_null, field}` The primary key is the pseudo-field `:_`. @@ -36,4 +37,6 @@ defmodule Efsql.Logical do def predicate_field({:like, field, _pattern}), do: field def predicate_field({:not_like, field, _pattern}), do: field def predicate_field({:in, field, _values}), do: field + def predicate_field({:is_null, field}), do: field + def predicate_field({:not_null, field}), do: field end diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 50a8967..d7dbc09 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -7,9 +7,75 @@ defmodule Efsql.Parser do alias Efsql.Exception.Unsupported alias Efsql.Logical + alias Efsql.Types @comparison_ops ~w[= >= <= > <]a + # Tokens that follow a column: an unquoted column named after a reserved + # word (day, at, date, user, ...) is recognised by its position here. + @column_followers ~w[= != <> < > <= >= in like ilike between is not asc desc]a + # Reserved words that can themselves precede one of those tokens. + @not_columns ~w[true false null unknown end case when then else where having on select is]a + + @doc """ + Lexes and parses SQL text into the SQL library's parse tree, as + `{:ok, context, parsed}`. Use this rather than calling the library + directly: it first rewrites token sequences the library mishandles. + """ + def parse(sql) do + {:ok, context, tokens} = SQL.Lexer.lex(sql) + SQL.Parser.parse(normalize_tokens(tokens), context) + end + + def sql_to_logical(sql) do + {:ok, _context, parsed} = parse(sql) + to_logical(parsed) + end + + # The library lexes a reserved word as its own token even where it names + # a column, and then misparses the predicate or never returns from + # `where day = 'x'`. A reserved word right before a column follower is + # a column: rewrite it to the identifier the library expects. + # + # The library assembles `x is null` into a node that composes with `and`, + # but leaves `x is not null` as loose tokens, and never returns from + # `... and x is not null`; its postfix `isnull`/`notnull` nodes hang the + # same way. Rewrite all of them to the `is null` shape, marking the + # negated ones on the `is` token's meta. The marker is appended because + # the library matches the leading meta entries by position. Lexer tokens + # are in reverse order. + defp normalize_tokens([{:null, _, []} = null, {:not, _, []}, {:is, meta, []} | rest]) do + [null | normalize_tokens([{:is, meta ++ [efsql_not_null: true], []} | rest])] + end + + defp normalize_tokens([{tag, meta, []} | rest]) when tag in ~w[isnull notnull]a do + is_meta = List.keyreplace(meta, :type, 0, {:type, :operator}) + is_meta = if tag == :notnull, do: is_meta ++ [efsql_not_null: true], else: is_meta + [{:null, meta, []} | normalize_tokens([{:is, is_meta, []} | rest])] + end + + defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do + [{tag, meta, normalize_tokens(data)} | normalize_tokens(rest)] + end + + defp normalize_tokens([token, {word, meta, []} | rest]) + when is_atom(word) and word not in @not_columns do + if column_follower?(token) and meta[:type] == :reserved do + ident = {:ident, List.keyreplace(meta, :type, 0, {:type, :literal}), Atom.to_charlist(word)} + [token | normalize_tokens([ident | rest])] + else + [token | normalize_tokens([{word, meta, []} | rest])] + end + end + + defp normalize_tokens([token | rest]), do: [token | normalize_tokens(rest)] + defp normalize_tokens([]), do: [] + + # Non-reserved words such as asc and desc lex as identifiers carrying + # the word in their meta's :tag. + defp column_follower?({:ident, meta, _}), do: meta[:tag] in @column_followers + defp column_follower?({tag, _meta, _}), do: tag in @column_followers + def to_logical(parsed) do parsed |> Enum.reject(fn @@ -130,6 +196,13 @@ defmodule Efsql.Parser do {:in, field_atom(field), values} end + defp conjunct({:is, meta, [field, {:null, _null_meta, []}]}) do + case field_atom(field) do + :_ -> raise Unsupported, "the primary key '_' is never NULL" + field -> if meta[:efsql_not_null], do: {:not_null, field}, else: {:is_null, field} + end + end + defp conjunct({token, _meta, args}) do raise Unsupported, "'#{token}'/#{length(args)} is not supported in the where clause." end @@ -149,13 +222,29 @@ defmodule Efsql.Parser do defp param({:integer, _meta, value}), do: :erlang.list_to_integer(value) + # sql 0.5.0 lexes integers as :numeric too. defp param({:numeric, _meta, value}) do - value |> :erlang.list_to_binary() |> String.to_float() + value = :erlang.list_to_binary(value) + + case Integer.parse(value) do + {integer, ""} -> integer + _ -> String.to_float(value) + end + end + + # 'value'::type and cast('value' as type) -- see Efsql.Types + defp param({:"::", _meta, [value, type]}), do: Types.cast(type_name(type), param(value)) + + defp param({:cast, _meta, [{:paren, _paren_meta, [{:as, _as_meta, [value, type]}]}]}) do + Types.cast(type_name(type), param(value)) end defp param({true, _meta, []}), do: true defp param({false, _meta, []}), do: false - defp param({nil, _meta, []}), do: nil + # SQL's `x = null` is never true, which is never what was meant. + defp param({:null, _meta, []}) do + raise Unsupported, "a comparison with NULL is never true; use IS NULL or IS NOT NULL" + end defp param({:paren, _meta, [{:quote, _, part}, {:comma, _, [{:*, _, []}]}]}) do {:erlang.list_to_binary(part), :*} @@ -167,6 +256,14 @@ defmodule Efsql.Parser do EctoFoundationDB.Versionstamp.from_integer(:erlang.list_to_integer(n))} end + # Non-reserved words lex as identifiers, reserved ones (e.g. `date`) as + # their own token. + defp type_name({tag, _meta, name}) when tag in ~w[ident double_quote]a do + name |> List.to_string() |> String.downcase() + end + + defp type_name({tag, _meta, []}) when is_atom(tag), do: Atom.to_string(tag) + defp sql_op(:=), do: :== defp sql_op(op), do: op diff --git a/lib/efsql/planner.ex b/lib/efsql/planner.ex index 766d751..f0bd294 100644 --- a/lib/efsql/planner.ex +++ b/lib/efsql/planner.ex @@ -22,6 +22,7 @@ defmodule Efsql.Planner do alias Efsql.Exception.Unsupported alias Efsql.Logical alias Efsql.Physical.Plan + alias Efsql.Types alias EctoFoundationDB.Layer.Metadata @pk_field :_ @@ -201,6 +202,8 @@ defmodule Efsql.Planner do defp kind({:in, _field, _values}), do: :in defp kind({:like, _field, _pattern}), do: :residual defp kind({:not_like, _field, _pattern}), do: :residual + defp kind({:is_null, _field}), do: :residual + defp kind({:not_null, _field}), do: :residual defp kind({:cmp, _op, @pk_field, _value}), do: :pk defp kind({:range, @pk_field, _lower, _upper}), do: :pk defp kind({:cmp, _op, _field, _value}), do: :pushable @@ -386,11 +389,12 @@ defmodule Efsql.Planner do end defp pred_to_expr({:cmp, op, field, value}) do - {op, [], [field_ref(field), value]} + {op, [], [field_ref(field), Types.index_key(value)]} end defp pred_to_expr({:range, field, {lower_op, lower}, {upper_op, upper}}) do - {{lower_op, [], [field_ref(field), lower]}, {upper_op, [], [field_ref(field), upper]}} + {{lower_op, [], [field_ref(field), Types.index_key(lower)]}, + {upper_op, [], [field_ref(field), Types.index_key(upper)]}} end defp pred_to_expr(pred) do diff --git a/lib/efsql/tui/session.ex b/lib/efsql/tui/session.ex index 0b2b062..481aae7 100644 --- a/lib/efsql/tui/session.ex +++ b/lib/efsql/tui/session.ex @@ -12,9 +12,7 @@ defmodule Efsql.Tui.Session do end def qall(sql, %{tenant: tenant, tenants: tenants}) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - logical = %Efsql.Logical.Select{} = Efsql.Parser.to_logical(parsed) + logical = %Efsql.Logical.Select{} = Efsql.Parser.sql_to_logical(sql) {logical, tenants} = case logical.prefix do diff --git a/lib/efsql/types.ex b/lib/efsql/types.ex new file mode 100644 index 0000000..e1ea03b --- /dev/null +++ b/lib/efsql/types.ex @@ -0,0 +1,159 @@ +defmodule Efsql.Types do + @moduledoc """ + Typed literals in the where clause, and how their values compare. + + A SQL literal is a string, number, or boolean. Elixir terms with no SQL + counterpart are written as a literal with a type annotation, using the + PostgreSQL cast operator or its standard SQL spelling: + + where status = 'active'::atom + where status = cast('active' as atom) + where inserted_at >= '2024-03-01 12:00:00'::timestamp + where birthday = '2024-03-01'::date + + Both forms parse to `cast(type, value)`, which converts the already-parsed + literal into the Elixir term the adapter compares against. Type names are + case-insensitive. + + Elixir has two datetime types, and Ecto stores each as-is, so each gets + its own SQL type. Use the one matching the column's Ecto type: + + * `timestamp` (or `naive_datetime`) — a `NaiveDateTime`, for + `:naive_datetime` and `:naive_datetime_usec` fields. A time zone + offset in the literal is rejected rather than silently dropped. + * `timestamptz` (or `utc_datetime`) — a UTC `DateTime`, for + `:utc_datetime` and `:utc_datetime_usec` fields. An offset in the + literal is applied; without one the literal is taken as UTC. + + Either accepts a bare date (`'2024-03-01'`) as midnight. + + `date` is a `Date` and `time` a `Time` (for `:time` and `:time_usec` + fields), both in ISO 8601. + + Numbers need no annotation: a numeric literal compares by value against a + `Decimal` field. + + To add a type, add its name to `@types` and a `cast/2` clause for it, plus + `compare/2` and `index_key/1` clauses if its terms need them. + """ + + alias Efsql.Exception.Unsupported + + @aliases %{"naive_datetime" => "timestamp", "utc_datetime" => "timestamptz"} + @types ~w[atom timestamp timestamptz date time] + + def types(), do: @types ++ Map.keys(@aliases) + + def cast(type, value), do: do_cast(Map.get(@aliases, type, type), type, value) + + # Atoms are created, not looked up: the value may only exist in stored + # data this node has not decoded yet, and the adapter needs the term + # itself to build index and primary-key ranges. + defp do_cast("atom", _as, value) when is_binary(value), do: String.to_atom(value) + + defp do_cast("timestamp", as, value) when is_binary(value) do + case DateTime.from_iso8601(value) do + {:ok, _datetime, _offset} -> + raise Unsupported, + "#{inspect(value)} has a time zone offset, which #{as} would drop; " <> + "use timestamptz for a DateTime" + + {:error, _} -> + naive_datetime!(value, as) + end + end + + defp do_cast("timestamptz", as, value) when is_binary(value) do + case DateTime.from_iso8601(value) do + {:ok, datetime, _offset} -> datetime + {:error, _} -> value |> naive_datetime!(as) |> DateTime.from_naive!("Etc/UTC") + end + end + + defp do_cast("date", as, value) when is_binary(value) do + case Date.from_iso8601(value) do + {:ok, date} -> date + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + + defp do_cast("time", as, value) when is_binary(value) do + case Time.from_iso8601(value) do + {:ok, time} -> time + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + + defp do_cast(type, as, value) when type in @types do + raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + + defp do_cast(_type, as, _value) do + raise Unsupported, "Unknown type '#{as}'. Supported types: #{Enum.join(types(), ", ")}" + end + + defp naive_datetime!(value, as) do + with {:error, _} <- NaiveDateTime.from_iso8601(value), + {:ok, date} <- Date.from_iso8601(value) do + NaiveDateTime.new!(date, ~T[00:00:00]) + else + {:ok, naive} -> naive + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + + @doc """ + Orders two field values, `:lt`, `:eq` or `:gt`. Dates and times compare + chronologically (term order would compare their struct fields, and term + equality their precision), and a `Decimal` by value against another + `Decimal` or a number (term order puts every number below every map); + everything else by Erlang term order. A `NaiveDateTime` never equals a + `DateTime`, as in an index lookup. + """ + def compare(%NaiveDateTime{} = a, %NaiveDateTime{} = b), do: NaiveDateTime.compare(a, b) + def compare(%DateTime{} = a, %DateTime{} = b), do: DateTime.compare(a, b) + def compare(%Date{} = a, %Date{} = b), do: Date.compare(a, b) + def compare(%Time{} = a, %Time{} = b), do: Time.compare(a, b) + + def compare(a, b) when is_struct(a, Decimal) or is_struct(b, Decimal) do + with %Decimal{} = da <- to_decimal(a), + %Decimal{} = db <- to_decimal(b) do + Decimal.compare(da, db) + else + nil -> term_compare(a, b) + end + end + + def compare(a, b), do: term_compare(a, b) + + defp term_compare(a, b) when a == b, do: :eq + defp term_compare(a, b) when a < b, do: :lt + defp term_compare(_a, _b), do: :gt + + # nil when the value has no ordered Decimal counterpart (NaN, or not a + # number), which Decimal.compare would raise on. + defp to_decimal(%Decimal{coef: :NaN}), do: nil + defp to_decimal(%Decimal{} = d), do: d + defp to_decimal(n) when is_integer(n), do: Decimal.new(n) + defp to_decimal(n) when is_float(n), do: Decimal.from_float(n) + defp to_decimal(_), do: nil + + @doc """ + The value to push to the adapter for an index lookup. efsql queries + schemaless, so the adapter can't encode a param by its field's Ecto type; + encode dates and times here exactly as the adapter's default indexer + does when it writes the index. + """ + def index_key(%NaiveDateTime{} = x), + do: x |> NaiveDateTime.add(0, :microsecond) |> NaiveDateTime.to_iso8601(:basic) + + def index_key(%DateTime{} = x), + do: x |> DateTime.add(0, :microsecond) |> DateTime.to_iso8601(:basic) + + def index_key(%Date{} = x), do: Date.to_iso8601(x, :basic) + + def index_key(%Time{} = x), + do: x |> Time.add(0, :microsecond) |> Time.to_iso8601(:basic) + + def index_key(x), do: x +end diff --git a/mix.exs b/mix.exs index a488139..c163be2 100644 --- a/mix.exs +++ b/mix.exs @@ -31,7 +31,12 @@ defmodule Efsql.MixProject do def application do [ mod: {Efsql.Application, []}, - extra_applications: [:logger] + extra_applications: [:logger], + # efsql uses only sql's lexer and parser, which need no running + # process. Including the app loads its code without starting its + # database client runtime (a :dets file in its priv dir, a startup + # sleep, and a stop/1 that crashes when no pool ever started). + included_applications: [:sql] ] end diff --git a/mix.lock b/mix.lock index 30e877e..6af86d3 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "local_cluster": {:hex, :local_cluster, "2.1.0", "1c847d69a927ef5a62db13236f93146e8a42377a9c9a5bb4cac3372cba69d683", [:mix], [{:global_flags, "~> 1.0", [hex: :global_flags, repo: "hexpm", optional: false]}], "hexpm", "dc1c3abb6fef00198dd53c855b39ea80c55b3a8059d8d9f17d50da46b1e3b858"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "owl": {:hex, :owl, "0.13.1", "1ec4a5dea170465f0e90c502c203079224516bc0cbd599281c8667b3c6ef8848", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "351e768af8f2edc575cdaab1a5a2f6d6381be591758a026c701c703145508a0c"}, - "sql": {:git, "https://github.com/elixir-dbvisor/sql.git", "da69080748c44d50869477f5d254504012ab7f6c", []}, + "sql": {:git, "https://github.com/elixir-dbvisor/sql.git", "f6097fff5b08efe9cd946a01cc2152c82d871cf7", []}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "unicode": {:hex, :unicode, "2.0.0", "d2d3ec27575fed3437fdcc0d383909378201843d27d2197c0c2742c973110b24", [:mix], [], "hexpm", "a8da44be6a2fc165e58ea332038686df79994b51bf2c7fe337bf2927ea148c13"}, "unicode_set": {:hex, :unicode_set, "1.7.0", "38c91095b35b90cf2e8f8aaa67d298937886293c5e0288953325014503888cd2", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode, "~> 2.0", [hex: :unicode, repo: "hexpm", optional: false]}], "hexpm", "66c8c02adadac58fcaf64e22eba90c243a6528d1d36fb014e2d8c0e062efbae6"}, diff --git a/test/complete_test.exs b/test/complete_test.exs index d5b3ca6..238046c 100644 --- a/test/complete_test.exs +++ b/test/complete_test.exs @@ -43,6 +43,20 @@ defmodule Efsql.CompleteTest do assert "between" in candidates end + test "is offers null and not null" do + {_, candidates} = Complete.complete("select id from users where name ", @context) + assert "is" in candidates + + assert {_, ["null", "not"]} = + Complete.complete("select id from users where name is ", @context) + + assert {_, ["null"]} = + Complete.complete("select id from users where name is not ", @context) + + {_, candidates} = Complete.complete("select id from users where name is null ", @context) + assert "and" in candidates + end + test "order by offers fields" do assert {_, ["by"]} = Complete.complete("select id from users order ", @context) {_, candidates} = Complete.complete("select id from users order by ", @context) diff --git a/test/null_test.exs b/test/null_test.exs new file mode 100644 index 0000000..fb9a30e --- /dev/null +++ b/test/null_test.exs @@ -0,0 +1,32 @@ +defmodule EfsqlTest.Integration.Null do + use EfsqlTest.Case, async: true + + test "is null matches a field that was never set", context do + tenant_id = context[:tenant_id] + + assert [%{id: "0003"}] = + Efsql.all("select id from #{tenant_id}.users where notes is null;") + end + + test "is not null", context do + tenant_id = context[:tenant_id] + + assert [%{id: "0001"}, %{id: "0002"}] = + Efsql.all("select id from #{tenant_id}.users where notes is not null;") + |> Enum.sort_by(& &1.id) + end + + test "is not null combined with an indexed equality", context do + tenant_id = context[:tenant_id] + + assert [] = + Efsql.all( + "select id from #{tenant_id}.users where name = 'Charles' and notes is not null;" + ) + + assert [%{id: "0001"}] = + Efsql.all( + "select id from #{tenant_id}.users where name = 'Alice' and notes is not null;" + ) + end +end diff --git a/test/parser_test.exs b/test/parser_test.exs index 8e02182..f93f9c9 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -1,14 +1,11 @@ defmodule Efsql.ParserTest do use ExUnit.Case, async: true + alias Efsql.Exception.Unsupported alias Efsql.Logical alias Efsql.Parser - defp parse(sql) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - Parser.to_logical(parsed) - end + defp parse(sql), do: Parser.sql_to_logical(sql) describe "select" do test "single field" do @@ -114,6 +111,206 @@ defmodule Efsql.ParserTest do parse("select id from t.users where qty in (1, 2, 3);") end + test "atom literal" do + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = 'active'::atom;") + + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = cast('active' as atom);") + + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = 'active'::ATOM;") + end + + test "atom literal is distinct from a string" do + assert %Logical.Select{predicates: [{:cmp, :==, :status, "active"}]} = + parse("select id from t.users where status = 'active';") + end + + test "atom literal keeps case and module names" do + assert %Logical.Select{predicates: [{:cmp, :==, :kind, Foo.Bar}]} = + parse("select id from t.users where kind = 'Elixir.Foo.Bar'::atom;") + end + + test "atom literals in compound predicates" do + assert %Logical.Select{ + predicates: [{:cmp, :==, :status, :active}, {:cmp, :==, :name, "Alice"}] + } = + parse("select id from t.users where status = 'active'::atom and name = 'Alice';") + + assert %Logical.Select{predicates: [{:in, :status, [:active, :pending]}]} = + parse("select id from t.users where status in ('active'::atom, 'pending'::atom);") + + assert %Logical.Select{predicates: [{:cmp, :>=, :status, :a}, {:cmp, :<, :status, :m}]} = + parse("select id from t.users where status >= 'a'::atom and status < 'm'::atom;") + end + + test "timestamp literal is a NaiveDateTime" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::timestamp;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56.123456]}]} = + parse( + "select id from t.users where at >= '2024-03-01T12:34:56.123456'::timestamp;" + ) + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 00:00:00]}]} = + parse("select id from t.users where at >= cast('2024-03-01' as timestamp);") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::naive_datetime;") + end + + test "timestamp literal rejects a time zone offset" do + assert_raise Unsupported, ~r/use timestamptz/, fn -> + parse("select id from t.users where at >= '2024-03-01 12:34:56Z'::timestamp;") + end + end + + test "timestamptz literal is a UTC DateTime" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 12:34:56Z]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56Z'::timestamptz;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 10:34:56Z]}]} = + parse( + "select id from t.users where at >= '2024-03-01T12:34:56+02:00'::timestamptz;" + ) + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 12:34:56Z]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::utc_datetime;") + + assert %Logical.Select{predicates: [{:cmp, :<, :at, ~U[2024-03-01 00:00:00Z]}]} = + parse("select id from t.users where at < '2024-03-01'::timestamptz;") + end + + test "date literal is a Date" do + assert %Logical.Select{predicates: [{:cmp, :==, :day, ~D[2024-03-01]}]} = + parse("select id from t.users where day = '2024-03-01'::date;") + + assert %Logical.Select{predicates: [{:in, :day, [~D[2024-03-01], ~D[2024-03-02]]}]} = + parse( + "select id from t.users where day in (cast('2024-03-01' as date), '2024-03-02'::date);" + ) + end + + test "time literal is a Time" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~T[12:34:56]}]} = + parse("select id from t.users where at >= '12:34:56'::time;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~T[12:34:56.123456]}]} = + parse("select id from t.users where at >= '12:34:56.123456'::time;") + end + + test "malformed date and time raise" do + assert_raise Unsupported, ~r/Cannot cast "2024-02-30" to date/, fn -> + parse("select id from t.users where day = '2024-02-30'::date;") + end + + assert_raise Unsupported, ~r/Cannot cast "25:00:00" to time/, fn -> + parse("select id from t.users where at = '25:00:00'::time;") + end + end + + test "malformed timestamp raises" do + assert_raise Unsupported, ~r/Cannot cast "yesterday" to timestamp/, fn -> + parse("select id from t.users where at >= 'yesterday'::timestamp;") + end + end + + test "unknown type raises" do + assert_raise Unsupported, ~r/Unknown type 'widget'/, fn -> + parse("select id from t.users where status = 'active'::widget;") + end + end + + test "atom cast of a non-string raises" do + assert_raise Unsupported, ~r/Cannot cast 1 to atom/, fn -> + parse("select id from t.users where status = 1::atom;") + end + end + + test "like on an atom raises" do + assert_raise Unsupported, fn -> + parse("select id from t.users where status like 'a%'::atom;") + end + end + + test "is null and is not null" do + assert %Logical.Select{predicates: [{:is_null, :notes}]} = + parse("select id from t.users where notes is null;") + + assert %Logical.Select{predicates: [{:not_null, :notes}]} = + parse("select id from t.users where notes is not null;") + + assert %Logical.Select{predicates: [{:is_null, :notes}]} = + parse("select id from t.users where notes isnull;") + + assert %Logical.Select{predicates: [{:not_null, :notes}]} = + parse("select id from t.users where notes notnull;") + end + + # The SQL library hangs on these without Efsql.Parser's token rewrite. + test "null tests combine with other predicates in any position" do + assert %Logical.Select{predicates: [{:cmp, :==, :name, "a"}, {:not_null, :notes}]} = + parse("select id from t.users where name = 'a' and notes is not null;") + + assert %Logical.Select{predicates: [{:not_null, :notes}, {:cmp, :==, :name, "a"}]} = + parse("select id from t.users where notes is not null and name = 'a';") + + assert %Logical.Select{ + predicates: [{:is_null, :a}, {:not_null, :b}, {:cmp, :==, :c, "x"}] + } = parse("select id from t.users where a is null and b notnull and c = 'x';") + + assert %Logical.Select{predicates: [{:cmp, :==, :c, "x"}, {:is_null, :a}]} = + parse("select id from t.users where c = 'x' and a isnull;") + end + + test "comparison with null raises" do + assert_raise Unsupported, ~r/use IS NULL or IS NOT NULL/, fn -> + parse("select id from t.users where notes = null;") + end + + assert_raise Unsupported, ~r/use IS NULL or IS NOT NULL/, fn -> + parse("select id from t.users where notes in ('a', null);") + end + end + + test "the primary key is never null" do + assert_raise Unsupported, ~r/never NULL/, fn -> + parse("select id from t.users where _ is null;") + end + end + + # The SQL library hangs or misparses on these without the token + # rewrite in Efsql.Parser.parse/1. + test "columns named after reserved words" do + assert %Logical.Select{predicates: [{:cmp, :==, :day, "x"}]} = + parse("select id from t.users where day = 'x';") + + assert %Logical.Select{predicates: [{:cmp, :==, :x, "a"}, {:cmp, :>=, :at, 1}]} = + parse("select id from t.users where x = 'a' and at >= 1;") + + assert %Logical.Select{ + predicates: [ + {:in, :day, ["a"]}, + {:like, :user, "a%"}, + {:not_like, :value, "b%"}, + {:range, :year, {:>=, 1}, {:<=, 2}} + ] + } = + parse( + "select id from t.users where day in ('a') and user like 'a%' and value not like 'b%' and year between 1 and 2;" + ) + + assert %Logical.Select{predicates: [{:is_null, :day}, {:not_null, :at}]} = + parse("select id from t.users where day is null and at is not null;") + end + + test "reserved-word columns keep their sort direction" do + assert %Logical.Select{order: [desc: :day, asc: :user]} = + parse("select id from t.users order by day desc, user asc;") + end + test "versionstamp partition scan value" do assert %Logical.Select{predicates: [{:cmp, :==, :_, {"p", :*}}]} = parse("select id from t.users where _ = ('p', *);") diff --git a/test/timestamp_test.exs b/test/timestamp_test.exs new file mode 100644 index 0000000..8063c9e --- /dev/null +++ b/test/timestamp_test.exs @@ -0,0 +1,31 @@ +defmodule EfsqlTest.Integration.Timestamp do + use EfsqlTest.Case, async: true + + test "timestamp range on a naive_datetime field", context do + tenant_id = context[:tenant_id] + + assert [_, _, _] = + Efsql.all( + "select id from #{tenant_id}.users where inserted_at > '2000-01-01'::timestamp;" + ) + + assert [] = + Efsql.all( + "select id from #{tenant_id}.users where inserted_at < '2000-01-01'::timestamp;" + ) + end + + test "timestamp equality ignores precision", context do + tenant_id = context[:tenant_id] + + [%{inserted_at: inserted_at} | _] = + Efsql.all("select id, inserted_at from #{tenant_id}.users where _ = '0001';") + + literal = inserted_at |> NaiveDateTime.add(0, :microsecond) |> NaiveDateTime.to_iso8601() + + assert [%{id: "0001"}] = + Efsql.all( + "select id from #{tenant_id}.users where _ = '0001' and inserted_at = '#{literal}'::timestamp;" + ) + end +end diff --git a/test/types_test.exs b/test/types_test.exs new file mode 100644 index 0000000..64b33b6 --- /dev/null +++ b/test/types_test.exs @@ -0,0 +1,68 @@ +defmodule Efsql.TypesTest do + use ExUnit.Case, async: true + + alias Efsql.Types + + describe "compare" do + test "datetimes compare chronologically, not by struct fields" do + # Term order compares the structs' fields alphabetically, so day 2 of + # a later year would sort before day 3 of an earlier one. + assert Types.compare(~N[2025-01-02 00:00:00], ~N[2024-01-03 00:00:00]) == :gt + assert Types.compare(~U[2025-01-02 00:00:00Z], ~U[2024-01-03 00:00:00Z]) == :gt + end + + test "datetimes equal regardless of precision" do + assert Types.compare(~N[2024-03-01 12:34:56.000000], ~N[2024-03-01 12:34:56]) == :eq + assert Types.compare(~U[2024-03-01 12:34:56.000000Z], ~U[2024-03-01 12:34:56Z]) == :eq + end + + test "a NaiveDateTime never equals a DateTime" do + refute Types.compare(~N[2024-03-01 12:34:56], ~U[2024-03-01 12:34:56Z]) == :eq + end + + test "dates and times compare chronologically" do + # Term order compares Date's fields alphabetically, year last. + assert Types.compare(~D[2025-01-02], ~D[2024-01-03]) == :gt + # ... and Time's microsecond before its minute. + assert Types.compare(~T[09:00:00.9], ~T[09:01:00]) == :lt + assert Types.compare(~T[09:00:00.000000], ~T[09:00:00]) == :eq + end + + test "decimals compare by value, against decimals and numbers" do + # Term order puts every number below every map. + assert Types.compare(Decimal.new("10.5"), 100) == :lt + assert Types.compare(100, Decimal.new("10.5")) == :gt + assert Types.compare(Decimal.new("10.5"), 10.5) == :eq + assert Types.compare(Decimal.new("0.1"), 0.1) == :eq + assert Types.compare(Decimal.new("1.50"), Decimal.new("1.5")) == :eq + assert Types.compare(Decimal.new("2"), 2) == :eq + assert Types.compare(Decimal.new("-3"), Decimal.new("2")) == :lt + end + + test "a decimal NaN or non-number falls back to term order instead of raising" do + assert Types.compare(Decimal.new("NaN"), 1) in [:lt, :gt] + assert Types.compare(Decimal.new("1"), "1") in [:lt, :gt] + end + + test "other values use term order" do + assert Types.compare("a", "b") == :lt + assert Types.compare(2, 1) == :gt + assert Types.compare(1, 1.0) == :eq + assert Types.compare(:a, :a) == :eq + end + end + + describe "index_key" do + test "datetimes encode as the adapter's index does" do + assert Types.index_key(~N[2024-03-01 12:34:56]) == "20240301T123456.000000" + assert Types.index_key(~U[2024-03-01 12:34:56Z]) == "20240301T123456.000000Z" + assert Types.index_key(~D[2024-03-01]) == "20240301" + assert Types.index_key(~T[12:34:56]) == "123456.000000" + end + + test "other values pass through" do + assert Types.index_key("x") == "x" + assert Types.index_key(:active) == :active + end + end +end