From 09b780655b20bd3f1ae65635a515c3c915d8ab49 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 23:48:33 +0000 Subject: [PATCH 01/10] Parse typed literals in the where clause, starting with atoms A quoted literal stays a string; an atom is written as a type-annotated literal using the PostgreSQL cast operator or standard CAST: where status = 'active'::atom where status = cast('active' as atom) Both forms dispatch through Efsql.Types.cast/2, so adding a type is a new clause there. Reserved type names (e.g. date) are handled as well as identifiers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 19 ++++++++++++++++ lib/efsql/parser.ex | 16 +++++++++++++ lib/efsql/types.ex | 38 +++++++++++++++++++++++++++++++ test/parser_test.exs | 53 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 lib/efsql/types.ex diff --git a/README.md b/README.md index d34497f..b68e7f8 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,25 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and Since efsql doesn't have access to the Ecto schema, type checking is loosened. For example, a `naive_datetime` indexed column must be queried using its string representation. +### Typed literals + +A quoted literal is always a string. To compare against an Elixir term that +has no SQL literal, annotate the literal with a type using the PostgreSQL cast +operator `::`, or the standard `CAST(... AS ...)`: + +```sql +-- atoms, e.g. an Ecto.Enum column +select col_a from tenant_id.table_name where status = 'active'::atom; +select col_a from tenant_id.table_name where status = cast('active' as atom); +select col_a from tenant_id.table_name where status in ('active'::atom, 'pending'::atom); + +-- module names are atoms too +select col_a from tenant_id.table_name where kind = 'Elixir.MyApp.Widget'::atom; +``` + +Supported types: `atom`. `BETWEEN` and `LIKE` don't accept typed literals; +write a range as `status >= 'a'::atom and status < 'm'::atom` instead. + ### Limit ```sql diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 50a8967..a8f0440 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -7,6 +7,7 @@ defmodule Efsql.Parser do alias Efsql.Exception.Unsupported alias Efsql.Logical + alias Efsql.Types @comparison_ops ~w[= >= <= > <]a @@ -153,6 +154,13 @@ defmodule Efsql.Parser do value |> :erlang.list_to_binary() |> String.to_float() end + # 'value'::type and cast('value' as type) -- see Efsql.Types + defp param({:"::", _meta, [value, type]}), do: Types.cast(type_name(type), param(value)) + + defp param({:cast, _meta, [{:paren, _paren_meta, [{:as, _as_meta, [value, type]}]}]}) do + Types.cast(type_name(type), param(value)) + end + defp param({true, _meta, []}), do: true defp param({false, _meta, []}), do: false defp param({nil, _meta, []}), do: nil @@ -167,6 +175,14 @@ defmodule Efsql.Parser do EctoFoundationDB.Versionstamp.from_integer(:erlang.list_to_integer(n))} end + # Non-reserved words lex as identifiers, reserved ones (e.g. `date`) as + # their own token. + defp type_name({tag, _meta, name}) when tag in ~w[ident double_quote]a do + name |> List.to_string() |> String.downcase() + end + + defp type_name({tag, _meta, []}) when is_atom(tag), do: Atom.to_string(tag) + defp sql_op(:=), do: :== defp sql_op(op), do: op diff --git a/lib/efsql/types.ex b/lib/efsql/types.ex new file mode 100644 index 0000000..214059c --- /dev/null +++ b/lib/efsql/types.ex @@ -0,0 +1,38 @@ +defmodule Efsql.Types do + @moduledoc """ + Typed literals in the where clause. + + A SQL literal is a string, number, or boolean. Elixir terms with no SQL + counterpart are written as a literal with a type annotation, using the + PostgreSQL cast operator or its standard SQL spelling: + + where status = 'active'::atom + where status = cast('active' as atom) + where status in ('active'::atom, 'pending'::atom) + + Both forms parse to `cast(type, value)`, which converts the already-parsed + literal into the Elixir term the adapter compares against. Type names are + case-insensitive. + + To add a type, add its name to `@types` and a `cast/2` clause for it. + """ + + alias Efsql.Exception.Unsupported + + @types ~w[atom] + + def types(), do: @types + + # Atoms are created, not looked up: the value may only exist in stored + # data this node has not decoded yet, and the adapter needs the term + # itself to build index and primary-key ranges. + def cast("atom", value) when is_binary(value), do: String.to_atom(value) + + def cast(type, value) when type in @types do + raise Unsupported, "Cannot cast #{inspect(value)} to #{type}" + end + + def cast(type, _value) do + raise Unsupported, "Unknown type '#{type}'. Supported types: #{Enum.join(@types, ", ")}" + end +end diff --git a/test/parser_test.exs b/test/parser_test.exs index 8e02182..af23542 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -1,6 +1,7 @@ defmodule Efsql.ParserTest do use ExUnit.Case, async: true + alias Efsql.Exception.Unsupported alias Efsql.Logical alias Efsql.Parser @@ -114,6 +115,58 @@ defmodule Efsql.ParserTest do parse("select id from t.users where qty in (1, 2, 3);") end + test "atom literal" do + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = 'active'::atom;") + + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = cast('active' as atom);") + + assert %Logical.Select{predicates: [{:cmp, :==, :status, :active}]} = + parse("select id from t.users where status = 'active'::ATOM;") + end + + test "atom literal is distinct from a string" do + assert %Logical.Select{predicates: [{:cmp, :==, :status, "active"}]} = + parse("select id from t.users where status = 'active';") + end + + test "atom literal keeps case and module names" do + assert %Logical.Select{predicates: [{:cmp, :==, :kind, Foo.Bar}]} = + parse("select id from t.users where kind = 'Elixir.Foo.Bar'::atom;") + end + + test "atom literals in compound predicates" do + assert %Logical.Select{ + predicates: [{:cmp, :==, :status, :active}, {:cmp, :==, :name, "Alice"}] + } = + parse("select id from t.users where status = 'active'::atom and name = 'Alice';") + + assert %Logical.Select{predicates: [{:in, :status, [:active, :pending]}]} = + parse("select id from t.users where status in ('active'::atom, 'pending'::atom);") + + assert %Logical.Select{predicates: [{:cmp, :>=, :status, :a}, {:cmp, :<, :status, :m}]} = + parse("select id from t.users where status >= 'a'::atom and status < 'm'::atom;") + end + + test "unknown type raises" do + assert_raise Unsupported, ~r/Unknown type 'widget'/, fn -> + parse("select id from t.users where status = 'active'::widget;") + end + end + + test "atom cast of a non-string raises" do + assert_raise Unsupported, ~r/Cannot cast 1 to atom/, fn -> + parse("select id from t.users where status = 1::atom;") + end + end + + test "like on an atom raises" do + assert_raise Unsupported, fn -> + parse("select id from t.users where status like 'a%'::atom;") + end + end + test "versionstamp partition scan value" do assert %Logical.Select{predicates: [{:cmp, :==, :_, {"p", :*}}]} = parse("select id from t.users where _ = ('p', *);") From f5077004185ca7cb8cacb6a858e7d1a4909ccacf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 00:02:14 +0000 Subject: [PATCH 02/10] Add timestamp and timestamptz typed literals Elixir has two datetime types and Ecto stores whichever a field declares, so each gets its own literal type: '2024-03-01 12:00:00'::timestamp -- NaiveDateTime (alias naive_datetime) '2024-03-01T12:00:00Z'::timestamptz -- UTC DateTime (alias utc_datetime) A timestamp literal with an offset is rejected rather than silently dropping it; a timestamptz literal without one is taken as UTC; a bare date means midnight. Supporting the type end to end also needed: - Executor filters, IN and sorting compare through Efsql.Types.compare/2. Kernel comparison on datetime structs orders by struct fields, not time, and == fails when only the precision differs. - Pushed index predicates encode datetimes the way the adapter's default indexer writes them. efsql queries schemaless, so the adapter would otherwise term_to_binary the param and never match the index key. Also corrects the README: Ecto.Enum fields are stored as strings, so they are queried with a string literal, not an atom. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 34 +++++++++++++-- lib/efsql/executor.ex | 33 +++++++++++---- lib/efsql/planner.ex | 6 ++- lib/efsql/types.ex | 91 ++++++++++++++++++++++++++++++++++++----- test/parser_test.exs | 44 ++++++++++++++++++++ test/timestamp_test.exs | 31 ++++++++++++++ test/types_test.exs | 42 +++++++++++++++++++ 7 files changed, 258 insertions(+), 23 deletions(-) create mode 100644 test/timestamp_test.exs create mode 100644 test/types_test.exs diff --git a/README.md b/README.md index b68e7f8..df3ddfe 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ select col_a, col_b from tenant_id.table_name where index_col >= 'baz' and index select col_a, col_b from tenant_id.table_name where index_col between 'baz' and 'zaz'; ``` -Since efsql doesn't have access to the Ecto schema, type checking is loosened. For example, a `naive_datetime` indexed column must be queried using its string representation. +Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals). ### Typed literals @@ -230,17 +230,43 @@ has no SQL literal, annotate the literal with a type using the PostgreSQL cast operator `::`, or the standard `CAST(... AS ...)`: ```sql --- atoms, e.g. an Ecto.Enum column +-- atoms select col_a from tenant_id.table_name where status = 'active'::atom; select col_a from tenant_id.table_name where status = cast('active' as atom); select col_a from tenant_id.table_name where status in ('active'::atom, 'pending'::atom); -- module names are atoms too select col_a from tenant_id.table_name where kind = 'Elixir.MyApp.Widget'::atom; + +-- NaiveDateTime, for :naive_datetime / :naive_datetime_usec fields +select col_a from tenant_id.table_name where inserted_at >= '2024-03-01 12:00:00'::timestamp; +select col_a from tenant_id.table_name where inserted_at >= '2024-03-01'::timestamp; + +-- DateTime, for :utc_datetime / :utc_datetime_usec fields +select col_a from tenant_id.table_name where seen_at >= '2024-03-01T12:00:00Z'::timestamptz; +select col_a from tenant_id.table_name where seen_at >= '2024-03-01T14:00:00+02:00'::timestamptz; ``` -Supported types: `atom`. `BETWEEN` and `LIKE` don't accept typed literals; -write a range as `status >= 'a'::atom and status < 'm'::atom` instead. +| Type | Elixir term | Alias | +| --- | --- | --- | +| `atom` | `Atom` | | +| `timestamp` | `NaiveDateTime` | `naive_datetime` | +| `timestamptz` | `DateTime` (UTC) | `utc_datetime` | + +Elixir has two datetime types and Ecto stores whichever the field declares, +so pick the literal type that matches the column: a `NaiveDateTime` never +equals a `DateTime`, and index lookups compare the exact encoding. A +`timestamp` literal with a time zone offset is rejected rather than having the +offset silently dropped; a `timestamptz` literal without one is taken as UTC. +A bare date means midnight. Fractional seconds are optional, and values +compare equal regardless of precision. + +Ecto stores `Ecto.Enum` fields as strings, so query those with a plain string +literal, not an atom. + +`BETWEEN` and `LIKE` don't accept typed literals; write a range as +`inserted_at >= '2024-01-01'::timestamp and inserted_at < '2025-01-01'::timestamp` +instead. ### Limit diff --git a/lib/efsql/executor.ex b/lib/efsql/executor.ex index a9c5074..3dadbf6 100644 --- a/lib/efsql/executor.ex +++ b/lib/efsql/executor.ex @@ -5,12 +5,21 @@ defmodule Efsql.Executor do SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is false — including NOT LIKE. Sorting places NULLs last ascending and first - descending, matching PostgreSQL's defaults. + descending, matching PostgreSQL's defaults. Values compare with + `Efsql.Types.compare/2`, so datetimes order chronologically. """ alias Efsql.Physical.Plan + alias Efsql.Types @cmp_ops ~w[== > >= < <=]a + @cmp_results %{ + ==: [:eq], + >: [:gt], + >=: [:gt, :eq], + <: [:lt], + <=: [:lt, :eq] + } def run(%Plan{access: access, ops: ops}) do Enum.reduce(ops, fetch(access), &apply_op/2) @@ -72,7 +81,7 @@ defmodule Efsql.Executor do defp eval({:cmp, op, field, param}, row) when op in @cmp_ops do case Map.get(row, field) do nil -> false - value -> apply(Kernel, op, [value, param]) + value -> Types.compare(value, param) in Map.fetch!(@cmp_results, op) end end @@ -83,7 +92,7 @@ defmodule Efsql.Executor do defp eval({:in, field, values}, row) do case Map.get(row, field) do nil -> false - value -> value in values + value -> Enum.any?(values, &(Types.compare(value, &1) == :eq)) end end @@ -117,10 +126,20 @@ defmodule Efsql.Executor do defp compare(a, b, [{dir, field} | rest]) do case {Map.get(a, field), Map.get(b, field)} do - {v, v} -> compare(a, b, rest) - {nil, _} -> if dir == :asc, do: :gt, else: :lt - {_, nil} -> if dir == :asc, do: :lt, else: :gt - {va, vb} -> order(if(va < vb, do: :lt, else: :gt), dir) + {v, v} -> + compare(a, b, rest) + + {nil, _} -> + if dir == :asc, do: :gt, else: :lt + + {_, nil} -> + if dir == :asc, do: :lt, else: :gt + + {va, vb} -> + case Types.compare(va, vb) do + :eq -> compare(a, b, rest) + cmp -> order(cmp, dir) + end end end diff --git a/lib/efsql/planner.ex b/lib/efsql/planner.ex index 766d751..ed33660 100644 --- a/lib/efsql/planner.ex +++ b/lib/efsql/planner.ex @@ -22,6 +22,7 @@ defmodule Efsql.Planner do alias Efsql.Exception.Unsupported alias Efsql.Logical alias Efsql.Physical.Plan + alias Efsql.Types alias EctoFoundationDB.Layer.Metadata @pk_field :_ @@ -386,11 +387,12 @@ defmodule Efsql.Planner do end defp pred_to_expr({:cmp, op, field, value}) do - {op, [], [field_ref(field), value]} + {op, [], [field_ref(field), Types.index_key(value)]} end defp pred_to_expr({:range, field, {lower_op, lower}, {upper_op, upper}}) do - {{lower_op, [], [field_ref(field), lower]}, {upper_op, [], [field_ref(field), upper]}} + {{lower_op, [], [field_ref(field), Types.index_key(lower)]}, + {upper_op, [], [field_ref(field), Types.index_key(upper)]}} end defp pred_to_expr(pred) do diff --git a/lib/efsql/types.ex b/lib/efsql/types.ex index 214059c..691d1a4 100644 --- a/lib/efsql/types.ex +++ b/lib/efsql/types.ex @@ -1,6 +1,6 @@ defmodule Efsql.Types do @moduledoc """ - Typed literals in the where clause. + Typed literals in the where clause, and how their values compare. A SQL literal is a string, number, or boolean. Elixir terms with no SQL counterpart are written as a literal with a type annotation, using the @@ -8,31 +8,102 @@ defmodule Efsql.Types do where status = 'active'::atom where status = cast('active' as atom) - where status in ('active'::atom, 'pending'::atom) + where inserted_at >= '2024-03-01 12:00:00'::timestamp Both forms parse to `cast(type, value)`, which converts the already-parsed literal into the Elixir term the adapter compares against. Type names are case-insensitive. - To add a type, add its name to `@types` and a `cast/2` clause for it. + Elixir has two datetime types, and Ecto stores each as-is, so each gets + its own SQL type. Use the one matching the column's Ecto type: + + * `timestamp` (or `naive_datetime`) — a `NaiveDateTime`, for + `:naive_datetime` and `:naive_datetime_usec` fields. A time zone + offset in the literal is rejected rather than silently dropped. + * `timestamptz` (or `utc_datetime`) — a UTC `DateTime`, for + `:utc_datetime` and `:utc_datetime_usec` fields. An offset in the + literal is applied; without one the literal is taken as UTC. + + Either accepts a bare date (`'2024-03-01'`) as midnight. + + To add a type, add its name to `@types` and a `cast/2` clause for it, plus + `compare/2` and `index_key/1` clauses if its terms need them. """ alias Efsql.Exception.Unsupported - @types ~w[atom] + @aliases %{"naive_datetime" => "timestamp", "utc_datetime" => "timestamptz"} + @types ~w[atom timestamp timestamptz] - def types(), do: @types + def types(), do: @types ++ Map.keys(@aliases) + + def cast(type, value), do: do_cast(Map.get(@aliases, type, type), type, value) # Atoms are created, not looked up: the value may only exist in stored # data this node has not decoded yet, and the adapter needs the term # itself to build index and primary-key ranges. - def cast("atom", value) when is_binary(value), do: String.to_atom(value) + defp do_cast("atom", _as, value) when is_binary(value), do: String.to_atom(value) + + defp do_cast("timestamp", as, value) when is_binary(value) do + case DateTime.from_iso8601(value) do + {:ok, _datetime, _offset} -> + raise Unsupported, + "#{inspect(value)} has a time zone offset, which #{as} would drop; " <> + "use timestamptz for a DateTime" + + {:error, _} -> + naive_datetime!(value, as) + end + end + + defp do_cast("timestamptz", as, value) when is_binary(value) do + case DateTime.from_iso8601(value) do + {:ok, datetime, _offset} -> datetime + {:error, _} -> value |> naive_datetime!(as) |> DateTime.from_naive!("Etc/UTC") + end + end - def cast(type, value) when type in @types do - raise Unsupported, "Cannot cast #{inspect(value)} to #{type}" + defp do_cast(type, as, value) when type in @types do + raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" end - def cast(type, _value) do - raise Unsupported, "Unknown type '#{type}'. Supported types: #{Enum.join(@types, ", ")}" + defp do_cast(_type, as, _value) do + raise Unsupported, "Unknown type '#{as}'. Supported types: #{Enum.join(types(), ", ")}" end + + defp naive_datetime!(value, as) do + with {:error, _} <- NaiveDateTime.from_iso8601(value), + {:ok, date} <- Date.from_iso8601(value) do + NaiveDateTime.new!(date, ~T[00:00:00]) + else + {:ok, naive} -> naive + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + + @doc """ + Orders two field values, `:lt`, `:eq` or `:gt`. Datetimes compare by the + instant they denote (term order would compare their struct fields, and + term equality their precision); everything else by Erlang term order. + A `NaiveDateTime` never equals a `DateTime`, as in an index lookup. + """ + def compare(%NaiveDateTime{} = a, %NaiveDateTime{} = b), do: NaiveDateTime.compare(a, b) + def compare(%DateTime{} = a, %DateTime{} = b), do: DateTime.compare(a, b) + def compare(a, b) when a == b, do: :eq + def compare(a, b) when a < b, do: :lt + def compare(_a, _b), do: :gt + + @doc """ + The value to push to the adapter for an index lookup. efsql queries + schemaless, so the adapter can't encode a param by its field's Ecto type; + encode datetimes here exactly as the adapter's default indexer does when + it writes the index. + """ + def index_key(%NaiveDateTime{} = x), + do: x |> NaiveDateTime.add(0, :microsecond) |> NaiveDateTime.to_iso8601(:basic) + + def index_key(%DateTime{} = x), + do: x |> DateTime.add(0, :microsecond) |> DateTime.to_iso8601(:basic) + + def index_key(x), do: x end diff --git a/test/parser_test.exs b/test/parser_test.exs index af23542..ca9aa17 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -149,6 +149,50 @@ defmodule Efsql.ParserTest do parse("select id from t.users where status >= 'a'::atom and status < 'm'::atom;") end + test "timestamp literal is a NaiveDateTime" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::timestamp;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56.123456]}]} = + parse( + "select id from t.users where at >= '2024-03-01T12:34:56.123456'::timestamp;" + ) + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 00:00:00]}]} = + parse("select id from t.users where at >= cast('2024-03-01' as timestamp);") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~N[2024-03-01 12:34:56]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::naive_datetime;") + end + + test "timestamp literal rejects a time zone offset" do + assert_raise Unsupported, ~r/use timestamptz/, fn -> + parse("select id from t.users where at >= '2024-03-01 12:34:56Z'::timestamp;") + end + end + + test "timestamptz literal is a UTC DateTime" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 12:34:56Z]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56Z'::timestamptz;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 10:34:56Z]}]} = + parse( + "select id from t.users where at >= '2024-03-01T12:34:56+02:00'::timestamptz;" + ) + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~U[2024-03-01 12:34:56Z]}]} = + parse("select id from t.users where at >= '2024-03-01 12:34:56'::utc_datetime;") + + assert %Logical.Select{predicates: [{:cmp, :<, :at, ~U[2024-03-01 00:00:00Z]}]} = + parse("select id from t.users where at < '2024-03-01'::timestamptz;") + end + + test "malformed timestamp raises" do + assert_raise Unsupported, ~r/Cannot cast "yesterday" to timestamp/, fn -> + parse("select id from t.users where at >= 'yesterday'::timestamp;") + end + end + test "unknown type raises" do assert_raise Unsupported, ~r/Unknown type 'widget'/, fn -> parse("select id from t.users where status = 'active'::widget;") diff --git a/test/timestamp_test.exs b/test/timestamp_test.exs new file mode 100644 index 0000000..8063c9e --- /dev/null +++ b/test/timestamp_test.exs @@ -0,0 +1,31 @@ +defmodule EfsqlTest.Integration.Timestamp do + use EfsqlTest.Case, async: true + + test "timestamp range on a naive_datetime field", context do + tenant_id = context[:tenant_id] + + assert [_, _, _] = + Efsql.all( + "select id from #{tenant_id}.users where inserted_at > '2000-01-01'::timestamp;" + ) + + assert [] = + Efsql.all( + "select id from #{tenant_id}.users where inserted_at < '2000-01-01'::timestamp;" + ) + end + + test "timestamp equality ignores precision", context do + tenant_id = context[:tenant_id] + + [%{inserted_at: inserted_at} | _] = + Efsql.all("select id, inserted_at from #{tenant_id}.users where _ = '0001';") + + literal = inserted_at |> NaiveDateTime.add(0, :microsecond) |> NaiveDateTime.to_iso8601() + + assert [%{id: "0001"}] = + Efsql.all( + "select id from #{tenant_id}.users where _ = '0001' and inserted_at = '#{literal}'::timestamp;" + ) + end +end diff --git a/test/types_test.exs b/test/types_test.exs new file mode 100644 index 0000000..5db1747 --- /dev/null +++ b/test/types_test.exs @@ -0,0 +1,42 @@ +defmodule Efsql.TypesTest do + use ExUnit.Case, async: true + + alias Efsql.Types + + describe "compare" do + test "datetimes compare chronologically, not by struct fields" do + # Term order compares the structs' fields alphabetically, so day 2 of + # a later year would sort before day 3 of an earlier one. + assert Types.compare(~N[2025-01-02 00:00:00], ~N[2024-01-03 00:00:00]) == :gt + assert Types.compare(~U[2025-01-02 00:00:00Z], ~U[2024-01-03 00:00:00Z]) == :gt + end + + test "datetimes equal regardless of precision" do + assert Types.compare(~N[2024-03-01 12:34:56.000000], ~N[2024-03-01 12:34:56]) == :eq + assert Types.compare(~U[2024-03-01 12:34:56.000000Z], ~U[2024-03-01 12:34:56Z]) == :eq + end + + test "a NaiveDateTime never equals a DateTime" do + refute Types.compare(~N[2024-03-01 12:34:56], ~U[2024-03-01 12:34:56Z]) == :eq + end + + test "other values use term order" do + assert Types.compare("a", "b") == :lt + assert Types.compare(2, 1) == :gt + assert Types.compare(1, 1.0) == :eq + assert Types.compare(:a, :a) == :eq + end + end + + describe "index_key" do + test "datetimes encode as the adapter's index does" do + assert Types.index_key(~N[2024-03-01 12:34:56]) == "20240301T123456.000000" + assert Types.index_key(~U[2024-03-01 12:34:56Z]) == "20240301T123456.000000Z" + end + + test "other values pass through" do + assert Types.index_key("x") == "x" + assert Types.index_key(:active) == :active + end + end +end From d737afd80f95863886d7fb3afc5a94c252e3a0eb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 00:16:45 +0000 Subject: [PATCH 03/10] Add date and time literals; compare Decimals by value '2024-03-01'::date -- Date, for :date fields '09:30:00'::time -- Time, for :time / :time_usec fields Like the timestamp types, these needed more than parsing: term order compares Date's struct fields with the year last, and Time's microsecond before its minute, so filters and ORDER BY were wrong. Both now compare chronologically, and pushed index predicates encode them as the adapter's default indexer does. Decimal fields need no literal type, but a number compared against one used term order, which puts every number below every map: `price < 100` never matched and `price > 100` matched everything. Types.compare/2 now compares a Decimal by value against a Decimal, integer or float. A NaN falls back to term order rather than raising. Indexes on decimal fields remain unusable for this, since the adapter keys them by term encoding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 11 +++++++ lib/efsql/types.ex | 70 +++++++++++++++++++++++++++++++++++++------- test/parser_test.exs | 28 ++++++++++++++++++ test/types_test.exs | 26 ++++++++++++++++ 4 files changed, 125 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index df3ddfe..e4a1f28 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,10 @@ select col_a from tenant_id.table_name where inserted_at >= '2024-03-01'::timest -- DateTime, for :utc_datetime / :utc_datetime_usec fields select col_a from tenant_id.table_name where seen_at >= '2024-03-01T12:00:00Z'::timestamptz; select col_a from tenant_id.table_name where seen_at >= '2024-03-01T14:00:00+02:00'::timestamptz; + +-- Date and Time, for :date and :time / :time_usec fields +select col_a from tenant_id.table_name where birthday = '2024-03-01'::date; +select col_a from tenant_id.table_name where opens_at < '09:30:00'::time; ``` | Type | Elixir term | Alias | @@ -252,6 +256,8 @@ select col_a from tenant_id.table_name where seen_at >= '2024-03-01T14:00:00+02: | `atom` | `Atom` | | | `timestamp` | `NaiveDateTime` | `naive_datetime` | | `timestamptz` | `DateTime` (UTC) | `utc_datetime` | +| `date` | `Date` | | +| `time` | `Time` | | Elixir has two datetime types and Ecto stores whichever the field declares, so pick the literal type that matches the column: a `NaiveDateTime` never @@ -261,6 +267,11 @@ offset silently dropped; a `timestamptz` literal without one is taken as UTC. A bare date means midnight. Fractional seconds are optional, and values compare equal regardless of precision. +Numbers need no type: a numeric literal compares by value against a +`:decimal` field, so `where price < 100` and `where price = 0.1` work. This +holds for filtering and sorting, but not for an index on a decimal field, +which the adapter keys by the `Decimal`'s term encoding rather than its value. + Ecto stores `Ecto.Enum` fields as strings, so query those with a plain string literal, not an atom. diff --git a/lib/efsql/types.ex b/lib/efsql/types.ex index 691d1a4..e1ea03b 100644 --- a/lib/efsql/types.ex +++ b/lib/efsql/types.ex @@ -9,6 +9,7 @@ defmodule Efsql.Types do where status = 'active'::atom where status = cast('active' as atom) where inserted_at >= '2024-03-01 12:00:00'::timestamp + where birthday = '2024-03-01'::date Both forms parse to `cast(type, value)`, which converts the already-parsed literal into the Elixir term the adapter compares against. Type names are @@ -26,6 +27,12 @@ defmodule Efsql.Types do Either accepts a bare date (`'2024-03-01'`) as midnight. + `date` is a `Date` and `time` a `Time` (for `:time` and `:time_usec` + fields), both in ISO 8601. + + Numbers need no annotation: a numeric literal compares by value against a + `Decimal` field. + To add a type, add its name to `@types` and a `cast/2` clause for it, plus `compare/2` and `index_key/1` clauses if its terms need them. """ @@ -33,7 +40,7 @@ defmodule Efsql.Types do alias Efsql.Exception.Unsupported @aliases %{"naive_datetime" => "timestamp", "utc_datetime" => "timestamptz"} - @types ~w[atom timestamp timestamptz] + @types ~w[atom timestamp timestamptz date time] def types(), do: @types ++ Map.keys(@aliases) @@ -63,6 +70,20 @@ defmodule Efsql.Types do end end + defp do_cast("date", as, value) when is_binary(value) do + case Date.from_iso8601(value) do + {:ok, date} -> date + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + + defp do_cast("time", as, value) when is_binary(value) do + case Time.from_iso8601(value) do + {:ok, time} -> time + {:error, _} -> raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" + end + end + defp do_cast(type, as, value) when type in @types do raise Unsupported, "Cannot cast #{inspect(value)} to #{as}" end @@ -82,22 +103,46 @@ defmodule Efsql.Types do end @doc """ - Orders two field values, `:lt`, `:eq` or `:gt`. Datetimes compare by the - instant they denote (term order would compare their struct fields, and - term equality their precision); everything else by Erlang term order. - A `NaiveDateTime` never equals a `DateTime`, as in an index lookup. + Orders two field values, `:lt`, `:eq` or `:gt`. Dates and times compare + chronologically (term order would compare their struct fields, and term + equality their precision), and a `Decimal` by value against another + `Decimal` or a number (term order puts every number below every map); + everything else by Erlang term order. A `NaiveDateTime` never equals a + `DateTime`, as in an index lookup. """ def compare(%NaiveDateTime{} = a, %NaiveDateTime{} = b), do: NaiveDateTime.compare(a, b) def compare(%DateTime{} = a, %DateTime{} = b), do: DateTime.compare(a, b) - def compare(a, b) when a == b, do: :eq - def compare(a, b) when a < b, do: :lt - def compare(_a, _b), do: :gt + def compare(%Date{} = a, %Date{} = b), do: Date.compare(a, b) + def compare(%Time{} = a, %Time{} = b), do: Time.compare(a, b) + + def compare(a, b) when is_struct(a, Decimal) or is_struct(b, Decimal) do + with %Decimal{} = da <- to_decimal(a), + %Decimal{} = db <- to_decimal(b) do + Decimal.compare(da, db) + else + nil -> term_compare(a, b) + end + end + + def compare(a, b), do: term_compare(a, b) + + defp term_compare(a, b) when a == b, do: :eq + defp term_compare(a, b) when a < b, do: :lt + defp term_compare(_a, _b), do: :gt + + # nil when the value has no ordered Decimal counterpart (NaN, or not a + # number), which Decimal.compare would raise on. + defp to_decimal(%Decimal{coef: :NaN}), do: nil + defp to_decimal(%Decimal{} = d), do: d + defp to_decimal(n) when is_integer(n), do: Decimal.new(n) + defp to_decimal(n) when is_float(n), do: Decimal.from_float(n) + defp to_decimal(_), do: nil @doc """ The value to push to the adapter for an index lookup. efsql queries schemaless, so the adapter can't encode a param by its field's Ecto type; - encode datetimes here exactly as the adapter's default indexer does when - it writes the index. + encode dates and times here exactly as the adapter's default indexer + does when it writes the index. """ def index_key(%NaiveDateTime{} = x), do: x |> NaiveDateTime.add(0, :microsecond) |> NaiveDateTime.to_iso8601(:basic) @@ -105,5 +150,10 @@ defmodule Efsql.Types do def index_key(%DateTime{} = x), do: x |> DateTime.add(0, :microsecond) |> DateTime.to_iso8601(:basic) + def index_key(%Date{} = x), do: Date.to_iso8601(x, :basic) + + def index_key(%Time{} = x), + do: x |> Time.add(0, :microsecond) |> Time.to_iso8601(:basic) + def index_key(x), do: x end diff --git a/test/parser_test.exs b/test/parser_test.exs index ca9aa17..349cb24 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -187,6 +187,34 @@ defmodule Efsql.ParserTest do parse("select id from t.users where at < '2024-03-01'::timestamptz;") end + test "date literal is a Date" do + assert %Logical.Select{predicates: [{:cmp, :==, :day, ~D[2024-03-01]}]} = + parse("select id from t.users where day = '2024-03-01'::date;") + + assert %Logical.Select{predicates: [{:in, :day, [~D[2024-03-01], ~D[2024-03-02]]}]} = + parse( + "select id from t.users where day in (cast('2024-03-01' as date), '2024-03-02'::date);" + ) + end + + test "time literal is a Time" do + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~T[12:34:56]}]} = + parse("select id from t.users where at >= '12:34:56'::time;") + + assert %Logical.Select{predicates: [{:cmp, :>=, :at, ~T[12:34:56.123456]}]} = + parse("select id from t.users where at >= '12:34:56.123456'::time;") + end + + test "malformed date and time raise" do + assert_raise Unsupported, ~r/Cannot cast "2024-02-30" to date/, fn -> + parse("select id from t.users where day = '2024-02-30'::date;") + end + + assert_raise Unsupported, ~r/Cannot cast "25:00:00" to time/, fn -> + parse("select id from t.users where at = '25:00:00'::time;") + end + end + test "malformed timestamp raises" do assert_raise Unsupported, ~r/Cannot cast "yesterday" to timestamp/, fn -> parse("select id from t.users where at >= 'yesterday'::timestamp;") diff --git a/test/types_test.exs b/test/types_test.exs index 5db1747..64b33b6 100644 --- a/test/types_test.exs +++ b/test/types_test.exs @@ -20,6 +20,30 @@ defmodule Efsql.TypesTest do refute Types.compare(~N[2024-03-01 12:34:56], ~U[2024-03-01 12:34:56Z]) == :eq end + test "dates and times compare chronologically" do + # Term order compares Date's fields alphabetically, year last. + assert Types.compare(~D[2025-01-02], ~D[2024-01-03]) == :gt + # ... and Time's microsecond before its minute. + assert Types.compare(~T[09:00:00.9], ~T[09:01:00]) == :lt + assert Types.compare(~T[09:00:00.000000], ~T[09:00:00]) == :eq + end + + test "decimals compare by value, against decimals and numbers" do + # Term order puts every number below every map. + assert Types.compare(Decimal.new("10.5"), 100) == :lt + assert Types.compare(100, Decimal.new("10.5")) == :gt + assert Types.compare(Decimal.new("10.5"), 10.5) == :eq + assert Types.compare(Decimal.new("0.1"), 0.1) == :eq + assert Types.compare(Decimal.new("1.50"), Decimal.new("1.5")) == :eq + assert Types.compare(Decimal.new("2"), 2) == :eq + assert Types.compare(Decimal.new("-3"), Decimal.new("2")) == :lt + end + + test "a decimal NaN or non-number falls back to term order instead of raising" do + assert Types.compare(Decimal.new("NaN"), 1) in [:lt, :gt] + assert Types.compare(Decimal.new("1"), "1") in [:lt, :gt] + end + test "other values use term order" do assert Types.compare("a", "b") == :lt assert Types.compare(2, 1) == :gt @@ -32,6 +56,8 @@ defmodule Efsql.TypesTest do test "datetimes encode as the adapter's index does" do assert Types.index_key(~N[2024-03-01 12:34:56]) == "20240301T123456.000000" assert Types.index_key(~U[2024-03-01 12:34:56Z]) == "20240301T123456.000000Z" + assert Types.index_key(~D[2024-03-01]) == "20240301" + assert Types.index_key(~T[12:34:56]) == "123456.000000" end test "other values pass through" do From 5bedf851a13b42a5df5a4b8ce6618cc8d813293e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 00:28:28 +0000 Subject: [PATCH 04/10] Support IS NULL and IS NOT NULL in the where clause where notes is null where notes is not null where notes isnull / notnull -- PostgreSQL shorthands A nil or absent field is NULL. Both are residual filters in the executor; an indexed equality alongside still uses the index. The SQL library only assembles `x is null` into a node that composes with `and`. It leaves `x is not null` as loose tokens, never returns from `... and x is not null`, and hangs the same way on its postfix isnull/notnull nodes. Efsql.Parser.parse/1 now rewrites all of these to the `is null` shape before parsing, marking negation on the `is` token's meta, and every lex/parse call site goes through it. `x = null` (never true in SQL) now raises Unsupported pointing to IS NULL, instead of a FunctionClauseError. Completion offers `is`, then `null`/`not`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 12 ++++++++++ lib/efsql.ex | 9 ++----- lib/efsql/complete.ex | 13 ++++++++-- lib/efsql/executor.ex | 5 +++- lib/efsql/logical.ex | 3 +++ lib/efsql/parser.ex | 51 ++++++++++++++++++++++++++++++++++++++- lib/efsql/planner.ex | 2 ++ lib/efsql/tui/session.ex | 4 +--- test/complete_test.exs | 14 +++++++++++ test/null_test.exs | 32 +++++++++++++++++++++++++ test/parser_test.exs | 52 ++++++++++++++++++++++++++++++++++++---- 11 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 test/null_test.exs diff --git a/README.md b/README.md index e4a1f28..038b15f 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,18 @@ select col_a, col_b from tenant_id.table_name where index_col between 'baz' and Since efsql doesn't have access to the Ecto schema, type checking is loosened: a value must be written as the type the column stores. For datetime columns, use a [typed literal](#typed-literals). +### Filter by NULL + +```sql +select col_a from tenant_id.table_name where col_b is null; +select col_a from tenant_id.table_name where col_b is not null; +``` + +A field that is `nil` or absent from the stored record is NULL. The +PostgreSQL shorthands `col_b isnull` and `col_b notnull` also work. As in SQL, +`col_b = null` would never be true, so efsql rejects it with a pointer to +`IS NULL`. + ### Typed literals A quoted literal is always a string. To compare against an Elixir term that diff --git a/lib/efsql.ex b/lib/efsql.ex index fab1145..ef0e7bc 100644 --- a/lib/efsql.ex +++ b/lib/efsql.ex @@ -14,10 +14,7 @@ defmodule Efsql do import Ecto.Query - def lex_and_parse(sql) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - SQL.Parser.parse(tokens, context) - end + def lex_and_parse(sql), do: Efsql.Parser.parse(sql) def hello() do tenant = EctoFoundationDB.Tenant.open!(Efsql.Repo, "localhost") @@ -48,9 +45,7 @@ defmodule Efsql do end def sql_to_logical(sql, tenants \\ %{}) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - logical = %Efsql.Logical.Select{} = Efsql.Parser.to_logical(parsed) + logical = %Efsql.Logical.Select{} = Efsql.Parser.sql_to_logical(sql) resolve_tenant(logical, tenants) end diff --git a/lib/efsql/complete.ex b/lib/efsql/complete.ex index 647f25d..a17a8a6 100644 --- a/lib/efsql/complete.ex +++ b/lib/efsql/complete.ex @@ -14,7 +14,7 @@ defmodule Efsql.Complete do @statement_start ~w(select) @post_expr ~w(and order limit) - @operators ~w(= > >= < <= like in between not) + @operators ~w(= > >= < <= like in between not is) def complete(input, context) do {word_start, word} = current_word(input) @@ -33,8 +33,17 @@ defmodule Efsql.Complete do defp candidates_for([], _context, _tokens), do: @statement_start - defp candidates_for([last | _], context, tokens) do + defp candidates_for([last | earlier], context, tokens) do cond do + last == "is" -> + ~w(null not) + + last == "not" and List.first(earlier) == "is" -> + ~w(null) + + last == "null" -> + @post_expr + last == "select" or last == "," -> fields(context, tokens) ++ ["*"] diff --git a/lib/efsql/executor.ex b/lib/efsql/executor.ex index 3dadbf6..1e9f6f0 100644 --- a/lib/efsql/executor.ex +++ b/lib/efsql/executor.ex @@ -4,7 +4,7 @@ defmodule Efsql.Executor do then folds the operator pipeline over them. SQL NULL semantics: a comparison, LIKE, or IN against a NULL (nil) field is - false — including NOT LIKE. Sorting places NULLs last ascending and first + false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first descending, matching PostgreSQL's defaults. Values compare with `Efsql.Types.compare/2`, so datetimes order chronologically. """ @@ -96,6 +96,9 @@ defmodule Efsql.Executor do end end + defp eval({:is_null, field}, row), do: Map.get(row, field) == nil + defp eval({:not_null, field}, row), do: Map.get(row, field) != nil + defp eval({:like, field, pattern}, row) do case Map.get(row, field) do nil -> false diff --git a/lib/efsql/logical.ex b/lib/efsql/logical.ex index 7d61767..384d346 100644 --- a/lib/efsql/logical.ex +++ b/lib/efsql/logical.ex @@ -10,6 +10,7 @@ defmodule Efsql.Logical do bound, `lower_op` in `:> :>=`, `upper_op` in `:< :<=` * `{:like, field, pattern}` / `{:not_like, field, pattern}` * `{:in, field, values}` + * `{:is_null, field}` / `{:not_null, field}` The primary key is the pseudo-field `:_`. @@ -36,4 +37,6 @@ defmodule Efsql.Logical do def predicate_field({:like, field, _pattern}), do: field def predicate_field({:not_like, field, _pattern}), do: field def predicate_field({:in, field, _values}), do: field + def predicate_field({:is_null, field}), do: field + def predicate_field({:not_null, field}), do: field end diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index a8f0440..2dd2723 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -11,6 +11,45 @@ defmodule Efsql.Parser do @comparison_ops ~w[= >= <= > <]a + @doc """ + Lexes and parses SQL text into the SQL library's parse tree, as + `{:ok, context, parsed}`. Use this rather than calling the library + directly: it first rewrites token sequences the library mishandles. + """ + def parse(sql) do + {:ok, context, tokens} = SQL.Lexer.lex(sql) + SQL.Parser.parse(normalize_tokens(tokens), context) + end + + def sql_to_logical(sql) do + {:ok, _context, parsed} = parse(sql) + to_logical(parsed) + end + + # The library assembles `x is null` into a node that composes with `and`, + # but leaves `x is not null` as loose tokens, and never returns from + # `... and x is not null`; its postfix `isnull`/`notnull` nodes hang the + # same way. Rewrite all of them to the `is null` shape, marking the + # negated ones on the `is` token's meta. The marker is appended because + # the library matches the leading meta entries by position. Lexer tokens + # are in reverse order. + defp normalize_tokens([{:null, _, []} = null, {:not, _, []}, {:is, meta, []} | rest]) do + [null, {:is, meta ++ [efsql_not_null: true], []} | normalize_tokens(rest)] + end + + defp normalize_tokens([{tag, meta, []} | rest]) when tag in ~w[isnull notnull]a do + is_meta = List.keyreplace(meta, :type, 0, {:type, :operator}) + is_meta = if tag == :notnull, do: is_meta ++ [efsql_not_null: true], else: is_meta + [{:null, meta, []}, {:is, is_meta, []} | normalize_tokens(rest)] + end + + defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do + [{tag, meta, normalize_tokens(data)} | normalize_tokens(rest)] + end + + defp normalize_tokens([token | rest]), do: [token | normalize_tokens(rest)] + defp normalize_tokens([]), do: [] + def to_logical(parsed) do parsed |> Enum.reject(fn @@ -131,6 +170,13 @@ defmodule Efsql.Parser do {:in, field_atom(field), values} end + defp conjunct({:is, meta, [field, {:null, _null_meta, []}]}) do + case field_atom(field) do + :_ -> raise Unsupported, "the primary key '_' is never NULL" + field -> if meta[:efsql_not_null], do: {:not_null, field}, else: {:is_null, field} + end + end + defp conjunct({token, _meta, args}) do raise Unsupported, "'#{token}'/#{length(args)} is not supported in the where clause." end @@ -163,7 +209,10 @@ defmodule Efsql.Parser do defp param({true, _meta, []}), do: true defp param({false, _meta, []}), do: false - defp param({nil, _meta, []}), do: nil + # SQL's `x = null` is never true, which is never what was meant. + defp param({:null, _meta, []}) do + raise Unsupported, "a comparison with NULL is never true; use IS NULL or IS NOT NULL" + end defp param({:paren, _meta, [{:quote, _, part}, {:comma, _, [{:*, _, []}]}]}) do {:erlang.list_to_binary(part), :*} diff --git a/lib/efsql/planner.ex b/lib/efsql/planner.ex index ed33660..f0bd294 100644 --- a/lib/efsql/planner.ex +++ b/lib/efsql/planner.ex @@ -202,6 +202,8 @@ defmodule Efsql.Planner do defp kind({:in, _field, _values}), do: :in defp kind({:like, _field, _pattern}), do: :residual defp kind({:not_like, _field, _pattern}), do: :residual + defp kind({:is_null, _field}), do: :residual + defp kind({:not_null, _field}), do: :residual defp kind({:cmp, _op, @pk_field, _value}), do: :pk defp kind({:range, @pk_field, _lower, _upper}), do: :pk defp kind({:cmp, _op, _field, _value}), do: :pushable diff --git a/lib/efsql/tui/session.ex b/lib/efsql/tui/session.ex index 0b2b062..481aae7 100644 --- a/lib/efsql/tui/session.ex +++ b/lib/efsql/tui/session.ex @@ -12,9 +12,7 @@ defmodule Efsql.Tui.Session do end def qall(sql, %{tenant: tenant, tenants: tenants}) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - logical = %Efsql.Logical.Select{} = Efsql.Parser.to_logical(parsed) + logical = %Efsql.Logical.Select{} = Efsql.Parser.sql_to_logical(sql) {logical, tenants} = case logical.prefix do diff --git a/test/complete_test.exs b/test/complete_test.exs index d5b3ca6..238046c 100644 --- a/test/complete_test.exs +++ b/test/complete_test.exs @@ -43,6 +43,20 @@ defmodule Efsql.CompleteTest do assert "between" in candidates end + test "is offers null and not null" do + {_, candidates} = Complete.complete("select id from users where name ", @context) + assert "is" in candidates + + assert {_, ["null", "not"]} = + Complete.complete("select id from users where name is ", @context) + + assert {_, ["null"]} = + Complete.complete("select id from users where name is not ", @context) + + {_, candidates} = Complete.complete("select id from users where name is null ", @context) + assert "and" in candidates + end + test "order by offers fields" do assert {_, ["by"]} = Complete.complete("select id from users order ", @context) {_, candidates} = Complete.complete("select id from users order by ", @context) diff --git a/test/null_test.exs b/test/null_test.exs new file mode 100644 index 0000000..fb9a30e --- /dev/null +++ b/test/null_test.exs @@ -0,0 +1,32 @@ +defmodule EfsqlTest.Integration.Null do + use EfsqlTest.Case, async: true + + test "is null matches a field that was never set", context do + tenant_id = context[:tenant_id] + + assert [%{id: "0003"}] = + Efsql.all("select id from #{tenant_id}.users where notes is null;") + end + + test "is not null", context do + tenant_id = context[:tenant_id] + + assert [%{id: "0001"}, %{id: "0002"}] = + Efsql.all("select id from #{tenant_id}.users where notes is not null;") + |> Enum.sort_by(& &1.id) + end + + test "is not null combined with an indexed equality", context do + tenant_id = context[:tenant_id] + + assert [] = + Efsql.all( + "select id from #{tenant_id}.users where name = 'Charles' and notes is not null;" + ) + + assert [%{id: "0001"}] = + Efsql.all( + "select id from #{tenant_id}.users where name = 'Alice' and notes is not null;" + ) + end +end diff --git a/test/parser_test.exs b/test/parser_test.exs index 349cb24..ac7695d 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -5,11 +5,7 @@ defmodule Efsql.ParserTest do alias Efsql.Logical alias Efsql.Parser - defp parse(sql) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - {:ok, _context, parsed} = SQL.Parser.parse(tokens, context) - Parser.to_logical(parsed) - end + defp parse(sql), do: Parser.sql_to_logical(sql) describe "select" do test "single field" do @@ -239,6 +235,52 @@ defmodule Efsql.ParserTest do end end + test "is null and is not null" do + assert %Logical.Select{predicates: [{:is_null, :notes}]} = + parse("select id from t.users where notes is null;") + + assert %Logical.Select{predicates: [{:not_null, :notes}]} = + parse("select id from t.users where notes is not null;") + + assert %Logical.Select{predicates: [{:is_null, :notes}]} = + parse("select id from t.users where notes isnull;") + + assert %Logical.Select{predicates: [{:not_null, :notes}]} = + parse("select id from t.users where notes notnull;") + end + + # The SQL library hangs on these without Efsql.Parser's token rewrite. + test "null tests combine with other predicates in any position" do + assert %Logical.Select{predicates: [{:cmp, :==, :name, "a"}, {:not_null, :notes}]} = + parse("select id from t.users where name = 'a' and notes is not null;") + + assert %Logical.Select{predicates: [{:not_null, :notes}, {:cmp, :==, :name, "a"}]} = + parse("select id from t.users where notes is not null and name = 'a';") + + assert %Logical.Select{ + predicates: [{:is_null, :a}, {:not_null, :b}, {:cmp, :==, :c, "x"}] + } = parse("select id from t.users where a is null and b notnull and c = 'x';") + + assert %Logical.Select{predicates: [{:cmp, :==, :c, "x"}, {:is_null, :a}]} = + parse("select id from t.users where c = 'x' and a isnull;") + end + + test "comparison with null raises" do + assert_raise Unsupported, ~r/use IS NULL or IS NOT NULL/, fn -> + parse("select id from t.users where notes = null;") + end + + assert_raise Unsupported, ~r/use IS NULL or IS NOT NULL/, fn -> + parse("select id from t.users where notes in ('a', null);") + end + end + + test "the primary key is never null" do + assert_raise Unsupported, ~r/never NULL/, fn -> + parse("select id from t.users where _ is null;") + end + end + test "versionstamp partition scan value" do assert %Logical.Select{predicates: [{:cmp, :==, :_, {"p", :*}}]} = parse("select id from t.users where _ = ('p', *);") From 8d7d630c4c5402a8843e0821c493dea96c5667be Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:09:49 +0000 Subject: [PATCH 05/10] Depend on sql from hex (0.5.0) instead of git 0.5.0 is the latest hex release; it predates the git commit efsql was locked to (da69080, one large refactor after v0.5.0). efsql only uses SQL.Lexer.lex/1 and SQL.Parser.parse/2, and its tests pass on both. The one incompatibility: 0.5.0 lexes integers as :numeric rather than :integer, so `where age > 40` raised in String.to_float/1. A :numeric literal with no fractional part now parses as an integer. The git entry is removed from mix.lock; the next `mix deps.get` writes the hex entry with its checksums (hex.pm was unreachable here). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- lib/efsql/parser.ex | 8 +++++++- mix.exs | 2 +- mix.lock | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 2dd2723..57c054b 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -196,8 +196,14 @@ defmodule Efsql.Parser do defp param({:integer, _meta, value}), do: :erlang.list_to_integer(value) + # sql 0.5.0 lexes integers as :numeric too. defp param({:numeric, _meta, value}) do - value |> :erlang.list_to_binary() |> String.to_float() + value = :erlang.list_to_binary(value) + + case Integer.parse(value) do + {integer, ""} -> integer + _ -> String.to_float(value) + end end # 'value'::type and cast('value' as type) -- see Efsql.Types diff --git a/mix.exs b/mix.exs index a488139..9fb457e 100644 --- a/mix.exs +++ b/mix.exs @@ -42,7 +42,7 @@ defmodule Efsql.MixProject do # advance them together. {:ecto, "~> 3.13.0"}, {:ex_fdbmonitor, github: "foundationdb-beam/ex_fdbmonitor", only: :dev, runtime: false}, - {:sql, github: "elixir-dbvisor/sql"}, + {:sql, "~> 0.5.0"}, {:owl, "~> 0.13"} ] end diff --git a/mix.lock b/mix.lock index 30e877e..fa5b03d 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,6 @@ "local_cluster": {:hex, :local_cluster, "2.1.0", "1c847d69a927ef5a62db13236f93146e8a42377a9c9a5bb4cac3372cba69d683", [:mix], [{:global_flags, "~> 1.0", [hex: :global_flags, repo: "hexpm", optional: false]}], "hexpm", "dc1c3abb6fef00198dd53c855b39ea80c55b3a8059d8d9f17d50da46b1e3b858"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "owl": {:hex, :owl, "0.13.1", "1ec4a5dea170465f0e90c502c203079224516bc0cbd599281c8667b3c6ef8848", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "351e768af8f2edc575cdaab1a5a2f6d6381be591758a026c701c703145508a0c"}, - "sql": {:git, "https://github.com/elixir-dbvisor/sql.git", "da69080748c44d50869477f5d254504012ab7f6c", []}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "unicode": {:hex, :unicode, "2.0.0", "d2d3ec27575fed3437fdcc0d383909378201843d27d2197c0c2742c973110b24", [:mix], [], "hexpm", "a8da44be6a2fc165e58ea332038686df79994b51bf2c7fe337bf2927ea148c13"}, "unicode_set": {:hex, :unicode_set, "1.7.0", "38c91095b35b90cf2e8f8aaa67d298937886293c5e0288953325014503888cd2", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode, "~> 2.0", [hex: :unicode, repo: "hexpm", optional: false]}], "hexpm", "66c8c02adadac58fcaf64e22eba90c243a6528d1d36fb014e2d8c0e062efbae6"}, From 8b8ccc3be76782846ff68fb0caf04e2051a11a4a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:14:12 +0000 Subject: [PATCH 06/10] Track sql from GitHub main (f6097ff) Back to the git dependency, now at the tip of main, one commit past the previously locked da69080. efsql uses only SQL.Lexer.lex/1 and SQL.Parser.parse/2; its tests pass unchanged. The :numeric literal fix from the hex switch stays, so parsing works on either library version. What the new commit brings for efsql: - The parser handles `... and (expr)`, which used to never return. efsql now raises Unsupported for a parenthesized predicate instead of hanging. - The library now compiles three C NIFs via a `make` compile alias, so building efsql needs make and a C compiler. The release runners (GitHub-hosted macOS and Ubuntu) have both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- mix.exs | 2 +- mix.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 9fb457e..a488139 100644 --- a/mix.exs +++ b/mix.exs @@ -42,7 +42,7 @@ defmodule Efsql.MixProject do # advance them together. {:ecto, "~> 3.13.0"}, {:ex_fdbmonitor, github: "foundationdb-beam/ex_fdbmonitor", only: :dev, runtime: false}, - {:sql, "~> 0.5.0"}, + {:sql, github: "elixir-dbvisor/sql"}, {:owl, "~> 0.13"} ] end diff --git a/mix.lock b/mix.lock index fa5b03d..6af86d3 100644 --- a/mix.lock +++ b/mix.lock @@ -11,6 +11,7 @@ "local_cluster": {:hex, :local_cluster, "2.1.0", "1c847d69a927ef5a62db13236f93146e8a42377a9c9a5bb4cac3372cba69d683", [:mix], [{:global_flags, "~> 1.0", [hex: :global_flags, repo: "hexpm", optional: false]}], "hexpm", "dc1c3abb6fef00198dd53c855b39ea80c55b3a8059d8d9f17d50da46b1e3b858"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "owl": {:hex, :owl, "0.13.1", "1ec4a5dea170465f0e90c502c203079224516bc0cbd599281c8667b3c6ef8848", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "351e768af8f2edc575cdaab1a5a2f6d6381be591758a026c701c703145508a0c"}, + "sql": {:git, "https://github.com/elixir-dbvisor/sql.git", "f6097fff5b08efe9cd946a01cc2152c82d871cf7", []}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "unicode": {:hex, :unicode, "2.0.0", "d2d3ec27575fed3437fdcc0d383909378201843d27d2197c0c2742c973110b24", [:mix], [], "hexpm", "a8da44be6a2fc165e58ea332038686df79994b51bf2c7fe337bf2927ea148c13"}, "unicode_set": {:hex, :unicode_set, "1.7.0", "38c91095b35b90cf2e8f8aaa67d298937886293c5e0288953325014503888cd2", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode, "~> 2.0", [hex: :unicode, repo: "hexpm", optional: false]}], "hexpm", "66c8c02adadac58fcaf64e22eba90c243a6528d1d36fb014e2d8c0e062efbae6"}, From 3b8ba05a80627dd6b984e182623285bb7b6625d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:26:07 +0000 Subject: [PATCH 07/10] Load sql without starting its client runtime efsql uses only sql's lexer and parser. Listing sql in included_applications keeps its code loaded (and in the release) but never runs SQL.Application, which since f6097ff opens a :dets file in its priv dir, sleeps 100 ms on every start, and crashes in stop/1 when no pool was ever started. Mix also loads sql's own dependencies without starting them. The NIFs are still compiled by sql's `make` alias and loaded at boot; a release in embedded mode fails to boot without them. Removing that build dependency means vendoring the lexer and parser. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- mix.exs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index a488139..c163be2 100644 --- a/mix.exs +++ b/mix.exs @@ -31,7 +31,12 @@ defmodule Efsql.MixProject do def application do [ mod: {Efsql.Application, []}, - extra_applications: [:logger] + extra_applications: [:logger], + # efsql uses only sql's lexer and parser, which need no running + # process. Including the app loads its code without starting its + # database client runtime (a :dets file in its priv dir, a startup + # sleep, and a stop/1 that crashes when no pool ever started). + included_applications: [:sql] ] end From 87c3231d0066c27a53899d2934b42feb18d4222d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:29:40 +0000 Subject: [PATCH 08/10] Add a GitHub Actions workflow that runs mix test Runs on pushes to main and on pull requests, on ubuntu-22.04 with the same Elixir, OTP and FoundationDB versions as release.yml. Installs the FDB server package as well as the client: EctoFoundationDB.Sandbox runs the tests against an fdbserver it starts itself. Mirrors ecto_foundationdb's own CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- .github/workflows/test.yml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c372a9a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +env: + # Keep in step with release.yml. + ELIXIR_VERSION: "1.19" + OTP_VERSION: "28" + FDB_VERSION: "7.3.69" + ERLFDB_COMPILE_API_VERSION: "730" + MIX_ENV: test + +jobs: + test: + name: mix test + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ env.OTP_VERSION }} + elixir-version: ${{ env.ELIXIR_VERSION }} + + # The client links the erlfdb NIF; the server package provides the + # fdbserver binary that EctoFoundationDB.Sandbox runs the tests against. + - name: Install FoundationDB + run: | + BASE="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}" + curl -fsSL -o fdb-clients.deb "$BASE/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" + curl -fsSL -o fdb-server.deb "$BASE/foundationdb-server_${FDB_VERSION}-1_amd64.deb" + sudo dpkg -i fdb-clients.deb fdb-server.deb + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: test-${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-fdb${{ env.FDB_VERSION }}-mix-${{ hashFiles('mix.lock') }} + restore-keys: test-${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-fdb${{ env.FDB_VERSION }}-mix- + + - run: mix deps.get --only test + - run: mix deps.compile + - run: mix compile + - run: mix test From 97e8c8f5b1a4bc7bb8e05a8da5afc995bc934f7d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:45:59 +0000 Subject: [PATCH 09/10] Parse columns named after SQL reserved words The SQL library lexes a reserved word as its own token even where it names a column. For `where day = 'x'` or `where at >= ...` its parser never returns; with in, like, is null or between it misparses, and `order by day desc` silently became [asc: :day, asc: :desc]. Only double-quoting the column avoided it. Efsql.Parser.parse/1 now rewrites a reserved word that directly precedes a predicate operator or asc/desc into an identifier, except for words that legitimately sit there (true, null, end, where, ...). This is why seven parser tests timed out in CI: their `at` and `day` columns are reserved in the real SQL grammar. New tests cover reserved-word columns across every predicate form and in order by. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- lib/efsql/parser.ex | 25 +++++++++++++++++++++++-- test/parser_test.exs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 57c054b..727140a 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -11,6 +11,12 @@ defmodule Efsql.Parser do @comparison_ops ~w[= >= <= > <]a + # Tokens that follow a column: an unquoted column named after a reserved + # word (day, at, date, user, ...) is recognised by its position here. + @column_followers ~w[= != <> < > <= >= in like ilike between is not asc desc]a + # Reserved words that can themselves precede one of those tokens. + @not_columns ~w[true false null unknown end case when then else where having on select is]a + @doc """ Lexes and parses SQL text into the SQL library's parse tree, as `{:ok, context, parsed}`. Use this rather than calling the library @@ -26,6 +32,11 @@ defmodule Efsql.Parser do to_logical(parsed) end + # The library lexes a reserved word as its own token even where it names + # a column, and then misparses the predicate or never returns from + # `where day = 'x'`. A reserved word right before a column follower is + # a column: rewrite it to the identifier the library expects. + # # The library assembles `x is null` into a node that composes with `and`, # but leaves `x is not null` as loose tokens, and never returns from # `... and x is not null`; its postfix `isnull`/`notnull` nodes hang the @@ -34,13 +45,23 @@ defmodule Efsql.Parser do # the library matches the leading meta entries by position. Lexer tokens # are in reverse order. defp normalize_tokens([{:null, _, []} = null, {:not, _, []}, {:is, meta, []} | rest]) do - [null, {:is, meta ++ [efsql_not_null: true], []} | normalize_tokens(rest)] + [null | normalize_tokens([{:is, meta ++ [efsql_not_null: true], []} | rest])] end defp normalize_tokens([{tag, meta, []} | rest]) when tag in ~w[isnull notnull]a do is_meta = List.keyreplace(meta, :type, 0, {:type, :operator}) is_meta = if tag == :notnull, do: is_meta ++ [efsql_not_null: true], else: is_meta - [{:null, meta, []}, {:is, is_meta, []} | normalize_tokens(rest)] + [{:null, meta, []} | normalize_tokens([{:is, is_meta, []} | rest])] + end + + defp normalize_tokens([{follower, _, _} = token, {word, meta, []} | rest]) + when follower in @column_followers and word not in @not_columns do + if meta[:type] == :reserved do + ident = {:ident, List.keyreplace(meta, :type, 0, {:type, :literal}), Atom.to_charlist(word)} + [token | normalize_tokens([ident | rest])] + else + [token | normalize_tokens([{word, meta, []} | rest])] + end end defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do diff --git a/test/parser_test.exs b/test/parser_test.exs index ac7695d..f93f9c9 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -281,6 +281,36 @@ defmodule Efsql.ParserTest do end end + # The SQL library hangs or misparses on these without the token + # rewrite in Efsql.Parser.parse/1. + test "columns named after reserved words" do + assert %Logical.Select{predicates: [{:cmp, :==, :day, "x"}]} = + parse("select id from t.users where day = 'x';") + + assert %Logical.Select{predicates: [{:cmp, :==, :x, "a"}, {:cmp, :>=, :at, 1}]} = + parse("select id from t.users where x = 'a' and at >= 1;") + + assert %Logical.Select{ + predicates: [ + {:in, :day, ["a"]}, + {:like, :user, "a%"}, + {:not_like, :value, "b%"}, + {:range, :year, {:>=, 1}, {:<=, 2}} + ] + } = + parse( + "select id from t.users where day in ('a') and user like 'a%' and value not like 'b%' and year between 1 and 2;" + ) + + assert %Logical.Select{predicates: [{:is_null, :day}, {:not_null, :at}]} = + parse("select id from t.users where day is null and at is not null;") + end + + test "reserved-word columns keep their sort direction" do + assert %Logical.Select{order: [desc: :day, asc: :user]} = + parse("select id from t.users order by day desc, user asc;") + end + test "versionstamp partition scan value" do assert %Logical.Select{predicates: [{:cmp, :==, :_, {"p", :*}}]} = parse("select id from t.users where _ = ('p', *);") From c06add08162304e52a9fd7537396dae83f6b5b01 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:48:26 +0000 Subject: [PATCH 10/10] Recognise asc/desc as column followers when non-reserved In the real SQL grammar asc and desc are non-reserved words, lexed as identifiers that carry the word in their meta's :tag, so the reserved-word column rewrite never fired before them and `order by day desc, user asc` reached efsql as a bare comma node. Followers are now matched by tag for identifier tokens too. The paren clause moves first so a leading (...) is still normalised. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- lib/efsql/parser.ex | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index 727140a..d7dbc09 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -54,9 +54,13 @@ defmodule Efsql.Parser do [{:null, meta, []} | normalize_tokens([{:is, is_meta, []} | rest])] end - defp normalize_tokens([{follower, _, _} = token, {word, meta, []} | rest]) - when follower in @column_followers and word not in @not_columns do - if meta[:type] == :reserved do + defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do + [{tag, meta, normalize_tokens(data)} | normalize_tokens(rest)] + end + + defp normalize_tokens([token, {word, meta, []} | rest]) + when is_atom(word) and word not in @not_columns do + if column_follower?(token) and meta[:type] == :reserved do ident = {:ident, List.keyreplace(meta, :type, 0, {:type, :literal}), Atom.to_charlist(word)} [token | normalize_tokens([ident | rest])] else @@ -64,13 +68,14 @@ defmodule Efsql.Parser do end end - defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do - [{tag, meta, normalize_tokens(data)} | normalize_tokens(rest)] - end - defp normalize_tokens([token | rest]), do: [token | normalize_tokens(rest)] defp normalize_tokens([]), do: [] + # Non-reserved words such as asc and desc lex as identifiers carrying + # the word in their meta's :tag. + defp column_follower?({:ident, meta, _}), do: meta[:tag] in @column_followers + defp column_follower?({tag, _meta, _}), do: tag in @column_followers + def to_logical(parsed) do parsed |> Enum.reject(fn