From 5963f3ac4e7a44770f6e2bab3b8355ce1eb83bda Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 03:09:39 +0000 Subject: [PATCH 1/5] Replace the sql dependency with our own lexer and parser efsql used elixir-dbvisor/sql only for lexing and parsing, and needed a growing set of token rewrites to work around it: `and x is not null` and any column named after a reserved word (`where day = 'x'`) never returned, and `order by day desc` silently flipped direction. The dependency also brought C NIFs and a runtime that efsql had to keep from starting. Efsql.SQL is a small front end written for the dialect efsql speaks: - Lexer: hand-written over binaries. Folds unquoted names, keeps quoted ones, handles '' and "" escapes, nested /* */ comments, CRLF, Unicode names and whitespace, and reports every error with line and column. - Parser: recursive descent into a documented AST (Efsql.SQL.AST). Keywords are only keywords where the grammar expects one, so day, user, date or value are plain names; the handful that must be quoted are listed and the error says to quote them. Unsupported features (OR, joins, functions, GROUP BY, subqueries...) are rejected by name, and nesting depth is capped. - Efsql.Parser now just translates the AST to Efsql.Logical. It also validates what used to crash: field names too long for an atom, and versionstamps outside 0..2^96-1. Typed literals now also work as BETWEEN bounds. The CLI prints exception messages instead of inspected structs, so syntax errors read cleanly. Tests: lexer and parser suites covering every token and construct with exact positions; the existing translation tests unchanged; every query on the TUI help page; and randomized tests that print random trees with randomized spelling (case, comments, whitespace, quoting, alternate syntax) and require the exact tree back, plus fuzzing of mangled queries and random bytes, which must yield a result or a positioned SyntaxError and never crash. The fuzzing found and fixed one crash: float literals beyond a double's range. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- README.md | 20 +- lib/efsql.ex | 2 - lib/efsql/cli.ex | 8 +- lib/efsql/parser.ex | 316 ++++++------------ lib/efsql/sql/ast.ex | 58 ++++ lib/efsql/sql/lexer.ex | 289 ++++++++++++++++ lib/efsql/sql/parser.ex | 516 +++++++++++++++++++++++++++++ lib/efsql/sql/syntax_error.ex | 13 + mix.exs | 8 +- mix.lock | 4 - test/by_pk_test.exs | 5 +- test/parser_test.exs | 84 ++++- test/sql/lexer_test.exs | 244 ++++++++++++++ test/sql/parser_test.exs | 418 +++++++++++++++++++++++ test/sql/property_test.exs | 126 +++++++ test/support/efsql_test/sql_gen.ex | 239 +++++++++++++ 16 files changed, 2107 insertions(+), 243 deletions(-) create mode 100644 lib/efsql/sql/ast.ex create mode 100644 lib/efsql/sql/lexer.ex create mode 100644 lib/efsql/sql/parser.ex create mode 100644 lib/efsql/sql/syntax_error.ex create mode 100644 test/sql/lexer_test.exs create mode 100644 test/sql/parser_test.exs create mode 100644 test/sql/property_test.exs create mode 100644 test/support/efsql_test/sql_gen.ex diff --git a/README.md b/README.md index 038b15f..dfb79ea 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,17 @@ Connected to /etc/foundationdb/fdb.cluster ## Supported SQL -All queries require at minimum a `tenant_id.table_name` form in the `FROM` clause. Column names that are reserved SQL words (e.g. `ref`) are supported. +All queries require at minimum a `tenant_id.table_name` form in the `FROM` clause. + +Keywords and unquoted names are case-insensitive; double-quote a name to keep +its case (`"CreatedAt"`). Most SQL keywords work as bare names, so columns like +`day`, `user`, `date` or `value` need no quoting. The few that can't, such as +`order`, `select`, `limit` or `and`, must be quoted (`"order"`). The error says +so if you forget. + +Comments (`-- ...` and `/* ... */`) can go anywhere. A syntax error reports the +line and column it was found at, and features efsql doesn't support (`OR`, +`NOT`, `<>`, joins, functions, `GROUP BY`) are rejected by name. ### Storage IDs @@ -287,9 +297,11 @@ which the adapter keys by the `Decimal`'s term encoding rather than its value. Ecto stores `Ecto.Enum` fields as strings, so query those with a plain string literal, not an atom. -`BETWEEN` and `LIKE` don't accept typed literals; write a range as -`inserted_at >= '2024-01-01'::timestamp and inserted_at < '2025-01-01'::timestamp` -instead. +Typed literals work anywhere a value does, including `IN` and `BETWEEN`: + +```sql +select col_a from tenant_id.table_name where inserted_at between '2024-01-01'::timestamp and '2025-01-01'::timestamp; +``` ### Limit diff --git a/lib/efsql.ex b/lib/efsql.ex index ef0e7bc..4c9e4b6 100644 --- a/lib/efsql.ex +++ b/lib/efsql.ex @@ -14,8 +14,6 @@ defmodule Efsql do import Ecto.Query - def lex_and_parse(sql), do: Efsql.Parser.parse(sql) - def hello() do tenant = EctoFoundationDB.Tenant.open!(Efsql.Repo, "localhost") diff --git a/lib/efsql/cli.ex b/lib/efsql/cli.ex index fd415f5..3c25577 100644 --- a/lib/efsql/cli.ex +++ b/lib/efsql/cli.ex @@ -283,7 +283,9 @@ defmodule Efsql.Cli do |> then(&"async union (#{length(nodes)}):\n#{&1}") end - defp print_error(term) do - Owl.IO.puts(Owl.Data.tag(inspect(term), :red)) - end + defp print_error(%{__exception__: true} = e), + do: Owl.IO.puts(Owl.Data.tag(Exception.message(e), :red)) + + defp print_error(term) when is_binary(term), do: Owl.IO.puts(Owl.Data.tag(term, :red)) + defp print_error(term), do: Owl.IO.puts(Owl.Data.tag(inspect(term), :red)) end diff --git a/lib/efsql/parser.ex b/lib/efsql/parser.ex index d7dbc09..802aed9 100644 --- a/lib/efsql/parser.ex +++ b/lib/efsql/parser.ex @@ -1,272 +1,156 @@ defmodule Efsql.Parser do @moduledoc """ - Converts the SQL library's parse tree into an `Efsql.Logical.Select`. - Purely syntactic — normalization (range merging, LIKE rewriting) happens - in `Efsql.Rewrite`. + SQL text to `Efsql.Logical.Select`: parses with `Efsql.SQL.Parser`, then + translates the syntax tree. Syntax problems raise `Efsql.SQL.SyntaxError`; + valid SQL that efsql can't execute (`OR`, comparing two columns, ...) + raises `Efsql.Exception.Unsupported`. + + Purely a translation — normalization (range merging, LIKE rewriting) + happens in `Efsql.Rewrite`. """ alias Efsql.Exception.Unsupported alias Efsql.Logical + alias Efsql.SQL.AST alias Efsql.Types - @comparison_ops ~w[= >= <= > <]a - - # Tokens that follow a column: an unquoted column named after a reserved - # word (day, at, date, user, ...) is recognised by its position here. - @column_followers ~w[= != <> < > <= >= in like ilike between is not asc desc]a - # Reserved words that can themselves precede one of those tokens. - @not_columns ~w[true false null unknown end case when then else where having on select is]a - - @doc """ - Lexes and parses SQL text into the SQL library's parse tree, as - `{:ok, context, parsed}`. Use this rather than calling the library - directly: it first rewrites token sequences the library mishandles. - """ - def parse(sql) do - {:ok, context, tokens} = SQL.Lexer.lex(sql) - SQL.Parser.parse(normalize_tokens(tokens), context) - end + @flipped %{:< => :>, :> => :<, :<= => :>=, :>= => :<=, := => :=} + @max_versionstamp Bitwise.bsl(1, 96) - 1 def sql_to_logical(sql) do - {:ok, _context, parsed} = parse(sql) - to_logical(parsed) - end - - # The library lexes a reserved word as its own token even where it names - # a column, and then misparses the predicate or never returns from - # `where day = 'x'`. A reserved word right before a column follower is - # a column: rewrite it to the identifier the library expects. - # - # The library assembles `x is null` into a node that composes with `and`, - # but leaves `x is not null` as loose tokens, and never returns from - # `... and x is not null`; its postfix `isnull`/`notnull` nodes hang the - # same way. Rewrite all of them to the `is null` shape, marking the - # negated ones on the `is` token's meta. The marker is appended because - # the library matches the leading meta entries by position. Lexer tokens - # are in reverse order. - defp normalize_tokens([{:null, _, []} = null, {:not, _, []}, {:is, meta, []} | rest]) do - [null | normalize_tokens([{:is, meta ++ [efsql_not_null: true], []} | rest])] - end - - defp normalize_tokens([{tag, meta, []} | rest]) when tag in ~w[isnull notnull]a do - is_meta = List.keyreplace(meta, :type, 0, {:type, :operator}) - is_meta = if tag == :notnull, do: is_meta ++ [efsql_not_null: true], else: is_meta - [{:null, meta, []} | normalize_tokens([{:is, is_meta, []} | rest])] - end - - defp normalize_tokens([{tag, meta, data} | rest]) when tag in ~w[paren bracket brace]a do - [{tag, meta, normalize_tokens(data)} | normalize_tokens(rest)] - end - - defp normalize_tokens([token, {word, meta, []} | rest]) - when is_atom(word) and word not in @not_columns do - if column_follower?(token) and meta[:type] == :reserved do - ident = {:ident, List.keyreplace(meta, :type, 0, {:type, :literal}), Atom.to_charlist(word)} - [token | normalize_tokens([ident | rest])] - else - [token | normalize_tokens([{word, meta, []} | rest])] - end - end - - defp normalize_tokens([token | rest]), do: [token | normalize_tokens(rest)] - defp normalize_tokens([]), do: [] - - # Non-reserved words such as asc and desc lex as identifiers carrying - # the word in their meta's :tag. - defp column_follower?({:ident, meta, _}), do: meta[:tag] in @column_followers - defp column_follower?({tag, _meta, _}), do: tag in @column_followers - - def to_logical(parsed) do - parsed - |> Enum.reject(fn - {:colon, _, _} -> true - [] -> true - _ -> false - end) - |> Enum.reduce(%Logical.Select{}, &clause/2) + sql |> Efsql.SQL.Parser.parse!() |> to_logical() end - defp clause({:select, _meta, fields}, %Logical.Select{} = select) do - %Logical.Select{select | projection: parse_select_fields(fields, [])} - end + def to_logical(%AST.Select{} = select) do + {prefix, source} = split_from(select.from) - defp clause({:from, _meta, [source_token | _]}, %Logical.Select{} = select) do %Logical.Select{ - select - | source: from_token_to_source(source_token), - prefix: from_token_to_prefix(source_token) + projection: projection(select.fields), + source: source, + prefix: prefix, + predicates: predicates(select.where), + order: Enum.map(select.order_by, fn {name, dir} -> {dir, field(name)} end), + limit: select.limit } end - defp clause({:where, _meta, [expr]}, %Logical.Select{} = select) do - %Logical.Select{select | predicates: expr |> flatten_and() |> Enum.map(&conjunct/1)} - end + defp projection(:star), do: :star + defp projection(fields), do: Enum.map(fields, &field/1) - defp clause({:order, _meta, [{:by, _by_meta, items}]}, %Logical.Select{} = select) do - %Logical.Select{select | order: Enum.map(items, &order_item/1)} - end + defp split_from([table]), do: {nil, table} + defp split_from([tenant, table]), do: {tenant, table} + defp split_from([storage, tenant, table]), do: {{storage, tenant}, table} - defp clause({:limit, _meta, [{tag, _nmeta, value}]}, %Logical.Select{} = select) - when tag in ~w[integer numeric]a do - %Logical.Select{select | limit: :erlang.list_to_integer(value)} - end + # -- WHERE -- - defp clause({token, _, _}, %Logical.Select{}) do - raise Unsupported, "'#{token}' is not supported" - end - - # SELECT - - defp parse_select_fields([], acc), do: Enum.reverse(acc) + defp predicates(nil), do: [] + defp predicates(expr), do: expr |> conjuncts([]) |> Enum.map(&predicate/1) - defp parse_select_fields([{:*, _, []} | _], _acc), do: :star + defp conjuncts({:and, left, right}, acc), do: conjuncts(left, conjuncts(right, acc)) + defp conjuncts(expr, acc), do: [expr | acc] - defp parse_select_fields([{:comma, _meta, [field]} | rest], acc) do - parse_select_fields(rest, [field_atom(field) | acc]) - end + defp predicate({:compare, :<>, _left, _right}), + do: raise(Unsupported, "<> and != are not supported") - defp parse_select_fields([field | rest], acc) do - parse_select_fields(rest, [field_atom(field) | acc]) - end + defp predicate({:compare, op, {:column, _} = column, {:column, _}}), + do: + raise(Unsupported, "comparing two fields (#{describe(column)} #{op} ...) is not supported") - defp field_atom({:ident, _meta, value}), do: charlist_to_atom(value) - defp field_atom({:double_quote, _meta, value}), do: charlist_to_atom(value) - defp field_atom({token, _meta, []}), do: token + defp predicate({:compare, op, {:column, field}, value}), + do: {:cmp, logical_op(op), field(field), value(value)} - defp field_atom({token, _meta, args}) do - raise Unsupported, "Expected an identifier, got #{token}/#{length(args)} instead." - end + # `1 < x` reads as `x > 1` + defp predicate({:compare, op, value, {:column, field}}), + do: {:cmp, logical_op(Map.fetch!(@flipped, op)), field(field), value(value)} - # FROM + defp predicate({:between, {:column, field}, low, high, false}), + do: {:range, field(field), {:>=, value(low)}, {:<=, value(high)}} - defp from_token_to_source({:dot, _meta, [_storage, {:dot, _, [_tenant, {:ident, _m, table}]}]}) do - :erlang.list_to_binary(table) - end + defp predicate({:in, {:column, field}, values, false}), + do: {:in, field(field), Enum.map(values, &value/1)} - defp from_token_to_source({:dot, _meta, [_schema, {:ident, _m, table}]}) do - :erlang.list_to_binary(table) - end + defp predicate({:like, {:column, field}, pattern, negated?}) do + pattern = + case value(pattern) do + pattern when is_binary(pattern) -> pattern + other -> raise Unsupported, "a LIKE pattern must be a string, got #{inspect(other)}" + end - defp from_token_to_source({:ident, _meta, table}) do - :erlang.list_to_binary(table) + {if(negated?, do: :not_like, else: :like), field(field), pattern} end - defp from_token_to_prefix( - {:dot, _meta, [{_st, _sm, storage}, {:dot, _, [{_tt, _tm, tenant}, _table]}]} - ) do - {:erlang.list_to_binary(storage), :erlang.list_to_binary(tenant)} - end + defp predicate({:is_null, {:column, "_"}, _negated?}), + do: raise(Unsupported, "the primary key '_' is never NULL") - defp from_token_to_prefix({:dot, _meta, [{_tag, _m, schema}, _table]}) do - :erlang.list_to_binary(schema) - end + defp predicate({:is_null, {:column, field}, negated?}), + do: {if(negated?, do: :not_null, else: :is_null), field(field)} - defp from_token_to_prefix(_), do: nil + defp predicate({:or, _, _}), do: raise(Unsupported, "OR is not supported") + defp predicate({:not, _}), do: raise(Unsupported, "NOT is not supported") - # WHERE + defp predicate({:between, _, _, _, true}), + do: raise(Unsupported, "NOT BETWEEN is not supported") - defp flatten_and({:and, _meta, [lhs, rhs]}), do: flatten_and(lhs) ++ flatten_and(rhs) - defp flatten_and(other), do: [other] + defp predicate({:in, _, _, true}), do: raise(Unsupported, "NOT IN is not supported") + defp predicate({:ilike, _, _, _}), do: raise(Unsupported, "ILIKE is not supported") - defp conjunct({operator, _meta, [lhs, rhs]}) when operator in @comparison_ops do - {:cmp, sql_op(operator), field_atom(lhs), param(rhs)} - end + defp predicate({kind, _, _, _}) when kind in [:in, :like], + do: raise(Unsupported, "#{String.upcase(to_string(kind))} needs a field on its left") - defp conjunct({:between, _meta, [field, {:and, _and_meta, [rhs1, rhs2]}]}) do - {:range, field_atom(field), {:>=, param(rhs1)}, {:<=, param(rhs2)}} - end + defp predicate({:between, _, _, _, _}), + do: raise(Unsupported, "BETWEEN needs a field on its left") - defp conjunct({:like, _meta, [{:not, _not_meta, [field]}, pattern]}) do - {:not_like, field_atom(field), param(pattern)} - end + defp predicate({:is_null, _, _}), do: raise(Unsupported, "IS NULL needs a field on its left") - defp conjunct({:like, _meta, [field, pattern]}) do - {:like, field_atom(field), param(pattern)} - end + defp predicate({:compare, _op, _left, _right}), + do: raise(Unsupported, "a comparison needs a field on one side") - defp conjunct({:in, _meta, [field, {:paren, _paren_meta, items}]}) do - values = - items - |> Enum.map(fn - {:comma, _, [item]} -> item - item -> item - end) - |> Enum.map(¶m/1) + defp predicate(expr), + do: raise(Unsupported, "#{describe(expr)} is not a condition; compare it to something") - {:in, field_atom(field), values} - end + defp logical_op(:=), do: :== + defp logical_op(op), do: op - defp conjunct({:is, meta, [field, {:null, _null_meta, []}]}) do - case field_atom(field) do - :_ -> raise Unsupported, "the primary key '_' is never NULL" - field -> if meta[:efsql_not_null], do: {:not_null, field}, else: {:is_null, field} + # Fields are atoms, which can't be longer than 255 characters. + defp field(name) do + if String.length(name) > 255 do + raise Unsupported, "field names are limited to 255 characters" end - end - - defp conjunct({token, _meta, args}) do - raise Unsupported, "'#{token}'/#{length(args)} is not supported in the where clause." - end - - # ORDER BY - defp order_item({:comma, _meta, [item]}), do: order_item(item) - defp order_item({:asc, _meta, [field]}), do: {:asc, field_atom(field)} - defp order_item({:desc, _meta, [field]}), do: {:desc, field_atom(field)} - defp order_item(field), do: {:asc, field_atom(field)} - - # values - - defp param({:quote, _meta, value}) do - :erlang.list_to_binary(value) + String.to_atom(name) end - defp param({:integer, _meta, value}), do: :erlang.list_to_integer(value) + # -- values -- - # sql 0.5.0 lexes integers as :numeric too. - defp param({:numeric, _meta, value}) do - value = :erlang.list_to_binary(value) + # SQL's `x = null` is never true, which is never what was meant. + defp value({:literal, nil}), + do: raise(Unsupported, "a comparison with NULL is never true; use IS NULL or IS NOT NULL") - case Integer.parse(value) do - {integer, ""} -> integer - _ -> String.to_float(value) - end - end + defp value({:literal, value}), do: value + defp value({:cast, expr, type}), do: Types.cast(type, value(expr)) - # 'value'::type and cast('value' as type) -- see Efsql.Types - defp param({:"::", _meta, [value, type]}), do: Types.cast(type_name(type), param(value)) + # Versionstamp primary keys partitioned by a field: ('partition', *) is + # the whole partition, ('partition', n) one versionstamp in it. + defp value({:tuple, [{:literal, part}, :star]}) when is_binary(part), do: {part, :*} - defp param({:cast, _meta, [{:paren, _paren_meta, [{:as, _as_meta, [value, type]}]}]}) do - Types.cast(type_name(type), param(value)) - end + defp value({:tuple, [{:literal, part}, {:literal, n}]}) + when is_binary(part) and is_integer(n) and n >= 0 and n <= @max_versionstamp, + do: {part, EctoFoundationDB.Versionstamp.from_integer(n)} - defp param({true, _meta, []}), do: true - defp param({false, _meta, []}), do: false - # SQL's `x = null` is never true, which is never what was meant. - defp param({:null, _meta, []}) do - raise Unsupported, "a comparison with NULL is never true; use IS NULL or IS NOT NULL" - end - - defp param({:paren, _meta, [{:quote, _, part}, {:comma, _, [{:*, _, []}]}]}) do - {:erlang.list_to_binary(part), :*} - end + defp value({:tuple, [{:literal, part}, {:literal, n}]}) when is_binary(part) and is_integer(n), + do: raise(Unsupported, "a versionstamp is a whole number from 0 to 2^96 - 1, got #{n}") - defp param({:paren, _meta, [{:quote, _, part}, {:comma, _, [{tag, _, n}]}]}) - when tag in ~w[integer numeric]a do - {:erlang.list_to_binary(part), - EctoFoundationDB.Versionstamp.from_integer(:erlang.list_to_integer(n))} - end - - # Non-reserved words lex as identifiers, reserved ones (e.g. `date`) as - # their own token. - defp type_name({tag, _meta, name}) when tag in ~w[ident double_quote]a do - name |> List.to_string() |> String.downcase() - end + defp value({:tuple, _}), + do: raise(Unsupported, "a tuple must be ('partition', *) or ('partition', versionstamp)") - defp type_name({tag, _meta, []}) when is_atom(tag), do: Atom.to_string(tag) + defp value({:column, _} = column), + do: raise(Unsupported, "expected a value, got #{describe(column)}") - defp sql_op(:=), do: :== - defp sql_op(op), do: op + defp value(expr), do: raise(Unsupported, "expected a value, got #{describe(expr)}") - defp charlist_to_atom(charlist) when is_list(charlist), do: :erlang.list_to_atom(charlist) - defp charlist_to_atom(atom) when is_atom(atom), do: atom + defp describe({:column, name}), do: "the field #{name}" + defp describe({:literal, value}), do: inspect(value) + defp describe({:cast, _, type}), do: "a #{type} value" + defp describe({kind, _, _}) when kind in [:and, :or], do: "a condition" + defp describe(_), do: "a condition" end diff --git a/lib/efsql/sql/ast.ex b/lib/efsql/sql/ast.ex new file mode 100644 index 0000000..f17c784 --- /dev/null +++ b/lib/efsql/sql/ast.ex @@ -0,0 +1,58 @@ +defmodule Efsql.SQL.AST do + @moduledoc """ + The syntax tree `Efsql.SQL.Parser` produces. It records what the SQL text + says, not what efsql can execute: `OR`, `NOT IN` and the like parse fine + and are rejected later, in `Efsql.Parser`, with a message about the + feature rather than the syntax. + + Names are strings. Unquoted names are folded to lower case; quoted names + keep their case. + """ + + defmodule Select do + @moduledoc "A `SELECT` statement." + + @type t :: %__MODULE__{ + fields: :star | [String.t()], + from: [String.t(), ...], + where: Efsql.SQL.AST.expr() | nil, + order_by: [{String.t(), :asc | :desc}], + limit: non_neg_integer() | nil + } + + # `from` holds the dotted name's parts: [table], [tenant, table] or + # [storage_id, tenant, table]. + defstruct fields: :star, from: [], where: nil, order_by: [], limit: nil + end + + @typedoc """ + An expression. + + * `{:column, name}` + * `{:literal, value}` — a string, integer, float, boolean, or `nil` + for `NULL` + * `{:cast, expr, type}` — `expr::type` or `CAST(expr AS type)`; `type` + is a lower-case name, with `timestamp with time zone` read as + `timestamptz` + * `{:tuple, [expr | :star]}` — `(a, b, ...)`, whose elements may be `*` + * `{:and, left, right}`, `{:or, left, right}`, `{:not, expr}` + * `{:compare, op, left, right}` — `op` is `:=`, `:<>`, `:<`, `:>`, + `:<=` or `:>=`; `!=` reads as `:<>` + * `{:between, expr, low, high, negated?}` + * `{:in, expr, [expr], negated?}` + * `{:like, expr, pattern, negated?}`, `{:ilike, expr, pattern, negated?}` + * `{:is_null, expr, negated?}` — also `ISNULL` and `NOTNULL` + """ + @type expr :: + {:column, String.t()} + | {:literal, String.t() | number() | boolean() | nil} + | {:cast, expr(), String.t()} + | {:tuple, [expr() | :star]} + | {:and | :or, expr(), expr()} + | {:not, expr()} + | {:compare, := | :<> | :< | :> | :<= | :>=, expr(), expr()} + | {:between, expr(), expr(), expr(), boolean()} + | {:in, expr(), [expr()], boolean()} + | {:like | :ilike, expr(), expr(), boolean()} + | {:is_null, expr(), boolean()} +end diff --git a/lib/efsql/sql/lexer.ex b/lib/efsql/sql/lexer.ex new file mode 100644 index 0000000..96a3ada --- /dev/null +++ b/lib/efsql/sql/lexer.ex @@ -0,0 +1,289 @@ +defmodule Efsql.SQL.Lexer do + @moduledoc """ + Turns SQL text into a list of tokens, ending with `{:eof, nil, position}`. + + A token is `{type, value, {line, column}}`, positioned at its first + character: + + * `:word` — an unquoted identifier or keyword, folded to lower case as + PostgreSQL does. Whether it is a keyword is up to the parser, so + `day` or `user` can name a column. + * `:quoted` — a `"double-quoted"` identifier, case kept; `""` is a + literal quote. + * `:string` — a `'single-quoted'` string; `''` is a literal quote. + * `:integer` / `:float` — unsigned numbers: `42`, `1.5`, `.5`, `1e-3`. + * `:op` — punctuation and operators, as atoms: `:=`, `:<>` (also + written `!=`), `:<`, `:>`, `:<=`, `:>=`, `:"::"`, `:"("`, `:")"`, + `:","`, `:.`, `:";"`, `:*`, `:+`, `:-`, `:/`, `:%`. + + Whitespace and comments (`-- to end of line`, and `/* ... */`, which + nest) are dropped. + """ + + alias Efsql.SQL.SyntaxError + + @type position :: {pos_integer(), pos_integer()} + @type token :: + {:word | :quoted | :string, String.t(), position()} + | {:integer, non_neg_integer(), position()} + | {:float, float(), position()} + | {:op, atom(), position()} + | {:eof, nil, position()} + + @spec tokenize(String.t()) :: {:ok, [token()]} | {:error, SyntaxError.t()} + def tokenize(sql) when is_binary(sql) do + {:ok, lex(sql, 1, 1, [])} + catch + {:syntax_error, reason, {line, column}} -> + {:error, %SyntaxError{reason: reason, line: line, column: column}} + end + + @two_char_ops %{ + "<>" => :<>, + "!=" => :<>, + "<=" => :<=, + ">=" => :>=, + "::" => :"::" + } + + @one_char_ops %{ + ?= => :=, + ?< => :<, + ?> => :>, + ?( => :"(", + ?) => :")", + ?, => :",", + ?. => :., + ?; => :";", + ?* => :*, + ?+ => :+, + ?- => :-, + ?/ => :/, + ?% => :% + } + + defp lex(<<>>, line, col, acc), do: Enum.reverse([{:eof, nil, {line, col}} | acc]) + + # newlines: \r\n, \n and a lone \r each end a line + defp lex(<>, line, _col, acc), do: lex(rest, line + 1, 1, acc) + + defp lex(<>, line, _col, acc) when c in [?\n, ?\r], + do: lex(rest, line + 1, 1, acc) + + defp lex(<>, line, col, acc) when c in [?\s, ?\t, ?\f, ?\v], + do: lex(rest, line, col + 1, acc) + + defp lex(<<"--", rest::binary>>, line, _col, acc), do: line_comment(rest, line, acc) + + defp lex(<<"/*", rest::binary>>, line, col, acc), + do: block_comment(rest, line, col + 2, 1, {line, col}, acc) + + defp lex(<>, line, col, acc) do + {value, rest, line2, col2} = quoted(rest, ?', line, col + 1, [], {line, col}, "string") + lex(rest, line2, col2, [{:string, value, {line, col}} | acc]) + end + + defp lex(<>, line, col, acc) do + {value, rest, line2, col2} = quoted(rest, ?", line, col + 1, [], {line, col}, "identifier") + + if value == "" do + throw({:syntax_error, "zero-length quoted identifier", {line, col}}) + end + + lex(rest, line2, col2, [{:quoted, value, {line, col}} | acc]) + end + + defp lex(<> = sql, line, col, acc) when c in ?0..?9, + do: number(sql, line, col, acc) + + defp lex(<> = sql, line, col, acc) when c in ?0..?9, + do: number(sql, line, col, acc) + + defp lex(<>, line, col, acc) + when is_map_key(@two_char_ops, <>) do + lex(rest, line, col + 2, [{:op, Map.fetch!(@two_char_ops, <>), {line, col}} | acc]) + end + + defp lex(<>, line, col, _acc) do + if word_start?(c) do + throw( + {:syntax_error, "unexpected ':' (write an atom as a cast, e.g. 'name'::atom)", + {line, col}} + ) + else + throw({:syntax_error, "unexpected ':'", {line, col}}) + end + end + + defp lex(<>, line, col, acc) when is_map_key(@one_char_ops, c) do + lex(rest, line, col + 1, [{:op, Map.fetch!(@one_char_ops, c), {line, col}} | acc]) + end + + defp lex(<> = sql, line, col, acc) do + cond do + word_start?(c) -> + {raw, rest, length} = take_word(sql, [], 0) + lex(rest, line, col + length, [{:word, String.downcase(raw), {line, col}} | acc]) + + unicode_space?(c) -> + lex(rest, line, col + 1, acc) + + true -> + throw({:syntax_error, "unexpected character #{inspect(<>)}", {line, col}}) + end + end + + defp lex(<>, line, col, _acc) do + throw({:syntax_error, "invalid UTF-8 byte 0x#{Integer.to_string(byte, 16)}", {line, col}}) + end + + # -- comments -- + + defp line_comment(<<>>, line, acc), do: lex(<<>>, line, 1, acc) + defp line_comment(<>, line, acc), do: lex(rest, line + 1, 1, acc) + + defp line_comment(<>, line, acc) when c in [?\n, ?\r], + do: lex(rest, line + 1, 1, acc) + + defp line_comment(<<_, rest::binary>>, line, acc), do: line_comment(rest, line, acc) + + defp block_comment(<<>>, _line, _col, _depth, start, _acc), + do: throw({:syntax_error, "unterminated /* comment", start}) + + defp block_comment(<<"*/", rest::binary>>, line, col, 1, _start, acc), + do: lex(rest, line, col + 2, acc) + + defp block_comment(<<"*/", rest::binary>>, line, col, depth, start, acc), + do: block_comment(rest, line, col + 2, depth - 1, start, acc) + + defp block_comment(<<"/*", rest::binary>>, line, col, depth, start, acc), + do: block_comment(rest, line, col + 2, depth + 1, start, acc) + + defp block_comment(<>, line, _col, depth, start, acc), + do: block_comment(rest, line + 1, 1, depth, start, acc) + + defp block_comment(<>, line, _col, depth, start, acc) when c in [?\n, ?\r], + do: block_comment(rest, line + 1, 1, depth, start, acc) + + defp block_comment(<<_::utf8, rest::binary>>, line, col, depth, start, acc), + do: block_comment(rest, line, col + 1, depth, start, acc) + + defp block_comment(<<_, rest::binary>>, line, col, depth, start, acc), + do: block_comment(rest, line, col + 1, depth, start, acc) + + # -- quoted strings and identifiers -- + + # A doubled quote is a literal quote; anything else, newlines included, + # is taken verbatim. + defp quoted(<<>>, q, _line, _col, _acc, start, what) do + throw({:syntax_error, "unterminated #{what} (missing closing #{<>})", start}) + end + + defp quoted(<>, q, line, col, acc, start, what), + do: quoted(rest, q, line, col + 2, [q | acc], start, what) + + defp quoted(<>, q, line, col, acc, _start, _what), + do: {acc |> Enum.reverse() |> List.to_string(), rest, line, col + 1} + + defp quoted(<>, q, line, _col, acc, start, what), + do: quoted(rest, q, line + 1, 1, [?\n, ?\r | acc], start, what) + + defp quoted(<>, q, line, _col, acc, start, what) when c in [?\n, ?\r], + do: quoted(rest, q, line + 1, 1, [c | acc], start, what) + + defp quoted(<>, q, line, col, acc, start, what), + do: quoted(rest, q, line, col + 1, [c | acc], start, what) + + defp quoted(<>, _q, line, col, _acc, _start, _what) do + throw({:syntax_error, "invalid UTF-8 byte 0x#{Integer.to_string(byte, 16)}", {line, col}}) + end + + # -- numbers -- + + defp number(sql, line, col, acc) do + {int, rest} = digits(sql, []) + {frac, rest} = fraction(rest) + {exp, rest} = exponent(rest, line, col) + text = int <> frac <> exp + length = String.length(text) + + case rest do + <> -> + if word_start?(c) or c in ?0..?9 or c == ?. do + throw({:syntax_error, "invalid number #{inspect(text <> <>)}", {line, col}}) + end + + _ -> + :ok + end + + token = + if frac == "" and exp == "" do + {:integer, String.to_integer(int), {line, col}} + else + int = if int == "", do: "0", else: int + frac = if frac in ["", "."], do: ".0", else: frac + {:float, to_float(int <> frac <> exp, text, {line, col}), {line, col}} + end + + lex(rest, line, col + length, [token | acc]) + end + + # Beyond a double's range, e.g. 1e400. + defp to_float(normalized, text, pos) do + String.to_float(normalized) + rescue + ArgumentError -> throw({:syntax_error, "number out of range: #{text}", pos}) + end + + defp digits(<>, acc) when c in ?0..?9, do: digits(rest, [c | acc]) + defp digits(rest, acc), do: {acc |> Enum.reverse() |> List.to_string(), rest} + + defp fraction(<>) do + {frac, rest} = digits(rest, []) + {"." <> frac, rest} + end + + defp fraction(rest), do: {"", rest} + + defp exponent(<>, line, col) when e in [?e, ?E] and sign in [?+, ?-] do + exponent_digits(<>, rest, line, col) + end + + defp exponent(<>, line, col) when e in [?e, ?E] do + exponent_digits(<>, rest, line, col) + end + + defp exponent(rest, _line, _col), do: {"", rest} + + defp exponent_digits(prefix, rest, line, col) do + case digits(rest, []) do + {"", _} -> throw({:syntax_error, "invalid number: exponent has no digits", {line, col}}) + {exp, rest} -> {prefix <> exp, rest} + end + end + + # -- words -- + + defp take_word(<> = sql, acc, length) do + if word_continue?(c) do + take_word(rest, [c | acc], length + 1) + else + {acc |> Enum.reverse() |> List.to_string(), sql, length} + end + end + + defp take_word(rest, acc, length), do: {acc |> Enum.reverse() |> List.to_string(), rest, length} + + defp word_start?(c) when c in ?a..?z or c in ?A..?Z or c == ?_, do: true + defp word_start?(c) when c < 128, do: false + defp word_start?(c), do: String.match?(<>, ~r/^\p{L}$/u) + + defp word_continue?(c) when c in ?0..?9 or c == ?$, do: true + defp word_continue?(c) when c < 128, do: word_start?(c) + defp word_continue?(c), do: String.match?(<>, ~r/^[\p{L}\p{M}\p{N}]$/u) + + # NBSP, the Unicode spaces, and a byte-order mark + defp unicode_space?(c), + do: c in [0xA0, 0x1680, 0x202F, 0x205F, 0x3000, 0xFEFF] or c in 0x2000..0x200A +end diff --git a/lib/efsql/sql/parser.ex b/lib/efsql/sql/parser.ex new file mode 100644 index 0000000..be2e189 --- /dev/null +++ b/lib/efsql/sql/parser.ex @@ -0,0 +1,516 @@ +defmodule Efsql.SQL.Parser do + @moduledoc """ + Parses one `SELECT` statement into an `Efsql.SQL.AST.Select`. + + SELECT fields FROM name [WHERE expr] [ORDER BY items] [LIMIT n] [;] + + A recursive-descent parser over `Efsql.SQL.Lexer` tokens. Keywords are + recognised by position, so a word is only a keyword where the grammar + expects one. The reserved words below can't be bare names anywhere, + because a name could be mistaken for them; quote them to use them as + names (`"order"`). Everything else (`day`, `user`, `date`, `value`, ...) + is an ordinary name. + + Precedence, loosest first: `OR`, `AND`, `NOT`, then a predicate + (comparison, `BETWEEN`, `IN`, `LIKE`, `IS NULL`), then `::` casts. + """ + + alias Efsql.SQL.AST + alias Efsql.SQL.Lexer + alias Efsql.SQL.SyntaxError + + @reserved ~w[all and as asc between by case cast cross desc distinct else end except + false from full group having ilike in inner intersect is isnull join + left like limit not notnull null offset on or order right select then + true union when where with] + + @not_select ~w[insert update delete create drop alter truncate grant revoke explain] + @unsupported_clauses %{ + "group" => "GROUP BY", + "having" => "HAVING", + "offset" => "OFFSET", + "join" => "JOIN", + "inner" => "JOIN", + "left" => "JOIN", + "right" => "JOIN", + "full" => "JOIN", + "cross" => "JOIN", + "union" => "UNION", + "intersect" => "INTERSECT", + "except" => "EXCEPT", + "fetch" => "FETCH" + } + + @comparison_ops [:=, :<>, :<, :>, :<=, :>=] + @arithmetic_ops [:+, :-, :*, :/, :%] + @max_depth 200 + + def reserved_words, do: @reserved + + @spec parse(String.t()) :: {:ok, AST.Select.t()} | {:error, SyntaxError.t()} + def parse(sql) when is_binary(sql) do + with {:ok, tokens} <- Lexer.tokenize(sql) do + {:ok, statement(tokens)} + end + catch + {:syntax_error, reason, {line, column}} -> + {:error, %SyntaxError{reason: reason, line: line, column: column}} + end + + @spec parse!(String.t()) :: AST.Select.t() + def parse!(sql) do + case parse(sql) do + {:ok, select} -> select + {:error, error} -> raise error + end + end + + # -- statement -- + + defp statement([{:eof, _, pos}]), do: error("empty statement", pos) + + defp statement(tokens) do + {select, rest} = select(tokens) + + case rest do + [{:eof, _, _}] -> select + [{:op, :";", _}, {:eof, _, _}] -> select + [{:op, :";", _}, {_, _, pos} | _] -> error("only one statement is supported", pos) + [token | _] -> end_of_statement_error(token) + end + end + + defp end_of_statement_error({:word, word, pos}) when is_map_key(@unsupported_clauses, word) do + error("#{Map.fetch!(@unsupported_clauses, word)} is not supported", pos) + end + + defp end_of_statement_error({:op, op, pos}) when op in @arithmetic_ops do + error("arithmetic is not supported", pos) + end + + defp end_of_statement_error({_, _, pos} = token) do + error("expected end of statement, got #{describe(token)}", pos) + end + + defp select([{:word, "select", _} | rest]) do + {fields, rest} = select_list(rest) + rest = expect_keyword(rest, "from", "FROM") + {from, rest} = table_name(rest) + + case rest do + [{:op, :",", pos} | _] -> error("selecting from more than one table is not supported", pos) + _ -> :ok + end + + {where, rest} = where(rest) + {order_by, rest} = order_by(rest) + {limit, rest} = limit(rest) + + {%AST.Select{fields: fields, from: from, where: where, order_by: order_by, limit: limit}, + rest} + end + + defp select([{:word, word, pos} | _]) when word in @not_select, + do: error("only SELECT statements are supported", pos) + + defp select([{_, _, pos} = token | _]), + do: error("expected SELECT, got #{describe(token)}", pos) + + # -- SELECT list -- + + defp select_list([{:op, :*, _} | rest]) do + case rest do + [{:op, :",", pos} | _] -> error("* can't be combined with other fields", pos) + _ -> {:star, rest} + end + end + + defp select_list([{:word, "distinct", pos} | _]), do: error("DISTINCT is not supported", pos) + + defp select_list(tokens) do + {name, rest} = name(tokens, "a field name or *") + rest = reject_qualified_or_call(rest) + + case rest do + [{:op, :",", _} | rest] -> + {names, rest} = select_list_rest(rest) + {[name | names], rest} + + rest -> + {[name], rest} + end + end + + defp select_list_rest(tokens) do + {name, rest} = name(tokens, "a field name") + rest = reject_qualified_or_call(rest) + + case rest do + [{:op, :",", _} | rest] -> + {names, rest} = select_list_rest(rest) + {[name | names], rest} + + rest -> + {[name], rest} + end + end + + # -- FROM -- + + defp table_name(tokens) do + {first, rest} = name(tokens, "a table name") + table_name_parts(rest, [first]) + end + + defp table_name_parts([{:op, :., pos} | rest], parts) do + if length(parts) == 3 do + error("a table name has at most three parts (storage_id.tenant.table)", pos) + end + + {part, rest} = name(rest, "a name after '.'") + table_name_parts(rest, parts ++ [part]) + end + + defp table_name_parts(rest, parts), do: {parts, rest} + + # -- WHERE -- + + defp where([{:word, "where", _} | rest]) do + {expr, rest} = expr(rest, 0) + {expr, rest} + end + + defp where(rest), do: {nil, rest} + + # -- ORDER BY -- + + defp order_by([{:word, "order", _} | rest]) do + rest = expect_keyword(rest, "by", "BY after ORDER") + order_items(rest) + end + + defp order_by(rest), do: {[], rest} + + defp order_items(tokens) do + {name, rest} = name(tokens, "a field name to order by") + rest = reject_qualified_or_call(rest) + + {dir, rest} = + case rest do + [{:word, "asc", _} | rest] -> {:asc, rest} + [{:word, "desc", _} | rest] -> {:desc, rest} + rest -> {:asc, rest} + end + + case rest do + [{:word, "nulls", pos} | _] -> + error("NULLS FIRST/LAST is not supported", pos) + + [{:op, :",", _} | rest] -> + {items, rest} = order_items(rest) + {[{name, dir} | items], rest} + + rest -> + {[{name, dir}], rest} + end + end + + # -- LIMIT -- + + defp limit([{:word, "limit", _} | rest]) do + case rest do + [{:integer, n, _} | rest] -> + {n, rest} + + [{_, _, pos} = token | _] -> + error("LIMIT expects a whole number, got #{describe(token)}", pos) + end + end + + defp limit(rest), do: {nil, rest} + + # -- expressions -- + + defp expr(tokens, depth), do: or_expr(tokens, depth) + + defp or_expr(tokens, depth) do + {left, rest} = and_expr(tokens, depth) + or_rest(left, rest, depth) + end + + defp or_rest(left, [{:word, "or", _} | rest], depth) do + {right, rest} = and_expr(rest, depth) + or_rest({:or, left, right}, rest, depth) + end + + defp or_rest(left, rest, _depth), do: {left, rest} + + defp and_expr(tokens, depth) do + {left, rest} = not_expr(tokens, depth) + and_rest(left, rest, depth) + end + + defp and_rest(left, [{:word, "and", _} | rest], depth) do + {right, rest} = not_expr(rest, depth) + and_rest({:and, left, right}, rest, depth) + end + + defp and_rest(left, rest, _depth), do: {left, rest} + + defp not_expr([{:word, "not", pos} | rest], depth) do + check_depth(depth, pos) + {expr, rest} = not_expr(rest, depth + 1) + {{:not, expr}, rest} + end + + defp not_expr(tokens, depth), do: predicate(tokens, depth) + + defp predicate(tokens, depth) do + {left, rest} = operand(tokens, depth) + + case rest do + [{:op, op, _} | rest] when op in @comparison_ops -> + {right, rest} = operand(rest, depth) + {{:compare, op, left, right}, rest} + + [{:op, op, pos} | _] when op in @arithmetic_ops -> + error("arithmetic is not supported", pos) + + [{:word, "not", _}, {:word, word, _} = next | rest] + when word in ~w[between in like ilike] -> + negatable(left, next, rest, true, depth) + + [{:word, word, _} = next | rest] when word in ~w[between in like ilike] -> + negatable(left, next, rest, false, depth) + + [{:word, "is", _} | rest] -> + is_null(left, rest) + + [{:word, "isnull", _} | rest] -> + {{:is_null, left, false}, rest} + + [{:word, "notnull", _} | rest] -> + {{:is_null, left, true}, rest} + + rest -> + {left, rest} + end + end + + defp negatable(left, {:word, "between", _}, rest, negated?, depth) do + {low, rest} = operand(rest, depth) + rest = expect_keyword(rest, "and", "AND in BETWEEN") + {high, rest} = operand(rest, depth) + {{:between, left, low, high, negated?}, rest} + end + + defp negatable(left, {:word, "in", _}, rest, negated?, depth) do + case rest do + [{:op, :"(", pos}, {:op, :")", _} | _] -> + error("IN needs at least one value", pos) + + [{:op, :"(", pos} | rest] -> + check_depth(depth, pos) + {values, rest} = expr_list(rest, depth + 1) + rest = expect_op(rest, :")", "')' to close the IN list") + {{:in, left, values, negated?}, rest} + + [{_, _, pos} = token | _] -> + error("expected '(' after IN, got #{describe(token)}", pos) + end + end + + defp negatable(left, {:word, like, _}, rest, negated?, depth) when like in ~w[like ilike] do + {pattern, rest} = operand(rest, depth) + {{String.to_atom(like), left, pattern, negated?}, rest} + end + + defp is_null(left, rest) do + {negated?, rest} = + case rest do + [{:word, "not", _} | rest] -> {true, rest} + rest -> {false, rest} + end + + case rest do + [{:word, "null", _} | rest] -> + {{:is_null, left, negated?}, rest} + + [{_, _, pos} = token | _] -> + error("expected NULL after IS#{if negated?, do: " NOT"}, got #{describe(token)}", pos) + end + end + + defp expr_list(tokens, depth) do + {expr, rest} = expr(tokens, depth) + + case rest do + [{:op, :",", _} | rest] -> + {exprs, rest} = expr_list(rest, depth) + {[expr | exprs], rest} + + rest -> + {[expr], rest} + end + end + + # An operand is a primary followed by any number of `::type` casts. + defp operand(tokens, depth) do + {primary, rest} = primary(tokens, depth) + casts(primary, rest) + end + + defp casts(expr, [{:op, :"::", _} | rest]) do + {type, rest} = type_name(rest) + casts({:cast, expr, type}, rest) + end + + defp casts(expr, rest), do: {expr, rest} + + defp primary([{:string, s, _} | rest], _depth), do: {{:literal, s}, rest} + defp primary([{:integer, n, _} | rest], _depth), do: {{:literal, n}, rest} + defp primary([{:float, f, _} | rest], _depth), do: {{:literal, f}, rest} + + defp primary([{:op, :-, _}, {type, n, _} | rest], _depth) when type in [:integer, :float], + do: {{:literal, -n}, rest} + + defp primary([{:op, :+, _}, {type, n, _} | rest], _depth) when type in [:integer, :float], + do: {{:literal, n}, rest} + + defp primary([{:op, sign, pos} | _], _depth) when sign in [:-, :+], + do: error("a sign is only supported on a number", pos) + + defp primary([{:word, "true", _} | rest], _depth), do: {{:literal, true}, rest} + defp primary([{:word, "false", _} | rest], _depth), do: {{:literal, false}, rest} + defp primary([{:word, "null", _} | rest], _depth), do: {{:literal, nil}, rest} + + defp primary([{:word, "select", pos} | _], _depth), + do: error("subqueries are not supported", pos) + + defp primary([{:word, "cast", _}, {:op, :"(", pos} | rest], depth) do + check_depth(depth, pos) + {expr, rest} = expr(rest, depth + 1) + rest = expect_keyword(rest, "as", "AS in CAST") + {type, rest} = type_name(rest) + rest = expect_op(rest, :")", "')' to close CAST") + {{:cast, expr, type}, rest} + end + + defp primary([{:op, :"(", pos} | rest], depth) do + check_depth(depth, pos) + {first, rest} = paren_element(rest, depth + 1) + + case rest do + [{:op, :",", _} | rest] -> + {elements, rest} = tuple_elements(rest, depth + 1) + rest = expect_op(rest, :")", "')' to close the tuple") + {{:tuple, [first | elements]}, rest} + + rest -> + if first == :star, do: error("'*' is only allowed in a tuple", pos) + rest = expect_op(rest, :")", "')'") + {first, rest} + end + end + + defp primary([{:word, word, pos} | _] = tokens, _depth) when word not in @reserved do + {name, rest} = name(tokens, "an expression") + + case rest do + [{:op, :"(", _} | _] -> error("functions are not supported", pos) + [{:op, :., dot} | _] -> error("qualified column names are not supported", dot) + rest -> {{:column, name}, rest} + end + end + + defp primary([{:quoted, _, _} | _] = tokens, _depth) do + {name, rest} = name(tokens, "an expression") + {{:column, name}, reject_qualified_or_call(rest)} + end + + defp primary([{_, _, pos} = token | _], _depth), + do: error("expected an expression, got #{describe(token)}", pos) + + defp paren_element([{:op, :*, _} | rest], _depth), do: {:star, rest} + defp paren_element(tokens, depth), do: expr(tokens, depth) + + defp tuple_elements(tokens, depth) do + {element, rest} = paren_element(tokens, depth) + + case rest do + [{:op, :",", _} | rest] -> + {elements, rest} = tuple_elements(rest, depth) + {[element | elements], rest} + + rest -> + {[element], rest} + end + end + + # A type is any word (reserved ones like date and time included), with + # PostgreSQL's `timestamp with[out] time zone` spelled out. + defp type_name([{:word, type, _} | rest]) when type in ~w[timestamp time] do + case rest do + [{:word, "with", _}, {:word, "time", _}, {:word, "zone", _} | rest] -> {type <> "tz", rest} + [{:word, "without", _}, {:word, "time", _}, {:word, "zone", _} | rest] -> {type, rest} + rest -> {type, rest} + end + end + + defp type_name([{:word, type, _} | rest]), do: {type, rest} + + defp type_name([{_, _, pos} = token | _]), + do: error("expected a type name, got #{describe(token)}", pos) + + # -- names -- + + defp name([{:word, word, pos} | _], what) when word in @reserved do + error( + "expected #{what}, got the keyword '#{word}' (quote it, \"#{word}\", to use it as a name)", + pos + ) + end + + defp name([{:word, word, _} | rest], _what), do: {word, rest} + defp name([{:quoted, name, _} | rest], _what), do: {name, rest} + + defp name([{_, _, pos} = token | _], what), + do: error("expected #{what}, got #{describe(token)}", pos) + + defp reject_qualified_or_call([{:op, :"(", pos} | _]), + do: error("functions are not supported", pos) + + defp reject_qualified_or_call([{:op, :., pos} | _]), + do: error("qualified column names are not supported", pos) + + defp reject_qualified_or_call(rest), do: rest + + # -- helpers -- + + defp expect_keyword([{:word, word, _} | rest], word, _what), do: rest + + defp expect_keyword([{_, _, pos} = token | _], _word, what), + do: error("expected #{what}, got #{describe(token)}", pos) + + defp expect_op([{:op, op, _} | rest], op, _what), do: rest + + defp expect_op([{_, _, pos} = token | _], _op, what), + do: error("expected #{what}, got #{describe(token)}", pos) + + defp check_depth(depth, pos) do + if depth >= @max_depth, do: error("expression is nested too deeply", pos) + end + + defp describe({:word, word, _}), do: "'#{word}'" + defp describe({:quoted, name, _}), do: inspect(name) + defp describe({:string, s, _}), do: "the string '#{truncate(s)}'" + defp describe({type, n, _}) when type in [:integer, :float], do: "the number #{n}" + defp describe({:op, op, _}), do: "'#{op}'" + defp describe({:eof, _, _}), do: "the end of the statement" + + defp truncate(s) do + if String.length(s) > 20, do: String.slice(s, 0, 20) <> "...", else: s + end + + defp error(reason, pos), do: throw({:syntax_error, reason, pos}) +end diff --git a/lib/efsql/sql/syntax_error.ex b/lib/efsql/sql/syntax_error.ex new file mode 100644 index 0000000..53a8777 --- /dev/null +++ b/lib/efsql/sql/syntax_error.ex @@ -0,0 +1,13 @@ +defmodule Efsql.SQL.SyntaxError do + @moduledoc """ + Raised (or returned) when SQL text can't be lexed or parsed. `line` and + `column` are 1-based and count characters, not bytes. + """ + + defexception [:reason, :line, :column] + + @impl true + def message(%__MODULE__{reason: reason, line: line, column: column}) do + "syntax error at line #{line}, column #{column}: #{reason}" + end +end diff --git a/mix.exs b/mix.exs index c163be2..c58219a 100644 --- a/mix.exs +++ b/mix.exs @@ -31,12 +31,7 @@ defmodule Efsql.MixProject do def application do [ mod: {Efsql.Application, []}, - extra_applications: [:logger], - # efsql uses only sql's lexer and parser, which need no running - # process. Including the app loads its code without starting its - # database client runtime (a :dets file in its priv dir, a startup - # sleep, and a stop/1 that crashes when no pool ever started). - included_applications: [:sql] + extra_applications: [:logger] ] end @@ -47,7 +42,6 @@ defmodule Efsql.MixProject do # advance them together. {:ecto, "~> 3.13.0"}, {:ex_fdbmonitor, github: "foundationdb-beam/ex_fdbmonitor", only: :dev, runtime: false}, - {:sql, github: "elixir-dbvisor/sql"}, {:owl, "~> 0.13"} ] end diff --git a/mix.lock b/mix.lock index 6af86d3..5471730 100644 --- a/mix.lock +++ b/mix.lock @@ -9,10 +9,6 @@ "global_flags": {:hex, :global_flags, "1.0.0", "ee6b864979a1fb38d1fbc67838565644baf632212bce864adca21042df036433", [:rebar3], [], "hexpm", "85d944cecd0f8f96b20ce70b5b16ebccedfcd25e744376b131e89ce61ba93176"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "local_cluster": {:hex, :local_cluster, "2.1.0", "1c847d69a927ef5a62db13236f93146e8a42377a9c9a5bb4cac3372cba69d683", [:mix], [{:global_flags, "~> 1.0", [hex: :global_flags, repo: "hexpm", optional: false]}], "hexpm", "dc1c3abb6fef00198dd53c855b39ea80c55b3a8059d8d9f17d50da46b1e3b858"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "owl": {:hex, :owl, "0.13.1", "1ec4a5dea170465f0e90c502c203079224516bc0cbd599281c8667b3c6ef8848", [:mix], [{:ucwidth, "~> 0.2", [hex: :ucwidth, repo: "hexpm", optional: true]}], "hexpm", "351e768af8f2edc575cdaab1a5a2f6d6381be591758a026c701c703145508a0c"}, - "sql": {:git, "https://github.com/elixir-dbvisor/sql.git", "f6097fff5b08efe9cd946a01cc2152c82d871cf7", []}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, - "unicode": {:hex, :unicode, "2.0.0", "d2d3ec27575fed3437fdcc0d383909378201843d27d2197c0c2742c973110b24", [:mix], [], "hexpm", "a8da44be6a2fc165e58ea332038686df79994b51bf2c7fe337bf2927ea148c13"}, - "unicode_set": {:hex, :unicode_set, "1.7.0", "38c91095b35b90cf2e8f8aaa67d298937886293c5e0288953325014503888cd2", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode, "~> 2.0", [hex: :unicode, repo: "hexpm", optional: false]}], "hexpm", "66c8c02adadac58fcaf64e22eba90c243a6528d1d36fb014e2d8c0e062efbae6"}, } diff --git a/test/by_pk_test.exs b/test/by_pk_test.exs index ce675d3..0c45118 100644 --- a/test/by_pk_test.exs +++ b/test/by_pk_test.exs @@ -1,11 +1,8 @@ defmodule EfsqlTest.Integration.SelectByPk do use EfsqlTest.Case, async: true - # Querying _ via - # 1. _ identifier - # 2. '_' keywords + # The primary key is queried through the `_` placeholder. - # https://github.com/elixir-dbvisor/sql/issues/20 test "select by pk equals", context do tenant_id = context[:tenant_id] diff --git a/test/parser_test.exs b/test/parser_test.exs index f93f9c9..1631015 100644 --- a/test/parser_test.exs +++ b/test/parser_test.exs @@ -249,7 +249,7 @@ defmodule Efsql.ParserTest do parse("select id from t.users where notes notnull;") end - # The SQL library hangs on these without Efsql.Parser's token rewrite. + # Regression: the previous parser library never returned on some of these. test "null tests combine with other predicates in any position" do assert %Logical.Select{predicates: [{:cmp, :==, :name, "a"}, {:not_null, :notes}]} = parse("select id from t.users where name = 'a' and notes is not null;") @@ -281,8 +281,7 @@ defmodule Efsql.ParserTest do end end - # The SQL library hangs or misparses on these without the token - # rewrite in Efsql.Parser.parse/1. + # Regression: the previous parser library hung or misparsed these. test "columns named after reserved words" do assert %Logical.Select{predicates: [{:cmp, :==, :day, "x"}]} = parse("select id from t.users where day = 'x';") @@ -338,4 +337,83 @@ defmodule Efsql.ParserTest do parse("select id from t.users order by name limit 2;") end end + + describe "translation" do + test "parentheses around ANDed conditions flatten" do + assert %Logical.Select{ + predicates: [{:cmp, :==, :a, 1}, {:cmp, :==, :b, 2}, {:cmp, :==, :c, 3}] + } = + parse("select id from t.users where (a = 1 and (b = 2)) and c = 3;") + end + + test "typed literals as BETWEEN bounds" do + assert %Logical.Select{ + predicates: [ + {:range, :at, {:>=, ~N[2024-01-01 00:00:00]}, {:<=, ~N[2025-01-01 00:00:00]}} + ] + } = + parse( + "select id from t.users where at between '2024-01-01'::timestamp and '2025-01-01'::timestamp;" + ) + end + + test "a value on the left is flipped" do + assert %Logical.Select{predicates: [{:cmp, :>, :age, 40}, {:cmp, :==, :name, "x"}]} = + parse("select id from t.users where 40 < age and 'x' = name;") + end + + test "unsupported conditions say what is unsupported" do + for {condition, message} <- [ + {"a = 1 or b = 2", "OR is not supported"}, + {"not a = 1", "NOT is not supported"}, + {"a not in (1)", "NOT IN is not supported"}, + {"a not between 1 and 2", "NOT BETWEEN is not supported"}, + {"a ilike 'x'", "ILIKE is not supported"}, + {"a <> 1", "<> and != are not supported"}, + {"a != 1", "<> and != are not supported"}, + {"a = b", "comparing two fields"}, + {"1 = 1", "needs a field on one side"}, + {"'x' in ('x')", "IN needs a field on its left"}, + {"a", "the field a is not a condition"}, + {"a = (1, 2)", "a tuple must be"}, + {"_ = ('p', -1)", "versionstamp is a whole number from 0 to 2^96 - 1"}, + {"_ = ('p', #{Bitwise.bsl(1, 96)})", "versionstamp is a whole number"} + ] do + assert_raise Unsupported, ~r/#{Regex.escape(message)}/, fn -> + parse("select id from t.users where #{condition};") + end + end + end + + test "field names longer than an atom allows" do + long = String.duplicate("x", 256) + + assert_raise Unsupported, ~r/limited to 255 characters/, fn -> + parse(~s(select "#{long}" from t.users;)) + end + + assert %Logical.Select{projection: [_]} = + parse(~s(select "#{String.duplicate("x", 255)}" from t.users;)) + end + + test "syntax errors are SyntaxErrors" do + assert_raise Efsql.SQL.SyntaxError, ~r/line 1, column 27: expected an expression/, fn -> + parse("select id from t where a =;") + end + end + + test "every example on the help page parses" do + examples = + for line <- Efsql.Tui.Help.lines(), + {:accent, " " <> text} <- line, + String.starts_with?(text, "select "), + do: text + + assert length(examples) >= 9 + + for sql <- examples do + assert %Logical.Select{} = parse(sql) + end + end + end end diff --git a/test/sql/lexer_test.exs b/test/sql/lexer_test.exs new file mode 100644 index 0000000..2dddf23 --- /dev/null +++ b/test/sql/lexer_test.exs @@ -0,0 +1,244 @@ +defmodule Efsql.SQL.LexerTest do + use ExUnit.Case, async: true + + alias Efsql.SQL.Lexer + alias Efsql.SQL.SyntaxError + + defp tokens(sql) do + {:ok, tokens} = Lexer.tokenize(sql) + tokens |> Enum.map(fn {type, value, _pos} -> {type, value} end) |> Enum.drop(-1) + end + + defp positions(sql) do + {:ok, tokens} = Lexer.tokenize(sql) + Enum.map(tokens, fn {_type, _value, pos} -> pos end) + end + + defp error(sql) do + assert {:error, %SyntaxError{} = e} = Lexer.tokenize(sql) + e + end + + test "empty and blank input is just end of input" do + assert {:ok, [{:eof, nil, {1, 1}}]} = Lexer.tokenize("") + assert {:ok, [{:eof, nil, {2, 3}}]} = Lexer.tokenize(" \n ") + end + + describe "words" do + test "fold to lower case" do + assert tokens("SELECT Name fRoM") == [word: "select", word: "name", word: "from"] + end + + test "take underscores, digits and dollars after the first character" do + assert tokens("_ _id a1 a$b created_at") == + [word: "_", word: "_id", word: "a1", word: "a$b", word: "created_at"] + end + + test "take Unicode letters and marks" do + assert tokens("café 日本語 Ωmega") == [word: "café", word: "日本語", word: "ωmega"] + end + + test "keywords are plain words; the parser decides" do + assert tokens("day at user order") == + [word: "day", word: "at", word: "user", word: "order"] + end + end + + describe "quoted identifiers" do + test "keep case and spaces" do + assert tokens(~s("Mixed Case")) == [quoted: "Mixed Case"] + end + + test "a doubled quote is a quote" do + assert tokens(~s("say ""hi""")) == [quoted: ~s(say "hi")] + end + + test "may span lines and hold keywords" do + assert tokens(~s("select\nfrom")) == [quoted: "select\nfrom"] + end + + test "can't be empty" do + assert %SyntaxError{reason: "zero-length quoted identifier", line: 1, column: 8} = + error(~s(select "")) + end + + test "must be closed" do + assert %SyntaxError{line: 1, column: 8, reason: reason} = error(~s(select "abc)) + assert reason =~ "unterminated identifier" + end + end + + describe "strings" do + test "keep their content verbatim" do + assert tokens("'Hello, World'") == [string: "Hello, World"] + end + + test "a doubled quote is a quote" do + assert tokens("'it''s'") == [string: "it's"] + assert tokens("''''") == [string: "'"] + assert tokens("''") == [string: ""] + end + + test "hide comment and operator characters" do + assert tokens("'-- not a comment /* nor this */ ; ::'") == + [string: "-- not a comment /* nor this */ ; ::"] + end + + test "may span lines, and positions continue after them" do + assert tokens("'a\nb' x") == [string: "a\nb", word: "x"] + assert positions("'a\nb' x") == [{1, 1}, {2, 4}, {2, 5}] + end + + test "take any Unicode" do + assert tokens("'naïve ☃ 𝄞'") == [string: "naïve ☃ 𝄞"] + end + + test "must be closed" do + e = error("where x = 'abc") + assert e.reason =~ "unterminated string" + assert {e.line, e.column} == {1, 11} + end + end + + describe "numbers" do + test "integers" do + assert tokens("0 42 007 123456789012345678901234567890") == + [ + integer: 0, + integer: 42, + integer: 7, + integer: 123_456_789_012_345_678_901_234_567_890 + ] + end + + test "floats in every spelling" do + assert tokens("1.5 .5 1. 1e3 1E3 1.5e-3 2.5E+2") == + [ + float: 1.5, + float: 0.5, + float: 1.0, + float: 1.0e3, + float: 1.0e3, + float: 1.5e-3, + float: 250.0 + ] + end + + test "are unsigned; a sign is a separate token" do + assert tokens("-1 +2") == [op: :-, integer: 1, op: :+, integer: 2] + end + + test "can't run into a word" do + assert error("123abc").reason =~ "invalid number" + assert error("1.5.2").reason =~ "invalid number" + end + + test "out of a double's range" do + assert error("1e400").reason == "number out of range: 1e400" + assert tokens("1e-400") == [float: 0.0] + end + + test "need exponent digits" do + assert error("1e").reason =~ "exponent has no digits" + assert error("1e+x").reason =~ "exponent has no digits" + end + end + + describe "operators" do + test "two-character operators win over one-character ones" do + assert tokens("<> != <= >= ::") == [op: :<>, op: :<>, op: :<=, op: :>=, op: :"::"] + end + + test "one-character operators" do + assert tokens("= < > ( ) , . ; * + - / %") == + [ + op: :=, + op: :<, + op: :>, + op: :"(", + op: :")", + op: :",", + op: :., + op: :";", + op: :*, + op: :+, + op: :-, + op: :/, + op: :% + ] + end + + test "need no surrounding space" do + assert tokens("a>=1") == [word: "a", op: :>=, integer: 1] + assert tokens("'x'::atom") == [string: "x", op: :"::", word: "atom"] + assert tokens("t.users") == [word: "t", op: :., word: "users"] + end + end + + describe "comments and whitespace" do + test "line comments run to the end of the line" do + assert tokens("a -- ignored ; 'x'\nb") == [word: "a", word: "b"] + assert tokens("a --") == [word: "a"] + end + + test "block comments, which nest" do + assert tokens("a /* one /* two */ still */ b") == [word: "a", word: "b"] + end + + test "a block comment can split tokens" do + assert tokens("a/**/b") == [word: "a", word: "b"] + end + + test "an unclosed block comment is an error at its start" do + assert %SyntaxError{line: 2, column: 3} = error("a\n /* /* */") + end + + test "tabs, CRLF and Unicode spaces separate tokens" do + assert tokens("a\tb\r\nc\rd e f") == + [word: "a", word: "b", word: "c", word: "d", word: "e", word: "f"] + end + + test "a byte-order mark is ignored" do + assert tokens("select") == [word: "select"] + end + end + + describe "positions" do + test "are line and column of the token's first character" do + assert positions("select id\nfrom t") == [{1, 1}, {1, 8}, {2, 1}, {2, 6}, {2, 7}] + end + + test "count characters, not bytes" do + assert positions("'é' x") == [{1, 1}, {1, 5}, {1, 6}] + end + + test "CRLF is one line break" do + assert positions("a\r\nb") == [{1, 1}, {2, 1}, {2, 2}] + end + + test "comments advance them" do + assert positions("/* a\nb */ x -- c\ny") == [{2, 6}, {3, 1}, {3, 2}] + end + end + + describe "errors" do + test "an unknown character, with its position" do + assert %SyntaxError{reason: ~s(unexpected character "@"), line: 2, column: 3} = + error("a\nb @") + end + + test "Elixir atom syntax gets a hint" do + assert error("where x = :active").reason =~ "'name'::atom" + end + + test "invalid UTF-8" do + assert error(<<"a ", 0xFF, " b">>).reason =~ "invalid UTF-8" + assert error(<<"'", 0xC3, "'">>).reason =~ "invalid UTF-8" + end + + test "format as a message" do + assert Exception.message(error("@")) == + ~s(syntax error at line 1, column 1: unexpected character "@") + end + end +end diff --git a/test/sql/parser_test.exs b/test/sql/parser_test.exs new file mode 100644 index 0000000..41df49e --- /dev/null +++ b/test/sql/parser_test.exs @@ -0,0 +1,418 @@ +defmodule Efsql.SQL.ParserTest do + use ExUnit.Case, async: true + + alias Efsql.SQL.AST.Select + alias Efsql.SQL.Parser + alias Efsql.SQL.SyntaxError + + defp parse(sql) do + assert {:ok, %Select{} = select} = Parser.parse(sql) + select + end + + defp where(condition), do: parse("select a from t where #{condition}").where + + defp error(sql) do + assert {:error, %SyntaxError{} = e} = Parser.parse(sql) + e + end + + defp reason(sql), do: error(sql).reason + + describe "statement" do + test "the smallest query" do + assert parse("select * from t") == + %Select{fields: :star, from: ["t"], where: nil, order_by: [], limit: nil} + end + + test "every clause" do + assert parse("select a, b from s.t where a = 1 order by b desc limit 5;") == + %Select{ + fields: ["a", "b"], + from: ["s", "t"], + where: {:compare, :=, {:column, "a"}, {:literal, 1}}, + order_by: [{"b", :desc}], + limit: 5 + } + end + + test "the semicolon is optional" do + assert parse("select * from t") == parse("select * from t;") + end + + test "keywords are case-insensitive, names fold to lower case" do + assert parse("SELECT Id FROM Tenant.Users WHERE Name = 'Al' ORDER BY Id DESC LIMIT 1") == + parse("select id from tenant.users where name = 'Al' order by id desc limit 1") + end + + test "quoted names keep their case" do + assert %Select{fields: ["Id"], from: ["My Tenant", "Users"]} = + parse(~s(select "Id" from "My Tenant"."Users")) + end + + test "comments and whitespace anywhere" do + assert parse(""" + -- leading comment + select /* fields */ a, + b + from t -- table + where a = 1 /* trailing */ ; + """) == parse("select a, b from t where a = 1") + end + end + + describe "FROM" do + test "one, two or three parts" do + assert %Select{from: ["t"]} = parse("select * from t") + assert %Select{from: ["tenant", "t"]} = parse("select * from tenant.t") + assert %Select{from: ["storage", "tenant", "t"]} = parse("select * from storage.tenant.t") + end + + test "any non-reserved word is a name" do + assert %Select{from: ["user", "date"]} = parse("select * from user.date") + end + + test "four parts is too many" do + assert reason("select * from a.b.c.d") =~ "at most three parts" + end + + test "more than one table" do + assert reason("select * from a, b") =~ "more than one table" + end + end + + describe "SELECT list" do + test "star or names" do + assert %Select{fields: :star} = parse("select * from t") + assert %Select{fields: ["a", "b", "_"]} = parse("select a, b, _ from t") + end + + test "star can't be mixed with names" do + assert reason("select *, a from t") =~ "can't be combined" + end + + test "unsupported shapes say what is unsupported" do + assert reason("select distinct a from t") == "DISTINCT is not supported" + assert reason("select count(a) from t") == "functions are not supported" + assert reason("select t.a from t") == "qualified column names are not supported" + end + + test "a missing FROM" do + assert %SyntaxError{reason: "expected FROM, got the end of the statement", column: 9} = + error("select a") + end + end + + describe "comparisons" do + for {sql, op} <- [ + {"=", :=}, + {"<>", :<>}, + {"!=", :<>}, + {"<", :<}, + {">", :>}, + {"<=", :<=}, + {">=", :>=} + ] do + test "#{sql}" do + assert where("a #{unquote(sql)} 1") == + {:compare, unquote(op), {:column, "a"}, {:literal, 1}} + end + end + + test "either side can be the value" do + assert where("1 < a") == {:compare, :<, {:literal, 1}, {:column, "a"}} + end + + test "literals" do + assert where("a = 'x'") == {:compare, :=, {:column, "a"}, {:literal, "x"}} + assert where("a = -3") == {:compare, :=, {:column, "a"}, {:literal, -3}} + assert where("a = +3") == {:compare, :=, {:column, "a"}, {:literal, 3}} + assert where("a = -1.5") == {:compare, :=, {:column, "a"}, {:literal, -1.5}} + assert where("a = true") == {:compare, :=, {:column, "a"}, {:literal, true}} + assert where("a = FALSE") == {:compare, :=, {:column, "a"}, {:literal, false}} + assert where("a = null") == {:compare, :=, {:column, "a"}, {:literal, nil}} + end + + test "a sign only goes on a number" do + assert reason("select a from t where a = -b") =~ "only supported on a number" + end + + test "arithmetic is refused, not misread" do + assert reason("select a from t where a = 1 + 2") == "arithmetic is not supported" + assert reason("select a from t where a + 1 = 2") == "arithmetic is not supported" + end + + test "comparisons don't chain" do + assert reason("select a from t where a = 1 = 2") =~ "expected end of statement, got '='" + end + end + + describe "predicates" do + test "BETWEEN binds its own AND" do + assert where("a between 1 and 2 and b = 3") == + {:and, {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, false}, + {:compare, :=, {:column, "b"}, {:literal, 3}}} + end + + test "NOT BETWEEN" do + assert where("a not between 1 and 2") == + {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, true} + end + + test "IN and NOT IN" do + assert where("a in (1, 'b', 2.5)") == + {:in, {:column, "a"}, [{:literal, 1}, {:literal, "b"}, {:literal, 2.5}], false} + + assert where("a not in ('x')") == {:in, {:column, "a"}, [{:literal, "x"}], true} + end + + test "subqueries are named as unsupported" do + assert reason("select a from t where a in (select b from u)") == + "subqueries are not supported" + end + + test "IN needs a parenthesized, non-empty list" do + assert reason("select a from t where a in ()") == "IN needs at least one value" + assert reason("select a from t where a in 1") =~ "expected '(' after IN" + assert reason("select a from t where a in (1, 2") =~ "')' to close the IN list" + end + + test "LIKE, NOT LIKE, ILIKE" do + assert where("a like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, false} + assert where("a not like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, true} + assert where("a ilike 'x%'") == {:ilike, {:column, "a"}, {:literal, "x%"}, false} + end + + test "IS [NOT] NULL, ISNULL, NOTNULL" do + assert where("a is null") == {:is_null, {:column, "a"}, false} + assert where("a is not null") == {:is_null, {:column, "a"}, true} + assert where("a isnull") == {:is_null, {:column, "a"}, false} + assert where("a notnull") == {:is_null, {:column, "a"}, true} + end + + test "IS takes only NULL" do + assert reason("select a from t where a is true") =~ "expected NULL after IS, got 'true'" + assert reason("select a from t where a is not 1") =~ "expected NULL after IS NOT" + end + + test "a bare operand is kept for the translator to reject" do + assert where("a") == {:column, "a"} + end + end + + describe "boolean structure" do + test "AND binds tighter than OR" do + assert where("a = 1 or b = 2 and c = 3") == + {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, + {:and, {:compare, :=, {:column, "b"}, {:literal, 2}}, + {:compare, :=, {:column, "c"}, {:literal, 3}}}} + end + + test "AND and OR are left-associative" do + assert {:and, {:and, _, _}, _} = where("a = 1 and b = 2 and c = 3") + assert {:or, {:or, _, _}, _} = where("a = 1 or b = 2 or c = 3") + end + + test "NOT binds tighter than AND" do + assert where("not a = 1 and b = 2") == + {:and, {:not, {:compare, :=, {:column, "a"}, {:literal, 1}}}, + {:compare, :=, {:column, "b"}, {:literal, 2}}} + end + + test "parentheses group and then vanish" do + assert where("(a = 1 or b = 2) and c = 3") == + {:and, + {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, + {:compare, :=, {:column, "b"}, {:literal, 2}}}, + {:compare, :=, {:column, "c"}, {:literal, 3}}} + + assert where("((a = 1))") == where("a = 1") + end + + test "unbalanced parentheses" do + assert reason("select a from t where (a = 1") =~ "expected ')'" + assert reason("select a from t where a = 1)") =~ "expected end of statement, got ')'" + end + + test "deep nesting is fine up to a limit" do + assert where(String.duplicate("(", 150) <> "a = 1" <> String.duplicate(")", 150)) == + where("a = 1") + + assert reason("select a from t where " <> String.duplicate("(", 10_000) <> "a") == + "expression is nested too deeply" + + assert reason("select a from t where " <> String.duplicate("not ", 10_000) <> "a") == + "expression is nested too deeply" + end + end + + describe "casts" do + test "postfix and CAST forms give the same tree" do + assert where("a = 'x'::atom") == + {:compare, :=, {:column, "a"}, {:cast, {:literal, "x"}, "atom"}} + + assert where("a = cast('x' as atom)") == where("a = 'x'::atom") + end + + test "type names fold and may be reserved words" do + assert {:compare, :=, _, {:cast, _, "date"}} = where("a = '2024-01-01'::DATE") + assert {:compare, :=, _, {:cast, _, "time"}} = where("a = '10:00:00'::time") + end + + test "timestamp with/without time zone" do + assert {:compare, :=, _, {:cast, _, "timestamptz"}} = + where("a = 'x'::timestamp with time zone") + + assert {:compare, :=, _, {:cast, _, "timestamp"}} = + where("a = cast('x' as timestamp without time zone)") + end + + test "casts chain, and apply to any operand" do + assert where("a = 1::text::atom") == + {:compare, :=, {:column, "a"}, {:cast, {:cast, {:literal, 1}, "text"}, "atom"}} + + assert where("a::text = 'x'") == + {:compare, :=, {:cast, {:column, "a"}, "text"}, {:literal, "x"}} + end + + test "casts inside IN and BETWEEN" do + assert where("a in ('x'::atom, 'y'::atom)") == + {:in, {:column, "a"}, + [{:cast, {:literal, "x"}, "atom"}, {:cast, {:literal, "y"}, "atom"}], false} + + assert {:between, _, {:cast, _, "date"}, {:cast, _, "date"}, false} = + where("a between '2024-01-01'::date and '2025-01-01'::date") + end + + test "a cast needs a type" do + assert reason("select a from t where a = 'x'::") =~ "expected a type name" + assert reason("select a from t where a = cast('x' atom)") =~ "expected AS in CAST" + assert reason("select a from t where a = cast('x' as atom") =~ "')' to close CAST" + end + end + + describe "tuples" do + test "a partition scan" do + assert where("_ = ('p', *)") == + {:compare, :=, {:column, "_"}, {:tuple, [{:literal, "p"}, :star]}} + end + + test "a versionstamp" do + assert where("_ >= ('p', 22348699227647901699)") == + {:compare, :>=, {:column, "_"}, + {:tuple, [{:literal, "p"}, {:literal, 22_348_699_227_647_901_699}]}} + end + + test "star only inside a tuple" do + assert reason("select a from t where a = (*)") =~ "only allowed in a tuple" + assert reason("select a from t where a = *") =~ "expected an expression, got '*'" + end + end + + describe "reserved words" do + test "common column names are not reserved" do + for name <- ~w[day at user value date time timestamp year month zone key type level] do + assert where("#{name} = 1") == {:compare, :=, {:column, name}, {:literal, 1}}, + "#{name} should be usable as a name" + + assert %Select{fields: [^name], order_by: [{^name, :desc}]} = + parse("select #{name} from t order by #{name} desc") + end + end + + test "reserved words need quotes, and the error says so" do + assert reason("select order from t") =~ ~s(got the keyword 'order' (quote it, "order") + + assert where(~s("order" = 1)) == {:compare, :=, {:column, "order"}, {:literal, 1}} + assert %Select{fields: ["select", "from"]} = parse(~s(select "select", "from" from t)) + end + + test "the reserved list" do + for word <- Parser.reserved_words() do + assert {:error, %SyntaxError{}} = Parser.parse("select #{word} from t") + end + end + end + + describe "ORDER BY and LIMIT" do + test "directions default to ascending" do + assert %Select{order_by: [{"a", :asc}, {"b", :desc}, {"c", :asc}]} = + parse("select * from t order by a, b desc, c asc") + end + + test "ORDER needs BY and a field" do + assert reason("select * from t order a") =~ "expected BY after ORDER" + assert reason("select * from t order by") =~ "expected a field name to order by" + + assert reason("select * from t order by a nulls first") == + "NULLS FIRST/LAST is not supported" + end + + test "LIMIT takes a whole number" do + assert %Select{limit: 0} = parse("select * from t limit 0") + assert reason("select * from t limit -1") =~ "LIMIT expects a whole number, got '-'" + assert reason("select * from t limit 1.5") =~ "got the number 1.5" + assert reason("select * from t limit") =~ "got the end of the statement" + end + + test "clauses in the wrong order" do + assert reason("select * from t limit 1 where a = 1") =~ + "expected end of statement, got 'where'" + + assert reason("select * from t order by a where a = 1") =~ "got 'where'" + end + end + + describe "statement errors" do + test "empty input" do + assert reason("") == "empty statement" + assert reason(" -- just a comment\n") == "empty statement" + assert reason(";") =~ "expected SELECT, got ';'" + end + + test "only SELECT" do + assert reason("delete from t") == "only SELECT statements are supported" + assert reason("update t set a = 1") == "only SELECT statements are supported" + assert reason("with x as (select 1) select * from x") =~ "expected SELECT" + end + + test "one statement" do + assert %SyntaxError{reason: "only one statement is supported", column: 18} = + error("select * from t; select * from u") + + assert reason("select * from t;;") == "only one statement is supported" + end + + test "unsupported clauses are named" do + assert reason("select * from t group by a") == "GROUP BY is not supported" + assert reason("select * from t where a = 1 having a > 1") == "HAVING is not supported" + assert reason("select * from t limit 1 offset 2") == "OFFSET is not supported" + assert reason("select * from t join u on t.a = u.a") == "JOIN is not supported" + assert reason("select * from t left join u on a = b") == "JOIN is not supported" + assert reason("select * from t union select * from u") == "UNION is not supported" + end + + test "errors point at the offending token" do + assert %SyntaxError{line: 3, column: 10, reason: "expected an expression, got '='"} = + error("select a\nfrom t\nwhere a == 1") + + assert %SyntaxError{ + line: 1, + column: 23, + reason: "expected an expression, got the end of the statement" + } = + error("select a from t where ") + end + + test "long strings are truncated in messages" do + assert reason("select * from t limit '#{String.duplicate("x", 100)}'") =~ + "the string 'xxxxxxxxxxxxxxxxxxxx...'" + end + + test "parse! raises the error" do + assert_raise SyntaxError, ~r/line 1, column 1: expected SELECT/, fn -> + Parser.parse!("selec * from t") + end + end + end +end diff --git a/test/sql/property_test.exs b/test/sql/property_test.exs new file mode 100644 index 0000000..e43878e --- /dev/null +++ b/test/sql/property_test.exs @@ -0,0 +1,126 @@ +defmodule Efsql.SQL.PropertyTest do + # Randomized tests. `:rand` is seeded per test from ExUnit's --seed, so a + # failure reproduces with `mix test --seed N`. + use ExUnit.Case, async: true + + alias Efsql.Exception.Unsupported + alias Efsql.SQL.AST.Select + alias Efsql.SQL.Parser + alias Efsql.SQL.SyntaxError + alias EfsqlTest.SQLGen + + @runs 1_000 + + test "printing a random tree and parsing it gives the tree back" do + for _ <- 1..@runs do + select = SQLGen.select() + sql = SQLGen.to_sql(select) + + assert Parser.parse(sql) == {:ok, select}, """ + SQL: + #{sql} + """ + end + end + + test "mangled queries parse or fail with a positioned SyntaxError, never crash" do + for _ <- 1..@runs do + sql = SQLGen.select() |> SQLGen.to_sql() |> mangle(Enum.random(1..4)) + check(sql) + end + end + + test "random bytes parse or fail with a positioned SyntaxError, never crash" do + alphabet = + String.graphemes("selctfromwhandi*,.;()'\"=<>!:-+/%_ \n\t0123456789eé") ++ + ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>] + + for _ <- 1..@runs do + sql = Enum.map_join(1..Enum.random(0..40), fn _ -> Enum.random(alphabet) end) + check(sql) + end + end + + test "valid queries translate to a logical plan or raise Unsupported" do + for _ <- 1..@runs do + sql = SQLGen.select() |> SQLGen.to_sql() + + try do + Efsql.Parser.sql_to_logical(sql) + rescue + _ in [Unsupported] -> + :ok + + # Versionstamps need ecto_foundationdb, which a run without the + # adapter doesn't have; everything else must be one of the above. + e in [UndefinedFunctionError] -> + assert e.module == EctoFoundationDB.Versionstamp + end + end + end + + test "big inputs parse and translate quickly" do + values = Enum.map_join(1..20_000, ", ", &"'v#{&1}'") + conditions = Enum.map_join(1..20_000, " and ", &"f#{&1} = #{&1}") + sql = "select a from t where a in (#{values}) and #{conditions}" + + {micros, %Efsql.Logical.Select{predicates: predicates}} = + :timer.tc(fn -> Efsql.Parser.sql_to_logical(sql) end) + + assert length(predicates) == 20_001 + assert micros < 5_000_000 + end + + defp check(sql) do + case Parser.parse(sql) do + {:ok, %Select{}} -> + :ok + + {:error, %SyntaxError{line: line, column: column, reason: reason}} -> + assert is_binary(reason) and reason != "" + lines = String.split(sql, ["\r\n", "\n", "\r"]) + assert line in 1..length(lines), "line #{line} out of range for #{inspect(sql)}" + assert column >= 1, "column #{column} for #{inspect(sql)}" + end + end + + # Random edits: delete, insert, replace, duplicate or truncate. + defp mangle(sql, 0), do: sql + + defp mangle(sql, n) do + chars = String.graphemes(sql) + size = length(chars) + at = Enum.random(0..size) + + junk = + Enum.random([ + "(", + ")", + "'", + "\"", + ",", + ";", + "-", + "*", + ":", + ".", + "=", + "<", + "not", + " and ", + "é", + <<0xC3>> + ]) + + chars = + case Enum.random(1..5) do + 1 -> List.delete_at(chars, at) + 2 -> List.insert_at(chars, at, junk) + 3 -> List.replace_at(chars, at, junk) + 4 -> Enum.take(chars, at) ++ Enum.slice(chars, at, 10) ++ Enum.drop(chars, at) + 5 -> Enum.take(chars, at) + end + + mangle(Enum.join(chars), n - 1) + end +end diff --git a/test/support/efsql_test/sql_gen.ex b/test/support/efsql_test/sql_gen.ex new file mode 100644 index 0000000..e85eaf6 --- /dev/null +++ b/test/support/efsql_test/sql_gen.ex @@ -0,0 +1,239 @@ +defmodule EfsqlTest.SQLGen do + @moduledoc false + # Random `Efsql.SQL.AST.Select` trees, and a printer that renders one as + # SQL text with randomized spelling: keyword case, whitespace and + # comments between tokens, optional quoting, `!=` for `<>`, `CAST(...)` + # for `::`, `ISNULL` for `IS NULL`. Parsing the text must give back the + # tree. Uses `:rand`, which ExUnit seeds per test from `--seed`. + + alias Efsql.SQL.AST.Select + + # Names that print unquoted. None is reserved, and each survives + # upcasing and folding back down unchanged. + @plain_names ~w[a b id name status day at user value date time zone created_at _ x1 a$b café 日本語] + @types ~w[atom timestamp timestamptz date time widget] + @name_chars String.graphemes("abcXYZ019 _-.,;'\"()*é日\n") ++ ["select", "FROM", "--", "/*"] + @string_chars String.graphemes("abc XYZ 019_-.,;:'\"()*%é☃\n\t") ++ ["--", "/*", "*/", "''"] + + def select() do + %Select{ + fields: if(one_in(4), do: :star, else: list(1..4, &name/0)), + from: list(1..3, &name/0), + where: maybe(fn -> expr(3) end), + order_by: list(0..3, fn -> {name(), Enum.random([:asc, :desc])} end), + limit: maybe(fn -> Enum.random([0, 1, 15, 1000, 123_456_789]) end) + } + end + + # -- trees -- + + def expr(0), do: predicate() + + def expr(depth) do + case Enum.random(1..6) do + 1 -> {:and, expr(depth - 1), expr(depth - 1)} + 2 -> {:or, expr(depth - 1), expr(depth - 1)} + 3 -> {:not, expr(depth - 1)} + _ -> predicate() + end + end + + def predicate() do + case Enum.random(1..8) do + 1 -> {:between, operand(2), operand(2), operand(2), bool()} + 2 -> {:in, operand(2), list(1..4, fn -> operand(2) end), bool()} + 3 -> {Enum.random([:like, :ilike]), operand(2), operand(2), bool()} + 4 -> {:is_null, operand(2), bool()} + 5 -> operand(2) + _ -> {:compare, Enum.random([:=, :<>, :<, :>, :<=, :>=]), operand(2), operand(2)} + end + end + + def operand(depth) do + case Enum.random(1..7) do + 1 when depth > 0 -> {:cast, operand(depth - 1), Enum.random(@types)} + 2 -> tuple() + n when n in 3..4 -> {:column, name()} + _ -> literal() + end + end + + defp tuple() do + {:tuple, list(2..3, fn -> if one_in(3), do: :star, else: literal() end)} + end + + def literal() do + value = + case Enum.random(1..7) do + 1 -> Enum.random(-1000..1000) + 2 -> Enum.random([0, 22_348_699_227_647_901_699, 12_345_678_901_234_567_890_123]) + 3 -> Float.round((:rand.uniform() - 0.5) * 2000, Enum.random(0..6)) + 0.5 + 4 -> Enum.random([true, false, nil]) + _ -> random_string(@string_chars, 0..8) + end + + {:literal, value} + end + + def name() do + if one_in(4), + do: random_string(@name_chars, 1..6), + else: Enum.random(@plain_names) + end + + # -- printing -- + + def to_sql(%Select{} = select) do + [ + kw("select"), + fields(select.fields), + kw("from"), + Enum.intersperse(Enum.map(select.from, &print_name/1), p(".")), + if(select.where, do: [kw("where"), print(select.where)], else: []), + order_by(select.order_by), + if(select.limit, do: [kw("limit"), w(Integer.to_string(select.limit))], else: []), + if(one_in(2), do: [p(";")], else: []) + ] + |> List.flatten() + |> join() + end + + defp fields(:star), do: [p("*")] + defp fields(names), do: comma(Enum.map(names, &print_name/1)) + + defp order_by([]), do: [] + + defp order_by(items) do + [ + kw("order"), + kw("by"), + comma( + Enum.map(items, fn + {name, :asc} -> [print_name(name) | Enum.random([[], [kw("asc")]])] + {name, :desc} -> [print_name(name), kw("desc")] + end) + ) + ] + end + + # Boolean operators always get parentheses, so the printed nesting is + # exactly the tree's whatever the precedence. + defp print({op, l, r}) when op in [:and, :or], + do: [p("("), print(l), kw(Atom.to_string(op)), print(r), p(")")] + + defp print({:not, e}), do: [kw("not"), p("("), print(e), p(")")] + + defp print({:compare, op, l, r}) do + op = if op == :<>, do: Enum.random(["<>", "!="]), else: Atom.to_string(op) + [print(l), p(op), print(r)] + end + + defp print({:between, e, low, high, negated?}), + do: [print(e), negated(negated?), kw("between"), print(low), kw("and"), print(high)] + + defp print({:in, e, values, negated?}), + do: [print(e), negated(negated?), kw("in"), p("("), comma(Enum.map(values, &print/1)), p(")")] + + defp print({like, e, pattern, negated?}) when like in [:like, :ilike], + do: [print(e), negated(negated?), kw(Atom.to_string(like)), print(pattern)] + + defp print({:is_null, e, negated?}) do + case {negated?, one_in(3)} do + {false, true} -> [print(e), kw("isnull")] + {true, true} -> [print(e), kw("notnull")] + _ -> [print(e), kw("is"), negated(negated?), kw("null")] + end + end + + defp print({:column, name}), do: print_name(name) + + defp print({:cast, e, type}) do + if one_in(3), + do: [kw("cast"), p("("), print(e), kw("as"), type(type), p(")")], + else: [print(e), p("::"), type(type)] + end + + defp print({:tuple, elements}) do + [ + p("("), + comma( + Enum.map(elements, fn + :star -> p("*") + e -> print(e) + end) + ), + p(")") + ] + end + + defp print({:literal, nil}), do: kw("null") + defp print({:literal, b}) when is_boolean(b), do: kw(Atom.to_string(b)) + defp print({:literal, n}) when is_integer(n), do: w(Integer.to_string(n)) + defp print({:literal, f}) when is_float(f), do: w(Float.to_string(f)) + defp print({:literal, s}) when is_binary(s), do: w("'" <> String.replace(s, "'", "''") <> "'") + + defp type("timestamptz") do + if one_in(2), + do: w("timestamptz"), + else: [kw("timestamp"), kw("with"), kw("time"), kw("zone")] + end + + defp type(type), do: kw(type) + + defp negated(true), do: [kw("not")] + defp negated(false), do: [] + + defp print_name(name) do + if name in @plain_names and not one_in(5), + do: w(random_case(name)), + else: w(~s(") <> String.replace(name, ~s("), ~s("")) <> ~s(")) + end + + defp comma(items), do: Enum.intersperse(items, p(",")) + + # A token is {:w, text} (needs a separator from its neighbours) or + # {:p, text} (punctuation, which may touch them). + defp kw(text), do: {:w, random_case(text)} + defp w(text), do: {:w, text} + defp p(text), do: {:p, text} + + @separators [ + " ", + " ", + "\n", + "\t", + "\r\n", + " ", + " -- note ; 'x' \"y\"\n", + " /* a /* b */ c */ " + ] + + defp join(tokens) do + tokens + |> Enum.chunk_every(2, 1) + |> Enum.map_join(fn + [{_, text}] -> text + [{:w, text}, {:w, _}] -> text <> Enum.random(@separators) + [{_, text}, _] -> text <> Enum.random(["" | @separators]) + end) + end + + defp random_case(text) do + case Enum.random(1..3) do + 1 -> String.upcase(text) + 2 -> String.capitalize(text) + 3 -> text + end + end + + # -- helpers -- + + defp random_string(pool, range) do + Enum.map_join(1..Enum.random(range)//1, fn _ -> Enum.random(pool) end) + end + + defp list(range, fun), do: Enum.map(1..Enum.random(range)//1, fn _ -> fun.() end) + defp maybe(fun), do: if(one_in(2), do: fun.(), else: nil) + defp bool(), do: one_in(2) + defp one_in(n), do: :rand.uniform(n) == 1 +end From 1575f2ef9baa5995e153643aef707a4425ed358b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 03:13:35 +0000 Subject: [PATCH 2/5] Let the random-bytes fuzz test generate empty input `1..Enum.random(0..40)` counts down to 0 when the length drawn is 0, so that case produced two characters instead of none and warned on Elixir 1.19. An explicit //1 step makes it an empty range. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- test/sql/property_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sql/property_test.exs b/test/sql/property_test.exs index e43878e..e87a098 100644 --- a/test/sql/property_test.exs +++ b/test/sql/property_test.exs @@ -36,7 +36,7 @@ defmodule Efsql.SQL.PropertyTest do ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>] for _ <- 1..@runs do - sql = Enum.map_join(1..Enum.random(0..40), fn _ -> Enum.random(alphabet) end) + sql = Enum.map_join(1..Enum.random(0..40)//1, fn _ -> Enum.random(alphabet) end) check(sql) end end From a967c0e57d486d6b2940ef21b7d1960a1e7c8fec Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 11:41:58 +0000 Subject: [PATCH 3/5] Add a leex/yecc lexer and parser alongside the hand-written ones Efsql.SQL.Leex and Efsql.SQL.Yecc keep the same contracts as Efsql.SQL.Lexer and Efsql.SQL.Parser, down to error messages and positions. They're built on src/efsql_sql_leex.xrl and src/efsql_sql_yecc.yrl, which Mix compiles automatically. yecc only reports the token where it stopped, so Efsql.SQL.Yecc.Explain works out the wording from the tokens that came before. The lexer, parser and property suites now run against both implementations. A new differential test checks that the two agree on random valid, mangled and garbage input. Also fixes the hand lexer's column for the end of input after a trailing line comment. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- .gitignore | 1 + lib/efsql/sql/leex.ex | 243 +++++++++++++++ lib/efsql/sql/lexer.ex | 14 +- lib/efsql/sql/yecc.ex | 52 ++++ lib/efsql/sql/yecc/explain.ex | 454 +++++++++++++++++++++++++++++ src/efsql_sql_leex.xrl | 57 ++++ src/efsql_sql_yecc.yrl | 200 +++++++++++++ test/sql/differential_test.exs | 28 ++ test/sql/lexer_test.exs | 202 +++++++------ test/sql/parser_test.exs | 401 +++++++++++++------------ test/sql/property_test.exs | 85 ++---- test/support/efsql_test/sql_gen.ex | 79 +++++ 12 files changed, 1467 insertions(+), 349 deletions(-) create mode 100644 lib/efsql/sql/leex.ex create mode 100644 lib/efsql/sql/yecc.ex create mode 100644 lib/efsql/sql/yecc/explain.ex create mode 100644 src/efsql_sql_leex.xrl create mode 100644 src/efsql_sql_yecc.yrl create mode 100644 test/sql/differential_test.exs diff --git a/.gitignore b/.gitignore index 9f1d749..f5b5e74 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ dev-efsql # Local dev launcher; also the name mix escript.build would write here. /efsql +/src/efsql_sql_*.erl diff --git a/lib/efsql/sql/leex.ex b/lib/efsql/sql/leex.ex new file mode 100644 index 0000000..461e5bb --- /dev/null +++ b/lib/efsql/sql/leex.ex @@ -0,0 +1,243 @@ +defmodule Efsql.SQL.Leex do + @moduledoc """ + The same contract as `Efsql.SQL.Lexer.tokenize/1`, built on the leex + tokenizer in `src/efsql_sql_leex.xrl`. + + leex matches the token shapes. This driver pulls one match at a time + (`token/2`) and does what the rules can't: it tracks positions in + characters (leex counts UTF-8 bytes), skips nested block comments, + applies the Unicode letter classes, and turns the raw matches into + tokens and errors. + """ + + alias Efsql.SQL.SyntaxError + + # Invalid UTF-8 bytes are passed to leex as integers above U+10FFFF. + @bad_byte 0x110000 + + @ops %{ + ~c"<>" => :<>, + ~c"!=" => :<>, + ~c"<=" => :<=, + ~c">=" => :>=, + ~c"::" => :"::", + ~c"=" => :=, + ~c"<" => :<, + ~c">" => :>, + ~c"(" => :"(", + ~c")" => :")", + ~c"," => :",", + ~c"." => :., + ~c";" => :";", + ~c"*" => :*, + ~c"+" => :+, + ~c"-" => :-, + ~c"/" => :/, + ~c"%" => :% + } + + @spec tokenize(String.t()) :: {:ok, [Efsql.SQL.Lexer.token()]} | {:error, SyntaxError.t()} + def tokenize(sql) when is_binary(sql) do + {:ok, lex(decode(sql, []), 1, 1, [])} + catch + {:syntax_error, reason, {line, column}} -> + {:error, %SyntaxError{reason: reason, line: line, column: column}} + end + + defp decode(<>, acc), do: decode(rest, [c | acc]) + defp decode(<>, acc), do: decode(rest, [@bad_byte + byte | acc]) + defp decode(<<>>, acc), do: Enum.reverse(acc) + + defp lex([], line, col, acc), do: Enum.reverse([{:eof, nil, {line, col}} | acc]) + + defp lex(chars, line, col, acc) do + case next(chars) do + {{:ok, {kind, matched}, _}, rest} -> token(kind, matched, rest, {line, col}, acc) + {{:error, _, _}, _} -> illegal(hd(chars), {line, col}) + end + end + + defp next(chars) do + case :efsql_sql_leex.token([], chars) do + {:done, result, rest} -> + {result, rest(rest)} + + {:more, cont} -> + cont |> :efsql_sql_leex.token(:eof) |> then(fn {:done, r, rest} -> {r, rest(rest)} end) + end + end + + defp rest(:eof), do: [] + defp rest(chars), do: chars + + defp token(:ws, matched, rest, pos, acc), do: continue(matched, rest, pos, acc) + + # A line comment runs to the end of the line, whatever it holds. + defp token(:line_comment, matched, rest, pos, acc) do + {comment, rest} = Enum.split_while(rest, &(&1 not in [?\n, ?\r])) + continue(matched ++ comment, rest, pos, acc) + end + + defp token(:block_comment, _matched, rest, {line, col} = start, acc) do + {rest, line, col} = block_comment(rest, line, col + 2, 1, start) + lex(rest, line, col, acc) + end + + defp token(kind, matched, rest, pos, acc) when kind in [:string, :quoted] do + check_bytes(matched, pos) + value = matched |> Enum.slice(1..-2//1) |> List.to_string() + + {type, value} = + case kind do + :string -> + {:string, String.replace(value, "''", "'")} + + :quoted -> + if value == "", do: error("zero-length quoted identifier", pos) + {:quoted, String.replace(value, ~s(""), ~s("))} + end + + continue(matched, rest, pos, [{type, value, pos} | acc]) + end + + defp token(:unterminated, [quote | _] = matched, _rest, pos, _acc) do + check_bytes(matched, pos) + what = if quote == ?', do: "string", else: "identifier" + error("unterminated #{what} (missing closing #{<>})", pos) + end + + defp token(:number, matched, rest, pos, acc) do + continue(matched, rest, pos, [number(List.to_string(matched), pos) | acc]) + end + + defp token(:bad_exponent, _matched, _rest, pos, _acc), + do: error("invalid number: exponent has no digits", pos) + + # A number run into another character is an error when that character + # could continue it; otherwise the number ends and the character is + # lexed on its own. + defp token(:number_junk, matched, rest, pos, acc) do + {number, [c]} = Enum.split(matched, -1) + + if word_start?(c) or c == ?. do + error("invalid number #{inspect(List.to_string(matched))}", pos) + end + + continue(number, [c | rest], pos, [number(List.to_string(number), pos) | acc]) + end + + # leex matches any run of letters and non-ASCII characters; the word is + # the part the Unicode classes allow, and the rest is lexed again. + defp token(:word, [c | tail] = matched, rest, pos, acc) do + cond do + word_start?(c) -> + {word, leftover} = Enum.split_while(matched, &word_continue?/1) + value = word |> List.to_string() |> String.downcase() + continue(word, leftover ++ rest, pos, [{:word, value, pos} | acc]) + + unicode_space?(c) -> + continue([c], tail ++ rest, pos, acc) + + true -> + illegal(c, pos) + end + end + + defp token(:op, matched, rest, pos, acc), + do: continue(matched, rest, pos, [{:op, Map.fetch!(@ops, matched), pos} | acc]) + + defp token(:colon, [?:, c], _rest, pos, _acc) do + if word_start?(c), + do: error("unexpected ':' (write an atom as a cast, e.g. 'name'::atom)", pos), + else: error("unexpected ':'", pos) + end + + defp token(:colon, [?:], _rest, pos, _acc), do: illegal(?:, pos) + + defp continue(consumed, rest, {line, col}, acc) do + {line, col} = advance(consumed, line, col) + lex(rest, line, col, acc) + end + + defp advance([?\r, ?\n | rest], line, _col), do: advance(rest, line + 1, 1) + defp advance([c | rest], line, _col) when c in [?\n, ?\r], do: advance(rest, line + 1, 1) + defp advance([_ | rest], line, col), do: advance(rest, line, col + 1) + defp advance([], line, col), do: {line, col} + + defp block_comment([], _line, _col, _depth, start), + do: error("unterminated /* comment", start) + + defp block_comment([?*, ?/ | rest], line, col, 1, _start), do: {rest, line, col + 2} + + defp block_comment([?*, ?/ | rest], line, col, depth, start), + do: block_comment(rest, line, col + 2, depth - 1, start) + + defp block_comment([?/, ?* | rest], line, col, depth, start), + do: block_comment(rest, line, col + 2, depth + 1, start) + + defp block_comment([?\r, ?\n | rest], line, _col, depth, start), + do: block_comment(rest, line + 1, 1, depth, start) + + defp block_comment([c | rest], line, _col, depth, start) when c in [?\n, ?\r], + do: block_comment(rest, line + 1, 1, depth, start) + + defp block_comment([_ | rest], line, col, depth, start), + do: block_comment(rest, line, col + 1, depth, start) + + # An invalid byte inside a string or quoted name is reported where it is. + defp check_bytes(matched, {line, col}) do + case Enum.split_while(matched, &(&1 < @bad_byte)) do + {_, []} -> :ok + {before, [bad | _]} -> illegal(bad, advance(before, line, col)) + end + end + + defp illegal(c, pos) when c >= @bad_byte, + do: error("invalid UTF-8 byte 0x#{Integer.to_string(c - @bad_byte, 16)}", pos) + + defp illegal(c, pos), do: error("unexpected character #{inspect(<>)}", pos) + + defp number(text, pos) do + {mantissa, exp} = + case :binary.match(text, ["e", "E"]) do + {at, _} -> {binary_part(text, 0, at), binary_part(text, at, byte_size(text) - at)} + :nomatch -> {text, ""} + end + + case String.split(mantissa, ".", parts: 2) do + [int] when exp == "" -> + {:integer, String.to_integer(int), pos} + + parts -> + {int, frac} = + case parts do + [int] -> {int, ""} + [int, frac] -> {int, frac} + end + + int = if int == "", do: "0", else: int + frac = if frac == "", do: "0", else: frac + {:float, to_float("#{int}.#{frac}#{exp}", text, pos), pos} + end + end + + defp to_float(normalized, text, pos) do + String.to_float(normalized) + rescue + ArgumentError -> error("number out of range: #{text}", pos) + end + + # The same character classes as Efsql.SQL.Lexer. + defp word_start?(c) when c in ?a..?z or c in ?A..?Z or c == ?_, do: true + defp word_start?(c) when c < 128 or c >= @bad_byte, do: false + defp word_start?(c), do: String.match?(<>, ~r/^\p{L}$/u) + + defp word_continue?(c) when c in ?0..?9 or c == ?$, do: true + defp word_continue?(c) when c < 128 or c >= @bad_byte, do: word_start?(c) + defp word_continue?(c), do: String.match?(<>, ~r/^[\p{L}\p{M}\p{N}]$/u) + + defp unicode_space?(c), + do: c in [0xA0, 0x1680, 0x202F, 0x205F, 0x3000, 0xFEFF] or c in 0x2000..0x200A + + defp error(reason, pos), do: throw({:syntax_error, reason, pos}) +end diff --git a/lib/efsql/sql/lexer.ex b/lib/efsql/sql/lexer.ex index 96a3ada..bfd3c78 100644 --- a/lib/efsql/sql/lexer.ex +++ b/lib/efsql/sql/lexer.ex @@ -73,7 +73,7 @@ defmodule Efsql.SQL.Lexer do defp lex(<>, line, col, acc) when c in [?\s, ?\t, ?\f, ?\v], do: lex(rest, line, col + 1, acc) - defp lex(<<"--", rest::binary>>, line, _col, acc), do: line_comment(rest, line, acc) + defp lex(<<"--", rest::binary>>, line, col, acc), do: line_comment(rest, line, col + 2, acc) defp lex(<<"/*", rest::binary>>, line, col, acc), do: block_comment(rest, line, col + 2, 1, {line, col}, acc) @@ -139,13 +139,17 @@ defmodule Efsql.SQL.Lexer do # -- comments -- - defp line_comment(<<>>, line, acc), do: lex(<<>>, line, 1, acc) - defp line_comment(<>, line, acc), do: lex(rest, line + 1, 1, acc) + defp line_comment(<<>>, line, col, acc), do: lex(<<>>, line, col, acc) + defp line_comment(<>, line, _col, acc), do: lex(rest, line + 1, 1, acc) - defp line_comment(<>, line, acc) when c in [?\n, ?\r], + defp line_comment(<>, line, _col, acc) when c in [?\n, ?\r], do: lex(rest, line + 1, 1, acc) - defp line_comment(<<_, rest::binary>>, line, acc), do: line_comment(rest, line, acc) + defp line_comment(<<_::utf8, rest::binary>>, line, col, acc), + do: line_comment(rest, line, col + 1, acc) + + defp line_comment(<<_, rest::binary>>, line, col, acc), + do: line_comment(rest, line, col + 1, acc) defp block_comment(<<>>, _line, _col, _depth, start, _acc), do: throw({:syntax_error, "unterminated /* comment", start}) diff --git a/lib/efsql/sql/yecc.ex b/lib/efsql/sql/yecc.ex new file mode 100644 index 0000000..0792200 --- /dev/null +++ b/lib/efsql/sql/yecc.ex @@ -0,0 +1,52 @@ +defmodule Efsql.SQL.Yecc do + @moduledoc """ + The same contract as `Efsql.SQL.Parser`, built on leex and yecc: the + tokens come from `Efsql.SQL.Leex` and the grammar is + `src/efsql_sql_yecc.yrl`. + + yecc reports only the token a syntax error stopped at, so the error's + wording is worked out by `Efsql.SQL.Yecc.Explain` from the tokens + before it. + """ + + alias Efsql.SQL.AST + alias Efsql.SQL.Leex + alias Efsql.SQL.SyntaxError + alias Efsql.SQL.Yecc.Explain + + # Words the grammar has as terminals. They must match its Terminals. + @keywords Efsql.SQL.Parser.reserved_words() ++ ~w[time timestamp without zone] + + @spec parse(String.t()) :: {:ok, AST.Select.t()} | {:error, SyntaxError.t()} + def parse(sql) when is_binary(sql) do + with {:ok, tokens} <- Leex.tokenize(sql) do + # A token's location is its index, so an error names its token + # exactly (end of input can share a position with another token). + ytokens = tokens |> Enum.with_index() |> Enum.map(fn {t, i} -> to_yecc(t, i) end) + + case :efsql_sql_yecc.parse(ytokens) do + {:ok, {:select, fields, from, where, order_by, limit}} -> + {:ok, + %AST.Select{fields: fields, from: from, where: where, order_by: order_by, limit: limit}} + + {:error, {index, :efsql_sql_yecc, _message}} -> + {reason, {line, column}} = Explain.explain(tokens, index) + {:error, %SyntaxError{reason: reason, line: line, column: column}} + end + end + end + + @spec parse!(String.t()) :: AST.Select.t() + def parse!(sql) do + case parse(sql) do + {:ok, select} -> select + {:error, error} -> raise error + end + end + + defp to_yecc({:word, word, _}, i) when word in @keywords, do: {String.to_atom(word), i, word} + defp to_yecc({:word, word, _}, i), do: {:ident, i, word} + defp to_yecc({:op, op, _}, i), do: {op, i, op} + defp to_yecc({:eof, nil, _}, i), do: {:"$end", i} + defp to_yecc({type, value, _}, i), do: {type, i, value} +end diff --git a/lib/efsql/sql/yecc/explain.ex b/lib/efsql/sql/yecc/explain.ex new file mode 100644 index 0000000..a126e76 --- /dev/null +++ b/lib/efsql/sql/yecc/explain.ex @@ -0,0 +1,454 @@ +defmodule Efsql.SQL.Yecc.Explain do + @moduledoc false + # Words for a yecc syntax error. yecc says only which token it stopped + # at; Efsql.SQL.Parser says what it expected there. This reconstructs + # that from the tokens before the error: which clause it is in, what + # the previous token asks for, and which parenthesis is still open. + # + # In a few places the LALR parser reads further than the hand-written + # one before failing (an infix `not` that isn't NOT IN/LIKE/BETWEEN, a + # half-written `timestamp with time zone`); there it steps back to the + # token Efsql.SQL.Parser would have stopped at. + + @reserved Efsql.SQL.Parser.reserved_words() + @not_select ~w[insert update delete create drop alter truncate grant revoke explain] + @unsupported_clauses %{ + "group" => "GROUP BY", + "having" => "HAVING", + "offset" => "OFFSET", + "join" => "JOIN", + "inner" => "JOIN", + "left" => "JOIN", + "right" => "JOIN", + "full" => "JOIN", + "cross" => "JOIN", + "union" => "UNION", + "intersect" => "INTERSECT", + "except" => "EXCEPT", + "fetch" => "FETCH" + } + @comparisons [:=, :<>, :<, :>, :<=, :>=] + @arithmetic [:+, :-, :*, :/, :%] + @clauses ~w[select from where order limit] + + @doc "The reason and position Efsql.SQL.Parser gives for the error yecc hit at token `index`." + def explain(tokens, index), do: at(List.to_tuple(tokens), index) + + defp at(t, 0) do + case elem(t, 0) do + {:eof, _, pos} -> {"empty statement", pos} + {:word, word, pos} when word in @not_select -> {"only SELECT statements are supported", pos} + token -> expected("SELECT", token) + end + end + + defp at(t, i) do + case elem(t, i - 1) do + {:op, :";", _} -> {"only one statement is supported", pos(elem(t, i))} + _ -> in_clause(t, i, clause(t, i)) + end + end + + # The clause the error is in: the last clause keyword before it, not + # counting type names (`::order` is a type). + defp clause(t, i) do + Enum.reduce(1..(i - 1)//1, {"select", 0}, fn j, acc -> + case elem(t, j) do + {:word, word, _} when word in @clauses -> + if type_position?(t, j), do: acc, else: {word, j} + + _ -> + acc + end + end) + end + + # -- SELECT list -- + + defp in_clause(t, i, {"select", _}) do + token = elem(t, i) + + case {i, elem(t, i - 1), token} do + {1, _, {:word, "distinct", pos}} -> {"DISTINCT is not supported", pos} + {1, _, _} -> name_error(token, "a field name or *") + {2, {:op, :*, _}, {:op, :",", pos}} -> {"* can't be combined with other fields", pos} + {2, {:op, :*, _}, _} -> expected("FROM", token) + {_, {:op, :",", _}, _} -> name_error(token, "a field name") + {_, _, {:op, :"(", pos}} -> {"functions are not supported", pos} + {_, _, {:op, :., pos}} -> {"qualified column names are not supported", pos} + _ -> expected("FROM", token) + end + end + + # -- FROM -- + + defp in_clause(t, i, {"from", f}) do + token = elem(t, i) + + case {elem(t, i - 1), token} do + {{:word, "from", _}, _} -> + name_error(token, "a table name") + + {{:op, :., _}, _} -> + name_error(token, "a name after '.'") + + {_, {:op, :., pos}} when i - f == 6 -> + {"a table name has at most three parts (storage_id.tenant.table)", pos} + + {_, {:op, :",", pos}} -> + {"selecting from more than one table is not supported", pos} + + _ -> + end_of_statement(token) + end + end + + # -- ORDER BY -- + + defp in_clause(t, i, {"order", o}) do + token = elem(t, i) + + case {elem(t, i - 1), token} do + {_, _} when i == o + 1 -> expected("BY after ORDER", token) + {{:word, "by", _}, _} -> name_error(token, "a field name to order by") + {{:op, :",", _}, _} -> name_error(token, "a field name to order by") + {{:word, dir, _}, {:word, "nulls", pos}} when dir in ~w[asc desc] -> nulls(pos) + {{:word, dir, _}, _} when dir in ~w[asc desc] -> end_of_statement(token) + {_, {:op, :"(", pos}} -> {"functions are not supported", pos} + {_, {:op, :., pos}} -> {"qualified column names are not supported", pos} + {_, {:word, "nulls", pos}} -> nulls(pos) + _ -> end_of_statement(token) + end + end + + # -- LIMIT -- + + defp in_clause(t, i, {"limit", l}) do + token = elem(t, i) + + if i == l + 1, + do: {"LIMIT expects a whole number, got #{describe(token)}", pos(token)}, + else: end_of_statement(token) + end + + # -- WHERE -- + + defp in_clause(t, i, {"where", w}) do + token = elem(t, i) + prev = elem(t, i - 1) + + cond do + k = half_type(t, i) -> + at(t, k) + + # A type name can be any word (`x::or`); it is never a keyword. + type_position?(t, i - 1) -> + after_operand(t, w, i) + + word?(prev, "is") -> + expected("NULL after IS", token) + + word?(prev, "not") and word?(elem(t, i - 2), "is") -> + expected("NULL after IS NOT", token) + + word?(prev, "not") and not operand_expected_after?(elem(t, i - 2)) and + not word?(token, ~w[between in like ilike]) -> + at(t, i - 1) + + word?(prev, "cast") and not op?(token, :"(") -> + {"expected an expression, got 'cast'", pos(prev)} + + op?(prev, [:-, :+]) and operand_expected_after?(elem(t, i - 2)) -> + {"a sign is only supported on a number", pos(prev)} + + op?(prev, :"::") or (word?(prev, "as") and in_cast?(t, w, i)) -> + expected("a type name", token) + + word?(prev, "in") -> + expected("'(' after IN", token) + + op?(prev, :"(") and op?(token, :")") and word?(elem(t, i - 2), "in") -> + {"IN needs at least one value", pos(prev)} + + operand_expected_after?(prev) -> + case token do + {:word, "select", pos} -> {"subqueries are not supported", pos} + _ -> expected("an expression", token) + end + + true -> + after_operand(t, w, i) + end + end + + # Something complete comes before the token: a name, a literal, a + # parenthesized expression, a whole predicate. + defp after_operand(t, w, i) do + token = elem(t, i) + prev = elem(t, i - 1) + start = operand_start(t, i - 1) + before = elem(t, start - 1) + + cond do + # A tuple's `*`, the type in CAST(x AS type), or the end of a whole + # predicate: what follows is the group's business. + op?(prev, :*) or type_after_as?(t, i - 1) or ends_predicate?(t, i - 1) -> + in_group(t, w, i) + + name_primary?(t, i - 1) and op?(token, :"(") -> + {"functions are not supported", + if(match?({:quoted, _, _}, prev), do: pos(token), else: pos(prev))} + + name_primary?(t, i - 1) and op?(token, :.) -> + {"qualified column names are not supported", pos(token)} + + word?(before, "between") -> + expected("AND in BETWEEN", token) + + op?(token, @arithmetic) and not right_operand?(t, start) -> + {"arithmetic is not supported", pos(token)} + + true -> + in_group(t, w, i) + end + end + + # The innermost parenthesis still open says what was expected. + defp in_group(t, w, i) do + token = elem(t, i) + + case open_group(t, w, i) do + nil -> + end_of_statement(token) + + {_, :in_list} -> + expected("')' to close the IN list", token) + + {_, :tuple} -> + expected("')' to close the tuple", token) + + {_, :cast} -> + expected("AS in CAST", token) + + {_, :cast_as} -> + expected("')' to close CAST", token) + + {k, :paren} -> + if op?(elem(t, k + 1), :*), + do: {"'*' is only allowed in a tuple", pos(elem(t, k))}, + else: expected("')'", token) + end + end + + defp open_group(t, w, i) do + Enum.reduce((w + 1)..(i - 1)//1, [], fn j, stack -> + case {elem(t, j), stack} do + {{:op, :"(", _}, _} -> + kind = + case elem(t, j - 1) do + {:word, "in", _} -> :in_list + {:word, "cast", _} -> :cast + _ -> :paren + end + + [{j, kind} | stack] + + {{:op, :")", _}, [_ | rest]} -> + rest + + {{:op, :",", _}, [{k, :paren} | rest]} -> + [{k, :tuple} | rest] + + {{:word, "as", _}, [{k, :cast} | rest]} -> + [{k, :cast_as} | rest] + + _ -> + stack + end + end) + |> List.first() + end + + defp in_cast?(t, w, i), do: match?({_, :cast}, open_group(t, w, i - 1)) + + # -- the operand before an error -- + + # Where the operand ending at `j` starts: step back over `::type` casts, + # then over one primary. + defp operand_start(t, j) do + case type_start(t, j) do + nil -> + case elem(t, j) do + {:op, :")", _} -> + k = matching_open(t, j - 1, 0) + if word?(elem(t, k - 1), "cast"), do: k - 1, else: k + + {type, _, _} when type in [:integer, :float] -> + if op?(elem(t, j - 1), [:-, :+]) and operand_expected_after?(elem(t, j - 2)), + do: j - 1, + else: j + + _ -> + j + end + + k -> + operand_start(t, k - 1) + end + end + + # If a `::type` ends at `j`, the index of its `::`. + defp type_start(t, j) when j >= tuple_size(t), do: nil + + defp type_start(t, j) do + cond do + j >= 1 and op?(elem(t, j - 1), :"::") and match?({:word, _, _}, elem(t, j)) -> + j - 1 + + j >= 4 and op?(elem(t, j - 4), :"::") and word?(elem(t, j - 3), ~w[timestamp time]) and + word?(elem(t, j - 2), ~w[with without]) and word?(elem(t, j - 1), "time") and + word?(elem(t, j), "zone") -> + j - 4 + + true -> + nil + end + end + + defp matching_open(t, j, depth) do + case elem(t, j) do + {:op, :")", _} -> matching_open(t, j - 1, depth + 1) + {:op, :"(", _} when depth == 0 -> j + {:op, :"(", _} -> matching_open(t, j - 1, depth - 1) + _ -> matching_open(t, j - 1, depth) + end + end + + # The right-hand side of a comparison, LIKE or BETWEEN ... AND: the + # operand the predicate ends with, so the error is the group's. + defp right_operand?(t, start) do + case elem(t, start - 1) do + {:op, op, _} when op in @comparisons -> true + {:word, like, _} when like in ~w[like ilike] -> true + {:word, "and", _} -> between_and?(t, start - 1) + _ -> false + end + end + + # An AND belongs to BETWEEN when the nearest boolean boundary before it + # at the same depth is the BETWEEN. + defp between_and?(t, j), do: between_and?(t, j - 1, 0) + + defp between_and?(t, j, depth) do + case elem(t, j) do + {:op, :")", _} -> between_and?(t, j - 1, depth + 1) + {:op, :"(", _} when depth > 0 -> between_and?(t, j - 1, depth - 1) + {:word, "between", _} when depth == 0 -> true + {:word, word, _} when depth == 0 and word in ~w[and or where] -> false + {:op, op, _} when depth == 0 and op in [:"(", :","] -> false + _ -> between_and?(t, j - 1, depth) + end + end + + # A name standing as an expression, not a type name. + defp name_primary?(t, j) do + case elem(t, j) do + {:quoted, _, _} -> true + {:word, word, _} -> word not in @reserved and not type_position?(t, j) + _ -> false + end + end + + # A word after `::` or CAST's AS, or inside `timestamp with time zone`. + defp type_position?(t, j) do + op?(elem(t, j - 1), :"::") or word?(elem(t, j - 1), "as") or type_start(t, j) != nil or + (j >= 2 and word?(elem(t, j), ~w[with without time zone]) and + (type_start(t, j + 1) != nil or type_start(t, j + 2) != nil or + type_start(t, j + 3) != nil)) + end + + # `timestamp with` / `timestamp with time` cut short: the hand parser + # takes the type as `timestamp` and stops at the `with`. + defp half_type(t, i) do + cond do + i >= 3 and word?(elem(t, i - 1), ~w[with without]) and + word?(elem(t, i - 2), ~w[timestamp time]) and + type_opener?(elem(t, i - 3)) -> + i - 1 + + i >= 4 and word?(elem(t, i - 1), "time") and word?(elem(t, i - 2), ~w[with without]) and + word?(elem(t, i - 3), ~w[timestamp time]) and type_opener?(elem(t, i - 4)) -> + i - 2 + + true -> + nil + end + end + + defp type_opener?(token), do: op?(token, :"::") or word?(token, "as") + + # The last token of IN (...), IS [NOT] NULL, ISNULL or NOTNULL. + defp ends_predicate?(t, j) do + case elem(t, j) do + {:op, :")", _} -> word?(elem(t, matching_open(t, j - 1, 0) - 1), "in") + {:word, "null", _} -> word?(elem(t, j - 1), "is") or word?(elem(t, j - 2), "is") + {:word, word, _} -> word in ~w[isnull notnull] + _ -> false + end + end + + defp type_after_as?(t, j) do + word?(elem(t, j - 1), "as") or + (j >= 4 and word?(elem(t, j - 4), "as") and word?(elem(t, j - 3), ~w[timestamp time]) and + word?(elem(t, j - 2), ~w[with without]) and word?(elem(t, j - 1), "time") and + word?(elem(t, j), "zone")) + end + + # Tokens after which an operand must come. + defp operand_expected_after?(token) do + word?(token, ~w[where and or not between like ilike]) or + op?(token, [:"(", :"," | @comparisons]) + end + + # -- messages -- + + defp end_of_statement({:word, word, pos}) when is_map_key(@unsupported_clauses, word), + do: {"#{Map.fetch!(@unsupported_clauses, word)} is not supported", pos} + + defp end_of_statement({:op, op, pos}) when op in @arithmetic, + do: {"arithmetic is not supported", pos} + + defp end_of_statement(token), do: expected("end of statement", token) + + defp nulls(pos), do: {"NULLS FIRST/LAST is not supported", pos} + + defp name_error({:word, word, pos}, what) when word in @reserved do + {"expected #{what}, got the keyword '#{word}' (quote it, \"#{word}\", to use it as a name)", + pos} + end + + defp name_error(token, what), do: expected(what, token) + + defp expected(what, token), do: {"expected #{what}, got #{describe(token)}", pos(token)} + + defp describe({:word, word, _}), do: "'#{word}'" + defp describe({:quoted, name, _}), do: inspect(name) + defp describe({:string, s, _}), do: "the string '#{truncate(s)}'" + defp describe({type, n, _}) when type in [:integer, :float], do: "the number #{n}" + defp describe({:op, op, _}), do: "'#{op}'" + defp describe({:eof, _, _}), do: "the end of the statement" + + defp truncate(s) do + if String.length(s) > 20, do: String.slice(s, 0, 20) <> "...", else: s + end + + defp pos({_, _, pos}), do: pos + + defp word?({:word, word, _}, words) when is_list(words), do: word in words + defp word?({:word, word, _}, word), do: true + defp word?(_, _), do: false + + defp op?({:op, op, _}, ops) when is_list(ops), do: op in ops + defp op?({:op, op, _}, op), do: true + defp op?(_, _), do: false +end diff --git a/src/efsql_sql_leex.xrl b/src/efsql_sql_leex.xrl new file mode 100644 index 0000000..1d993ce --- /dev/null +++ b/src/efsql_sql_leex.xrl @@ -0,0 +1,57 @@ +%% Tokenizer for efsql's SQL dialect, driven by Efsql.SQL.Leex. +%% +%% Every rule returns a token carrying the matched characters, and nothing +%% is skipped: the driver needs each match to track character-accurate +%% positions (leex's own columns count UTF-8 bytes), and it finishes the +%% work leex can't express: nested block comments, Unicode letter classes, +%% and the checks on numbers and words. Tokens are +%% {Kind, Chars} with Kind one of: ws, line_comment, block_comment, +%% string, quoted, unterminated, number, bad_exponent, number_junk, word, +%% op, colon. +%% +%% Invalid UTF-8 bytes reach this lexer as integers above 16#10FFFF; no +%% positive class includes them, so outside strings and comments they are +%% illegal characters. + +Definitions. + +D = [0-9] +NUM = ({D}+\.?{D}*|\.{D}+) +%% Macros are pasted in as text, so a macro used with a postfix operator +%% needs its own parentheses. +EXP = ([eE][+-]?{D}+) +WS = [\s\t\f\v\r\n\x{A0}\x{1680}\x{2000}-\x{200A}\x{202F}\x{205F}\x{3000}\x{FEFF}] +ID = [A-Za-z_\x{80}-\x{10FFFF}][A-Za-z0-9_$\x{80}-\x{10FFFF}]* + +Rules. + +{WS}+ : {token, {ws, TokenChars}}. + +-- : {token, {line_comment, TokenChars}}. +/\* : {token, {block_comment, TokenChars}}. + +'([^']|'')*' : {token, {string, TokenChars}}. +"([^"]|"")*" : {token, {quoted, TokenChars}}. +'([^']|'')* : {token, {unterminated, TokenChars}}. +"([^"]|"")* : {token, {unterminated, TokenChars}}. + +%% Longest match picks the number, then an exponent with no digits, then +%% a number running into another character (which the driver checks). +{NUM}{EXP}? : {token, {number, TokenChars}}. +{NUM}[eE][+-]? : {token, {bad_exponent, TokenChars}}. +{NUM}{EXP}?[A-Za-z_.0-9\x{80}-\x{10FFFF}] : {token, {number_junk, TokenChars}}. + +{ID} : {token, {word, TokenChars}}. + +<> : {token, {op, TokenChars}}. +!= : {token, {op, TokenChars}}. +<= : {token, {op, TokenChars}}. +>= : {token, {op, TokenChars}}. +:: : {token, {op, TokenChars}}. +[=<>(),.;*+\-/%] : {token, {op, TokenChars}}. + +%% A lone colon, or one followed by anything: the driver explains it. +:[\x{0}-\x{10FFFF}] : {token, {colon, TokenChars}}. +: : {token, {colon, TokenChars}}. + +Erlang code. diff --git a/src/efsql_sql_yecc.yrl b/src/efsql_sql_yecc.yrl new file mode 100644 index 0000000..af0c489 --- /dev/null +++ b/src/efsql_sql_yecc.yrl @@ -0,0 +1,200 @@ +%% Grammar for efsql's SQL dialect, driven by Efsql.SQL.Yecc. +%% +%% SELECT fields FROM name [WHERE expr] [ORDER BY items] [LIMIT n] [;] +%% +%% Tokens are {Category, {Line, Column}, Value}; `/` and `%` have no +%% terminal, so they are syntax errors wherever they appear. Words the driver finds in +%% the reserved list come as their own category; `time`, `timestamp`, +%% `without` and `zone` do too, but stay usable as names. Everything else +%% is `ident`. +%% +%% The grammar accepts exactly the valid statements; explaining an error +%% in words is the driver's job, since yecc only reports the token it +%% stopped at. Actions build the same tree as Efsql.SQL.Parser (see +%% Efsql.SQL.AST), with the statement as a tuple the driver makes a struct. + +Nonterminals +statement select_stmt fields field_list table where_clause order_clause order_items order_item limit_clause +expr or_expr and_expr not_expr predicate operand primary name +type_name type_word keyword element elements expr_list comparison. + +Terminals +ident quoted string integer float +'all' 'and' 'as' 'asc' 'between' 'by' 'case' 'cast' 'cross' 'desc' 'distinct' +'else' 'end' 'except' 'false' 'from' 'full' 'group' 'having' 'ilike' 'in' +'inner' 'intersect' 'is' 'isnull' 'join' 'left' 'like' 'limit' 'not' 'notnull' +'null' 'offset' 'on' 'or' 'order' 'right' 'select' 'then' 'true' 'union' +'when' 'where' 'with' +'time' 'timestamp' 'without' 'zone' +'=' '<>' '<' '>' '<=' '>=' '::' '(' ')' ',' '.' ';' '*' '+' '-'. + +Rootsymbol statement. +Endsymbol '$end'. + +statement -> select_stmt : '$1'. +statement -> select_stmt ';' : '$1'. + +select_stmt -> 'select' fields 'from' table where_clause order_clause limit_clause : + {select, '$2', '$4', '$5', '$6', '$7'}. + +fields -> '*' : star. +fields -> field_list : '$1'. + +field_list -> name : ['$1']. +field_list -> name ',' field_list : ['$1' | '$3']. + +table -> name : ['$1']. +table -> name '.' name : ['$1', '$3']. +table -> name '.' name '.' name : ['$1', '$3', '$5']. + +where_clause -> '$empty' : nil. +where_clause -> 'where' expr : '$2'. + +order_clause -> '$empty' : []. +order_clause -> 'order' 'by' order_items : '$3'. + +order_items -> order_item : ['$1']. +order_items -> order_item ',' order_items : ['$1' | '$3']. + +order_item -> name : {'$1', asc}. +order_item -> name 'asc' : {'$1', asc}. +order_item -> name 'desc' : {'$1', desc}. + +limit_clause -> '$empty' : nil. +limit_clause -> 'limit' integer : value('$2'). + +%% Loosest first: OR, AND, NOT, then a predicate. +expr -> or_expr : '$1'. + +or_expr -> or_expr 'or' and_expr : {'or', '$1', '$3'}. +or_expr -> and_expr : '$1'. + +and_expr -> and_expr 'and' not_expr : {'and', '$1', '$3'}. +and_expr -> not_expr : '$1'. + +not_expr -> 'not' not_expr : {'not', '$2'}. +not_expr -> predicate : '$1'. + +predicate -> operand comparison operand : {compare, '$2', '$1', '$3'}. +predicate -> operand 'between' operand 'and' operand : {between, '$1', '$3', '$5', false}. +predicate -> operand 'not' 'between' operand 'and' operand : {between, '$1', '$4', '$6', true}. +predicate -> operand 'in' '(' expr_list ')' : {in, '$1', '$4', false}. +predicate -> operand 'not' 'in' '(' expr_list ')' : {in, '$1', '$5', true}. +predicate -> operand 'like' operand : {like, '$1', '$3', false}. +predicate -> operand 'not' 'like' operand : {like, '$1', '$4', true}. +predicate -> operand 'ilike' operand : {ilike, '$1', '$3', false}. +predicate -> operand 'not' 'ilike' operand : {ilike, '$1', '$4', true}. +predicate -> operand 'is' 'null' : {is_null, '$1', false}. +predicate -> operand 'is' 'not' 'null' : {is_null, '$1', true}. +predicate -> operand 'isnull' : {is_null, '$1', false}. +predicate -> operand 'notnull' : {is_null, '$1', true}. +predicate -> operand : '$1'. + +comparison -> '=' : '='. +comparison -> '<>' : '<>'. +comparison -> '<' : '<'. +comparison -> '>' : '>'. +comparison -> '<=' : '<='. +comparison -> '>=' : '>='. + +operand -> primary : '$1'. +operand -> operand '::' type_name : {cast, '$1', '$3'}. + +primary -> string : {literal, value('$1')}. +primary -> integer : {literal, value('$1')}. +primary -> float : {literal, value('$1')}. +primary -> '-' integer : {literal, -value('$2')}. +primary -> '-' float : {literal, -value('$2')}. +primary -> '+' integer : {literal, value('$2')}. +primary -> '+' float : {literal, value('$2')}. +primary -> 'true' : {literal, true}. +primary -> 'false' : {literal, false}. +primary -> 'null' : {literal, nil}. +primary -> name : {column, '$1'}. +primary -> 'cast' '(' expr 'as' type_name ')' : {cast, '$3', '$5'}. +primary -> '(' expr ')' : '$2'. +primary -> '(' element ',' elements ')' : {tuple, ['$2' | '$4']}. + +%% A tuple element may be `*`: ('partition', *). +element -> expr : '$1'. +element -> '*' : star. + +elements -> element : ['$1']. +elements -> element ',' elements : ['$1' | '$3']. + +expr_list -> expr : ['$1']. +expr_list -> expr ',' expr_list : ['$1' | '$3']. + +%% Names: identifiers, quoted names, and the keywords that aren't reserved. +name -> ident : value('$1'). +name -> quoted : value('$1'). +name -> 'time' : <<"time">>. +name -> 'timestamp' : <<"timestamp">>. +name -> 'without' : <<"without">>. +name -> 'zone' : <<"zone">>. + +%% A type is any word, reserved ones included, with PostgreSQL's +%% `timestamp with[out] time zone` spelled out. +type_name -> type_word : '$1'. +type_name -> 'timestamp' 'with' 'time' 'zone' : <<"timestamptz">>. +type_name -> 'timestamp' 'without' 'time' 'zone' : <<"timestamp">>. +type_name -> 'time' 'with' 'time' 'zone' : <<"timetz">>. +type_name -> 'time' 'without' 'time' 'zone' : <<"time">>. + +type_word -> ident : value('$1'). +type_word -> keyword : atom_to_binary(category('$1')). + +keyword -> 'all' : '$1'. +keyword -> 'and' : '$1'. +keyword -> 'as' : '$1'. +keyword -> 'asc' : '$1'. +keyword -> 'between' : '$1'. +keyword -> 'by' : '$1'. +keyword -> 'case' : '$1'. +keyword -> 'cast' : '$1'. +keyword -> 'cross' : '$1'. +keyword -> 'desc' : '$1'. +keyword -> 'distinct' : '$1'. +keyword -> 'else' : '$1'. +keyword -> 'end' : '$1'. +keyword -> 'except' : '$1'. +keyword -> 'false' : '$1'. +keyword -> 'from' : '$1'. +keyword -> 'full' : '$1'. +keyword -> 'group' : '$1'. +keyword -> 'having' : '$1'. +keyword -> 'ilike' : '$1'. +keyword -> 'in' : '$1'. +keyword -> 'inner' : '$1'. +keyword -> 'intersect' : '$1'. +keyword -> 'is' : '$1'. +keyword -> 'isnull' : '$1'. +keyword -> 'join' : '$1'. +keyword -> 'left' : '$1'. +keyword -> 'like' : '$1'. +keyword -> 'limit' : '$1'. +keyword -> 'not' : '$1'. +keyword -> 'notnull' : '$1'. +keyword -> 'null' : '$1'. +keyword -> 'offset' : '$1'. +keyword -> 'on' : '$1'. +keyword -> 'or' : '$1'. +keyword -> 'order' : '$1'. +keyword -> 'right' : '$1'. +keyword -> 'select' : '$1'. +keyword -> 'then' : '$1'. +keyword -> 'true' : '$1'. +keyword -> 'union' : '$1'. +keyword -> 'when' : '$1'. +keyword -> 'where' : '$1'. +keyword -> 'with' : '$1'. +keyword -> 'time' : '$1'. +keyword -> 'timestamp' : '$1'. +keyword -> 'without' : '$1'. +keyword -> 'zone' : '$1'. + +Erlang code. + +value({_Category, _Location, Value}) -> Value. + +category({Category, _Location, _Value}) -> Category. diff --git a/test/sql/differential_test.exs b/test/sql/differential_test.exs new file mode 100644 index 0000000..46c9238 --- /dev/null +++ b/test/sql/differential_test.exs @@ -0,0 +1,28 @@ +defmodule Efsql.SQL.DifferentialTest do + # The two front ends side by side: the hand-written lexer and parser, and + # the ones built on leex and yecc, must give the same result for any + # input: the same tokens, the same tree, or the same error at the same + # position. Inputs are valid queries, mangled ones and garbage in equal + # measure; reproduce a failure with `mix test --seed N`. + use ExUnit.Case, async: true + + alias EfsqlTest.SQLGen + + @runs 3_000 + + test "the lexers agree on every input" do + for _ <- 1..@runs do + sql = SQLGen.any_input() + + assert Efsql.SQL.Leex.tokenize(sql) == Efsql.SQL.Lexer.tokenize(sql), + "input: #{inspect(sql)}" + end + end + + test "the parsers agree on every input" do + for _ <- 1..@runs do + sql = SQLGen.any_input() + assert Efsql.SQL.Yecc.parse(sql) == Efsql.SQL.Parser.parse(sql), "input: #{inspect(sql)}" + end + end +end diff --git a/test/sql/lexer_test.exs b/test/sql/lexer_test.exs index 2dddf23..02d54ce 100644 --- a/test/sql/lexer_test.exs +++ b/test/sql/lexer_test.exs @@ -1,108 +1,111 @@ defmodule Efsql.SQL.LexerTest do - use ExUnit.Case, async: true + # Every test runs against both lexers: the hand-written one and the one + # built on leex. They share a contract, token for token. + use ExUnit.Case, + async: true, + parameterize: [%{lexer: Efsql.SQL.Lexer}, %{lexer: Efsql.SQL.Leex}] - alias Efsql.SQL.Lexer alias Efsql.SQL.SyntaxError - defp tokens(sql) do - {:ok, tokens} = Lexer.tokenize(sql) + defp tokens(lexer, sql) do + {:ok, tokens} = lexer.tokenize(sql) tokens |> Enum.map(fn {type, value, _pos} -> {type, value} end) |> Enum.drop(-1) end - defp positions(sql) do - {:ok, tokens} = Lexer.tokenize(sql) + defp positions(lexer, sql) do + {:ok, tokens} = lexer.tokenize(sql) Enum.map(tokens, fn {_type, _value, pos} -> pos end) end - defp error(sql) do - assert {:error, %SyntaxError{} = e} = Lexer.tokenize(sql) + defp error(lexer, sql) do + assert {:error, %SyntaxError{} = e} = lexer.tokenize(sql) e end - test "empty and blank input is just end of input" do - assert {:ok, [{:eof, nil, {1, 1}}]} = Lexer.tokenize("") - assert {:ok, [{:eof, nil, {2, 3}}]} = Lexer.tokenize(" \n ") + test "empty and blank input is just end of input", %{lexer: lexer} do + assert {:ok, [{:eof, nil, {1, 1}}]} = lexer.tokenize("") + assert {:ok, [{:eof, nil, {2, 3}}]} = lexer.tokenize(" \n ") end describe "words" do - test "fold to lower case" do - assert tokens("SELECT Name fRoM") == [word: "select", word: "name", word: "from"] + test "fold to lower case", %{lexer: lexer} do + assert tokens(lexer, "SELECT Name fRoM") == [word: "select", word: "name", word: "from"] end - test "take underscores, digits and dollars after the first character" do - assert tokens("_ _id a1 a$b created_at") == + test "take underscores, digits and dollars after the first character", %{lexer: lexer} do + assert tokens(lexer, "_ _id a1 a$b created_at") == [word: "_", word: "_id", word: "a1", word: "a$b", word: "created_at"] end - test "take Unicode letters and marks" do - assert tokens("café 日本語 Ωmega") == [word: "café", word: "日本語", word: "ωmega"] + test "take Unicode letters and marks", %{lexer: lexer} do + assert tokens(lexer, "café 日本語 Ωmega") == [word: "café", word: "日本語", word: "ωmega"] end - test "keywords are plain words; the parser decides" do - assert tokens("day at user order") == + test "keywords are plain words; the parser decides", %{lexer: lexer} do + assert tokens(lexer, "day at user order") == [word: "day", word: "at", word: "user", word: "order"] end end describe "quoted identifiers" do - test "keep case and spaces" do - assert tokens(~s("Mixed Case")) == [quoted: "Mixed Case"] + test "keep case and spaces", %{lexer: lexer} do + assert tokens(lexer, ~s("Mixed Case")) == [quoted: "Mixed Case"] end - test "a doubled quote is a quote" do - assert tokens(~s("say ""hi""")) == [quoted: ~s(say "hi")] + test "a doubled quote is a quote", %{lexer: lexer} do + assert tokens(lexer, ~s("say ""hi""")) == [quoted: ~s(say "hi")] end - test "may span lines and hold keywords" do - assert tokens(~s("select\nfrom")) == [quoted: "select\nfrom"] + test "may span lines and hold keywords", %{lexer: lexer} do + assert tokens(lexer, ~s("select\nfrom")) == [quoted: "select\nfrom"] end - test "can't be empty" do + test "can't be empty", %{lexer: lexer} do assert %SyntaxError{reason: "zero-length quoted identifier", line: 1, column: 8} = - error(~s(select "")) + error(lexer, ~s(select "")) end - test "must be closed" do - assert %SyntaxError{line: 1, column: 8, reason: reason} = error(~s(select "abc)) + test "must be closed", %{lexer: lexer} do + assert %SyntaxError{line: 1, column: 8, reason: reason} = error(lexer, ~s(select "abc)) assert reason =~ "unterminated identifier" end end describe "strings" do - test "keep their content verbatim" do - assert tokens("'Hello, World'") == [string: "Hello, World"] + test "keep their content verbatim", %{lexer: lexer} do + assert tokens(lexer, "'Hello, World'") == [string: "Hello, World"] end - test "a doubled quote is a quote" do - assert tokens("'it''s'") == [string: "it's"] - assert tokens("''''") == [string: "'"] - assert tokens("''") == [string: ""] + test "a doubled quote is a quote", %{lexer: lexer} do + assert tokens(lexer, "'it''s'") == [string: "it's"] + assert tokens(lexer, "''''") == [string: "'"] + assert tokens(lexer, "''") == [string: ""] end - test "hide comment and operator characters" do - assert tokens("'-- not a comment /* nor this */ ; ::'") == + test "hide comment and operator characters", %{lexer: lexer} do + assert tokens(lexer, "'-- not a comment /* nor this */ ; ::'") == [string: "-- not a comment /* nor this */ ; ::"] end - test "may span lines, and positions continue after them" do - assert tokens("'a\nb' x") == [string: "a\nb", word: "x"] - assert positions("'a\nb' x") == [{1, 1}, {2, 4}, {2, 5}] + test "may span lines, and positions continue after them", %{lexer: lexer} do + assert tokens(lexer, "'a\nb' x") == [string: "a\nb", word: "x"] + assert positions(lexer, "'a\nb' x") == [{1, 1}, {2, 4}, {2, 5}] end - test "take any Unicode" do - assert tokens("'naïve ☃ 𝄞'") == [string: "naïve ☃ 𝄞"] + test "take any Unicode", %{lexer: lexer} do + assert tokens(lexer, "'naïve ☃ 𝄞'") == [string: "naïve ☃ 𝄞"] end - test "must be closed" do - e = error("where x = 'abc") + test "must be closed", %{lexer: lexer} do + e = error(lexer, "where x = 'abc") assert e.reason =~ "unterminated string" assert {e.line, e.column} == {1, 11} end end describe "numbers" do - test "integers" do - assert tokens("0 42 007 123456789012345678901234567890") == + test "integers", %{lexer: lexer} do + assert tokens(lexer, "0 42 007 123456789012345678901234567890") == [ integer: 0, integer: 42, @@ -111,8 +114,8 @@ defmodule Efsql.SQL.LexerTest do ] end - test "floats in every spelling" do - assert tokens("1.5 .5 1. 1e3 1E3 1.5e-3 2.5E+2") == + test "floats in every spelling", %{lexer: lexer} do + assert tokens(lexer, "1.5 .5 1. 1e3 1E3 1.5e-3 2.5E+2") == [ float: 1.5, float: 0.5, @@ -124,33 +127,33 @@ defmodule Efsql.SQL.LexerTest do ] end - test "are unsigned; a sign is a separate token" do - assert tokens("-1 +2") == [op: :-, integer: 1, op: :+, integer: 2] + test "are unsigned; a sign is a separate token", %{lexer: lexer} do + assert tokens(lexer, "-1 +2") == [op: :-, integer: 1, op: :+, integer: 2] end - test "can't run into a word" do - assert error("123abc").reason =~ "invalid number" - assert error("1.5.2").reason =~ "invalid number" + test "can't run into a word", %{lexer: lexer} do + assert error(lexer, "123abc").reason =~ "invalid number" + assert error(lexer, "1.5.2").reason =~ "invalid number" end - test "out of a double's range" do - assert error("1e400").reason == "number out of range: 1e400" - assert tokens("1e-400") == [float: 0.0] + test "out of a double's range", %{lexer: lexer} do + assert error(lexer, "1e400").reason == "number out of range: 1e400" + assert tokens(lexer, "1e-400") == [float: 0.0] end - test "need exponent digits" do - assert error("1e").reason =~ "exponent has no digits" - assert error("1e+x").reason =~ "exponent has no digits" + test "need exponent digits", %{lexer: lexer} do + assert error(lexer, "1e").reason =~ "exponent has no digits" + assert error(lexer, "1e+x").reason =~ "exponent has no digits" end end describe "operators" do - test "two-character operators win over one-character ones" do - assert tokens("<> != <= >= ::") == [op: :<>, op: :<>, op: :<=, op: :>=, op: :"::"] + test "two-character operators win over one-character ones", %{lexer: lexer} do + assert tokens(lexer, "<> != <= >= ::") == [op: :<>, op: :<>, op: :<=, op: :>=, op: :"::"] end - test "one-character operators" do - assert tokens("= < > ( ) , . ; * + - / %") == + test "one-character operators", %{lexer: lexer} do + assert tokens(lexer, "= < > ( ) , . ; * + - / %") == [ op: :=, op: :<, @@ -168,76 +171,81 @@ defmodule Efsql.SQL.LexerTest do ] end - test "need no surrounding space" do - assert tokens("a>=1") == [word: "a", op: :>=, integer: 1] - assert tokens("'x'::atom") == [string: "x", op: :"::", word: "atom"] - assert tokens("t.users") == [word: "t", op: :., word: "users"] + test "need no surrounding space", %{lexer: lexer} do + assert tokens(lexer, "a>=1") == [word: "a", op: :>=, integer: 1] + assert tokens(lexer, "'x'::atom") == [string: "x", op: :"::", word: "atom"] + assert tokens(lexer, "t.users") == [word: "t", op: :., word: "users"] end end describe "comments and whitespace" do - test "line comments run to the end of the line" do - assert tokens("a -- ignored ; 'x'\nb") == [word: "a", word: "b"] - assert tokens("a --") == [word: "a"] + test "line comments run to the end of the line", %{lexer: lexer} do + assert tokens(lexer, "a -- ignored ; 'x'\nb") == [word: "a", word: "b"] + assert tokens(lexer, "a --") == [word: "a"] end - test "block comments, which nest" do - assert tokens("a /* one /* two */ still */ b") == [word: "a", word: "b"] + test "the end of input after a line comment is positioned after it", %{lexer: lexer} do + assert positions(lexer, "a -- é\xFFz") == [{1, 1}, {1, 9}] + assert positions(lexer, "a --\nb") == [{1, 1}, {2, 1}, {2, 2}] end - test "a block comment can split tokens" do - assert tokens("a/**/b") == [word: "a", word: "b"] + test "block comments, which nest", %{lexer: lexer} do + assert tokens(lexer, "a /* one /* two */ still */ b") == [word: "a", word: "b"] end - test "an unclosed block comment is an error at its start" do - assert %SyntaxError{line: 2, column: 3} = error("a\n /* /* */") + test "a block comment can split tokens", %{lexer: lexer} do + assert tokens(lexer, "a/**/b") == [word: "a", word: "b"] end - test "tabs, CRLF and Unicode spaces separate tokens" do - assert tokens("a\tb\r\nc\rd e f") == + test "an unclosed block comment is an error at its start", %{lexer: lexer} do + assert %SyntaxError{line: 2, column: 3} = error(lexer, "a\n /* /* */") + end + + test "tabs, CRLF and Unicode spaces separate tokens", %{lexer: lexer} do + assert tokens(lexer, "a\tb\r\nc\rd e f") == [word: "a", word: "b", word: "c", word: "d", word: "e", word: "f"] end - test "a byte-order mark is ignored" do - assert tokens("select") == [word: "select"] + test "a byte-order mark is ignored", %{lexer: lexer} do + assert tokens(lexer, "select") == [word: "select"] end end describe "positions" do - test "are line and column of the token's first character" do - assert positions("select id\nfrom t") == [{1, 1}, {1, 8}, {2, 1}, {2, 6}, {2, 7}] + test "are line and column of the token's first character", %{lexer: lexer} do + assert positions(lexer, "select id\nfrom t") == [{1, 1}, {1, 8}, {2, 1}, {2, 6}, {2, 7}] end - test "count characters, not bytes" do - assert positions("'é' x") == [{1, 1}, {1, 5}, {1, 6}] + test "count characters, not bytes", %{lexer: lexer} do + assert positions(lexer, "'é' x") == [{1, 1}, {1, 5}, {1, 6}] end - test "CRLF is one line break" do - assert positions("a\r\nb") == [{1, 1}, {2, 1}, {2, 2}] + test "CRLF is one line break", %{lexer: lexer} do + assert positions(lexer, "a\r\nb") == [{1, 1}, {2, 1}, {2, 2}] end - test "comments advance them" do - assert positions("/* a\nb */ x -- c\ny") == [{2, 6}, {3, 1}, {3, 2}] + test "comments advance them", %{lexer: lexer} do + assert positions(lexer, "/* a\nb */ x -- c\ny") == [{2, 6}, {3, 1}, {3, 2}] end end describe "errors" do - test "an unknown character, with its position" do + test "an unknown character, with its position", %{lexer: lexer} do assert %SyntaxError{reason: ~s(unexpected character "@"), line: 2, column: 3} = - error("a\nb @") + error(lexer, "a\nb @") end - test "Elixir atom syntax gets a hint" do - assert error("where x = :active").reason =~ "'name'::atom" + test "Elixir atom syntax gets a hint", %{lexer: lexer} do + assert error(lexer, "where x = :active").reason =~ "'name'::atom" end - test "invalid UTF-8" do - assert error(<<"a ", 0xFF, " b">>).reason =~ "invalid UTF-8" - assert error(<<"'", 0xC3, "'">>).reason =~ "invalid UTF-8" + test "invalid UTF-8", %{lexer: lexer} do + assert error(lexer, <<"a ", 0xFF, " b">>).reason =~ "invalid UTF-8" + assert error(lexer, <<"'", 0xC3, "'">>).reason =~ "invalid UTF-8" end - test "format as a message" do - assert Exception.message(error("@")) == + test "format as a message", %{lexer: lexer} do + assert Exception.message(error(lexer, "@")) == ~s(syntax error at line 1, column 1: unexpected character "@") end end diff --git a/test/sql/parser_test.exs b/test/sql/parser_test.exs index 41df49e..b120959 100644 --- a/test/sql/parser_test.exs +++ b/test/sql/parser_test.exs @@ -1,32 +1,37 @@ defmodule Efsql.SQL.ParserTest do - use ExUnit.Case, async: true + # Every test runs against both parsers: the hand-written recursive + # descent one and the one built on leex and yecc. They share a contract: + # the same tree, and the same error message at the same position. + use ExUnit.Case, + async: true, + parameterize: [%{parser: Efsql.SQL.Parser}, %{parser: Efsql.SQL.Yecc}] alias Efsql.SQL.AST.Select alias Efsql.SQL.Parser alias Efsql.SQL.SyntaxError - defp parse(sql) do - assert {:ok, %Select{} = select} = Parser.parse(sql) + defp parse(parser, sql) do + assert {:ok, %Select{} = select} = parser.parse(sql) select end - defp where(condition), do: parse("select a from t where #{condition}").where + defp where(parser, condition), do: parse(parser, "select a from t where #{condition}").where - defp error(sql) do - assert {:error, %SyntaxError{} = e} = Parser.parse(sql) + defp error(parser, sql) do + assert {:error, %SyntaxError{} = e} = parser.parse(sql) e end - defp reason(sql), do: error(sql).reason + defp reason(parser, sql), do: error(parser, sql).reason describe "statement" do - test "the smallest query" do - assert parse("select * from t") == + test "the smallest query", %{parser: parser} do + assert parse(parser, "select * from t") == %Select{fields: :star, from: ["t"], where: nil, order_by: [], limit: nil} end - test "every clause" do - assert parse("select a, b from s.t where a = 1 order by b desc limit 5;") == + test "every clause", %{parser: parser} do + assert parse(parser, "select a, b from s.t where a = 1 order by b desc limit 5;") == %Select{ fields: ["a", "b"], from: ["s", "t"], @@ -36,70 +41,78 @@ defmodule Efsql.SQL.ParserTest do } end - test "the semicolon is optional" do - assert parse("select * from t") == parse("select * from t;") + test "the semicolon is optional", %{parser: parser} do + assert parse(parser, "select * from t") == parse(parser, "select * from t;") end - test "keywords are case-insensitive, names fold to lower case" do - assert parse("SELECT Id FROM Tenant.Users WHERE Name = 'Al' ORDER BY Id DESC LIMIT 1") == - parse("select id from tenant.users where name = 'Al' order by id desc limit 1") + test "keywords are case-insensitive, names fold to lower case", %{parser: parser} do + assert parse( + parser, + "SELECT Id FROM Tenant.Users WHERE Name = 'Al' ORDER BY Id DESC LIMIT 1" + ) == + parse( + parser, + "select id from tenant.users where name = 'Al' order by id desc limit 1" + ) end - test "quoted names keep their case" do + test "quoted names keep their case", %{parser: parser} do assert %Select{fields: ["Id"], from: ["My Tenant", "Users"]} = - parse(~s(select "Id" from "My Tenant"."Users")) + parse(parser, ~s(select "Id" from "My Tenant"."Users")) end - test "comments and whitespace anywhere" do - assert parse(""" + test "comments and whitespace anywhere", %{parser: parser} do + assert parse(parser, """ -- leading comment select /* fields */ a, b from t -- table where a = 1 /* trailing */ ; - """) == parse("select a, b from t where a = 1") + """) == parse(parser, "select a, b from t where a = 1") end end describe "FROM" do - test "one, two or three parts" do - assert %Select{from: ["t"]} = parse("select * from t") - assert %Select{from: ["tenant", "t"]} = parse("select * from tenant.t") - assert %Select{from: ["storage", "tenant", "t"]} = parse("select * from storage.tenant.t") + test "one, two or three parts", %{parser: parser} do + assert %Select{from: ["t"]} = parse(parser, "select * from t") + assert %Select{from: ["tenant", "t"]} = parse(parser, "select * from tenant.t") + + assert %Select{from: ["storage", "tenant", "t"]} = + parse(parser, "select * from storage.tenant.t") end - test "any non-reserved word is a name" do - assert %Select{from: ["user", "date"]} = parse("select * from user.date") + test "any non-reserved word is a name", %{parser: parser} do + assert %Select{from: ["user", "date"]} = parse(parser, "select * from user.date") end - test "four parts is too many" do - assert reason("select * from a.b.c.d") =~ "at most three parts" + test "four parts is too many", %{parser: parser} do + assert reason(parser, "select * from a.b.c.d") =~ "at most three parts" end - test "more than one table" do - assert reason("select * from a, b") =~ "more than one table" + test "more than one table", %{parser: parser} do + assert reason(parser, "select * from a, b") =~ "more than one table" end end describe "SELECT list" do - test "star or names" do - assert %Select{fields: :star} = parse("select * from t") - assert %Select{fields: ["a", "b", "_"]} = parse("select a, b, _ from t") + test "star or names", %{parser: parser} do + assert %Select{fields: :star} = parse(parser, "select * from t") + assert %Select{fields: ["a", "b", "_"]} = parse(parser, "select a, b, _ from t") end - test "star can't be mixed with names" do - assert reason("select *, a from t") =~ "can't be combined" + test "star can't be mixed with names", %{parser: parser} do + assert reason(parser, "select *, a from t") =~ "can't be combined" end - test "unsupported shapes say what is unsupported" do - assert reason("select distinct a from t") == "DISTINCT is not supported" - assert reason("select count(a) from t") == "functions are not supported" - assert reason("select t.a from t") == "qualified column names are not supported" + test "unsupported shapes say what is unsupported", %{parser: parser} do + assert reason(parser, "select distinct a from t") == "DISTINCT is not supported" + assert reason(parser, "select count(a) from t") == "functions are not supported" + assert reason(parser, "select t.a from t") == "qualified column names are not supported" end - test "a missing FROM" do + test "a missing FROM", %{parser: parser} do assert %SyntaxError{reason: "expected FROM, got the end of the statement", column: 9} = - error("select a") + error(parser, "select a") end end @@ -113,305 +126,329 @@ defmodule Efsql.SQL.ParserTest do {"<=", :<=}, {">=", :>=} ] do - test "#{sql}" do - assert where("a #{unquote(sql)} 1") == + test "#{sql}", %{parser: parser} do + assert where(parser, "a #{unquote(sql)} 1") == {:compare, unquote(op), {:column, "a"}, {:literal, 1}} end end - test "either side can be the value" do - assert where("1 < a") == {:compare, :<, {:literal, 1}, {:column, "a"}} + test "either side can be the value", %{parser: parser} do + assert where(parser, "1 < a") == {:compare, :<, {:literal, 1}, {:column, "a"}} end - test "literals" do - assert where("a = 'x'") == {:compare, :=, {:column, "a"}, {:literal, "x"}} - assert where("a = -3") == {:compare, :=, {:column, "a"}, {:literal, -3}} - assert where("a = +3") == {:compare, :=, {:column, "a"}, {:literal, 3}} - assert where("a = -1.5") == {:compare, :=, {:column, "a"}, {:literal, -1.5}} - assert where("a = true") == {:compare, :=, {:column, "a"}, {:literal, true}} - assert where("a = FALSE") == {:compare, :=, {:column, "a"}, {:literal, false}} - assert where("a = null") == {:compare, :=, {:column, "a"}, {:literal, nil}} + test "literals", %{parser: parser} do + assert where(parser, "a = 'x'") == {:compare, :=, {:column, "a"}, {:literal, "x"}} + assert where(parser, "a = -3") == {:compare, :=, {:column, "a"}, {:literal, -3}} + assert where(parser, "a = +3") == {:compare, :=, {:column, "a"}, {:literal, 3}} + assert where(parser, "a = -1.5") == {:compare, :=, {:column, "a"}, {:literal, -1.5}} + assert where(parser, "a = true") == {:compare, :=, {:column, "a"}, {:literal, true}} + assert where(parser, "a = FALSE") == {:compare, :=, {:column, "a"}, {:literal, false}} + assert where(parser, "a = null") == {:compare, :=, {:column, "a"}, {:literal, nil}} end - test "a sign only goes on a number" do - assert reason("select a from t where a = -b") =~ "only supported on a number" + test "a sign only goes on a number", %{parser: parser} do + assert reason(parser, "select a from t where a = -b") =~ "only supported on a number" end - test "arithmetic is refused, not misread" do - assert reason("select a from t where a = 1 + 2") == "arithmetic is not supported" - assert reason("select a from t where a + 1 = 2") == "arithmetic is not supported" + test "arithmetic is refused, not misread", %{parser: parser} do + assert reason(parser, "select a from t where a = 1 + 2") == "arithmetic is not supported" + assert reason(parser, "select a from t where a + 1 = 2") == "arithmetic is not supported" end - test "comparisons don't chain" do - assert reason("select a from t where a = 1 = 2") =~ "expected end of statement, got '='" + test "comparisons don't chain", %{parser: parser} do + assert reason(parser, "select a from t where a = 1 = 2") =~ + "expected end of statement, got '='" end end describe "predicates" do - test "BETWEEN binds its own AND" do - assert where("a between 1 and 2 and b = 3") == + test "BETWEEN binds its own AND", %{parser: parser} do + assert where(parser, "a between 1 and 2 and b = 3") == {:and, {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, false}, {:compare, :=, {:column, "b"}, {:literal, 3}}} end - test "NOT BETWEEN" do - assert where("a not between 1 and 2") == + test "NOT BETWEEN", %{parser: parser} do + assert where(parser, "a not between 1 and 2") == {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, true} end - test "IN and NOT IN" do - assert where("a in (1, 'b', 2.5)") == + test "IN and NOT IN", %{parser: parser} do + assert where(parser, "a in (1, 'b', 2.5)") == {:in, {:column, "a"}, [{:literal, 1}, {:literal, "b"}, {:literal, 2.5}], false} - assert where("a not in ('x')") == {:in, {:column, "a"}, [{:literal, "x"}], true} + assert where(parser, "a not in ('x')") == {:in, {:column, "a"}, [{:literal, "x"}], true} end - test "subqueries are named as unsupported" do - assert reason("select a from t where a in (select b from u)") == + test "subqueries are named as unsupported", %{parser: parser} do + assert reason(parser, "select a from t where a in (select b from u)") == "subqueries are not supported" end - test "IN needs a parenthesized, non-empty list" do - assert reason("select a from t where a in ()") == "IN needs at least one value" - assert reason("select a from t where a in 1") =~ "expected '(' after IN" - assert reason("select a from t where a in (1, 2") =~ "')' to close the IN list" + test "IN needs a parenthesized, non-empty list", %{parser: parser} do + assert reason(parser, "select a from t where a in ()") == "IN needs at least one value" + assert reason(parser, "select a from t where a in 1") =~ "expected '(' after IN" + assert reason(parser, "select a from t where a in (1, 2") =~ "')' to close the IN list" end - test "LIKE, NOT LIKE, ILIKE" do - assert where("a like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, false} - assert where("a not like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, true} - assert where("a ilike 'x%'") == {:ilike, {:column, "a"}, {:literal, "x%"}, false} + test "LIKE, NOT LIKE, ILIKE", %{parser: parser} do + assert where(parser, "a like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, false} + assert where(parser, "a not like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, true} + assert where(parser, "a ilike 'x%'") == {:ilike, {:column, "a"}, {:literal, "x%"}, false} end - test "IS [NOT] NULL, ISNULL, NOTNULL" do - assert where("a is null") == {:is_null, {:column, "a"}, false} - assert where("a is not null") == {:is_null, {:column, "a"}, true} - assert where("a isnull") == {:is_null, {:column, "a"}, false} - assert where("a notnull") == {:is_null, {:column, "a"}, true} + test "IS [NOT] NULL, ISNULL, NOTNULL", %{parser: parser} do + assert where(parser, "a is null") == {:is_null, {:column, "a"}, false} + assert where(parser, "a is not null") == {:is_null, {:column, "a"}, true} + assert where(parser, "a isnull") == {:is_null, {:column, "a"}, false} + assert where(parser, "a notnull") == {:is_null, {:column, "a"}, true} end - test "IS takes only NULL" do - assert reason("select a from t where a is true") =~ "expected NULL after IS, got 'true'" - assert reason("select a from t where a is not 1") =~ "expected NULL after IS NOT" + test "IS takes only NULL", %{parser: parser} do + assert reason(parser, "select a from t where a is true") =~ + "expected NULL after IS, got 'true'" + + assert reason(parser, "select a from t where a is not 1") =~ "expected NULL after IS NOT" end - test "a bare operand is kept for the translator to reject" do - assert where("a") == {:column, "a"} + test "a bare operand is kept for the translator to reject", %{parser: parser} do + assert where(parser, "a") == {:column, "a"} end end describe "boolean structure" do - test "AND binds tighter than OR" do - assert where("a = 1 or b = 2 and c = 3") == + test "AND binds tighter than OR", %{parser: parser} do + assert where(parser, "a = 1 or b = 2 and c = 3") == {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, {:and, {:compare, :=, {:column, "b"}, {:literal, 2}}, {:compare, :=, {:column, "c"}, {:literal, 3}}}} end - test "AND and OR are left-associative" do - assert {:and, {:and, _, _}, _} = where("a = 1 and b = 2 and c = 3") - assert {:or, {:or, _, _}, _} = where("a = 1 or b = 2 or c = 3") + test "AND and OR are left-associative", %{parser: parser} do + assert {:and, {:and, _, _}, _} = where(parser, "a = 1 and b = 2 and c = 3") + assert {:or, {:or, _, _}, _} = where(parser, "a = 1 or b = 2 or c = 3") end - test "NOT binds tighter than AND" do - assert where("not a = 1 and b = 2") == + test "NOT binds tighter than AND", %{parser: parser} do + assert where(parser, "not a = 1 and b = 2") == {:and, {:not, {:compare, :=, {:column, "a"}, {:literal, 1}}}, {:compare, :=, {:column, "b"}, {:literal, 2}}} end - test "parentheses group and then vanish" do - assert where("(a = 1 or b = 2) and c = 3") == + test "parentheses group and then vanish", %{parser: parser} do + assert where(parser, "(a = 1 or b = 2) and c = 3") == {:and, {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, {:compare, :=, {:column, "b"}, {:literal, 2}}}, {:compare, :=, {:column, "c"}, {:literal, 3}}} - assert where("((a = 1))") == where("a = 1") + assert where(parser, "((a = 1))") == where(parser, "a = 1") end - test "unbalanced parentheses" do - assert reason("select a from t where (a = 1") =~ "expected ')'" - assert reason("select a from t where a = 1)") =~ "expected end of statement, got ')'" + test "unbalanced parentheses", %{parser: parser} do + assert reason(parser, "select a from t where (a = 1") =~ "expected ')'" + + assert reason(parser, "select a from t where a = 1)") =~ + "expected end of statement, got ')'" end - test "deep nesting is fine up to a limit" do - assert where(String.duplicate("(", 150) <> "a = 1" <> String.duplicate(")", 150)) == - where("a = 1") + test "deep nesting", %{parser: parser} do + assert where(parser, String.duplicate("(", 150) <> "a = 1" <> String.duplicate(")", 150)) == + where(parser, "a = 1") + + case parser do + # Recursive descent caps its depth rather than risk the stack. + Parser -> + assert reason(parser, "select a from t where " <> String.duplicate("(", 10_000) <> "a") == + "expression is nested too deeply" - assert reason("select a from t where " <> String.duplicate("(", 10_000) <> "a") == - "expression is nested too deeply" + assert reason( + parser, + "select a from t where " <> String.duplicate("not ", 10_000) <> "a" + ) == + "expression is nested too deeply" - assert reason("select a from t where " <> String.duplicate("not ", 10_000) <> "a") == - "expression is nested too deeply" + # yecc keeps its own stack, so it needs no limit. + Efsql.SQL.Yecc -> + deep = String.duplicate("(", 10_000) <> "a = 1" <> String.duplicate(")", 10_000) + assert where(parser, deep) == where(parser, "a = 1") + assert {:not, {:not, _}} = where(parser, String.duplicate("not ", 10_000) <> "a") + end end end describe "casts" do - test "postfix and CAST forms give the same tree" do - assert where("a = 'x'::atom") == + test "postfix and CAST forms give the same tree", %{parser: parser} do + assert where(parser, "a = 'x'::atom") == {:compare, :=, {:column, "a"}, {:cast, {:literal, "x"}, "atom"}} - assert where("a = cast('x' as atom)") == where("a = 'x'::atom") + assert where(parser, "a = cast('x' as atom)") == where(parser, "a = 'x'::atom") end - test "type names fold and may be reserved words" do - assert {:compare, :=, _, {:cast, _, "date"}} = where("a = '2024-01-01'::DATE") - assert {:compare, :=, _, {:cast, _, "time"}} = where("a = '10:00:00'::time") + test "type names fold and may be reserved words", %{parser: parser} do + assert {:compare, :=, _, {:cast, _, "date"}} = where(parser, "a = '2024-01-01'::DATE") + assert {:compare, :=, _, {:cast, _, "time"}} = where(parser, "a = '10:00:00'::time") end - test "timestamp with/without time zone" do + test "timestamp with/without time zone", %{parser: parser} do assert {:compare, :=, _, {:cast, _, "timestamptz"}} = - where("a = 'x'::timestamp with time zone") + where(parser, "a = 'x'::timestamp with time zone") assert {:compare, :=, _, {:cast, _, "timestamp"}} = - where("a = cast('x' as timestamp without time zone)") + where(parser, "a = cast('x' as timestamp without time zone)") end - test "casts chain, and apply to any operand" do - assert where("a = 1::text::atom") == + test "casts chain, and apply to any operand", %{parser: parser} do + assert where(parser, "a = 1::text::atom") == {:compare, :=, {:column, "a"}, {:cast, {:cast, {:literal, 1}, "text"}, "atom"}} - assert where("a::text = 'x'") == + assert where(parser, "a::text = 'x'") == {:compare, :=, {:cast, {:column, "a"}, "text"}, {:literal, "x"}} end - test "casts inside IN and BETWEEN" do - assert where("a in ('x'::atom, 'y'::atom)") == + test "casts inside IN and BETWEEN", %{parser: parser} do + assert where(parser, "a in ('x'::atom, 'y'::atom)") == {:in, {:column, "a"}, [{:cast, {:literal, "x"}, "atom"}, {:cast, {:literal, "y"}, "atom"}], false} assert {:between, _, {:cast, _, "date"}, {:cast, _, "date"}, false} = - where("a between '2024-01-01'::date and '2025-01-01'::date") + where(parser, "a between '2024-01-01'::date and '2025-01-01'::date") end - test "a cast needs a type" do - assert reason("select a from t where a = 'x'::") =~ "expected a type name" - assert reason("select a from t where a = cast('x' atom)") =~ "expected AS in CAST" - assert reason("select a from t where a = cast('x' as atom") =~ "')' to close CAST" + test "a cast needs a type", %{parser: parser} do + assert reason(parser, "select a from t where a = 'x'::") =~ "expected a type name" + assert reason(parser, "select a from t where a = cast('x' atom)") =~ "expected AS in CAST" + assert reason(parser, "select a from t where a = cast('x' as atom") =~ "')' to close CAST" end end describe "tuples" do - test "a partition scan" do - assert where("_ = ('p', *)") == + test "a partition scan", %{parser: parser} do + assert where(parser, "_ = ('p', *)") == {:compare, :=, {:column, "_"}, {:tuple, [{:literal, "p"}, :star]}} end - test "a versionstamp" do - assert where("_ >= ('p', 22348699227647901699)") == + test "a versionstamp", %{parser: parser} do + assert where(parser, "_ >= ('p', 22348699227647901699)") == {:compare, :>=, {:column, "_"}, {:tuple, [{:literal, "p"}, {:literal, 22_348_699_227_647_901_699}]}} end - test "star only inside a tuple" do - assert reason("select a from t where a = (*)") =~ "only allowed in a tuple" - assert reason("select a from t where a = *") =~ "expected an expression, got '*'" + test "star only inside a tuple", %{parser: parser} do + assert reason(parser, "select a from t where a = (*)") =~ "only allowed in a tuple" + assert reason(parser, "select a from t where a = *") =~ "expected an expression, got '*'" end end describe "reserved words" do - test "common column names are not reserved" do + test "common column names are not reserved", %{parser: parser} do for name <- ~w[day at user value date time timestamp year month zone key type level] do - assert where("#{name} = 1") == {:compare, :=, {:column, name}, {:literal, 1}}, + assert where(parser, "#{name} = 1") == {:compare, :=, {:column, name}, {:literal, 1}}, "#{name} should be usable as a name" assert %Select{fields: [^name], order_by: [{^name, :desc}]} = - parse("select #{name} from t order by #{name} desc") + parse(parser, "select #{name} from t order by #{name} desc") end end - test "reserved words need quotes, and the error says so" do - assert reason("select order from t") =~ ~s(got the keyword 'order' (quote it, "order") + test "reserved words need quotes, and the error says so", %{parser: parser} do + assert reason(parser, "select order from t") =~ + ~s(got the keyword 'order' (quote it, "order") + + assert where(parser, ~s("order" = 1)) == {:compare, :=, {:column, "order"}, {:literal, 1}} - assert where(~s("order" = 1)) == {:compare, :=, {:column, "order"}, {:literal, 1}} - assert %Select{fields: ["select", "from"]} = parse(~s(select "select", "from" from t)) + assert %Select{fields: ["select", "from"]} = + parse(parser, ~s(select "select", "from" from t)) end - test "the reserved list" do + test "the reserved list", %{parser: parser} do for word <- Parser.reserved_words() do - assert {:error, %SyntaxError{}} = Parser.parse("select #{word} from t") + assert {:error, %SyntaxError{}} = parser.parse("select #{word} from t") end end end describe "ORDER BY and LIMIT" do - test "directions default to ascending" do + test "directions default to ascending", %{parser: parser} do assert %Select{order_by: [{"a", :asc}, {"b", :desc}, {"c", :asc}]} = - parse("select * from t order by a, b desc, c asc") + parse(parser, "select * from t order by a, b desc, c asc") end - test "ORDER needs BY and a field" do - assert reason("select * from t order a") =~ "expected BY after ORDER" - assert reason("select * from t order by") =~ "expected a field name to order by" + test "ORDER needs BY and a field", %{parser: parser} do + assert reason(parser, "select * from t order a") =~ "expected BY after ORDER" + assert reason(parser, "select * from t order by") =~ "expected a field name to order by" - assert reason("select * from t order by a nulls first") == + assert reason(parser, "select * from t order by a nulls first") == "NULLS FIRST/LAST is not supported" end - test "LIMIT takes a whole number" do - assert %Select{limit: 0} = parse("select * from t limit 0") - assert reason("select * from t limit -1") =~ "LIMIT expects a whole number, got '-'" - assert reason("select * from t limit 1.5") =~ "got the number 1.5" - assert reason("select * from t limit") =~ "got the end of the statement" + test "LIMIT takes a whole number", %{parser: parser} do + assert %Select{limit: 0} = parse(parser, "select * from t limit 0") + assert reason(parser, "select * from t limit -1") =~ "LIMIT expects a whole number, got '-'" + assert reason(parser, "select * from t limit 1.5") =~ "got the number 1.5" + assert reason(parser, "select * from t limit") =~ "got the end of the statement" end - test "clauses in the wrong order" do - assert reason("select * from t limit 1 where a = 1") =~ + test "clauses in the wrong order", %{parser: parser} do + assert reason(parser, "select * from t limit 1 where a = 1") =~ "expected end of statement, got 'where'" - assert reason("select * from t order by a where a = 1") =~ "got 'where'" + assert reason(parser, "select * from t order by a where a = 1") =~ "got 'where'" end end describe "statement errors" do - test "empty input" do - assert reason("") == "empty statement" - assert reason(" -- just a comment\n") == "empty statement" - assert reason(";") =~ "expected SELECT, got ';'" + test "empty input", %{parser: parser} do + assert reason(parser, "") == "empty statement" + assert reason(parser, " -- just a comment\n") == "empty statement" + assert reason(parser, ";") =~ "expected SELECT, got ';'" end - test "only SELECT" do - assert reason("delete from t") == "only SELECT statements are supported" - assert reason("update t set a = 1") == "only SELECT statements are supported" - assert reason("with x as (select 1) select * from x") =~ "expected SELECT" + test "only SELECT", %{parser: parser} do + assert reason(parser, "delete from t") == "only SELECT statements are supported" + assert reason(parser, "update t set a = 1") == "only SELECT statements are supported" + assert reason(parser, "with x as (select 1) select * from x") =~ "expected SELECT" end - test "one statement" do + test "one statement", %{parser: parser} do assert %SyntaxError{reason: "only one statement is supported", column: 18} = - error("select * from t; select * from u") + error(parser, "select * from t; select * from u") - assert reason("select * from t;;") == "only one statement is supported" + assert reason(parser, "select * from t;;") == "only one statement is supported" end - test "unsupported clauses are named" do - assert reason("select * from t group by a") == "GROUP BY is not supported" - assert reason("select * from t where a = 1 having a > 1") == "HAVING is not supported" - assert reason("select * from t limit 1 offset 2") == "OFFSET is not supported" - assert reason("select * from t join u on t.a = u.a") == "JOIN is not supported" - assert reason("select * from t left join u on a = b") == "JOIN is not supported" - assert reason("select * from t union select * from u") == "UNION is not supported" + test "unsupported clauses are named", %{parser: parser} do + assert reason(parser, "select * from t group by a") == "GROUP BY is not supported" + + assert reason(parser, "select * from t where a = 1 having a > 1") == + "HAVING is not supported" + + assert reason(parser, "select * from t limit 1 offset 2") == "OFFSET is not supported" + assert reason(parser, "select * from t join u on t.a = u.a") == "JOIN is not supported" + assert reason(parser, "select * from t left join u on a = b") == "JOIN is not supported" + assert reason(parser, "select * from t union select * from u") == "UNION is not supported" end - test "errors point at the offending token" do + test "errors point at the offending token", %{parser: parser} do assert %SyntaxError{line: 3, column: 10, reason: "expected an expression, got '='"} = - error("select a\nfrom t\nwhere a == 1") + error(parser, "select a\nfrom t\nwhere a == 1") assert %SyntaxError{ line: 1, column: 23, reason: "expected an expression, got the end of the statement" } = - error("select a from t where ") + error(parser, "select a from t where ") end - test "long strings are truncated in messages" do - assert reason("select * from t limit '#{String.duplicate("x", 100)}'") =~ + test "long strings are truncated in messages", %{parser: parser} do + assert reason(parser, "select * from t limit '#{String.duplicate("x", 100)}'") =~ "the string 'xxxxxxxxxxxxxxxxxxxx...'" end - test "parse! raises the error" do + test "parse! raises the error", %{parser: parser} do assert_raise SyntaxError, ~r/line 1, column 1: expected SELECT/, fn -> - Parser.parse!("selec * from t") + parser.parse!("selec * from t") end end end diff --git a/test/sql/property_test.exs b/test/sql/property_test.exs index e87a098..c621efa 100644 --- a/test/sql/property_test.exs +++ b/test/sql/property_test.exs @@ -1,52 +1,47 @@ defmodule Efsql.SQL.PropertyTest do - # Randomized tests. `:rand` is seeded per test from ExUnit's --seed, so a - # failure reproduces with `mix test --seed N`. - use ExUnit.Case, async: true + # Randomized tests, run against both parsers. `:rand` is seeded per test + # from ExUnit's --seed, so a failure reproduces with `mix test --seed N`. + use ExUnit.Case, + async: true, + parameterize: [%{parser: Efsql.SQL.Parser}, %{parser: Efsql.SQL.Yecc}] alias Efsql.Exception.Unsupported alias Efsql.SQL.AST.Select - alias Efsql.SQL.Parser alias Efsql.SQL.SyntaxError alias EfsqlTest.SQLGen @runs 1_000 - test "printing a random tree and parsing it gives the tree back" do + test "printing a random tree and parsing it gives the tree back", %{parser: parser} do for _ <- 1..@runs do select = SQLGen.select() sql = SQLGen.to_sql(select) - assert Parser.parse(sql) == {:ok, select}, """ + assert parser.parse(sql) == {:ok, select}, """ SQL: #{sql} """ end end - test "mangled queries parse or fail with a positioned SyntaxError, never crash" do + test "mangled queries parse or fail with a positioned SyntaxError, never crash", + %{parser: parser} do for _ <- 1..@runs do - sql = SQLGen.select() |> SQLGen.to_sql() |> mangle(Enum.random(1..4)) - check(sql) + check(parser, SQLGen.select() |> SQLGen.to_sql() |> SQLGen.mangle(Enum.random(1..4))) end end - test "random bytes parse or fail with a positioned SyntaxError, never crash" do - alphabet = - String.graphemes("selctfromwhandi*,.;()'\"=<>!:-+/%_ \n\t0123456789eé") ++ - ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>] - - for _ <- 1..@runs do - sql = Enum.map_join(1..Enum.random(0..40)//1, fn _ -> Enum.random(alphabet) end) - check(sql) - end + test "random bytes parse or fail with a positioned SyntaxError, never crash", + %{parser: parser} do + for _ <- 1..@runs, do: check(parser, SQLGen.garbage()) end - test "valid queries translate to a logical plan or raise Unsupported" do + test "valid queries translate to a logical plan or raise Unsupported", %{parser: parser} do for _ <- 1..@runs do - sql = SQLGen.select() |> SQLGen.to_sql() + {:ok, select} = parser.parse(SQLGen.select() |> SQLGen.to_sql()) try do - Efsql.Parser.sql_to_logical(sql) + Efsql.Parser.to_logical(select) rescue _ in [Unsupported] -> :ok @@ -59,20 +54,20 @@ defmodule Efsql.SQL.PropertyTest do end end - test "big inputs parse and translate quickly" do + test "big inputs parse and translate quickly", %{parser: parser} do values = Enum.map_join(1..20_000, ", ", &"'v#{&1}'") conditions = Enum.map_join(1..20_000, " and ", &"f#{&1} = #{&1}") sql = "select a from t where a in (#{values}) and #{conditions}" {micros, %Efsql.Logical.Select{predicates: predicates}} = - :timer.tc(fn -> Efsql.Parser.sql_to_logical(sql) end) + :timer.tc(fn -> sql |> parser.parse!() |> Efsql.Parser.to_logical() end) assert length(predicates) == 20_001 assert micros < 5_000_000 end - defp check(sql) do - case Parser.parse(sql) do + defp check(parser, sql) do + case parser.parse(sql) do {:ok, %Select{}} -> :ok @@ -83,44 +78,4 @@ defmodule Efsql.SQL.PropertyTest do assert column >= 1, "column #{column} for #{inspect(sql)}" end end - - # Random edits: delete, insert, replace, duplicate or truncate. - defp mangle(sql, 0), do: sql - - defp mangle(sql, n) do - chars = String.graphemes(sql) - size = length(chars) - at = Enum.random(0..size) - - junk = - Enum.random([ - "(", - ")", - "'", - "\"", - ",", - ";", - "-", - "*", - ":", - ".", - "=", - "<", - "not", - " and ", - "é", - <<0xC3>> - ]) - - chars = - case Enum.random(1..5) do - 1 -> List.delete_at(chars, at) - 2 -> List.insert_at(chars, at, junk) - 3 -> List.replace_at(chars, at, junk) - 4 -> Enum.take(chars, at) ++ Enum.slice(chars, at, 10) ++ Enum.drop(chars, at) - 5 -> Enum.take(chars, at) - end - - mangle(Enum.join(chars), n - 1) - end end diff --git a/test/support/efsql_test/sql_gen.ex b/test/support/efsql_test/sql_gen.ex index e85eaf6..90fbf05 100644 --- a/test/support/efsql_test/sql_gen.ex +++ b/test/support/efsql_test/sql_gen.ex @@ -226,6 +226,85 @@ defmodule EfsqlTest.SQLGen do end end + # -- broken input -- + + @junk [ + "(", + ")", + "'", + "\"", + ",", + ";", + "-", + "+", + "*", + ":", + "::", + ".", + "=", + "<", + "not", + " and ", + " or ", + " is ", + " in ", + " between ", + "é", + <<0xC3>>, + " select ", + " x ", + "1", + " -- c\n", + "/*", + "*/", + " timestamp ", + " with ", + " time ", + " zone ", + " cast ", + " as ", + " order ", + " by ", + " limit ", + " desc ", + " nulls " + ] + + @doc "`n` random edits to `sql`: deletes, inserts, replacements, duplications, truncation." + def mangle(sql, 0), do: sql + + def mangle(sql, n) do + chars = String.graphemes(sql) + at = Enum.random(0..length(chars)) + junk = Enum.random(@junk) + + chars = + case Enum.random(1..5) do + 1 -> List.delete_at(chars, at) + 2 -> List.insert_at(chars, at, junk) + 3 -> List.replace_at(chars, at, junk) + 4 -> Enum.take(chars, at) ++ Enum.slice(chars, at, 10) ++ Enum.drop(chars, at) + 5 -> Enum.take(chars, at) + end + + mangle(Enum.join(chars), n - 1) + end + + @alphabet String.graphemes("selctfromwhandi*,.;()'\"=<>!:-+/%_ \n\t0123456789eé") ++ + ["select ", " from ", " where ", " and ", "--", "/*", "*/", "::", <<0xFF>>, <<0>>] + + @doc "Up to 40 random SQL-ish fragments and bytes, invalid UTF-8 included." + def garbage(), do: random_string(@alphabet, 0..40) + + @doc "A valid query, a mangled one, or garbage, in equal measure." + def any_input() do + case Enum.random(1..3) do + 1 -> select() |> to_sql() + 2 -> select() |> to_sql() |> mangle(Enum.random(1..4)) + 3 -> garbage() + end + end + # -- helpers -- defp random_string(pool, range) do From 943e66fbc2e0c3259f352ffb7ce2362308050d29 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 11:44:35 +0000 Subject: [PATCH 4/5] Pass invalid UTF-8 bytes to leex as surrogates, not above U+10FFFF OTP 28's leex tracks columns by UTF-8 width, and its adjust_col/3 has no clause for integers above U+10FFFF, so input with an invalid byte crashed the leex lexer on CI (OTP 25, used locally, has no column tracking). Surrogates never come out of UTF-8 decoding, and leex can count them. The positive character classes skip the surrogate block, so the bytes are still illegal outside strings and comments. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- lib/efsql/sql/leex.ex | 16 ++++++++++------ src/efsql_sql_leex.xrl | 13 +++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/efsql/sql/leex.ex b/lib/efsql/sql/leex.ex index 461e5bb..49af15a 100644 --- a/lib/efsql/sql/leex.ex +++ b/lib/efsql/sql/leex.ex @@ -12,8 +12,12 @@ defmodule Efsql.SQL.Leex do alias Efsql.SQL.SyntaxError - # Invalid UTF-8 bytes are passed to leex as integers above U+10FFFF. - @bad_byte 0x110000 + # Invalid UTF-8 bytes are passed to leex as UTF-16 surrogates, which + # decoded UTF-8 never contains (and which leex, unlike integers above + # U+10FFFF, can count columns for). + @bad_byte 0xD800 + + defguardp bad_byte?(c) when c in @bad_byte..(@bad_byte + 0xFF)//1 @ops %{ ~c"<>" => :<>, @@ -186,13 +190,13 @@ defmodule Efsql.SQL.Leex do # An invalid byte inside a string or quoted name is reported where it is. defp check_bytes(matched, {line, col}) do - case Enum.split_while(matched, &(&1 < @bad_byte)) do + case Enum.split_while(matched, &(not bad_byte?(&1))) do {_, []} -> :ok {before, [bad | _]} -> illegal(bad, advance(before, line, col)) end end - defp illegal(c, pos) when c >= @bad_byte, + defp illegal(c, pos) when bad_byte?(c), do: error("invalid UTF-8 byte 0x#{Integer.to_string(c - @bad_byte, 16)}", pos) defp illegal(c, pos), do: error("unexpected character #{inspect(<>)}", pos) @@ -229,11 +233,11 @@ defmodule Efsql.SQL.Leex do # The same character classes as Efsql.SQL.Lexer. defp word_start?(c) when c in ?a..?z or c in ?A..?Z or c == ?_, do: true - defp word_start?(c) when c < 128 or c >= @bad_byte, do: false + defp word_start?(c) when c < 128 or bad_byte?(c), do: false defp word_start?(c), do: String.match?(<>, ~r/^\p{L}$/u) defp word_continue?(c) when c in ?0..?9 or c == ?$, do: true - defp word_continue?(c) when c < 128 or c >= @bad_byte, do: word_start?(c) + defp word_continue?(c) when c < 128 or bad_byte?(c), do: word_start?(c) defp word_continue?(c), do: String.match?(<>, ~r/^[\p{L}\p{M}\p{N}]$/u) defp unicode_space?(c), diff --git a/src/efsql_sql_leex.xrl b/src/efsql_sql_leex.xrl index 1d993ce..5d3f4c2 100644 --- a/src/efsql_sql_leex.xrl +++ b/src/efsql_sql_leex.xrl @@ -9,9 +9,10 @@ %% string, quoted, unterminated, number, bad_exponent, number_junk, word, %% op, colon. %% -%% Invalid UTF-8 bytes reach this lexer as integers above 16#10FFFF; no -%% positive class includes them, so outside strings and comments they are -%% illegal characters. +%% Invalid UTF-8 bytes reach this lexer as UTF-16 surrogates (16#D800 + +%% byte), which decoded UTF-8 never contains and leex's column counting +%% still accepts. No positive class includes them, so outside strings and +%% comments they are illegal characters. Definitions. @@ -21,7 +22,7 @@ NUM = ({D}+\.?{D}*|\.{D}+) %% needs its own parentheses. EXP = ([eE][+-]?{D}+) WS = [\s\t\f\v\r\n\x{A0}\x{1680}\x{2000}-\x{200A}\x{202F}\x{205F}\x{3000}\x{FEFF}] -ID = [A-Za-z_\x{80}-\x{10FFFF}][A-Za-z0-9_$\x{80}-\x{10FFFF}]* +ID = [A-Za-z_\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}][A-Za-z0-9_$\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}]* Rules. @@ -39,7 +40,7 @@ Rules. %% a number running into another character (which the driver checks). {NUM}{EXP}? : {token, {number, TokenChars}}. {NUM}[eE][+-]? : {token, {bad_exponent, TokenChars}}. -{NUM}{EXP}?[A-Za-z_.0-9\x{80}-\x{10FFFF}] : {token, {number_junk, TokenChars}}. +{NUM}{EXP}?[A-Za-z_.0-9\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}] : {token, {number_junk, TokenChars}}. {ID} : {token, {word, TokenChars}}. @@ -51,7 +52,7 @@ Rules. [=<>(),.;*+\-/%] : {token, {op, TokenChars}}. %% A lone colon, or one followed by anything: the driver explains it. -:[\x{0}-\x{10FFFF}] : {token, {colon, TokenChars}}. +:[\x{0}-\x{D7FF}\x{E000}-\x{10FFFF}] : {token, {colon, TokenChars}}. : : {token, {colon, TokenChars}}. Erlang code. From 4be360ab99616d5f8863efc9991377a124e47db7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 12:00:32 +0000 Subject: [PATCH 5/5] Parse with a yecc grammar and explain its errors from the grammar efsql's SQL is about to grow CTEs, joins and GROUP BY. A grammar that yecc checks for conflicts scales better than more recursive descent, so Efsql.SQL.Parser now drives src/efsql_sql_grammar.yrl with the hand-written lexer's tokens. yecc only reports the token where it stopped. To say what it expected, the parser replays the tokens before that point with each terminal and keeps the ones the grammar accepts: "expected BY, got 'a'". New rules need no extra work to get error messages. A short list of hints still names unsupported features (GROUP BY, JOIN, WITH, functions, ...); each hint goes away when its feature lands. Nesting no longer has a depth limit. Removes the leex lexer, the hand-written parser, the error explainer and the differential test. The lexer stays hand-written, since leex can't do nested comments, Unicode classes or character columns, and its behaviour differs between OTP versions. Error assertions now match the new messages. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFBZUrerge6bbWVCHpS5Ko --- lib/efsql/sql/leex.ex | 247 -------- lib/efsql/sql/parser.ex | 564 +++++------------- lib/efsql/sql/yecc.ex | 52 -- lib/efsql/sql/yecc/explain.ex | 454 -------------- ...sql_sql_yecc.yrl => efsql_sql_grammar.yrl} | 23 +- src/efsql_sql_leex.xrl | 58 -- test/sql/differential_test.exs | 28 - test/sql/lexer_test.exs | 203 ++++--- test/sql/parser_test.exs | 406 ++++++------- test/sql/property_test.exs | 33 +- test/support/efsql_test/sql_gen.ex | 9 - 11 files changed, 479 insertions(+), 1598 deletions(-) delete mode 100644 lib/efsql/sql/leex.ex delete mode 100644 lib/efsql/sql/yecc.ex delete mode 100644 lib/efsql/sql/yecc/explain.ex rename src/{efsql_sql_yecc.yrl => efsql_sql_grammar.yrl} (88%) delete mode 100644 src/efsql_sql_leex.xrl delete mode 100644 test/sql/differential_test.exs diff --git a/lib/efsql/sql/leex.ex b/lib/efsql/sql/leex.ex deleted file mode 100644 index 49af15a..0000000 --- a/lib/efsql/sql/leex.ex +++ /dev/null @@ -1,247 +0,0 @@ -defmodule Efsql.SQL.Leex do - @moduledoc """ - The same contract as `Efsql.SQL.Lexer.tokenize/1`, built on the leex - tokenizer in `src/efsql_sql_leex.xrl`. - - leex matches the token shapes. This driver pulls one match at a time - (`token/2`) and does what the rules can't: it tracks positions in - characters (leex counts UTF-8 bytes), skips nested block comments, - applies the Unicode letter classes, and turns the raw matches into - tokens and errors. - """ - - alias Efsql.SQL.SyntaxError - - # Invalid UTF-8 bytes are passed to leex as UTF-16 surrogates, which - # decoded UTF-8 never contains (and which leex, unlike integers above - # U+10FFFF, can count columns for). - @bad_byte 0xD800 - - defguardp bad_byte?(c) when c in @bad_byte..(@bad_byte + 0xFF)//1 - - @ops %{ - ~c"<>" => :<>, - ~c"!=" => :<>, - ~c"<=" => :<=, - ~c">=" => :>=, - ~c"::" => :"::", - ~c"=" => :=, - ~c"<" => :<, - ~c">" => :>, - ~c"(" => :"(", - ~c")" => :")", - ~c"," => :",", - ~c"." => :., - ~c";" => :";", - ~c"*" => :*, - ~c"+" => :+, - ~c"-" => :-, - ~c"/" => :/, - ~c"%" => :% - } - - @spec tokenize(String.t()) :: {:ok, [Efsql.SQL.Lexer.token()]} | {:error, SyntaxError.t()} - def tokenize(sql) when is_binary(sql) do - {:ok, lex(decode(sql, []), 1, 1, [])} - catch - {:syntax_error, reason, {line, column}} -> - {:error, %SyntaxError{reason: reason, line: line, column: column}} - end - - defp decode(<>, acc), do: decode(rest, [c | acc]) - defp decode(<>, acc), do: decode(rest, [@bad_byte + byte | acc]) - defp decode(<<>>, acc), do: Enum.reverse(acc) - - defp lex([], line, col, acc), do: Enum.reverse([{:eof, nil, {line, col}} | acc]) - - defp lex(chars, line, col, acc) do - case next(chars) do - {{:ok, {kind, matched}, _}, rest} -> token(kind, matched, rest, {line, col}, acc) - {{:error, _, _}, _} -> illegal(hd(chars), {line, col}) - end - end - - defp next(chars) do - case :efsql_sql_leex.token([], chars) do - {:done, result, rest} -> - {result, rest(rest)} - - {:more, cont} -> - cont |> :efsql_sql_leex.token(:eof) |> then(fn {:done, r, rest} -> {r, rest(rest)} end) - end - end - - defp rest(:eof), do: [] - defp rest(chars), do: chars - - defp token(:ws, matched, rest, pos, acc), do: continue(matched, rest, pos, acc) - - # A line comment runs to the end of the line, whatever it holds. - defp token(:line_comment, matched, rest, pos, acc) do - {comment, rest} = Enum.split_while(rest, &(&1 not in [?\n, ?\r])) - continue(matched ++ comment, rest, pos, acc) - end - - defp token(:block_comment, _matched, rest, {line, col} = start, acc) do - {rest, line, col} = block_comment(rest, line, col + 2, 1, start) - lex(rest, line, col, acc) - end - - defp token(kind, matched, rest, pos, acc) when kind in [:string, :quoted] do - check_bytes(matched, pos) - value = matched |> Enum.slice(1..-2//1) |> List.to_string() - - {type, value} = - case kind do - :string -> - {:string, String.replace(value, "''", "'")} - - :quoted -> - if value == "", do: error("zero-length quoted identifier", pos) - {:quoted, String.replace(value, ~s(""), ~s("))} - end - - continue(matched, rest, pos, [{type, value, pos} | acc]) - end - - defp token(:unterminated, [quote | _] = matched, _rest, pos, _acc) do - check_bytes(matched, pos) - what = if quote == ?', do: "string", else: "identifier" - error("unterminated #{what} (missing closing #{<>})", pos) - end - - defp token(:number, matched, rest, pos, acc) do - continue(matched, rest, pos, [number(List.to_string(matched), pos) | acc]) - end - - defp token(:bad_exponent, _matched, _rest, pos, _acc), - do: error("invalid number: exponent has no digits", pos) - - # A number run into another character is an error when that character - # could continue it; otherwise the number ends and the character is - # lexed on its own. - defp token(:number_junk, matched, rest, pos, acc) do - {number, [c]} = Enum.split(matched, -1) - - if word_start?(c) or c == ?. do - error("invalid number #{inspect(List.to_string(matched))}", pos) - end - - continue(number, [c | rest], pos, [number(List.to_string(number), pos) | acc]) - end - - # leex matches any run of letters and non-ASCII characters; the word is - # the part the Unicode classes allow, and the rest is lexed again. - defp token(:word, [c | tail] = matched, rest, pos, acc) do - cond do - word_start?(c) -> - {word, leftover} = Enum.split_while(matched, &word_continue?/1) - value = word |> List.to_string() |> String.downcase() - continue(word, leftover ++ rest, pos, [{:word, value, pos} | acc]) - - unicode_space?(c) -> - continue([c], tail ++ rest, pos, acc) - - true -> - illegal(c, pos) - end - end - - defp token(:op, matched, rest, pos, acc), - do: continue(matched, rest, pos, [{:op, Map.fetch!(@ops, matched), pos} | acc]) - - defp token(:colon, [?:, c], _rest, pos, _acc) do - if word_start?(c), - do: error("unexpected ':' (write an atom as a cast, e.g. 'name'::atom)", pos), - else: error("unexpected ':'", pos) - end - - defp token(:colon, [?:], _rest, pos, _acc), do: illegal(?:, pos) - - defp continue(consumed, rest, {line, col}, acc) do - {line, col} = advance(consumed, line, col) - lex(rest, line, col, acc) - end - - defp advance([?\r, ?\n | rest], line, _col), do: advance(rest, line + 1, 1) - defp advance([c | rest], line, _col) when c in [?\n, ?\r], do: advance(rest, line + 1, 1) - defp advance([_ | rest], line, col), do: advance(rest, line, col + 1) - defp advance([], line, col), do: {line, col} - - defp block_comment([], _line, _col, _depth, start), - do: error("unterminated /* comment", start) - - defp block_comment([?*, ?/ | rest], line, col, 1, _start), do: {rest, line, col + 2} - - defp block_comment([?*, ?/ | rest], line, col, depth, start), - do: block_comment(rest, line, col + 2, depth - 1, start) - - defp block_comment([?/, ?* | rest], line, col, depth, start), - do: block_comment(rest, line, col + 2, depth + 1, start) - - defp block_comment([?\r, ?\n | rest], line, _col, depth, start), - do: block_comment(rest, line + 1, 1, depth, start) - - defp block_comment([c | rest], line, _col, depth, start) when c in [?\n, ?\r], - do: block_comment(rest, line + 1, 1, depth, start) - - defp block_comment([_ | rest], line, col, depth, start), - do: block_comment(rest, line, col + 1, depth, start) - - # An invalid byte inside a string or quoted name is reported where it is. - defp check_bytes(matched, {line, col}) do - case Enum.split_while(matched, &(not bad_byte?(&1))) do - {_, []} -> :ok - {before, [bad | _]} -> illegal(bad, advance(before, line, col)) - end - end - - defp illegal(c, pos) when bad_byte?(c), - do: error("invalid UTF-8 byte 0x#{Integer.to_string(c - @bad_byte, 16)}", pos) - - defp illegal(c, pos), do: error("unexpected character #{inspect(<>)}", pos) - - defp number(text, pos) do - {mantissa, exp} = - case :binary.match(text, ["e", "E"]) do - {at, _} -> {binary_part(text, 0, at), binary_part(text, at, byte_size(text) - at)} - :nomatch -> {text, ""} - end - - case String.split(mantissa, ".", parts: 2) do - [int] when exp == "" -> - {:integer, String.to_integer(int), pos} - - parts -> - {int, frac} = - case parts do - [int] -> {int, ""} - [int, frac] -> {int, frac} - end - - int = if int == "", do: "0", else: int - frac = if frac == "", do: "0", else: frac - {:float, to_float("#{int}.#{frac}#{exp}", text, pos), pos} - end - end - - defp to_float(normalized, text, pos) do - String.to_float(normalized) - rescue - ArgumentError -> error("number out of range: #{text}", pos) - end - - # The same character classes as Efsql.SQL.Lexer. - defp word_start?(c) when c in ?a..?z or c in ?A..?Z or c == ?_, do: true - defp word_start?(c) when c < 128 or bad_byte?(c), do: false - defp word_start?(c), do: String.match?(<>, ~r/^\p{L}$/u) - - defp word_continue?(c) when c in ?0..?9 or c == ?$, do: true - defp word_continue?(c) when c < 128 or bad_byte?(c), do: word_start?(c) - defp word_continue?(c), do: String.match?(<>, ~r/^[\p{L}\p{M}\p{N}]$/u) - - defp unicode_space?(c), - do: c in [0xA0, 0x1680, 0x202F, 0x205F, 0x3000, 0xFEFF] or c in 0x2000..0x200A - - defp error(reason, pos), do: throw({:syntax_error, reason, pos}) -end diff --git a/lib/efsql/sql/parser.ex b/lib/efsql/sql/parser.ex index be2e189..f97340e 100644 --- a/lib/efsql/sql/parser.ex +++ b/lib/efsql/sql/parser.ex @@ -4,28 +4,58 @@ defmodule Efsql.SQL.Parser do SELECT fields FROM name [WHERE expr] [ORDER BY items] [LIMIT n] [;] - A recursive-descent parser over `Efsql.SQL.Lexer` tokens. Keywords are - recognised by position, so a word is only a keyword where the grammar - expects one. The reserved words below can't be bare names anywhere, - because a name could be mistaken for them; quote them to use them as - names (`"order"`). Everything else (`day`, `user`, `date`, `value`, ...) - is an ordinary name. + The grammar is `src/efsql_sql_grammar.yrl`, compiled by yecc; this + module feeds it `Efsql.SQL.Lexer` tokens and explains its errors. + + The reserved words below can't be bare names, because a name could be + mistaken for them; quote them to use them as names (`"order"`). + Everything else (`day`, `user`, `date`, `value`, ...) is an ordinary + name, and any word, reserved or not, can be a type (`'x'::date`). Precedence, loosest first: `OR`, `AND`, `NOT`, then a predicate (comparison, `BETWEEN`, `IN`, `LIKE`, `IS NULL`), then `::` casts. + + ## Errors + + yecc only reports the token it stopped at. To say what it wanted + instead, the parser replays the tokens before it with each terminal in + turn and keeps those the grammar accepts, so messages follow the + grammar with no upkeep: `expected BY, got the end of the statement`. + A few hints on top name features that aren't supported yet, such as + `GROUP BY` or functions; drop a hint when its feature lands. """ alias Efsql.SQL.AST alias Efsql.SQL.Lexer alias Efsql.SQL.SyntaxError + # These must match the grammar's keyword Terminals. @reserved ~w[all and as asc between by case cast cross desc distinct else end except false from full group having ilike in inner intersect is isnull join left like limit not notnull null offset on or order right select then true union when where with] + # Keywords that are also names. + @soft ~w[time timestamp without zone] + + @keywords @reserved ++ @soft + @ops [:=, :<>, :<, :>, :<=, :>=, :"::", :"(", :")", :",", :., :";", :*, :+, :-] + + # Every terminal of the grammar, plus the end of input: the candidates + # for "what could have come next". + @terminals [:ident, :quoted, :string, :integer, :float] ++ + Enum.map(@keywords, &String.to_atom/1) ++ @ops ++ [:"$end"] + + @name_starts [:ident, :quoted | Enum.map(@soft, &String.to_atom/1)] + @expression_starts @name_starts ++ + [:string, :integer, :float, :cast, true, false, :null, :"(", :+, :-] + @comparisons [:=, :<>, :<, :>, :<=, :>=] + @operators @comparisons ++ + [:and, :or, :not, :between, :in, :like, :ilike, :is, :isnull, :notnull, :"::"] + @not_select ~w[insert update delete create drop alter truncate grant revoke explain] - @unsupported_clauses %{ + @unsupported %{ + "distinct" => "DISTINCT", "group" => "GROUP BY", "having" => "HAVING", "offset" => "OFFSET", @@ -36,25 +66,31 @@ defmodule Efsql.SQL.Parser do "full" => "JOIN", "cross" => "JOIN", "union" => "UNION", + "with" => "WITH", "intersect" => "INTERSECT", "except" => "EXCEPT", "fetch" => "FETCH" } - @comparison_ops [:=, :<>, :<, :>, :<=, :>=] - @arithmetic_ops [:+, :-, :*, :/, :%] - @max_depth 200 - def reserved_words, do: @reserved @spec parse(String.t()) :: {:ok, AST.Select.t()} | {:error, SyntaxError.t()} def parse(sql) when is_binary(sql) do with {:ok, tokens} <- Lexer.tokenize(sql) do - {:ok, statement(tokens)} + grammar_tokens = tokens |> Enum.with_index() |> Enum.map(&to_grammar/1) + + case :efsql_sql_grammar.parse(grammar_tokens) do + {:ok, {:select, fields, from, where, order_by, limit}} -> + {:ok, + %AST.Select{fields: fields, from: from, where: where, order_by: order_by, limit: limit}} + + {:error, {at, :efsql_sql_grammar, _}} -> + tokens = List.to_tuple(tokens) + {_, _, {line, column}} = elem(tokens, at) + reason = explain(tokens, at, expected(grammar_tokens, at)) + {:error, %SyntaxError{reason: reason, line: line, column: column}} + end end - catch - {:syntax_error, reason, {line, column}} -> - {:error, %SyntaxError{reason: reason, line: line, column: column}} end @spec parse!(String.t()) :: AST.Select.t() @@ -65,441 +101,161 @@ defmodule Efsql.SQL.Parser do end end - # -- statement -- - - defp statement([{:eof, _, pos}]), do: error("empty statement", pos) - - defp statement(tokens) do - {select, rest} = select(tokens) - - case rest do - [{:eof, _, _}] -> select - [{:op, :";", _}, {:eof, _, _}] -> select - [{:op, :";", _}, {_, _, pos} | _] -> error("only one statement is supported", pos) - [token | _] -> end_of_statement_error(token) - end - end - - defp end_of_statement_error({:word, word, pos}) when is_map_key(@unsupported_clauses, word) do - error("#{Map.fetch!(@unsupported_clauses, word)} is not supported", pos) - end - - defp end_of_statement_error({:op, op, pos}) when op in @arithmetic_ops do - error("arithmetic is not supported", pos) - end - - defp end_of_statement_error({_, _, pos} = token) do - error("expected end of statement, got #{describe(token)}", pos) - end - - defp select([{:word, "select", _} | rest]) do - {fields, rest} = select_list(rest) - rest = expect_keyword(rest, "from", "FROM") - {from, rest} = table_name(rest) - - case rest do - [{:op, :",", pos} | _] -> error("selecting from more than one table is not supported", pos) - _ -> :ok - end - - {where, rest} = where(rest) - {order_by, rest} = order_by(rest) - {limit, rest} = limit(rest) - - {%AST.Select{fields: fields, from: from, where: where, order_by: order_by, limit: limit}, - rest} - end - - defp select([{:word, word, pos} | _]) when word in @not_select, - do: error("only SELECT statements are supported", pos) - - defp select([{_, _, pos} = token | _]), - do: error("expected SELECT, got #{describe(token)}", pos) - - # -- SELECT list -- - - defp select_list([{:op, :*, _} | rest]) do - case rest do - [{:op, :",", pos} | _] -> error("* can't be combined with other fields", pos) - _ -> {:star, rest} - end - end - - defp select_list([{:word, "distinct", pos} | _]), do: error("DISTINCT is not supported", pos) - - defp select_list(tokens) do - {name, rest} = name(tokens, "a field name or *") - rest = reject_qualified_or_call(rest) - - case rest do - [{:op, :",", _} | rest] -> - {names, rest} = select_list_rest(rest) - {[name | names], rest} - - rest -> - {[name], rest} - end - end - - defp select_list_rest(tokens) do - {name, rest} = name(tokens, "a field name") - rest = reject_qualified_or_call(rest) - - case rest do - [{:op, :",", _} | rest] -> - {names, rest} = select_list_rest(rest) - {[name | names], rest} - - rest -> - {[name], rest} - end - end - - # -- FROM -- - - defp table_name(tokens) do - {first, rest} = name(tokens, "a table name") - table_name_parts(rest, [first]) - end - - defp table_name_parts([{:op, :., pos} | rest], parts) do - if length(parts) == 3 do - error("a table name has at most three parts (storage_id.tenant.table)", pos) - end - - {part, rest} = name(rest, "a name after '.'") - table_name_parts(rest, parts ++ [part]) - end - - defp table_name_parts(rest, parts), do: {parts, rest} - - # -- WHERE -- - - defp where([{:word, "where", _} | rest]) do - {expr, rest} = expr(rest, 0) - {expr, rest} - end - - defp where(rest), do: {nil, rest} - - # -- ORDER BY -- - - defp order_by([{:word, "order", _} | rest]) do - rest = expect_keyword(rest, "by", "BY after ORDER") - order_items(rest) - end - - defp order_by(rest), do: {[], rest} - - defp order_items(tokens) do - {name, rest} = name(tokens, "a field name to order by") - rest = reject_qualified_or_call(rest) - - {dir, rest} = - case rest do - [{:word, "asc", _} | rest] -> {:asc, rest} - [{:word, "desc", _} | rest] -> {:desc, rest} - rest -> {:asc, rest} - end - - case rest do - [{:word, "nulls", pos} | _] -> - error("NULLS FIRST/LAST is not supported", pos) - - [{:op, :",", _} | rest] -> - {items, rest} = order_items(rest) - {[{name, dir} | items], rest} + defp to_grammar({{:word, word, _}, i}) when word in @keywords, + do: {String.to_atom(word), i, word} - rest -> - {[{name, dir}], rest} - end - end + defp to_grammar({{:word, word, _}, i}), do: {:ident, i, word} + defp to_grammar({{:op, op, _}, i}), do: {op, i, op} + defp to_grammar({{:eof, nil, _}, i}), do: {:"$end", i} + defp to_grammar({{type, value, _}, i}), do: {type, i, value} - # -- LIMIT -- + # -- errors -- - defp limit([{:word, "limit", _} | rest]) do - case rest do - [{:integer, n, _} | rest] -> - {n, rest} + # The terminals the grammar accepts after the first `at` tokens. An LR + # parser never shifts a token it can't use, so a candidate is refused + # exactly when the parse fails at the candidate itself. + defp expected(grammar_tokens, at) do + prefix = Enum.take(grammar_tokens, at) - [{_, _, pos} = token | _] -> - error("LIMIT expects a whole number, got #{describe(token)}", pos) + for terminal <- @terminals, accepts?(prefix, at, terminal), into: MapSet.new() do + terminal end end - defp limit(rest), do: {nil, rest} + defp accepts?(prefix, at, :"$end"), do: accepts?(prefix ++ [{:"$end", at}], at) - # -- expressions -- + defp accepts?(prefix, at, terminal), + do: accepts?(prefix ++ [{terminal, at, sample(terminal)}, {:"$end", at + 1}], at) - defp expr(tokens, depth), do: or_expr(tokens, depth) - - defp or_expr(tokens, depth) do - {left, rest} = and_expr(tokens, depth) - or_rest(left, rest, depth) - end - - defp or_rest(left, [{:word, "or", _} | rest], depth) do - {right, rest} = and_expr(rest, depth) - or_rest({:or, left, right}, rest, depth) - end + defp accepts?(tokens, at), + do: not match?({:error, {^at, _, _}}, :efsql_sql_grammar.parse(tokens)) - defp or_rest(left, rest, _depth), do: {left, rest} + defp sample(:integer), do: 1 + defp sample(:float), do: 1.0 + defp sample(terminal) when terminal in [:ident, :quoted, :string], do: "x" + defp sample(terminal) when is_atom(terminal), do: Atom.to_string(terminal) - defp and_expr(tokens, depth) do - {left, rest} = not_expr(tokens, depth) - and_rest(left, rest, depth) - end + defp explain(tokens, at, expected) do + token = elem(tokens, at) + previous = if at > 0, do: elem(tokens, at - 1) - defp and_rest(left, [{:word, "and", _} | rest], depth) do - {right, rest} = not_expr(rest, depth) - and_rest({:and, left, right}, rest, depth) + table_part_limit(tokens, at) || hint(token, previous, at, expected) || + "expected #{describe_expected(expected)}, got #{describe(token)}" end - defp and_rest(left, rest, _depth), do: {left, rest} - - defp not_expr([{:word, "not", pos} | rest], depth) do - check_depth(depth, pos) - {expr, rest} = not_expr(rest, depth + 1) - {{:not, expr}, rest} + # Only a table name has parts (storage_id.tenant.table), so a dot after + # a two-dot name is one part too many. + defp table_part_limit(tokens, at) do + if at >= 4 and match?({:op, :., _}, elem(tokens, at)) and + match?({:op, :., _}, elem(tokens, at - 2)) and + match?({:op, :., _}, elem(tokens, at - 4)), + do: "a table name has at most three parts (storage_id.tenant.table)" end - defp not_expr(tokens, depth), do: predicate(tokens, depth) - - defp predicate(tokens, depth) do - {left, rest} = operand(tokens, depth) - - case rest do - [{:op, op, _} | rest] when op in @comparison_ops -> - {right, rest} = operand(rest, depth) - {{:compare, op, left, right}, rest} + # Hints for what the grammar doesn't cover yet, and one for reserved + # words used as names. + defp hint({:eof, _, _}, nil, 0, _), do: "empty statement" - [{:op, op, pos} | _] when op in @arithmetic_ops -> - error("arithmetic is not supported", pos) + defp hint({:word, word, _}, nil, 0, _) when word in @not_select, + do: "only SELECT statements are supported" - [{:word, "not", _}, {:word, word, _} = next | rest] - when word in ~w[between in like ilike] -> - negatable(left, next, rest, true, depth) + defp hint(_, {:op, :";", _}, _, _), do: "only one statement is supported" - [{:word, word, _} = next | rest] when word in ~w[between in like ilike] -> - negatable(left, next, rest, false, depth) + defp hint({:word, "select", _}, {:op, :"(", _}, _, _), do: "subqueries are not supported" - [{:word, "is", _} | rest] -> - is_null(left, rest) - - [{:word, "isnull", _} | rest] -> - {{:is_null, left, false}, rest} - - [{:word, "notnull", _} | rest] -> - {{:is_null, left, true}, rest} - - rest -> - {left, rest} - end - end - - defp negatable(left, {:word, "between", _}, rest, negated?, depth) do - {low, rest} = operand(rest, depth) - rest = expect_keyword(rest, "and", "AND in BETWEEN") - {high, rest} = operand(rest, depth) - {{:between, left, low, high, negated?}, rest} - end - - defp negatable(left, {:word, "in", _}, rest, negated?, depth) do - case rest do - [{:op, :"(", pos}, {:op, :")", _} | _] -> - error("IN needs at least one value", pos) - - [{:op, :"(", pos} | rest] -> - check_depth(depth, pos) - {values, rest} = expr_list(rest, depth + 1) - rest = expect_op(rest, :")", "')' to close the IN list") - {{:in, left, values, negated?}, rest} - - [{_, _, pos} = token | _] -> - error("expected '(' after IN, got #{describe(token)}", pos) - end - end - - defp negatable(left, {:word, like, _}, rest, negated?, depth) when like in ~w[like ilike] do - {pattern, rest} = operand(rest, depth) - {{String.to_atom(like), left, pattern, negated?}, rest} - end - - defp is_null(left, rest) do - {negated?, rest} = - case rest do - [{:word, "not", _} | rest] -> {true, rest} - rest -> {false, rest} - end - - case rest do - [{:word, "null", _} | rest] -> - {{:is_null, left, negated?}, rest} - - [{_, _, pos} = token | _] -> - error("expected NULL after IS#{if negated?, do: " NOT"}, got #{describe(token)}", pos) - end - end + defp hint({:word, word, _}, _, _, expected) when word in @reserved do + cond do + word == "distinct" -> + "DISTINCT is not supported" - defp expr_list(tokens, depth) do - {expr, rest} = expr(tokens, depth) + MapSet.member?(expected, :ident) -> + "expected #{describe_expected(expected)}, got the keyword '#{word}' " <> + "(quote it, \"#{word}\", to use it as a name)" - case rest do - [{:op, :",", _} | rest] -> - {exprs, rest} = expr_list(rest, depth) - {[expr | exprs], rest} + Map.has_key?(@unsupported, word) -> + "#{@unsupported[word]} is not supported" - rest -> - {[expr], rest} + true -> + nil end end - # An operand is a primary followed by any number of `::type` casts. - defp operand(tokens, depth) do - {primary, rest} = primary(tokens, depth) - casts(primary, rest) - end - - defp casts(expr, [{:op, :"::", _} | rest]) do - {type, rest} = type_name(rest) - casts({:cast, expr, type}, rest) - end - - defp casts(expr, rest), do: {expr, rest} - - defp primary([{:string, s, _} | rest], _depth), do: {{:literal, s}, rest} - defp primary([{:integer, n, _} | rest], _depth), do: {{:literal, n}, rest} - defp primary([{:float, f, _} | rest], _depth), do: {{:literal, f}, rest} - - defp primary([{:op, :-, _}, {type, n, _} | rest], _depth) when type in [:integer, :float], - do: {{:literal, -n}, rest} - - defp primary([{:op, :+, _}, {type, n, _} | rest], _depth) when type in [:integer, :float], - do: {{:literal, n}, rest} + defp hint({:word, "fetch", _}, _, _, _), do: "FETCH is not supported" - defp primary([{:op, sign, pos} | _], _depth) when sign in [:-, :+], - do: error("a sign is only supported on a number", pos) + defp hint({:op, :"(", _}, {:word, word, _}, _, _) when word not in @keywords, + do: "functions are not supported" - defp primary([{:word, "true", _} | rest], _depth), do: {{:literal, true}, rest} - defp primary([{:word, "false", _} | rest], _depth), do: {{:literal, false}, rest} - defp primary([{:word, "null", _} | rest], _depth), do: {{:literal, nil}, rest} + defp hint({:op, :"(", _}, {:quoted, _, _}, _, _), do: "functions are not supported" - defp primary([{:word, "select", pos} | _], _depth), - do: error("subqueries are not supported", pos) + defp hint({:op, :., _}, {type, _, _}, _, _) when type in [:word, :quoted], + do: "qualified column names are not supported" - defp primary([{:word, "cast", _}, {:op, :"(", pos} | rest], depth) do - check_depth(depth, pos) - {expr, rest} = expr(rest, depth + 1) - rest = expect_keyword(rest, "as", "AS in CAST") - {type, rest} = type_name(rest) - rest = expect_op(rest, :")", "')' to close CAST") - {{:cast, expr, type}, rest} + defp hint({:op, op, _}, previous, _, _) + when op in [:+, :-, :*, :/, :%] and previous != nil do + if ends_operand?(previous), do: "arithmetic is not supported" end - defp primary([{:op, :"(", pos} | rest], depth) do - check_depth(depth, pos) - {first, rest} = paren_element(rest, depth + 1) - - case rest do - [{:op, :",", _} | rest] -> - {elements, rest} = tuple_elements(rest, depth + 1) - rest = expect_op(rest, :")", "')' to close the tuple") - {{:tuple, [first | elements]}, rest} - - rest -> - if first == :star, do: error("'*' is only allowed in a tuple", pos) - rest = expect_op(rest, :")", "')'") - {first, rest} - end + defp hint(_, {:op, sign, _}, _, expected) when sign in [:+, :-] do + if MapSet.member?(expected, :integer), do: "a sign is only supported on a number" end - defp primary([{:word, word, pos} | _] = tokens, _depth) when word not in @reserved do - {name, rest} = name(tokens, "an expression") + defp hint(_, _, _, _), do: nil - case rest do - [{:op, :"(", _} | _] -> error("functions are not supported", pos) - [{:op, :., dot} | _] -> error("qualified column names are not supported", dot) - rest -> {{:column, name}, rest} - end - end + defp ends_operand?({:word, word, _}), do: word not in @reserved - defp primary([{:quoted, _, _} | _] = tokens, _depth) do - {name, rest} = name(tokens, "an expression") - {{:column, name}, reject_qualified_or_call(rest)} - end + defp ends_operand?({type, _, _}) when type in [:quoted, :string, :integer, :float], + do: true - defp primary([{_, _, pos} = token | _], _depth), - do: error("expected an expression, got #{describe(token)}", pos) + defp ends_operand?({:op, :")", _}), do: true + defp ends_operand?(_), do: false - defp paren_element([{:op, :*, _} | rest], _depth), do: {:star, rest} - defp paren_element(tokens, depth), do: expr(tokens, depth) + # Collapses the accepted terminals into a few words: "an expression", + # "a name", "an operator", "a type name", then any keywords and symbols. + defp describe_expected(expected) do + cond do + MapSet.member?(expected, :"$end") -> + "the end of the statement" - defp tuple_elements(tokens, depth) do - {element, rest} = paren_element(tokens, depth) + MapSet.member?(expected, :from) and MapSet.member?(expected, :ident) -> + "a type name" - case rest do - [{:op, :",", _} | rest] -> - {elements, rest} = tuple_elements(rest, depth) - {[element | elements], rest} + true -> + {groups, rest} = + Enum.reduce( + [ + {"an expression", @expression_starts ++ [:not]}, + {"a name", @name_starts}, + {"an operator", @operators} + ], + {[], expected}, + fn {label, members}, {groups, rest} -> + if group_present?(label, members, rest), + do: {[label | groups], MapSet.difference(rest, MapSet.new(members))}, + else: {groups, rest} + end + ) - rest -> - {[element], rest} + (Enum.reverse(groups) ++ Enum.map(Enum.filter(@terminals, &(&1 in rest)), &terminal/1)) + |> join_or() end end - # A type is any word (reserved ones like date and time included), with - # PostgreSQL's `timestamp with[out] time zone` spelled out. - defp type_name([{:word, type, _} | rest]) when type in ~w[timestamp time] do - case rest do - [{:word, "with", _}, {:word, "time", _}, {:word, "zone", _} | rest] -> {type <> "tz", rest} - [{:word, "without", _}, {:word, "time", _}, {:word, "zone", _} | rest] -> {type, rest} - rest -> {type, rest} - end - end + defp group_present?("an operator", _members, rest), + do: Enum.all?(@comparisons, &MapSet.member?(rest, &1)) - defp type_name([{:word, type, _} | rest]), do: {type, rest} + defp group_present?(_label, members, rest), + do: Enum.all?(members -- [:not], &MapSet.member?(rest, &1)) - defp type_name([{_, _, pos} = token | _]), - do: error("expected a type name, got #{describe(token)}", pos) + defp terminal(:integer), do: "a whole number" + defp terminal(:float), do: "a number" + defp terminal(:string), do: "a string" + defp terminal(:ident), do: "a name" + defp terminal(:quoted), do: "a quoted name" + defp terminal(op) when op in @ops, do: "'#{op}'" + defp terminal(keyword), do: keyword |> Atom.to_string() |> String.upcase() - # -- names -- - - defp name([{:word, word, pos} | _], what) when word in @reserved do - error( - "expected #{what}, got the keyword '#{word}' (quote it, \"#{word}\", to use it as a name)", - pos - ) - end - - defp name([{:word, word, _} | rest], _what), do: {word, rest} - defp name([{:quoted, name, _} | rest], _what), do: {name, rest} - - defp name([{_, _, pos} = token | _], what), - do: error("expected #{what}, got #{describe(token)}", pos) - - defp reject_qualified_or_call([{:op, :"(", pos} | _]), - do: error("functions are not supported", pos) - - defp reject_qualified_or_call([{:op, :., pos} | _]), - do: error("qualified column names are not supported", pos) - - defp reject_qualified_or_call(rest), do: rest - - # -- helpers -- - - defp expect_keyword([{:word, word, _} | rest], word, _what), do: rest - - defp expect_keyword([{_, _, pos} = token | _], _word, what), - do: error("expected #{what}, got #{describe(token)}", pos) - - defp expect_op([{:op, op, _} | rest], op, _what), do: rest - - defp expect_op([{_, _, pos} = token | _], _op, what), - do: error("expected #{what}, got #{describe(token)}", pos) - - defp check_depth(depth, pos) do - if depth >= @max_depth, do: error("expression is nested too deeply", pos) - end + defp join_or([one]), do: one + defp join_or(items), do: Enum.join(Enum.drop(items, -1), ", ") <> " or " <> List.last(items) defp describe({:word, word, _}), do: "'#{word}'" defp describe({:quoted, name, _}), do: inspect(name) @@ -511,6 +267,4 @@ defmodule Efsql.SQL.Parser do defp truncate(s) do if String.length(s) > 20, do: String.slice(s, 0, 20) <> "...", else: s end - - defp error(reason, pos), do: throw({:syntax_error, reason, pos}) end diff --git a/lib/efsql/sql/yecc.ex b/lib/efsql/sql/yecc.ex deleted file mode 100644 index 0792200..0000000 --- a/lib/efsql/sql/yecc.ex +++ /dev/null @@ -1,52 +0,0 @@ -defmodule Efsql.SQL.Yecc do - @moduledoc """ - The same contract as `Efsql.SQL.Parser`, built on leex and yecc: the - tokens come from `Efsql.SQL.Leex` and the grammar is - `src/efsql_sql_yecc.yrl`. - - yecc reports only the token a syntax error stopped at, so the error's - wording is worked out by `Efsql.SQL.Yecc.Explain` from the tokens - before it. - """ - - alias Efsql.SQL.AST - alias Efsql.SQL.Leex - alias Efsql.SQL.SyntaxError - alias Efsql.SQL.Yecc.Explain - - # Words the grammar has as terminals. They must match its Terminals. - @keywords Efsql.SQL.Parser.reserved_words() ++ ~w[time timestamp without zone] - - @spec parse(String.t()) :: {:ok, AST.Select.t()} | {:error, SyntaxError.t()} - def parse(sql) when is_binary(sql) do - with {:ok, tokens} <- Leex.tokenize(sql) do - # A token's location is its index, so an error names its token - # exactly (end of input can share a position with another token). - ytokens = tokens |> Enum.with_index() |> Enum.map(fn {t, i} -> to_yecc(t, i) end) - - case :efsql_sql_yecc.parse(ytokens) do - {:ok, {:select, fields, from, where, order_by, limit}} -> - {:ok, - %AST.Select{fields: fields, from: from, where: where, order_by: order_by, limit: limit}} - - {:error, {index, :efsql_sql_yecc, _message}} -> - {reason, {line, column}} = Explain.explain(tokens, index) - {:error, %SyntaxError{reason: reason, line: line, column: column}} - end - end - end - - @spec parse!(String.t()) :: AST.Select.t() - def parse!(sql) do - case parse(sql) do - {:ok, select} -> select - {:error, error} -> raise error - end - end - - defp to_yecc({:word, word, _}, i) when word in @keywords, do: {String.to_atom(word), i, word} - defp to_yecc({:word, word, _}, i), do: {:ident, i, word} - defp to_yecc({:op, op, _}, i), do: {op, i, op} - defp to_yecc({:eof, nil, _}, i), do: {:"$end", i} - defp to_yecc({type, value, _}, i), do: {type, i, value} -end diff --git a/lib/efsql/sql/yecc/explain.ex b/lib/efsql/sql/yecc/explain.ex deleted file mode 100644 index a126e76..0000000 --- a/lib/efsql/sql/yecc/explain.ex +++ /dev/null @@ -1,454 +0,0 @@ -defmodule Efsql.SQL.Yecc.Explain do - @moduledoc false - # Words for a yecc syntax error. yecc says only which token it stopped - # at; Efsql.SQL.Parser says what it expected there. This reconstructs - # that from the tokens before the error: which clause it is in, what - # the previous token asks for, and which parenthesis is still open. - # - # In a few places the LALR parser reads further than the hand-written - # one before failing (an infix `not` that isn't NOT IN/LIKE/BETWEEN, a - # half-written `timestamp with time zone`); there it steps back to the - # token Efsql.SQL.Parser would have stopped at. - - @reserved Efsql.SQL.Parser.reserved_words() - @not_select ~w[insert update delete create drop alter truncate grant revoke explain] - @unsupported_clauses %{ - "group" => "GROUP BY", - "having" => "HAVING", - "offset" => "OFFSET", - "join" => "JOIN", - "inner" => "JOIN", - "left" => "JOIN", - "right" => "JOIN", - "full" => "JOIN", - "cross" => "JOIN", - "union" => "UNION", - "intersect" => "INTERSECT", - "except" => "EXCEPT", - "fetch" => "FETCH" - } - @comparisons [:=, :<>, :<, :>, :<=, :>=] - @arithmetic [:+, :-, :*, :/, :%] - @clauses ~w[select from where order limit] - - @doc "The reason and position Efsql.SQL.Parser gives for the error yecc hit at token `index`." - def explain(tokens, index), do: at(List.to_tuple(tokens), index) - - defp at(t, 0) do - case elem(t, 0) do - {:eof, _, pos} -> {"empty statement", pos} - {:word, word, pos} when word in @not_select -> {"only SELECT statements are supported", pos} - token -> expected("SELECT", token) - end - end - - defp at(t, i) do - case elem(t, i - 1) do - {:op, :";", _} -> {"only one statement is supported", pos(elem(t, i))} - _ -> in_clause(t, i, clause(t, i)) - end - end - - # The clause the error is in: the last clause keyword before it, not - # counting type names (`::order` is a type). - defp clause(t, i) do - Enum.reduce(1..(i - 1)//1, {"select", 0}, fn j, acc -> - case elem(t, j) do - {:word, word, _} when word in @clauses -> - if type_position?(t, j), do: acc, else: {word, j} - - _ -> - acc - end - end) - end - - # -- SELECT list -- - - defp in_clause(t, i, {"select", _}) do - token = elem(t, i) - - case {i, elem(t, i - 1), token} do - {1, _, {:word, "distinct", pos}} -> {"DISTINCT is not supported", pos} - {1, _, _} -> name_error(token, "a field name or *") - {2, {:op, :*, _}, {:op, :",", pos}} -> {"* can't be combined with other fields", pos} - {2, {:op, :*, _}, _} -> expected("FROM", token) - {_, {:op, :",", _}, _} -> name_error(token, "a field name") - {_, _, {:op, :"(", pos}} -> {"functions are not supported", pos} - {_, _, {:op, :., pos}} -> {"qualified column names are not supported", pos} - _ -> expected("FROM", token) - end - end - - # -- FROM -- - - defp in_clause(t, i, {"from", f}) do - token = elem(t, i) - - case {elem(t, i - 1), token} do - {{:word, "from", _}, _} -> - name_error(token, "a table name") - - {{:op, :., _}, _} -> - name_error(token, "a name after '.'") - - {_, {:op, :., pos}} when i - f == 6 -> - {"a table name has at most three parts (storage_id.tenant.table)", pos} - - {_, {:op, :",", pos}} -> - {"selecting from more than one table is not supported", pos} - - _ -> - end_of_statement(token) - end - end - - # -- ORDER BY -- - - defp in_clause(t, i, {"order", o}) do - token = elem(t, i) - - case {elem(t, i - 1), token} do - {_, _} when i == o + 1 -> expected("BY after ORDER", token) - {{:word, "by", _}, _} -> name_error(token, "a field name to order by") - {{:op, :",", _}, _} -> name_error(token, "a field name to order by") - {{:word, dir, _}, {:word, "nulls", pos}} when dir in ~w[asc desc] -> nulls(pos) - {{:word, dir, _}, _} when dir in ~w[asc desc] -> end_of_statement(token) - {_, {:op, :"(", pos}} -> {"functions are not supported", pos} - {_, {:op, :., pos}} -> {"qualified column names are not supported", pos} - {_, {:word, "nulls", pos}} -> nulls(pos) - _ -> end_of_statement(token) - end - end - - # -- LIMIT -- - - defp in_clause(t, i, {"limit", l}) do - token = elem(t, i) - - if i == l + 1, - do: {"LIMIT expects a whole number, got #{describe(token)}", pos(token)}, - else: end_of_statement(token) - end - - # -- WHERE -- - - defp in_clause(t, i, {"where", w}) do - token = elem(t, i) - prev = elem(t, i - 1) - - cond do - k = half_type(t, i) -> - at(t, k) - - # A type name can be any word (`x::or`); it is never a keyword. - type_position?(t, i - 1) -> - after_operand(t, w, i) - - word?(prev, "is") -> - expected("NULL after IS", token) - - word?(prev, "not") and word?(elem(t, i - 2), "is") -> - expected("NULL after IS NOT", token) - - word?(prev, "not") and not operand_expected_after?(elem(t, i - 2)) and - not word?(token, ~w[between in like ilike]) -> - at(t, i - 1) - - word?(prev, "cast") and not op?(token, :"(") -> - {"expected an expression, got 'cast'", pos(prev)} - - op?(prev, [:-, :+]) and operand_expected_after?(elem(t, i - 2)) -> - {"a sign is only supported on a number", pos(prev)} - - op?(prev, :"::") or (word?(prev, "as") and in_cast?(t, w, i)) -> - expected("a type name", token) - - word?(prev, "in") -> - expected("'(' after IN", token) - - op?(prev, :"(") and op?(token, :")") and word?(elem(t, i - 2), "in") -> - {"IN needs at least one value", pos(prev)} - - operand_expected_after?(prev) -> - case token do - {:word, "select", pos} -> {"subqueries are not supported", pos} - _ -> expected("an expression", token) - end - - true -> - after_operand(t, w, i) - end - end - - # Something complete comes before the token: a name, a literal, a - # parenthesized expression, a whole predicate. - defp after_operand(t, w, i) do - token = elem(t, i) - prev = elem(t, i - 1) - start = operand_start(t, i - 1) - before = elem(t, start - 1) - - cond do - # A tuple's `*`, the type in CAST(x AS type), or the end of a whole - # predicate: what follows is the group's business. - op?(prev, :*) or type_after_as?(t, i - 1) or ends_predicate?(t, i - 1) -> - in_group(t, w, i) - - name_primary?(t, i - 1) and op?(token, :"(") -> - {"functions are not supported", - if(match?({:quoted, _, _}, prev), do: pos(token), else: pos(prev))} - - name_primary?(t, i - 1) and op?(token, :.) -> - {"qualified column names are not supported", pos(token)} - - word?(before, "between") -> - expected("AND in BETWEEN", token) - - op?(token, @arithmetic) and not right_operand?(t, start) -> - {"arithmetic is not supported", pos(token)} - - true -> - in_group(t, w, i) - end - end - - # The innermost parenthesis still open says what was expected. - defp in_group(t, w, i) do - token = elem(t, i) - - case open_group(t, w, i) do - nil -> - end_of_statement(token) - - {_, :in_list} -> - expected("')' to close the IN list", token) - - {_, :tuple} -> - expected("')' to close the tuple", token) - - {_, :cast} -> - expected("AS in CAST", token) - - {_, :cast_as} -> - expected("')' to close CAST", token) - - {k, :paren} -> - if op?(elem(t, k + 1), :*), - do: {"'*' is only allowed in a tuple", pos(elem(t, k))}, - else: expected("')'", token) - end - end - - defp open_group(t, w, i) do - Enum.reduce((w + 1)..(i - 1)//1, [], fn j, stack -> - case {elem(t, j), stack} do - {{:op, :"(", _}, _} -> - kind = - case elem(t, j - 1) do - {:word, "in", _} -> :in_list - {:word, "cast", _} -> :cast - _ -> :paren - end - - [{j, kind} | stack] - - {{:op, :")", _}, [_ | rest]} -> - rest - - {{:op, :",", _}, [{k, :paren} | rest]} -> - [{k, :tuple} | rest] - - {{:word, "as", _}, [{k, :cast} | rest]} -> - [{k, :cast_as} | rest] - - _ -> - stack - end - end) - |> List.first() - end - - defp in_cast?(t, w, i), do: match?({_, :cast}, open_group(t, w, i - 1)) - - # -- the operand before an error -- - - # Where the operand ending at `j` starts: step back over `::type` casts, - # then over one primary. - defp operand_start(t, j) do - case type_start(t, j) do - nil -> - case elem(t, j) do - {:op, :")", _} -> - k = matching_open(t, j - 1, 0) - if word?(elem(t, k - 1), "cast"), do: k - 1, else: k - - {type, _, _} when type in [:integer, :float] -> - if op?(elem(t, j - 1), [:-, :+]) and operand_expected_after?(elem(t, j - 2)), - do: j - 1, - else: j - - _ -> - j - end - - k -> - operand_start(t, k - 1) - end - end - - # If a `::type` ends at `j`, the index of its `::`. - defp type_start(t, j) when j >= tuple_size(t), do: nil - - defp type_start(t, j) do - cond do - j >= 1 and op?(elem(t, j - 1), :"::") and match?({:word, _, _}, elem(t, j)) -> - j - 1 - - j >= 4 and op?(elem(t, j - 4), :"::") and word?(elem(t, j - 3), ~w[timestamp time]) and - word?(elem(t, j - 2), ~w[with without]) and word?(elem(t, j - 1), "time") and - word?(elem(t, j), "zone") -> - j - 4 - - true -> - nil - end - end - - defp matching_open(t, j, depth) do - case elem(t, j) do - {:op, :")", _} -> matching_open(t, j - 1, depth + 1) - {:op, :"(", _} when depth == 0 -> j - {:op, :"(", _} -> matching_open(t, j - 1, depth - 1) - _ -> matching_open(t, j - 1, depth) - end - end - - # The right-hand side of a comparison, LIKE or BETWEEN ... AND: the - # operand the predicate ends with, so the error is the group's. - defp right_operand?(t, start) do - case elem(t, start - 1) do - {:op, op, _} when op in @comparisons -> true - {:word, like, _} when like in ~w[like ilike] -> true - {:word, "and", _} -> between_and?(t, start - 1) - _ -> false - end - end - - # An AND belongs to BETWEEN when the nearest boolean boundary before it - # at the same depth is the BETWEEN. - defp between_and?(t, j), do: between_and?(t, j - 1, 0) - - defp between_and?(t, j, depth) do - case elem(t, j) do - {:op, :")", _} -> between_and?(t, j - 1, depth + 1) - {:op, :"(", _} when depth > 0 -> between_and?(t, j - 1, depth - 1) - {:word, "between", _} when depth == 0 -> true - {:word, word, _} when depth == 0 and word in ~w[and or where] -> false - {:op, op, _} when depth == 0 and op in [:"(", :","] -> false - _ -> between_and?(t, j - 1, depth) - end - end - - # A name standing as an expression, not a type name. - defp name_primary?(t, j) do - case elem(t, j) do - {:quoted, _, _} -> true - {:word, word, _} -> word not in @reserved and not type_position?(t, j) - _ -> false - end - end - - # A word after `::` or CAST's AS, or inside `timestamp with time zone`. - defp type_position?(t, j) do - op?(elem(t, j - 1), :"::") or word?(elem(t, j - 1), "as") or type_start(t, j) != nil or - (j >= 2 and word?(elem(t, j), ~w[with without time zone]) and - (type_start(t, j + 1) != nil or type_start(t, j + 2) != nil or - type_start(t, j + 3) != nil)) - end - - # `timestamp with` / `timestamp with time` cut short: the hand parser - # takes the type as `timestamp` and stops at the `with`. - defp half_type(t, i) do - cond do - i >= 3 and word?(elem(t, i - 1), ~w[with without]) and - word?(elem(t, i - 2), ~w[timestamp time]) and - type_opener?(elem(t, i - 3)) -> - i - 1 - - i >= 4 and word?(elem(t, i - 1), "time") and word?(elem(t, i - 2), ~w[with without]) and - word?(elem(t, i - 3), ~w[timestamp time]) and type_opener?(elem(t, i - 4)) -> - i - 2 - - true -> - nil - end - end - - defp type_opener?(token), do: op?(token, :"::") or word?(token, "as") - - # The last token of IN (...), IS [NOT] NULL, ISNULL or NOTNULL. - defp ends_predicate?(t, j) do - case elem(t, j) do - {:op, :")", _} -> word?(elem(t, matching_open(t, j - 1, 0) - 1), "in") - {:word, "null", _} -> word?(elem(t, j - 1), "is") or word?(elem(t, j - 2), "is") - {:word, word, _} -> word in ~w[isnull notnull] - _ -> false - end - end - - defp type_after_as?(t, j) do - word?(elem(t, j - 1), "as") or - (j >= 4 and word?(elem(t, j - 4), "as") and word?(elem(t, j - 3), ~w[timestamp time]) and - word?(elem(t, j - 2), ~w[with without]) and word?(elem(t, j - 1), "time") and - word?(elem(t, j), "zone")) - end - - # Tokens after which an operand must come. - defp operand_expected_after?(token) do - word?(token, ~w[where and or not between like ilike]) or - op?(token, [:"(", :"," | @comparisons]) - end - - # -- messages -- - - defp end_of_statement({:word, word, pos}) when is_map_key(@unsupported_clauses, word), - do: {"#{Map.fetch!(@unsupported_clauses, word)} is not supported", pos} - - defp end_of_statement({:op, op, pos}) when op in @arithmetic, - do: {"arithmetic is not supported", pos} - - defp end_of_statement(token), do: expected("end of statement", token) - - defp nulls(pos), do: {"NULLS FIRST/LAST is not supported", pos} - - defp name_error({:word, word, pos}, what) when word in @reserved do - {"expected #{what}, got the keyword '#{word}' (quote it, \"#{word}\", to use it as a name)", - pos} - end - - defp name_error(token, what), do: expected(what, token) - - defp expected(what, token), do: {"expected #{what}, got #{describe(token)}", pos(token)} - - defp describe({:word, word, _}), do: "'#{word}'" - defp describe({:quoted, name, _}), do: inspect(name) - defp describe({:string, s, _}), do: "the string '#{truncate(s)}'" - defp describe({type, n, _}) when type in [:integer, :float], do: "the number #{n}" - defp describe({:op, op, _}), do: "'#{op}'" - defp describe({:eof, _, _}), do: "the end of the statement" - - defp truncate(s) do - if String.length(s) > 20, do: String.slice(s, 0, 20) <> "...", else: s - end - - defp pos({_, _, pos}), do: pos - - defp word?({:word, word, _}, words) when is_list(words), do: word in words - defp word?({:word, word, _}, word), do: true - defp word?(_, _), do: false - - defp op?({:op, op, _}, ops) when is_list(ops), do: op in ops - defp op?({:op, op, _}, op), do: true - defp op?(_, _), do: false -end diff --git a/src/efsql_sql_yecc.yrl b/src/efsql_sql_grammar.yrl similarity index 88% rename from src/efsql_sql_yecc.yrl rename to src/efsql_sql_grammar.yrl index af0c489..5db49a1 100644 --- a/src/efsql_sql_yecc.yrl +++ b/src/efsql_sql_grammar.yrl @@ -1,17 +1,20 @@ -%% Grammar for efsql's SQL dialect, driven by Efsql.SQL.Yecc. +%% Grammar for efsql's SQL dialect. Efsql.SQL.Parser drives it with +%% tokens from Efsql.SQL.Lexer: %% %% SELECT fields FROM name [WHERE expr] [ORDER BY items] [LIMIT n] [;] %% -%% Tokens are {Category, {Line, Column}, Value}; `/` and `%` have no -%% terminal, so they are syntax errors wherever they appear. Words the driver finds in -%% the reserved list come as their own category; `time`, `timestamp`, -%% `without` and `zone` do too, but stay usable as names. Everything else -%% is `ident`. +%% A token is {Category, Index, Value}, where Index is its place in the +%% token list, so an error names its token exactly. Words in the reserved +%% list (Efsql.SQL.Parser.reserved_words/0) come as their own category and +%% must match the keyword Terminals below; `time`, `timestamp`, `without` +%% and `zone` do too, but stay usable as names. Every other word is +%% `ident`. `/` and `%` have no terminal, so they are errors anywhere. %% -%% The grammar accepts exactly the valid statements; explaining an error -%% in words is the driver's job, since yecc only reports the token it -%% stopped at. Actions build the same tree as Efsql.SQL.Parser (see -%% Efsql.SQL.AST), with the statement as a tuple the driver makes a struct. +%% The grammar accepts exactly the valid statements. Error messages are +%% worked out from it by the driver, which asks which tokens could have +%% come next, so a new rule needs nothing else to be explained. Actions +%% build Efsql.SQL.AST, with the statement as a tuple the driver makes a +%% struct. Nonterminals statement select_stmt fields field_list table where_clause order_clause order_items order_item limit_clause diff --git a/src/efsql_sql_leex.xrl b/src/efsql_sql_leex.xrl deleted file mode 100644 index 5d3f4c2..0000000 --- a/src/efsql_sql_leex.xrl +++ /dev/null @@ -1,58 +0,0 @@ -%% Tokenizer for efsql's SQL dialect, driven by Efsql.SQL.Leex. -%% -%% Every rule returns a token carrying the matched characters, and nothing -%% is skipped: the driver needs each match to track character-accurate -%% positions (leex's own columns count UTF-8 bytes), and it finishes the -%% work leex can't express: nested block comments, Unicode letter classes, -%% and the checks on numbers and words. Tokens are -%% {Kind, Chars} with Kind one of: ws, line_comment, block_comment, -%% string, quoted, unterminated, number, bad_exponent, number_junk, word, -%% op, colon. -%% -%% Invalid UTF-8 bytes reach this lexer as UTF-16 surrogates (16#D800 + -%% byte), which decoded UTF-8 never contains and leex's column counting -%% still accepts. No positive class includes them, so outside strings and -%% comments they are illegal characters. - -Definitions. - -D = [0-9] -NUM = ({D}+\.?{D}*|\.{D}+) -%% Macros are pasted in as text, so a macro used with a postfix operator -%% needs its own parentheses. -EXP = ([eE][+-]?{D}+) -WS = [\s\t\f\v\r\n\x{A0}\x{1680}\x{2000}-\x{200A}\x{202F}\x{205F}\x{3000}\x{FEFF}] -ID = [A-Za-z_\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}][A-Za-z0-9_$\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}]* - -Rules. - -{WS}+ : {token, {ws, TokenChars}}. - --- : {token, {line_comment, TokenChars}}. -/\* : {token, {block_comment, TokenChars}}. - -'([^']|'')*' : {token, {string, TokenChars}}. -"([^"]|"")*" : {token, {quoted, TokenChars}}. -'([^']|'')* : {token, {unterminated, TokenChars}}. -"([^"]|"")* : {token, {unterminated, TokenChars}}. - -%% Longest match picks the number, then an exponent with no digits, then -%% a number running into another character (which the driver checks). -{NUM}{EXP}? : {token, {number, TokenChars}}. -{NUM}[eE][+-]? : {token, {bad_exponent, TokenChars}}. -{NUM}{EXP}?[A-Za-z_.0-9\x{80}-\x{D7FF}\x{E000}-\x{10FFFF}] : {token, {number_junk, TokenChars}}. - -{ID} : {token, {word, TokenChars}}. - -<> : {token, {op, TokenChars}}. -!= : {token, {op, TokenChars}}. -<= : {token, {op, TokenChars}}. ->= : {token, {op, TokenChars}}. -:: : {token, {op, TokenChars}}. -[=<>(),.;*+\-/%] : {token, {op, TokenChars}}. - -%% A lone colon, or one followed by anything: the driver explains it. -:[\x{0}-\x{D7FF}\x{E000}-\x{10FFFF}] : {token, {colon, TokenChars}}. -: : {token, {colon, TokenChars}}. - -Erlang code. diff --git a/test/sql/differential_test.exs b/test/sql/differential_test.exs deleted file mode 100644 index 46c9238..0000000 --- a/test/sql/differential_test.exs +++ /dev/null @@ -1,28 +0,0 @@ -defmodule Efsql.SQL.DifferentialTest do - # The two front ends side by side: the hand-written lexer and parser, and - # the ones built on leex and yecc, must give the same result for any - # input: the same tokens, the same tree, or the same error at the same - # position. Inputs are valid queries, mangled ones and garbage in equal - # measure; reproduce a failure with `mix test --seed N`. - use ExUnit.Case, async: true - - alias EfsqlTest.SQLGen - - @runs 3_000 - - test "the lexers agree on every input" do - for _ <- 1..@runs do - sql = SQLGen.any_input() - - assert Efsql.SQL.Leex.tokenize(sql) == Efsql.SQL.Lexer.tokenize(sql), - "input: #{inspect(sql)}" - end - end - - test "the parsers agree on every input" do - for _ <- 1..@runs do - sql = SQLGen.any_input() - assert Efsql.SQL.Yecc.parse(sql) == Efsql.SQL.Parser.parse(sql), "input: #{inspect(sql)}" - end - end -end diff --git a/test/sql/lexer_test.exs b/test/sql/lexer_test.exs index 02d54ce..2452519 100644 --- a/test/sql/lexer_test.exs +++ b/test/sql/lexer_test.exs @@ -1,111 +1,108 @@ defmodule Efsql.SQL.LexerTest do - # Every test runs against both lexers: the hand-written one and the one - # built on leex. They share a contract, token for token. - use ExUnit.Case, - async: true, - parameterize: [%{lexer: Efsql.SQL.Lexer}, %{lexer: Efsql.SQL.Leex}] + use ExUnit.Case, async: true + alias Efsql.SQL.Lexer alias Efsql.SQL.SyntaxError - defp tokens(lexer, sql) do - {:ok, tokens} = lexer.tokenize(sql) + defp tokens(sql) do + {:ok, tokens} = Lexer.tokenize(sql) tokens |> Enum.map(fn {type, value, _pos} -> {type, value} end) |> Enum.drop(-1) end - defp positions(lexer, sql) do - {:ok, tokens} = lexer.tokenize(sql) + defp positions(sql) do + {:ok, tokens} = Lexer.tokenize(sql) Enum.map(tokens, fn {_type, _value, pos} -> pos end) end - defp error(lexer, sql) do - assert {:error, %SyntaxError{} = e} = lexer.tokenize(sql) + defp error(sql) do + assert {:error, %SyntaxError{} = e} = Lexer.tokenize(sql) e end - test "empty and blank input is just end of input", %{lexer: lexer} do - assert {:ok, [{:eof, nil, {1, 1}}]} = lexer.tokenize("") - assert {:ok, [{:eof, nil, {2, 3}}]} = lexer.tokenize(" \n ") + test "empty and blank input is just end of input" do + assert {:ok, [{:eof, nil, {1, 1}}]} = Lexer.tokenize("") + assert {:ok, [{:eof, nil, {2, 3}}]} = Lexer.tokenize(" \n ") end describe "words" do - test "fold to lower case", %{lexer: lexer} do - assert tokens(lexer, "SELECT Name fRoM") == [word: "select", word: "name", word: "from"] + test "fold to lower case" do + assert tokens("SELECT Name fRoM") == [word: "select", word: "name", word: "from"] end - test "take underscores, digits and dollars after the first character", %{lexer: lexer} do - assert tokens(lexer, "_ _id a1 a$b created_at") == + test "take underscores, digits and dollars after the first character" do + assert tokens("_ _id a1 a$b created_at") == [word: "_", word: "_id", word: "a1", word: "a$b", word: "created_at"] end - test "take Unicode letters and marks", %{lexer: lexer} do - assert tokens(lexer, "café 日本語 Ωmega") == [word: "café", word: "日本語", word: "ωmega"] + test "take Unicode letters and marks" do + assert tokens("café 日本語 Ωmega") == [word: "café", word: "日本語", word: "ωmega"] end - test "keywords are plain words; the parser decides", %{lexer: lexer} do - assert tokens(lexer, "day at user order") == + test "keywords are plain words; the parser decides" do + assert tokens("day at user order") == [word: "day", word: "at", word: "user", word: "order"] end end describe "quoted identifiers" do - test "keep case and spaces", %{lexer: lexer} do - assert tokens(lexer, ~s("Mixed Case")) == [quoted: "Mixed Case"] + test "keep case and spaces" do + assert tokens(~s("Mixed Case")) == [quoted: "Mixed Case"] end - test "a doubled quote is a quote", %{lexer: lexer} do - assert tokens(lexer, ~s("say ""hi""")) == [quoted: ~s(say "hi")] + test "a doubled quote is a quote" do + assert tokens(~s("say ""hi""")) == [quoted: ~s(say "hi")] end - test "may span lines and hold keywords", %{lexer: lexer} do - assert tokens(lexer, ~s("select\nfrom")) == [quoted: "select\nfrom"] + test "may span lines and hold keywords" do + assert tokens(~s("select\nfrom")) == [quoted: "select\nfrom"] end - test "can't be empty", %{lexer: lexer} do + test "can't be empty" do assert %SyntaxError{reason: "zero-length quoted identifier", line: 1, column: 8} = - error(lexer, ~s(select "")) + error(~s(select "")) end - test "must be closed", %{lexer: lexer} do - assert %SyntaxError{line: 1, column: 8, reason: reason} = error(lexer, ~s(select "abc)) + test "must be closed" do + assert %SyntaxError{line: 1, column: 8, reason: reason} = error(~s(select "abc)) assert reason =~ "unterminated identifier" end end describe "strings" do - test "keep their content verbatim", %{lexer: lexer} do - assert tokens(lexer, "'Hello, World'") == [string: "Hello, World"] + test "keep their content verbatim" do + assert tokens("'Hello, World'") == [string: "Hello, World"] end - test "a doubled quote is a quote", %{lexer: lexer} do - assert tokens(lexer, "'it''s'") == [string: "it's"] - assert tokens(lexer, "''''") == [string: "'"] - assert tokens(lexer, "''") == [string: ""] + test "a doubled quote is a quote" do + assert tokens("'it''s'") == [string: "it's"] + assert tokens("''''") == [string: "'"] + assert tokens("''") == [string: ""] end - test "hide comment and operator characters", %{lexer: lexer} do - assert tokens(lexer, "'-- not a comment /* nor this */ ; ::'") == + test "hide comment and operator characters" do + assert tokens("'-- not a comment /* nor this */ ; ::'") == [string: "-- not a comment /* nor this */ ; ::"] end - test "may span lines, and positions continue after them", %{lexer: lexer} do - assert tokens(lexer, "'a\nb' x") == [string: "a\nb", word: "x"] - assert positions(lexer, "'a\nb' x") == [{1, 1}, {2, 4}, {2, 5}] + test "may span lines, and positions continue after them" do + assert tokens("'a\nb' x") == [string: "a\nb", word: "x"] + assert positions("'a\nb' x") == [{1, 1}, {2, 4}, {2, 5}] end - test "take any Unicode", %{lexer: lexer} do - assert tokens(lexer, "'naïve ☃ 𝄞'") == [string: "naïve ☃ 𝄞"] + test "take any Unicode" do + assert tokens("'naïve ☃ 𝄞'") == [string: "naïve ☃ 𝄞"] end - test "must be closed", %{lexer: lexer} do - e = error(lexer, "where x = 'abc") + test "must be closed" do + e = error("where x = 'abc") assert e.reason =~ "unterminated string" assert {e.line, e.column} == {1, 11} end end describe "numbers" do - test "integers", %{lexer: lexer} do - assert tokens(lexer, "0 42 007 123456789012345678901234567890") == + test "integers" do + assert tokens("0 42 007 123456789012345678901234567890") == [ integer: 0, integer: 42, @@ -114,8 +111,8 @@ defmodule Efsql.SQL.LexerTest do ] end - test "floats in every spelling", %{lexer: lexer} do - assert tokens(lexer, "1.5 .5 1. 1e3 1E3 1.5e-3 2.5E+2") == + test "floats in every spelling" do + assert tokens("1.5 .5 1. 1e3 1E3 1.5e-3 2.5E+2") == [ float: 1.5, float: 0.5, @@ -127,33 +124,33 @@ defmodule Efsql.SQL.LexerTest do ] end - test "are unsigned; a sign is a separate token", %{lexer: lexer} do - assert tokens(lexer, "-1 +2") == [op: :-, integer: 1, op: :+, integer: 2] + test "are unsigned; a sign is a separate token" do + assert tokens("-1 +2") == [op: :-, integer: 1, op: :+, integer: 2] end - test "can't run into a word", %{lexer: lexer} do - assert error(lexer, "123abc").reason =~ "invalid number" - assert error(lexer, "1.5.2").reason =~ "invalid number" + test "can't run into a word" do + assert error("123abc").reason =~ "invalid number" + assert error("1.5.2").reason =~ "invalid number" end - test "out of a double's range", %{lexer: lexer} do - assert error(lexer, "1e400").reason == "number out of range: 1e400" - assert tokens(lexer, "1e-400") == [float: 0.0] + test "out of a double's range" do + assert error("1e400").reason == "number out of range: 1e400" + assert tokens("1e-400") == [float: 0.0] end - test "need exponent digits", %{lexer: lexer} do - assert error(lexer, "1e").reason =~ "exponent has no digits" - assert error(lexer, "1e+x").reason =~ "exponent has no digits" + test "need exponent digits" do + assert error("1e").reason =~ "exponent has no digits" + assert error("1e+x").reason =~ "exponent has no digits" end end describe "operators" do - test "two-character operators win over one-character ones", %{lexer: lexer} do - assert tokens(lexer, "<> != <= >= ::") == [op: :<>, op: :<>, op: :<=, op: :>=, op: :"::"] + test "two-character operators win over one-character ones" do + assert tokens("<> != <= >= ::") == [op: :<>, op: :<>, op: :<=, op: :>=, op: :"::"] end - test "one-character operators", %{lexer: lexer} do - assert tokens(lexer, "= < > ( ) , . ; * + - / %") == + test "one-character operators" do + assert tokens("= < > ( ) , . ; * + - / %") == [ op: :=, op: :<, @@ -171,81 +168,81 @@ defmodule Efsql.SQL.LexerTest do ] end - test "need no surrounding space", %{lexer: lexer} do - assert tokens(lexer, "a>=1") == [word: "a", op: :>=, integer: 1] - assert tokens(lexer, "'x'::atom") == [string: "x", op: :"::", word: "atom"] - assert tokens(lexer, "t.users") == [word: "t", op: :., word: "users"] + test "need no surrounding space" do + assert tokens("a>=1") == [word: "a", op: :>=, integer: 1] + assert tokens("'x'::atom") == [string: "x", op: :"::", word: "atom"] + assert tokens("t.users") == [word: "t", op: :., word: "users"] end end describe "comments and whitespace" do - test "line comments run to the end of the line", %{lexer: lexer} do - assert tokens(lexer, "a -- ignored ; 'x'\nb") == [word: "a", word: "b"] - assert tokens(lexer, "a --") == [word: "a"] + test "line comments run to the end of the line" do + assert tokens("a -- ignored ; 'x'\nb") == [word: "a", word: "b"] + assert tokens("a --") == [word: "a"] end - test "the end of input after a line comment is positioned after it", %{lexer: lexer} do - assert positions(lexer, "a -- é\xFFz") == [{1, 1}, {1, 9}] - assert positions(lexer, "a --\nb") == [{1, 1}, {2, 1}, {2, 2}] + test "the end of input after a line comment is positioned after it" do + assert positions("a -- é\xFFz") == [{1, 1}, {1, 9}] + assert positions("a --\nb") == [{1, 1}, {2, 1}, {2, 2}] end - test "block comments, which nest", %{lexer: lexer} do - assert tokens(lexer, "a /* one /* two */ still */ b") == [word: "a", word: "b"] + test "block comments, which nest" do + assert tokens("a /* one /* two */ still */ b") == [word: "a", word: "b"] end - test "a block comment can split tokens", %{lexer: lexer} do - assert tokens(lexer, "a/**/b") == [word: "a", word: "b"] + test "a block comment can split tokens" do + assert tokens("a/**/b") == [word: "a", word: "b"] end - test "an unclosed block comment is an error at its start", %{lexer: lexer} do - assert %SyntaxError{line: 2, column: 3} = error(lexer, "a\n /* /* */") + test "an unclosed block comment is an error at its start" do + assert %SyntaxError{line: 2, column: 3} = error("a\n /* /* */") end - test "tabs, CRLF and Unicode spaces separate tokens", %{lexer: lexer} do - assert tokens(lexer, "a\tb\r\nc\rd e f") == + test "tabs, CRLF and Unicode spaces separate tokens" do + assert tokens("a\tb\r\nc\rd e f") == [word: "a", word: "b", word: "c", word: "d", word: "e", word: "f"] end - test "a byte-order mark is ignored", %{lexer: lexer} do - assert tokens(lexer, "select") == [word: "select"] + test "a byte-order mark is ignored" do + assert tokens("select") == [word: "select"] end end describe "positions" do - test "are line and column of the token's first character", %{lexer: lexer} do - assert positions(lexer, "select id\nfrom t") == [{1, 1}, {1, 8}, {2, 1}, {2, 6}, {2, 7}] + test "are line and column of the token's first character" do + assert positions("select id\nfrom t") == [{1, 1}, {1, 8}, {2, 1}, {2, 6}, {2, 7}] end - test "count characters, not bytes", %{lexer: lexer} do - assert positions(lexer, "'é' x") == [{1, 1}, {1, 5}, {1, 6}] + test "count characters, not bytes" do + assert positions("'é' x") == [{1, 1}, {1, 5}, {1, 6}] end - test "CRLF is one line break", %{lexer: lexer} do - assert positions(lexer, "a\r\nb") == [{1, 1}, {2, 1}, {2, 2}] + test "CRLF is one line break" do + assert positions("a\r\nb") == [{1, 1}, {2, 1}, {2, 2}] end - test "comments advance them", %{lexer: lexer} do - assert positions(lexer, "/* a\nb */ x -- c\ny") == [{2, 6}, {3, 1}, {3, 2}] + test "comments advance them" do + assert positions("/* a\nb */ x -- c\ny") == [{2, 6}, {3, 1}, {3, 2}] end end describe "errors" do - test "an unknown character, with its position", %{lexer: lexer} do + test "an unknown character, with its position" do assert %SyntaxError{reason: ~s(unexpected character "@"), line: 2, column: 3} = - error(lexer, "a\nb @") + error("a\nb @") end - test "Elixir atom syntax gets a hint", %{lexer: lexer} do - assert error(lexer, "where x = :active").reason =~ "'name'::atom" + test "Elixir atom syntax gets a hint" do + assert error("where x = :active").reason =~ "'name'::atom" end - test "invalid UTF-8", %{lexer: lexer} do - assert error(lexer, <<"a ", 0xFF, " b">>).reason =~ "invalid UTF-8" - assert error(lexer, <<"'", 0xC3, "'">>).reason =~ "invalid UTF-8" + test "invalid UTF-8" do + assert error(<<"a ", 0xFF, " b">>).reason =~ "invalid UTF-8" + assert error(<<"'", 0xC3, "'">>).reason =~ "invalid UTF-8" end - test "format as a message", %{lexer: lexer} do - assert Exception.message(error(lexer, "@")) == + test "format as a message" do + assert Exception.message(error("@")) == ~s(syntax error at line 1, column 1: unexpected character "@") end end diff --git a/test/sql/parser_test.exs b/test/sql/parser_test.exs index b120959..6f91355 100644 --- a/test/sql/parser_test.exs +++ b/test/sql/parser_test.exs @@ -1,37 +1,32 @@ defmodule Efsql.SQL.ParserTest do - # Every test runs against both parsers: the hand-written recursive - # descent one and the one built on leex and yecc. They share a contract: - # the same tree, and the same error message at the same position. - use ExUnit.Case, - async: true, - parameterize: [%{parser: Efsql.SQL.Parser}, %{parser: Efsql.SQL.Yecc}] + use ExUnit.Case, async: true alias Efsql.SQL.AST.Select alias Efsql.SQL.Parser alias Efsql.SQL.SyntaxError - defp parse(parser, sql) do - assert {:ok, %Select{} = select} = parser.parse(sql) + defp parse(sql) do + assert {:ok, %Select{} = select} = Parser.parse(sql) select end - defp where(parser, condition), do: parse(parser, "select a from t where #{condition}").where + defp where(condition), do: parse("select a from t where #{condition}").where - defp error(parser, sql) do - assert {:error, %SyntaxError{} = e} = parser.parse(sql) + defp error(sql) do + assert {:error, %SyntaxError{} = e} = Parser.parse(sql) e end - defp reason(parser, sql), do: error(parser, sql).reason + defp reason(sql), do: error(sql).reason describe "statement" do - test "the smallest query", %{parser: parser} do - assert parse(parser, "select * from t") == + test "the smallest query" do + assert parse("select * from t") == %Select{fields: :star, from: ["t"], where: nil, order_by: [], limit: nil} end - test "every clause", %{parser: parser} do - assert parse(parser, "select a, b from s.t where a = 1 order by b desc limit 5;") == + test "every clause" do + assert parse("select a, b from s.t where a = 1 order by b desc limit 5;") == %Select{ fields: ["a", "b"], from: ["s", "t"], @@ -41,78 +36,72 @@ defmodule Efsql.SQL.ParserTest do } end - test "the semicolon is optional", %{parser: parser} do - assert parse(parser, "select * from t") == parse(parser, "select * from t;") + test "the semicolon is optional" do + assert parse("select * from t") == parse("select * from t;") end - test "keywords are case-insensitive, names fold to lower case", %{parser: parser} do - assert parse( - parser, - "SELECT Id FROM Tenant.Users WHERE Name = 'Al' ORDER BY Id DESC LIMIT 1" - ) == - parse( - parser, - "select id from tenant.users where name = 'Al' order by id desc limit 1" - ) + test "keywords are case-insensitive, names fold to lower case" do + assert parse("SELECT Id FROM Tenant.Users WHERE Name = 'Al' ORDER BY Id DESC LIMIT 1") == + parse("select id from tenant.users where name = 'Al' order by id desc limit 1") end - test "quoted names keep their case", %{parser: parser} do + test "quoted names keep their case" do assert %Select{fields: ["Id"], from: ["My Tenant", "Users"]} = - parse(parser, ~s(select "Id" from "My Tenant"."Users")) + parse(~s(select "Id" from "My Tenant"."Users")) end - test "comments and whitespace anywhere", %{parser: parser} do - assert parse(parser, """ + test "comments and whitespace anywhere" do + assert parse(""" -- leading comment select /* fields */ a, b from t -- table where a = 1 /* trailing */ ; - """) == parse(parser, "select a, b from t where a = 1") + """) == parse("select a, b from t where a = 1") end end describe "FROM" do - test "one, two or three parts", %{parser: parser} do - assert %Select{from: ["t"]} = parse(parser, "select * from t") - assert %Select{from: ["tenant", "t"]} = parse(parser, "select * from tenant.t") + test "one, two or three parts" do + assert %Select{from: ["t"]} = parse("select * from t") + assert %Select{from: ["tenant", "t"]} = parse("select * from tenant.t") assert %Select{from: ["storage", "tenant", "t"]} = - parse(parser, "select * from storage.tenant.t") + parse("select * from storage.tenant.t") end - test "any non-reserved word is a name", %{parser: parser} do - assert %Select{from: ["user", "date"]} = parse(parser, "select * from user.date") + test "any non-reserved word is a name" do + assert %Select{from: ["user", "date"]} = parse("select * from user.date") end - test "four parts is too many", %{parser: parser} do - assert reason(parser, "select * from a.b.c.d") =~ "at most three parts" + test "four parts is too many" do + assert reason("select * from a.b.c.d") =~ "at most three parts" end - test "more than one table", %{parser: parser} do - assert reason(parser, "select * from a, b") =~ "more than one table" + test "more than one table" do + assert reason("select * from a, b") == "expected the end of the statement, got ','" end end describe "SELECT list" do - test "star or names", %{parser: parser} do - assert %Select{fields: :star} = parse(parser, "select * from t") - assert %Select{fields: ["a", "b", "_"]} = parse(parser, "select a, b, _ from t") + test "star or names" do + assert %Select{fields: :star} = parse("select * from t") + assert %Select{fields: ["a", "b", "_"]} = parse("select a, b, _ from t") end - test "star can't be mixed with names", %{parser: parser} do - assert reason(parser, "select *, a from t") =~ "can't be combined" + test "star can't be mixed with names" do + assert reason("select *, a from t") == "expected FROM, got ','" end - test "unsupported shapes say what is unsupported", %{parser: parser} do - assert reason(parser, "select distinct a from t") == "DISTINCT is not supported" - assert reason(parser, "select count(a) from t") == "functions are not supported" - assert reason(parser, "select t.a from t") == "qualified column names are not supported" + test "unsupported shapes say what is unsupported" do + assert reason("select distinct a from t") == "DISTINCT is not supported" + assert reason("select count(a) from t") == "functions are not supported" + assert reason("select t.a from t") == "qualified column names are not supported" end - test "a missing FROM", %{parser: parser} do - assert %SyntaxError{reason: "expected FROM, got the end of the statement", column: 9} = - error(parser, "select a") + test "a missing FROM" do + assert %SyntaxError{reason: "expected FROM or ',', got the end of the statement", column: 9} = + error("select a") end end @@ -126,329 +115,318 @@ defmodule Efsql.SQL.ParserTest do {"<=", :<=}, {">=", :>=} ] do - test "#{sql}", %{parser: parser} do - assert where(parser, "a #{unquote(sql)} 1") == + test "#{sql}" do + assert where("a #{unquote(sql)} 1") == {:compare, unquote(op), {:column, "a"}, {:literal, 1}} end end - test "either side can be the value", %{parser: parser} do - assert where(parser, "1 < a") == {:compare, :<, {:literal, 1}, {:column, "a"}} + test "either side can be the value" do + assert where("1 < a") == {:compare, :<, {:literal, 1}, {:column, "a"}} end - test "literals", %{parser: parser} do - assert where(parser, "a = 'x'") == {:compare, :=, {:column, "a"}, {:literal, "x"}} - assert where(parser, "a = -3") == {:compare, :=, {:column, "a"}, {:literal, -3}} - assert where(parser, "a = +3") == {:compare, :=, {:column, "a"}, {:literal, 3}} - assert where(parser, "a = -1.5") == {:compare, :=, {:column, "a"}, {:literal, -1.5}} - assert where(parser, "a = true") == {:compare, :=, {:column, "a"}, {:literal, true}} - assert where(parser, "a = FALSE") == {:compare, :=, {:column, "a"}, {:literal, false}} - assert where(parser, "a = null") == {:compare, :=, {:column, "a"}, {:literal, nil}} + test "literals" do + assert where("a = 'x'") == {:compare, :=, {:column, "a"}, {:literal, "x"}} + assert where("a = -3") == {:compare, :=, {:column, "a"}, {:literal, -3}} + assert where("a = +3") == {:compare, :=, {:column, "a"}, {:literal, 3}} + assert where("a = -1.5") == {:compare, :=, {:column, "a"}, {:literal, -1.5}} + assert where("a = true") == {:compare, :=, {:column, "a"}, {:literal, true}} + assert where("a = FALSE") == {:compare, :=, {:column, "a"}, {:literal, false}} + assert where("a = null") == {:compare, :=, {:column, "a"}, {:literal, nil}} end - test "a sign only goes on a number", %{parser: parser} do - assert reason(parser, "select a from t where a = -b") =~ "only supported on a number" + test "a sign only goes on a number" do + assert reason("select a from t where a = -b") =~ "only supported on a number" end - test "arithmetic is refused, not misread", %{parser: parser} do - assert reason(parser, "select a from t where a = 1 + 2") == "arithmetic is not supported" - assert reason(parser, "select a from t where a + 1 = 2") == "arithmetic is not supported" + test "arithmetic is refused, not misread" do + assert reason("select a from t where a = 1 + 2") == "arithmetic is not supported" + assert reason("select a from t where a + 1 = 2") == "arithmetic is not supported" end - test "comparisons don't chain", %{parser: parser} do - assert reason(parser, "select a from t where a = 1 = 2") =~ - "expected end of statement, got '='" + test "comparisons don't chain" do + assert reason("select a from t where a = 1 = 2") =~ + "expected the end of the statement, got '='" end end describe "predicates" do - test "BETWEEN binds its own AND", %{parser: parser} do - assert where(parser, "a between 1 and 2 and b = 3") == + test "BETWEEN binds its own AND" do + assert where("a between 1 and 2 and b = 3") == {:and, {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, false}, {:compare, :=, {:column, "b"}, {:literal, 3}}} end - test "NOT BETWEEN", %{parser: parser} do - assert where(parser, "a not between 1 and 2") == + test "NOT BETWEEN" do + assert where("a not between 1 and 2") == {:between, {:column, "a"}, {:literal, 1}, {:literal, 2}, true} end - test "IN and NOT IN", %{parser: parser} do - assert where(parser, "a in (1, 'b', 2.5)") == + test "IN and NOT IN" do + assert where("a in (1, 'b', 2.5)") == {:in, {:column, "a"}, [{:literal, 1}, {:literal, "b"}, {:literal, 2.5}], false} - assert where(parser, "a not in ('x')") == {:in, {:column, "a"}, [{:literal, "x"}], true} + assert where("a not in ('x')") == {:in, {:column, "a"}, [{:literal, "x"}], true} end - test "subqueries are named as unsupported", %{parser: parser} do - assert reason(parser, "select a from t where a in (select b from u)") == + test "subqueries are named as unsupported" do + assert reason("select a from t where a in (select b from u)") == "subqueries are not supported" end - test "IN needs a parenthesized, non-empty list", %{parser: parser} do - assert reason(parser, "select a from t where a in ()") == "IN needs at least one value" - assert reason(parser, "select a from t where a in 1") =~ "expected '(' after IN" - assert reason(parser, "select a from t where a in (1, 2") =~ "')' to close the IN list" + test "IN needs a parenthesized, non-empty list" do + assert reason("select a from t where a in ()") == "expected an expression, got ')'" + assert reason("select a from t where a in 1") == "expected '(', got the number 1" + + assert reason("select a from t where a in (1, 2") == + "expected an operator, ')' or ',', got the end of the statement" end - test "LIKE, NOT LIKE, ILIKE", %{parser: parser} do - assert where(parser, "a like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, false} - assert where(parser, "a not like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, true} - assert where(parser, "a ilike 'x%'") == {:ilike, {:column, "a"}, {:literal, "x%"}, false} + test "LIKE, NOT LIKE, ILIKE" do + assert where("a like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, false} + assert where("a not like 'x%'") == {:like, {:column, "a"}, {:literal, "x%"}, true} + assert where("a ilike 'x%'") == {:ilike, {:column, "a"}, {:literal, "x%"}, false} end - test "IS [NOT] NULL, ISNULL, NOTNULL", %{parser: parser} do - assert where(parser, "a is null") == {:is_null, {:column, "a"}, false} - assert where(parser, "a is not null") == {:is_null, {:column, "a"}, true} - assert where(parser, "a isnull") == {:is_null, {:column, "a"}, false} - assert where(parser, "a notnull") == {:is_null, {:column, "a"}, true} + test "IS [NOT] NULL, ISNULL, NOTNULL" do + assert where("a is null") == {:is_null, {:column, "a"}, false} + assert where("a is not null") == {:is_null, {:column, "a"}, true} + assert where("a isnull") == {:is_null, {:column, "a"}, false} + assert where("a notnull") == {:is_null, {:column, "a"}, true} end - test "IS takes only NULL", %{parser: parser} do - assert reason(parser, "select a from t where a is true") =~ - "expected NULL after IS, got 'true'" + test "IS takes only NULL" do + assert reason("select a from t where a is true") == + "expected NOT or NULL, got 'true'" - assert reason(parser, "select a from t where a is not 1") =~ "expected NULL after IS NOT" + assert reason("select a from t where a is not 1") == "expected NULL, got the number 1" end - test "a bare operand is kept for the translator to reject", %{parser: parser} do - assert where(parser, "a") == {:column, "a"} + test "a bare operand is kept for the translator to reject" do + assert where("a") == {:column, "a"} end end describe "boolean structure" do - test "AND binds tighter than OR", %{parser: parser} do - assert where(parser, "a = 1 or b = 2 and c = 3") == + test "AND binds tighter than OR" do + assert where("a = 1 or b = 2 and c = 3") == {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, {:and, {:compare, :=, {:column, "b"}, {:literal, 2}}, {:compare, :=, {:column, "c"}, {:literal, 3}}}} end - test "AND and OR are left-associative", %{parser: parser} do - assert {:and, {:and, _, _}, _} = where(parser, "a = 1 and b = 2 and c = 3") - assert {:or, {:or, _, _}, _} = where(parser, "a = 1 or b = 2 or c = 3") + test "AND and OR are left-associative" do + assert {:and, {:and, _, _}, _} = where("a = 1 and b = 2 and c = 3") + assert {:or, {:or, _, _}, _} = where("a = 1 or b = 2 or c = 3") end - test "NOT binds tighter than AND", %{parser: parser} do - assert where(parser, "not a = 1 and b = 2") == + test "NOT binds tighter than AND" do + assert where("not a = 1 and b = 2") == {:and, {:not, {:compare, :=, {:column, "a"}, {:literal, 1}}}, {:compare, :=, {:column, "b"}, {:literal, 2}}} end - test "parentheses group and then vanish", %{parser: parser} do - assert where(parser, "(a = 1 or b = 2) and c = 3") == + test "parentheses group and then vanish" do + assert where("(a = 1 or b = 2) and c = 3") == {:and, {:or, {:compare, :=, {:column, "a"}, {:literal, 1}}, {:compare, :=, {:column, "b"}, {:literal, 2}}}, {:compare, :=, {:column, "c"}, {:literal, 3}}} - assert where(parser, "((a = 1))") == where(parser, "a = 1") + assert where("((a = 1))") == where("a = 1") end - test "unbalanced parentheses", %{parser: parser} do - assert reason(parser, "select a from t where (a = 1") =~ "expected ')'" + test "unbalanced parentheses" do + assert reason("select a from t where (a = 1") =~ "')'" - assert reason(parser, "select a from t where a = 1)") =~ - "expected end of statement, got ')'" + assert reason("select a from t where a = 1)") =~ + "expected the end of the statement, got ')'" end - test "deep nesting", %{parser: parser} do - assert where(parser, String.duplicate("(", 150) <> "a = 1" <> String.duplicate(")", 150)) == - where(parser, "a = 1") - - case parser do - # Recursive descent caps its depth rather than risk the stack. - Parser -> - assert reason(parser, "select a from t where " <> String.duplicate("(", 10_000) <> "a") == - "expression is nested too deeply" - - assert reason( - parser, - "select a from t where " <> String.duplicate("not ", 10_000) <> "a" - ) == - "expression is nested too deeply" - - # yecc keeps its own stack, so it needs no limit. - Efsql.SQL.Yecc -> - deep = String.duplicate("(", 10_000) <> "a = 1" <> String.duplicate(")", 10_000) - assert where(parser, deep) == where(parser, "a = 1") - assert {:not, {:not, _}} = where(parser, String.duplicate("not ", 10_000) <> "a") - end + # yecc keeps its own stack, so nesting has no limit. + test "deep nesting" do + deep = String.duplicate("(", 10_000) <> "a = 1" <> String.duplicate(")", 10_000) + assert where(deep) == where("a = 1") + assert {:not, {:not, _}} = where(String.duplicate("not ", 10_000) <> "a") end end describe "casts" do - test "postfix and CAST forms give the same tree", %{parser: parser} do - assert where(parser, "a = 'x'::atom") == + test "postfix and CAST forms give the same tree" do + assert where("a = 'x'::atom") == {:compare, :=, {:column, "a"}, {:cast, {:literal, "x"}, "atom"}} - assert where(parser, "a = cast('x' as atom)") == where(parser, "a = 'x'::atom") + assert where("a = cast('x' as atom)") == where("a = 'x'::atom") end - test "type names fold and may be reserved words", %{parser: parser} do - assert {:compare, :=, _, {:cast, _, "date"}} = where(parser, "a = '2024-01-01'::DATE") - assert {:compare, :=, _, {:cast, _, "time"}} = where(parser, "a = '10:00:00'::time") + test "type names fold and may be reserved words" do + assert {:compare, :=, _, {:cast, _, "date"}} = where("a = '2024-01-01'::DATE") + assert {:compare, :=, _, {:cast, _, "time"}} = where("a = '10:00:00'::time") end - test "timestamp with/without time zone", %{parser: parser} do + test "timestamp with/without time zone" do assert {:compare, :=, _, {:cast, _, "timestamptz"}} = - where(parser, "a = 'x'::timestamp with time zone") + where("a = 'x'::timestamp with time zone") assert {:compare, :=, _, {:cast, _, "timestamp"}} = - where(parser, "a = cast('x' as timestamp without time zone)") + where("a = cast('x' as timestamp without time zone)") end - test "casts chain, and apply to any operand", %{parser: parser} do - assert where(parser, "a = 1::text::atom") == + test "casts chain, and apply to any operand" do + assert where("a = 1::text::atom") == {:compare, :=, {:column, "a"}, {:cast, {:cast, {:literal, 1}, "text"}, "atom"}} - assert where(parser, "a::text = 'x'") == + assert where("a::text = 'x'") == {:compare, :=, {:cast, {:column, "a"}, "text"}, {:literal, "x"}} end - test "casts inside IN and BETWEEN", %{parser: parser} do - assert where(parser, "a in ('x'::atom, 'y'::atom)") == + test "casts inside IN and BETWEEN" do + assert where("a in ('x'::atom, 'y'::atom)") == {:in, {:column, "a"}, [{:cast, {:literal, "x"}, "atom"}, {:cast, {:literal, "y"}, "atom"}], false} assert {:between, _, {:cast, _, "date"}, {:cast, _, "date"}, false} = - where(parser, "a between '2024-01-01'::date and '2025-01-01'::date") + where("a between '2024-01-01'::date and '2025-01-01'::date") end - test "a cast needs a type", %{parser: parser} do - assert reason(parser, "select a from t where a = 'x'::") =~ "expected a type name" - assert reason(parser, "select a from t where a = cast('x' atom)") =~ "expected AS in CAST" - assert reason(parser, "select a from t where a = cast('x' as atom") =~ "')' to close CAST" + test "a cast needs a type" do + assert reason("select a from t where a = 'x'::") =~ "expected a type name" + + assert reason("select a from t where a = cast('x' atom)") == + "expected an operator or AS, got 'atom'" + + assert reason("select a from t where a = cast('x' as atom") == + "expected ')', got the end of the statement" end end describe "tuples" do - test "a partition scan", %{parser: parser} do - assert where(parser, "_ = ('p', *)") == + test "a partition scan" do + assert where("_ = ('p', *)") == {:compare, :=, {:column, "_"}, {:tuple, [{:literal, "p"}, :star]}} end - test "a versionstamp", %{parser: parser} do - assert where(parser, "_ >= ('p', 22348699227647901699)") == + test "a versionstamp" do + assert where("_ >= ('p', 22348699227647901699)") == {:compare, :>=, {:column, "_"}, {:tuple, [{:literal, "p"}, {:literal, 22_348_699_227_647_901_699}]}} end - test "star only inside a tuple", %{parser: parser} do - assert reason(parser, "select a from t where a = (*)") =~ "only allowed in a tuple" - assert reason(parser, "select a from t where a = *") =~ "expected an expression, got '*'" + test "star only inside a tuple" do + assert reason("select a from t where a = (*)") == "expected ',', got ')'" + assert reason("select a from t where a = *") =~ "expected an expression, got '*'" end end describe "reserved words" do - test "common column names are not reserved", %{parser: parser} do + test "common column names are not reserved" do for name <- ~w[day at user value date time timestamp year month zone key type level] do - assert where(parser, "#{name} = 1") == {:compare, :=, {:column, name}, {:literal, 1}}, + assert where("#{name} = 1") == {:compare, :=, {:column, name}, {:literal, 1}}, "#{name} should be usable as a name" assert %Select{fields: [^name], order_by: [{^name, :desc}]} = - parse(parser, "select #{name} from t order by #{name} desc") + parse("select #{name} from t order by #{name} desc") end end - test "reserved words need quotes, and the error says so", %{parser: parser} do - assert reason(parser, "select order from t") =~ + test "reserved words need quotes, and the error says so" do + assert reason("select order from t") =~ ~s(got the keyword 'order' (quote it, "order") - assert where(parser, ~s("order" = 1)) == {:compare, :=, {:column, "order"}, {:literal, 1}} + assert where(~s("order" = 1)) == {:compare, :=, {:column, "order"}, {:literal, 1}} assert %Select{fields: ["select", "from"]} = - parse(parser, ~s(select "select", "from" from t)) + parse(~s(select "select", "from" from t)) end - test "the reserved list", %{parser: parser} do + test "the reserved list" do for word <- Parser.reserved_words() do - assert {:error, %SyntaxError{}} = parser.parse("select #{word} from t") + assert {:error, %SyntaxError{}} = Parser.parse("select #{word} from t") end end end describe "ORDER BY and LIMIT" do - test "directions default to ascending", %{parser: parser} do + test "directions default to ascending" do assert %Select{order_by: [{"a", :asc}, {"b", :desc}, {"c", :asc}]} = - parse(parser, "select * from t order by a, b desc, c asc") + parse("select * from t order by a, b desc, c asc") end - test "ORDER needs BY and a field", %{parser: parser} do - assert reason(parser, "select * from t order a") =~ "expected BY after ORDER" - assert reason(parser, "select * from t order by") =~ "expected a field name to order by" + test "ORDER needs BY and a field" do + assert reason("select * from t order a") == "expected BY, got 'a'" + assert reason("select * from t order by") == "expected a name, got the end of the statement" - assert reason(parser, "select * from t order by a nulls first") == - "NULLS FIRST/LAST is not supported" + assert reason("select * from t order by a nulls first") == + "expected the end of the statement, got 'nulls'" end - test "LIMIT takes a whole number", %{parser: parser} do - assert %Select{limit: 0} = parse(parser, "select * from t limit 0") - assert reason(parser, "select * from t limit -1") =~ "LIMIT expects a whole number, got '-'" - assert reason(parser, "select * from t limit 1.5") =~ "got the number 1.5" - assert reason(parser, "select * from t limit") =~ "got the end of the statement" + test "LIMIT takes a whole number" do + assert %Select{limit: 0} = parse("select * from t limit 0") + assert reason("select * from t limit -1") == "expected a whole number, got '-'" + assert reason("select * from t limit 1.5") =~ "got the number 1.5" + assert reason("select * from t limit") =~ "got the end of the statement" end - test "clauses in the wrong order", %{parser: parser} do - assert reason(parser, "select * from t limit 1 where a = 1") =~ - "expected end of statement, got 'where'" + test "clauses in the wrong order" do + assert reason("select * from t limit 1 where a = 1") =~ + "expected the end of the statement, got 'where'" - assert reason(parser, "select * from t order by a where a = 1") =~ "got 'where'" + assert reason("select * from t order by a where a = 1") =~ "got 'where'" end end describe "statement errors" do - test "empty input", %{parser: parser} do - assert reason(parser, "") == "empty statement" - assert reason(parser, " -- just a comment\n") == "empty statement" - assert reason(parser, ";") =~ "expected SELECT, got ';'" + test "empty input" do + assert reason("") == "empty statement" + assert reason(" -- just a comment\n") == "empty statement" + assert reason(";") =~ "expected SELECT, got ';'" end - test "only SELECT", %{parser: parser} do - assert reason(parser, "delete from t") == "only SELECT statements are supported" - assert reason(parser, "update t set a = 1") == "only SELECT statements are supported" - assert reason(parser, "with x as (select 1) select * from x") =~ "expected SELECT" + test "only SELECT" do + assert reason("delete from t") == "only SELECT statements are supported" + assert reason("update t set a = 1") == "only SELECT statements are supported" + assert reason("with x as (select 1) select * from x") == "WITH is not supported" end - test "one statement", %{parser: parser} do + test "one statement" do assert %SyntaxError{reason: "only one statement is supported", column: 18} = - error(parser, "select * from t; select * from u") + error("select * from t; select * from u") - assert reason(parser, "select * from t;;") == "only one statement is supported" + assert reason("select * from t;;") == "only one statement is supported" end - test "unsupported clauses are named", %{parser: parser} do - assert reason(parser, "select * from t group by a") == "GROUP BY is not supported" + test "unsupported clauses are named" do + assert reason("select * from t group by a") == "GROUP BY is not supported" - assert reason(parser, "select * from t where a = 1 having a > 1") == + assert reason("select * from t where a = 1 having a > 1") == "HAVING is not supported" - assert reason(parser, "select * from t limit 1 offset 2") == "OFFSET is not supported" - assert reason(parser, "select * from t join u on t.a = u.a") == "JOIN is not supported" - assert reason(parser, "select * from t left join u on a = b") == "JOIN is not supported" - assert reason(parser, "select * from t union select * from u") == "UNION is not supported" + assert reason("select * from t limit 1 offset 2") == "OFFSET is not supported" + assert reason("select * from t join u on t.a = u.a") == "JOIN is not supported" + assert reason("select * from t left join u on a = b") == "JOIN is not supported" + assert reason("select * from t union select * from u") == "UNION is not supported" end - test "errors point at the offending token", %{parser: parser} do + test "errors point at the offending token" do assert %SyntaxError{line: 3, column: 10, reason: "expected an expression, got '='"} = - error(parser, "select a\nfrom t\nwhere a == 1") + error("select a\nfrom t\nwhere a == 1") assert %SyntaxError{ line: 1, column: 23, reason: "expected an expression, got the end of the statement" } = - error(parser, "select a from t where ") + error("select a from t where ") end - test "long strings are truncated in messages", %{parser: parser} do - assert reason(parser, "select * from t limit '#{String.duplicate("x", 100)}'") =~ + test "long strings are truncated in messages" do + assert reason("select * from t limit '#{String.duplicate("x", 100)}'") =~ "the string 'xxxxxxxxxxxxxxxxxxxx...'" end - test "parse! raises the error", %{parser: parser} do + test "parse! raises the error" do assert_raise SyntaxError, ~r/line 1, column 1: expected SELECT/, fn -> - parser.parse!("selec * from t") + Parser.parse!("selec * from t") end end end diff --git a/test/sql/property_test.exs b/test/sql/property_test.exs index c621efa..c8a4561 100644 --- a/test/sql/property_test.exs +++ b/test/sql/property_test.exs @@ -1,44 +1,41 @@ defmodule Efsql.SQL.PropertyTest do - # Randomized tests, run against both parsers. `:rand` is seeded per test + # Randomized tests. `:rand` is seeded per test # from ExUnit's --seed, so a failure reproduces with `mix test --seed N`. - use ExUnit.Case, - async: true, - parameterize: [%{parser: Efsql.SQL.Parser}, %{parser: Efsql.SQL.Yecc}] + use ExUnit.Case, async: true alias Efsql.Exception.Unsupported alias Efsql.SQL.AST.Select + alias Efsql.SQL.Parser alias Efsql.SQL.SyntaxError alias EfsqlTest.SQLGen @runs 1_000 - test "printing a random tree and parsing it gives the tree back", %{parser: parser} do + test "printing a random tree and parsing it gives the tree back" do for _ <- 1..@runs do select = SQLGen.select() sql = SQLGen.to_sql(select) - assert parser.parse(sql) == {:ok, select}, """ + assert Parser.parse(sql) == {:ok, select}, """ SQL: #{sql} """ end end - test "mangled queries parse or fail with a positioned SyntaxError, never crash", - %{parser: parser} do + test "mangled queries parse or fail with a positioned SyntaxError, never crash" do for _ <- 1..@runs do - check(parser, SQLGen.select() |> SQLGen.to_sql() |> SQLGen.mangle(Enum.random(1..4))) + check(SQLGen.select() |> SQLGen.to_sql() |> SQLGen.mangle(Enum.random(1..4))) end end - test "random bytes parse or fail with a positioned SyntaxError, never crash", - %{parser: parser} do - for _ <- 1..@runs, do: check(parser, SQLGen.garbage()) + test "random bytes parse or fail with a positioned SyntaxError, never crash" do + for _ <- 1..@runs, do: check(SQLGen.garbage()) end - test "valid queries translate to a logical plan or raise Unsupported", %{parser: parser} do + test "valid queries translate to a logical plan or raise Unsupported" do for _ <- 1..@runs do - {:ok, select} = parser.parse(SQLGen.select() |> SQLGen.to_sql()) + {:ok, select} = Parser.parse(SQLGen.select() |> SQLGen.to_sql()) try do Efsql.Parser.to_logical(select) @@ -54,20 +51,20 @@ defmodule Efsql.SQL.PropertyTest do end end - test "big inputs parse and translate quickly", %{parser: parser} do + test "big inputs parse and translate quickly" do values = Enum.map_join(1..20_000, ", ", &"'v#{&1}'") conditions = Enum.map_join(1..20_000, " and ", &"f#{&1} = #{&1}") sql = "select a from t where a in (#{values}) and #{conditions}" {micros, %Efsql.Logical.Select{predicates: predicates}} = - :timer.tc(fn -> sql |> parser.parse!() |> Efsql.Parser.to_logical() end) + :timer.tc(fn -> sql |> Parser.parse!() |> Efsql.Parser.to_logical() end) assert length(predicates) == 20_001 assert micros < 5_000_000 end - defp check(parser, sql) do - case parser.parse(sql) do + defp check(sql) do + case Parser.parse(sql) do {:ok, %Select{}} -> :ok diff --git a/test/support/efsql_test/sql_gen.ex b/test/support/efsql_test/sql_gen.ex index 90fbf05..ac9b4a3 100644 --- a/test/support/efsql_test/sql_gen.ex +++ b/test/support/efsql_test/sql_gen.ex @@ -296,15 +296,6 @@ defmodule EfsqlTest.SQLGen do @doc "Up to 40 random SQL-ish fragments and bytes, invalid UTF-8 included." def garbage(), do: random_string(@alphabet, 0..40) - @doc "A valid query, a mangled one, or garbage, in equal measure." - def any_input() do - case Enum.random(1..3) do - 1 -> select() |> to_sql() - 2 -> select() |> to_sql() |> mangle(Enum.random(1..4)) - 3 -> garbage() - end - end - # -- helpers -- defp random_string(pool, range) do