Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,48 @@ select * from admins.engineering.users;

The two-part `tenant_id.table_name` form continues to use the storage ID set at startup via `--storage-id` (or the default if none was given).

### Query every tenant

The tenants of one storage ID share a schema, so a query can read all of
them at once. Write `*` for the tenant:

```sql
select _tenant, count(*) from customer.*.orders group by _tenant;
select id, total, _tenant from *.orders where status = 'paid' order by total desc limit 20;
select * from *.orders where _tenant like 'acme%';
```

`*.table` uses the startup storage ID (in the TUI, the one you're browsing),
and `storage_id.*.table` names one. Every row gets a `_tenant` field with its
tenant's name, which you can select, filter, group and sort by like any
other.

A condition on `_tenant` decides which tenants are read at all, so
`where _tenant = 'acme'` never touches the others. Everything else runs in
each tenant, with that tenant's own indexes, and grouping, ordering and
`LIMIT` then apply to all the rows together.

The tenants are read in a single FoundationDB transaction, so the result is
one consistent snapshot. That also means the whole read has to finish within
FoundationDB's five-second transaction limit; a query that reads too much
says so, and a `_tenant` or `WHERE` condition narrows it. One transaction
reads at most 100 tenants unless the `:efsql, :max_tenants` setting says
otherwise.

When the tenants don't fit in one transaction, read them in batches:

```
\set tenant_batch 25
```

Each batch of 25 tenants is then its own transaction, and there is no
tenant limit. Up to four batches are read at once (the
`:efsql, :batch_concurrency` setting, or `batch_concurrency:` from Elixir). Grouping, ordering and `LIMIT` still apply to all the rows
together, but the result is no longer one snapshot: each batch sees the
database at a slightly different moment, and the result says how many
transactions it came from. `\set tenant_batch off` goes back to one
transaction. (From Elixir, pass `tenant_batch: 25` to `Efsql.qall/3`.)

### Select rows

```sql
Expand Down
78 changes: 56 additions & 22 deletions lib/efsql.ex
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,23 @@ defmodule Efsql do
end

def qall(sql, options \\ [], tenants \\ %{}) do
{logical, tenants} = sql_to_logical(sql, tenants)
sql |> Efsql.Parser.sql_to_logical() |> run_logical(options, tenants)
end

@doc """
Plans and runs a translated query, resolving its tenant (or, for
`*.table`, every tenant of the storage id) through the `tenants` cache.
Returns `{plan, rows, tenants}`.
"""
def run_logical(%Efsql.Logical.Select{prefix: {:all_tenants, _}} = logical, options, tenants) do
{plan, tenants} =
logical |> Efsql.Rewrite.normalize() |> Efsql.Fanout.plan(tenants, options)

{plan, Efsql.Executor.run(plan), tenants}
end

def run_logical(%Efsql.Logical.Select{} = logical, options, tenants) do
{logical, tenants} = resolve_tenant(logical, tenants)
plan = logical |> Efsql.Rewrite.normalize() |> Efsql.Planner.plan(options)
{plan, Efsql.Executor.run(plan), tenants}
end
Expand All @@ -47,33 +63,51 @@ defmodule Efsql do
resolve_tenant(logical, tenants)
end

def resolve_tenant(%Efsql.Logical.Select{prefix: {:all_tenants, _}}, _tenants) do
raise Efsql.Exception.Unsupported, "a query across tenants can't be streamed"
end

# Already resolved, e.g. to the TUI session's active tenant.
def resolve_tenant(%Efsql.Logical.Select{prefix: nil, tenant: tenant} = logical, tenants)
when tenant != nil,
do: {logical, tenants}

def resolve_tenant(%Efsql.Logical.Select{} = logical, tenants) do
{tenant_name, open_opts} =
{tenant_name, storage_id} =
case logical.prefix do
{storage_id, tenant_name} -> {tenant_name, [storage_id: storage_id]}
{storage_id, tenant_name} -> {tenant_name, storage_id}
nil -> raise "Tenant required"
tenant_name -> {tenant_name, []}
tenant_name -> {tenant_name, nil}
end

cache_key = {tenant_name, open_opts[:storage_id]}

{tenant, tenants} =
case Map.fetch(tenants, cache_key) do
{:ok, cached} ->
{cached, tenants}

:error ->
unless EctoFoundationDB.Tenant.exists?(Efsql.Repo, tenant_name) do
raise Efsql.Exception.Unsupported, "Tenant '#{tenant_name}' does not exist"
end

# migrate: false keeps this read-only. Tenant.open/3 already requires
# the tenant to exist (only open!/3 creates), so with the migration
# step skipped efsql never writes to the database it is exploring.
t = EctoFoundationDB.Tenant.open(Efsql.Repo, tenant_name, [migrate: false] ++ open_opts)
{t, Map.put(tenants, cache_key, t)}
end
if not Map.has_key?(tenants, {tenant_name, storage_id}) and
not EctoFoundationDB.Tenant.exists?(Efsql.Repo, tenant_name) do
raise Efsql.Exception.Unsupported, "Tenant '#{tenant_name}' does not exist"
end

{tenant, tenants} = open_tenant(tenants, tenant_name, storage_id)
{%Efsql.Logical.Select{logical | tenant: tenant}, tenants}
end

@doc """
Opens a tenant through the session's cache of open tenants, keyed by
name and storage id (nil for the Repo's own).
"""
def open_tenant(tenants, tenant_name, storage_id) do
case Map.fetch(tenants, {tenant_name, storage_id}) do
{:ok, tenant} ->
{tenant, tenants}

:error ->
open_opts = if storage_id, do: [storage_id: storage_id], else: []

# migrate: false keeps this read-only. Tenant.open/3 already requires
# the tenant to exist (only open!/3 creates), so with the migration
# step skipped efsql never writes to the database it is exploring.
tenant =
EctoFoundationDB.Tenant.open(Efsql.Repo, tenant_name, [migrate: false] ++ open_opts)

{tenant, Map.put(tenants, {tenant_name, storage_id}, tenant)}
end
end
end
30 changes: 28 additions & 2 deletions lib/efsql/cli.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Efsql.Cli do
defstruct args: [], history: [], debug: false, tenants: %{}, limit: 15
defstruct args: [], history: [], debug: false, tenants: %{}, limit: 15, tenant_batch: nil

use GenServer

Expand Down Expand Up @@ -93,6 +93,8 @@ defmodule Efsql.Cli do
Meta-commands:
\\tenants [storage_id] list tenants (optionally for a specific storage id)
\\set limit N set the default row limit (currently #{state.limit})
\\set tenant_batch N|off read *.table queries N tenants per transaction
(currently #{state.tenant_batch || "off"})
\\? show this help
""",
:light_black
Expand Down Expand Up @@ -150,6 +152,22 @@ defmodule Efsql.Cli do
end
end

defp handle_input("\\set tenant_batch " <> rest, state = %__MODULE__{}) do
case {String.trim(rest), Integer.parse(String.trim(rest))} do
{"off", _} ->
Owl.IO.puts(Owl.Data.tag("tenant_batch off: one transaction", :light_black))
%__MODULE__{state | tenant_batch: nil}

{_, {n, ""}} when n > 0 ->
Owl.IO.puts(Owl.Data.tag("tenant_batch set to #{n}", :light_black))
%__MODULE__{state | tenant_batch: n}

_ ->
print_error("Usage: \\set tenant_batch <positive integer> | off")
state
end
end

defp handle_input(data, state = %__MODULE__{}) do
limit_sql = "limit #{state.limit + 1}"

Expand All @@ -160,9 +178,11 @@ defmodule Efsql.Cli do
do: {data, :all},
else: {String.replace(data, ~r/;\s*$/, " #{limit_sql};"), state.limit}

{call, rows, tenants} = Efsql.qall(sql, [], state.tenants)
options = if state.tenant_batch, do: [tenant_batch: state.tenant_batch], else: []
{call, rows, tenants} = Efsql.qall(sql, options, state.tenants)
if state.debug, do: print_debug(call)
print_table(rows, display_limit, call.columns)
print_snapshots(Efsql.Fanout.transactions(call))
{tenants}
rescue
e ->
Expand Down Expand Up @@ -257,6 +277,12 @@ defmodule Efsql.Cli do
Owl.IO.puts(Owl.Data.tag(label, :light_black))
end

defp print_snapshots(1), do: :ok

defp print_snapshots(n) do
Owl.IO.puts(Owl.Data.tag("(read in #{n} transactions)", :yellow))
end

defp column_sorter(nil), do: :asc

defp column_sorter(columns) do
Expand Down
100 changes: 100 additions & 0 deletions lib/efsql/executor.ex
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,21 @@ defmodule Efsql.Executor do
false — including NOT LIKE. IS NULL matches a nil or absent field. Sorting places NULLs last ascending and first
descending, matching PostgreSQL's defaults. Values compare with
`Efsql.Types.compare/2`, so datetimes order chronologically.

A query across tenants (`{:fan_out, [{tenant_name, plan}]}`) reads every
tenant in one FoundationDB transaction on the database: each tenant's
reads are started, then all awaited together, so they are pipelined and
see one snapshot. Each tenant's own operators run on its rows, which
then get `_tenant`, before the shared operators run on them all.
"""

alias Efsql.Exception.Unsupported
alias Efsql.Physical.Plan
alias Efsql.Types

# transaction_too_old (retryable) and transaction_timed_out
@too_long [1007, 1031]

@cmp_ops ~w[== > >= < <=]a
@cmp_results %{
==: [:eq],
Expand All @@ -25,6 +35,42 @@ defmodule Efsql.Executor do
Enum.reduce(ops, fetch(access), &apply_op/2)
end

@doc """
Maps `fun` over `items`, up to `max` at a time, each in a process of its
own, and returns the results in order. The first failure stops the rest
and is raised (or thrown, or exited) here as it was there, so a caller's
`rescue` sees the original exception rather than a linked process's
exit. With `max` 1 it all runs in the calling process.
"""
def map_concurrently(items, 1, fun), do: Enum.map(items, fun)

def map_concurrently(items, max, fun) do
items
|> Task.async_stream(
fn item ->
try do
{:ok, fun.(item)}
catch
kind, reason -> {:failed, kind, reason, __STACKTRACE__}
end
end,
max_concurrency: max,
ordered: true,
timeout: :infinity
)
|> Enum.reduce_while([], fn
{:ok, {:ok, result}}, results -> {:cont, [result | results]}
{:ok, failed}, _results -> {:halt, failed}
end)
|> case do
{:failed, kind, reason, stacktrace} -> :erlang.raise(kind, reason, stacktrace)
results -> Enum.reverse(results)
end
end

@doc "Whether `row` satisfies every one of `predicates`, with SQL NULL semantics."
def matches?(row, predicates), do: Enum.all?(predicates, &eval(&1, row))

# -- access nodes --

defp fetch({:pk_range, query, id_start, id_end, options}) do
Expand All @@ -39,13 +85,67 @@ defmodule Efsql.Executor do
Efsql.Repo.all_from_source(query, options)
end

# One transaction per batch, up to `concurrency` at a time.
defp fetch({:batches, batches, concurrency}),
do: batches |> map_concurrently(concurrency, &fetch/1) |> Enum.concat()

defp fetch({:fan_out, []}), do: []

defp fetch({:fan_out, tenant_plans}) do
db = Ecto.Adapters.FoundationDB.db(Efsql.Repo)
Ecto.Adapters.FoundationDB.transactional(db, fn -> read_tenants(tenant_plans) end)
end

defp fetch({:union, nodes}) do
nodes
|> Enum.map(&async_fetch/1)
|> Efsql.Repo.await()
|> List.flatten()
end

# Runs inside the transaction. A read too big for one transaction fails
# with transaction_too_old, which erlfdb's transactional loop would retry,
# forever and each time as slowly. So that error is turned into one the
# loop doesn't catch (it only retries erlfdb errors) and the query fails
# at once; any other error still retries as usual.
defp read_tenants(tenant_plans) do
started =
for {name, %Plan{access: access} = plan} <- tenant_plans,
do: {name, plan, start(access)}

results = started |> Enum.flat_map(&elem(&1, 2)) |> Efsql.Repo.await()

{rows, []} =
Enum.flat_map_reduce(started, results, fn {name, plan, futures}, results ->
{mine, rest} = Enum.split(results, length(futures))

rows =
plan.ops
|> Enum.reduce(List.flatten(mine), &apply_op/2)
|> Enum.map(&Map.put(&1, :_tenant, name))

{rows, rest}
end)

rows
rescue
e in ErlangError ->
case e.original do
{:erlfdb_error, code} when code in @too_long ->
raise Unsupported,
"the query read too much to finish in one transaction across " <>
"#{length(tenant_plans)} tenants; narrow it with a condition on _tenant " <>
"or the WHERE clause, or read fewer tenants per transaction " <>
"(\\set tenant_batch N)"

_ ->
reraise e, __STACKTRACE__
end
end

defp start({:union, nodes}), do: Enum.map(nodes, &async_fetch/1)
defp start(access), do: [async_fetch(access)]

defp async_fetch({:pk_range, query, id_start, id_end, options}) do
Efsql.Repo.async_all_range(query, id_start, id_end, options)
end
Expand Down
Loading
Loading