graph of links

initdb/01-create-databases.sh

@@ -3,4 +3,5 @@ set -e
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "postgres" <<-EOSQL
     CREATE DATABASE tiled_catalog;
     CREATE DATABASE tiled_storage;
+    CRAEATE DATABASE tiled_graph;
 EOSQL

pixi.toml

@@ -110,6 +110,9 @@ starlette = ">=0.48.0"
 uvicorn = "*"
 zarr = "*"
 
+[feature.server.pypi-dependencies]
+strawberry-graphql = {version = ">=0.315.3", extras = ["fastapi"]}
+
 [feature.sparse.dependencies]
 ndindex = "*"
 pyarrow-all = ">=14.0.1"  # includes fix to CVE 2023-47248

pyproject.toml

@@ -101,6 +101,7 @@ all = [
     "sqlalchemy[asyncio] >=2",
     "stamina",
     "starlette >=0.48.0",
+    "strawberry-graphql[fastapi]>=0.315.3",
     "tifffile",
     "uvicorn[standard]",
     "watchfiles",
@@ -233,6 +234,7 @@ server = [
     "python-multipart",
     "sparse >=0.15.5",
     "stamina",
+    "strawberry-graphql[fastapi]>=0.315.3",
     "redis",
     "sqlalchemy[asyncio] >=2",
     "starlette >=0.48.0",

share/tiled/templates/index.html

@@ -45,6 +45,17 @@ <h2 class="subtitle">
       </button>
     </a>
   </section>
+  <section class="section">
+    <h1 class="title">Explore the Graph API</h1>
+    <h2 class="subtitle">
+      Use the interactive <em>GraphQL</em> playground to query the links graph.
+    </h2>
+    <a href="{{ root_url }}/graphql" target="_blank" rel="noreferrer">
+      <button class="button is-large is-responsive is-link">
+        Try it
+      </button>
+    </a>
+  </section>
   <section class="section">
     <h1 class="title">Learn</h1>
     <h2 class="subtitle">

tiled/commandline/_links.py

@@ -0,0 +1,115 @@
+from typing import Optional
+
+import typer
+
+graph_app = typer.Typer(no_args_is_help=True)
+
+
+@graph_app.command("initialize-database")
+def initialize_database(database_uri: str):
+    """
+    Initialize the graph database for use by Tiled.
+    """
+    import asyncio
+
+    from sqlalchemy.ext.asyncio import create_async_engine
+
+    from ..alembic_utils import UninitializedDatabase, check_database, stamp_head
+    from ..graph.alembic_constants import ALEMBIC_DIR, ALEMBIC_INI_TEMPLATE_PATH
+    from ..graph.core import ALL_REVISIONS, REQUIRED_REVISION, initialize_database
+    from ..utils import ensure_specified_sql_driver
+
+    database_uri = ensure_specified_sql_driver(database_uri)
+
+    async def do_setup():
+        engine = create_async_engine(database_uri)
+        redacted_url = engine.url._replace(password="[redacted]")
+        try:
+            await check_database(engine, REQUIRED_REVISION, ALL_REVISIONS)
+        except UninitializedDatabase:
+            typer.echo(
+                f"Database {redacted_url} is new. Creating tables and marking revision {REQUIRED_REVISION}.",
+                err=True,
+            )
+            await initialize_database(engine)
+            typer.echo("Database initialized.", err=True)
+        else:
+            typer.echo(f"Database at {redacted_url} is already initialized.", err=True)
+            raise typer.Abort()
+        await engine.dispose()
+
+    asyncio.run(do_setup())
+    stamp_head(ALEMBIC_INI_TEMPLATE_PATH, ALEMBIC_DIR, database_uri)
+
+
+@graph_app.command("upgrade-database")
+def upgrade_database(
+    database_uri: str,
+    revision: Optional[str] = typer.Argument(
+        None,
+        help="The ID of a revision to upgrade to. By default, upgrade to the latest one.",
+    ),
+):
+    """
+    Upgrade the graph database schema to the latest version.
+    """
+    import asyncio
+
+    from sqlalchemy.ext.asyncio import create_async_engine
+
+    from ..alembic_utils import get_current_revision, upgrade
+    from ..graph.alembic_constants import ALEMBIC_DIR, ALEMBIC_INI_TEMPLATE_PATH
+    from ..graph.core import ALL_REVISIONS
+    from ..utils import ensure_specified_sql_driver
+
+    database_uri = ensure_specified_sql_driver(database_uri)
+
+    async def do_setup():
+        engine = create_async_engine(database_uri)
+        redacted_url = engine.url._replace(password="[redacted]")
+        current_revision = await get_current_revision(engine, ALL_REVISIONS)
+        await engine.dispose()
+        if current_revision is None:
+            typer.echo(
+                f"Database {redacted_url} has not been initialized. Use `tiled graph initialize-database`.",
+                err=True,
+            )
+            raise typer.Abort()
+
+    asyncio.run(do_setup())
+    upgrade(ALEMBIC_INI_TEMPLATE_PATH, ALEMBIC_DIR, database_uri, revision or "head")
+
+
+@graph_app.command("downgrade-database")
+def downgrade_database(
+    database_uri: str,
+    revision: str = typer.Argument(..., help="The ID of a revision to downgrade to."),
+):
+    """
+    Downgrade the graph database schema to a previous version.
+    """
+    import asyncio
+
+    from sqlalchemy.ext.asyncio import create_async_engine
+
+    from ..alembic_utils import downgrade, get_current_revision
+    from ..graph.alembic_constants import ALEMBIC_DIR, ALEMBIC_INI_TEMPLATE_PATH
+    from ..graph.core import ALL_REVISIONS
+    from ..utils import ensure_specified_sql_driver
+
+    database_uri = ensure_specified_sql_driver(database_uri)
+
+    async def do_setup():
+        engine = create_async_engine(database_uri)
+        redacted_url = engine.url._replace(password="[redacted]")
+        current_revision = await get_current_revision(engine, ALL_REVISIONS)
+        await engine.dispose()
+        if current_revision is None:
+            typer.echo(
+                f"Database {redacted_url} has not been initialized. Use `tiled graph initialize-database`.",
+                err=True,
+            )
+            raise typer.Abort()
+
+    asyncio.run(do_setup())
+    downgrade(ALEMBIC_INI_TEMPLATE_PATH, ALEMBIC_DIR, database_uri, revision)

tiled/commandline/main.py

@@ -32,6 +32,7 @@
 from ._admin import admin_app  # noqa: E402
 from ._api_key import api_key_app  # noqa: E402
 from ._catalog import catalog_app  # noqa: E402
+from ._links import graph_app  # noqa: E402
 from ._profile import profile_app  # noqa: E402
 from ._register import register  # noqa: E402
 from ._serve import serve_app  # noqa: E402
@@ -52,6 +53,11 @@
     name="admin",
     help="Administrative utilities for managing large deployments.",
 )
+cli_app.add_typer(
+    graph_app,
+    name="graph",
+    help="Manage the graph (links) database.",
+)
 
 
 @cli_app.command("login")

tiled/config.py

@@ -238,6 +238,13 @@ class StreamingCacheConfig(BaseSettings):
     settings_customise_sources = classmethod(settings_customise_sources)
 
 
+class LinksDatabase(BaseSettings):
+    uri: Optional[str] = None
+
+    model_config = SettingsConfigDict(env_prefix="TILED_LINKS_DATABASE_")
+    settings_customise_sources = classmethod(settings_customise_sources)
+
+
 class WebhooksConfig(BaseSettings):
     """Configuration for the optional webhooks feature.
 
@@ -274,6 +281,7 @@ class Config(BaseSettings):
     file_extensions: dict[str, str] = {}
     authentication: Authentication = Authentication()
     database: Optional[Database] = None
+    links_database: Optional[LinksDatabase] = None
     # TODO: Replace Any with AccessPolicy when #1044 is merged
     access_policy: Annotated[Optional[Any], Field(alias="access_control")] = None
     response_bytesize_limit: int = 300_000_000
@@ -502,6 +510,7 @@ def construct_build_app_kwargs(config: Config):
         response_bytesize_limit=config.response_bytesize_limit,
         exact_count_limit=config.exact_count_limit,
         database=config.database,
+        links_database=config.links_database,
         reject_undeclared_specs=config.reject_undeclared_specs,
         expose_raw_assets=config.expose_raw_assets,
         metrics=config.metrics,

tiled/graph/__main__.py

@@ -0,0 +1,32 @@
+import os
+
+from ..alembic_utils import temp_alembic_ini
+from .alembic_constants import ALEMBIC_DIR, ALEMBIC_INI_TEMPLATE_PATH
+
+
+def main(args=None):
+    """
+    This is runs the alembic CLI with a dynamically genericated config file.
+
+    A database can be specified via TILED_DATABASE_URI, but it is not necessary to set
+    it for operations that do not connect to any database, such as defining new database
+    revisions (i.e. migrations).
+
+    To define a new revision:
+
+    $ python -m tiled.authn_database revision -m "description..."
+
+    """
+    import subprocess
+    import sys
+
+    if args is None:
+        args = sys.argv[1:]
+    with temp_alembic_ini(
+        ALEMBIC_INI_TEMPLATE_PATH, ALEMBIC_DIR, os.getenv("TILED_DATABASE_URI", "")
+    ) as config_file:
+        return subprocess.check_output(["alembic", "-c", config_file, *args])
+
+
+if __name__ == "__main__":
+    main()

tiled/graph/alembic.ini.template

@@ -0,0 +1,99 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = {migration_script_directory}
+sqlalchemy.url = {database_uri}
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator"
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. Valid values are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # default: use os.pathsep
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

tiled/graph/alembic_constants.py

@@ -0,0 +1,5 @@
+import os
+
+_here = os.path.abspath(os.path.dirname(__file__))
+ALEMBIC_INI_TEMPLATE_PATH = os.path.join(_here, "alembic.ini.template")
+ALEMBIC_DIR = os.path.join(_here, "migrations")

tiled/graph/core.py

@@ -0,0 +1,11 @@
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from .store import _metadata
+
+ALL_REVISIONS = ["7f3a9d1c0b25"]
+REQUIRED_REVISION = ALL_REVISIONS[0]
+
+
+async def initialize_database(engine: AsyncEngine) -> None:
+    async with engine.begin() as conn:
+        await conn.run_sync(_metadata.create_all)

tiled/graph/migrations/README

@@ -0,0 +1,7 @@
+Generic single-database configuration.
+
+To generate a new revision file, run:
+
+```
+python -m tiled.authn_database revision -m "description..."
+```