From 4d04c9648325ebbedcd25b24fb9fd777576177b8 Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 09:27:18 +1000 Subject: [PATCH 1/7] Add docs on DB management to README --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index a9a95459..abbdfa12 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,23 @@ uv run -- ruff check . --fix pre-commit run --all-files ``` +# Database management + +The deployed service uses a Postgres database on AWS RDS. +In order to generate migrations for the database locally, +we use a Postgres docker container to generate migrations against. + +After making any changes to the database models, run the +`generate_migrations.py` script to create migrations: + +```shell +python generate_migrations.py -m migration_name +``` + +and commit them to git. Once your updated code has been +deployed on AWS, you can use `aws ecs execute-command` +to run the migrations. + # Deployment Currently the service is deployed to AWS via the CDK scripts in `deploy/`, From 1a1246433a138e10d0f83b1cc8263ad660235d05 Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 09:29:48 +1000 Subject: [PATCH 2/7] Add migration generation script --- generate_migrations.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 generate_migrations.py diff --git a/generate_migrations.py b/generate_migrations.py new file mode 100644 index 00000000..cb3db5bf --- /dev/null +++ b/generate_migrations.py @@ -0,0 +1,52 @@ +import os +import subprocess +import time + +import click + +DB_CONTAINER_NAME = "temp_alembic_db" +DEFAULT_POSTGRES_IMAGE = "postgres:17" +DEFAULT_PORT = 5433 +POSTGRES_USER = "testuser" +POSTGRES_PASSWORD = "testpass" + + +def run(cmd, env=None): + print(f"> {cmd}") + subprocess.run(cmd, shell=True, check=True, env=env or os.environ) + + +@click.command() +@click.option('--revision-message', '-m', required=True, help="Message for Alembic revision.") +def generate_migrations(revision_message): + """Spin up a temp Postgres DB, apply migrations, and generate a new Alembic revision.""" + database_url = f"localhost:{DEFAULT_PORT}" + os.environ["DB_HOST"] = database_url + os.environ["DB_USER"] = POSTGRES_USER + os.environ["DB_PASSWORD"] = POSTGRES_PASSWORD + + try: + print("๐Ÿš€ Starting temporary Postgres container...") + run( + f"docker run --rm -d --name {DB_CONTAINER_NAME} " + f"-e POSTGRES_USER={POSTGRES_USER} " + f"-e POSTGRES_PASSWORD={POSTGRES_PASSWORD} " + f"-p {DEFAULT_PORT}:5432 {DEFAULT_POSTGRES_IMAGE}" + ) + + print("โณ Waiting for database to be ready...") + time.sleep(5) # Optionally: poll with pg_isready + + print("๐Ÿงฑ Applying existing Alembic migrations...") + run("alembic upgrade head") + + print("๐Ÿ“ Generating new Alembic revision...") + run(f"alembic revision --autogenerate -m \"{revision_message}\"") + + finally: + print("๐Ÿงน Cleaning up: stopping container...") + subprocess.run(f"docker stop {DB_CONTAINER_NAME}", shell=True) + + +if __name__ == "__main__": + generate_migrations() From 9f5dfec989208b3d521f575864490d25234f7199 Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 09:33:49 +1000 Subject: [PATCH 3/7] Enable command execution for the ECS container --- deploy/aai_backend_deploy/aai_backend_deploy_stack.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/aai_backend_deploy/aai_backend_deploy_stack.py b/deploy/aai_backend_deploy/aai_backend_deploy_stack.py index 0236fdfd..28df66d4 100644 --- a/deploy/aai_backend_deploy/aai_backend_deploy_stack.py +++ b/deploy/aai_backend_deploy/aai_backend_deploy_stack.py @@ -22,6 +22,7 @@ from aws_cdk import ( aws_elasticloadbalancingv2 as elbv2, ) +from aws_cdk import aws_iam as iam from aws_cdk import ( aws_route53 as route53, ) @@ -60,6 +61,10 @@ def __init__(self, scope: Construct, construct_id: str, config: dict, **kwargs) task_definition = ecs.FargateTaskDefinition(self, "AaiBackendTaskDef", memory_limit_mib=1024, cpu=512) + # Allow executing comands in the ECS container + task_definition.task_role.add_managed_policy( + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore") + ) container = task_definition.add_container( "FastAPIContainer", @@ -101,6 +106,7 @@ def __init__(self, scope: Construct, construct_id: str, config: dict, **kwargs) domain_zone=route53.HostedZone.from_lookup( self, "AaiBackendZone", domain_name=self.zone_domain ), + enable_execute_command=True ) service.target_group.configure_health_check(path="/", healthy_http_codes="200-399") From 48ba5af951714dbb57e6e342df0984887428ec15 Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 09:37:44 +1000 Subject: [PATCH 4/7] Add initial migration for GroupMembership model --- .../101f45395233_add_group_membership.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 migrations/versions/101f45395233_add_group_membership.py diff --git a/migrations/versions/101f45395233_add_group_membership.py b/migrations/versions/101f45395233_add_group_membership.py new file mode 100644 index 00000000..6d302439 --- /dev/null +++ b/migrations/versions/101f45395233_add_group_membership.py @@ -0,0 +1,41 @@ +"""add_group_membership + +Revision ID: 101f45395233 +Revises: +Create Date: 2025-07-01 16:29:48.072722 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '101f45395233' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('groupmembership', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('group', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('user_email', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('approval_status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('updated_by_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('updated_by_email', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('groupmembership') + # ### end Alembic commands ### From 5f6ea2534344f98223a94ac9cbc3f20a77a3411d Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 09:55:49 +1000 Subject: [PATCH 5/7] Update generate_migrations to allow checking revisions --- generate_migrations.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/generate_migrations.py b/generate_migrations.py index cb3db5bf..5b0e7e58 100644 --- a/generate_migrations.py +++ b/generate_migrations.py @@ -17,9 +17,13 @@ def run(cmd, env=None): @click.command() -@click.option('--revision-message', '-m', required=True, help="Message for Alembic revision.") -def generate_migrations(revision_message): - """Spin up a temp Postgres DB, apply migrations, and generate a new Alembic revision.""" +@click.option('--revision-message', '-m', required=False, help="Message for Alembic revision.") +@click.option('--check', is_flag=True, help="Only run 'alembic check' after DB container is up.") +def generate_migrations(revision_message, check): + """Spin up a temp Postgres DB, apply migrations or run alembic check.""" + if not check and not revision_message: + raise click.UsageError("Missing option '-m' / '--revision-message'. Required unless using --check.") + database_url = f"localhost:{DEFAULT_PORT}" os.environ["DB_HOST"] = database_url os.environ["DB_USER"] = POSTGRES_USER @@ -35,13 +39,17 @@ def generate_migrations(revision_message): ) print("โณ Waiting for database to be ready...") - time.sleep(5) # Optionally: poll with pg_isready + time.sleep(5) # Could be enhanced with pg_isready - print("๐Ÿงฑ Applying existing Alembic migrations...") - run("alembic upgrade head") + if check: + print("๐Ÿ” Running 'alembic check'...") + run("alembic check") + else: + print("๐Ÿงฑ Applying existing Alembic migrations...") + run("alembic upgrade head") - print("๐Ÿ“ Generating new Alembic revision...") - run(f"alembic revision --autogenerate -m \"{revision_message}\"") + print("๐Ÿ“ Generating new Alembic revision...") + run(f"alembic revision --autogenerate -m \"{revision_message}\"") finally: print("๐Ÿงน Cleaning up: stopping container...") From 46032017058fa043afd8ae19e726f5af4ff16288 Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 10:10:41 +1000 Subject: [PATCH 6/7] Apply existing migrations when checking --- generate_migrations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generate_migrations.py b/generate_migrations.py index 5b0e7e58..7bd1c8d0 100644 --- a/generate_migrations.py +++ b/generate_migrations.py @@ -42,6 +42,8 @@ def generate_migrations(revision_message, check): time.sleep(5) # Could be enhanced with pg_isready if check: + print("๐Ÿงฑ Applying existing Alembic migrations...") + run("alembic upgrade head") print("๐Ÿ” Running 'alembic check'...") run("alembic check") else: From 3ac671a5d13fda2d4e851f73bdac635d58ce499d Mon Sep 17 00:00:00 2001 From: marius-mather Date: Wed, 2 Jul 2025 10:16:23 +1000 Subject: [PATCH 7/7] Fix DB_HOST in deploy script: not a secret, just an env variable --- deploy/aai_backend_deploy/aai_backend_deploy_stack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/aai_backend_deploy/aai_backend_deploy_stack.py b/deploy/aai_backend_deploy/aai_backend_deploy_stack.py index 28df66d4..af979901 100644 --- a/deploy/aai_backend_deploy/aai_backend_deploy_stack.py +++ b/deploy/aai_backend_deploy/aai_backend_deploy_stack.py @@ -75,12 +75,12 @@ def __init__(self, scope: Construct, construct_id: str, config: dict, **kwargs) # Set an env variable to the current time to force redeploy - # might be better to use an image tag in future environment={ - "FORCE_REDEPLOY": str(datetime.datetime.now()) + "FORCE_REDEPLOY": str(datetime.datetime.now()), + "DB_HOST": self.db_host, }, secrets={ "DB_USER": ecs.Secret.from_secrets_manager(db_secret, field="username"), "DB_PASSWORD": ecs.Secret.from_secrets_manager(db_secret, field="password"), - "DB_HOST": self.db_host, }, logging=ecs.LogDrivers.aws_logs(stream_prefix="FastAPI"), )