Fixing issues with Traefik config, adding backup to update script

Author: hunvreus

app/services/deployment.py

@@ -1,10 +1,11 @@
 import os
 import re
+import tempfile
 import yaml
 import aiodocker
 import logging
 from datetime import datetime, timezone
-from sqlalchemy import select
+from sqlalchemy import or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from redis.asyncio import Redis
 from arq.connections import ArqRedis
@@ -247,17 +248,28 @@ async def update_traefik_config(
         project: Project,
         db: AsyncSession,
         settings: Settings,
+        *,
+        include_deployment_ids: set[str] | None = None,
     ) -> None:
         """Update Traefik config for a project including domains."""
         path = os.path.join(settings.traefik_dir, f"project_{project.id}.yml")
 
         # Get aliases
+        include_ids = include_deployment_ids or set()
+        if include_ids:
+            where_clause = or_(
+                Deployment.conclusion == "succeeded",
+                Deployment.id.in_(list(include_ids)),
+            )
+        else:
+            where_clause = Deployment.conclusion == "succeeded"
+
         result = await db.execute(
             select(Alias)
             .join(Deployment, Alias.deployment_id == Deployment.id)
             .filter(
                 Deployment.project_id == project.id,
-                Deployment.conclusion == "succeeded",
+                where_clause,
             )
         )
         aliases = result.scalars().all()
@@ -339,6 +351,12 @@ async def update_traefik_config(
                     }
                 }
 
+        # If there is nothing to configure, remove any stale config file.
+        if not routers and not services and not middlewares:
+            if os.path.exists(path):
+                os.remove(path)
+            return
+
         # Write config
         os.makedirs(settings.traefik_dir, exist_ok=True)
         config = {"http": {"routers": routers}}
@@ -347,8 +365,20 @@ async def update_traefik_config(
         if middlewares:
             config["http"]["middlewares"] = middlewares
 
-        with open(path, "w") as f:
-            yaml.dump(config, f, sort_keys=False, indent=2)
+        # Write atomically so Traefik's file watcher never reads a partially-written YAML.
+        fd, tmp_path = tempfile.mkstemp(
+            prefix=f".{os.path.basename(path)}.", dir=settings.traefik_dir
+        )
+        try:
+            with os.fdopen(fd, "w") as f:
+                yaml.safe_dump(config, f, sort_keys=False, indent=2)
+            os.replace(tmp_path, path)
+        finally:
+            try:
+                if os.path.exists(tmp_path):
+                    os.remove(tmp_path)
+            except OSError:
+                pass
 
     async def create(
         self,

app/workers/tasks/deployment.py

@@ -331,14 +331,6 @@ async def finalize_deployment(ctx, deployment_id: str):
                 )
                 return
 
-            await service.update_status(
-                db,
-                deployment,
-                status="finalize",
-                error=None,
-                redis_client=redis_client,
-            )
-
             # Log a success message
             async with aiodocker.Docker(url=settings.docker_host) as docker_client:
                 container = await docker_client.containers.get(deployment.container_id)
@@ -353,7 +345,10 @@ async def finalize_deployment(ctx, deployment_id: str):
             # Update Traefik dynamic config
             try:
                 await DeploymentService().update_traefik_config(
-                    deployment.project, db, settings
+                    deployment.project,
+                    db,
+                    settings,
+                    include_deployment_ids={deployment.id},
                 )
             except Exception as e:
                 logger.error(f"{log_prefix} Failed to update Traefik config: {e}")

scripts/update-apply.sh

@@ -20,6 +20,7 @@ Apply a fetched update: validate, pull images, rollout, migrate, and record vers
   --components <csv>
                     Comma-separated list of services to update (${VALID_COMPONENTS//|/, })
   --full            Full stack update (down whole stack, then up). Causes downtime
+  --backup          Create a backup before applying the update (recommended)
   --no-migrate      Do not run DB migrations after app update
   --no-telemetry    Do not send telemetry
   --yes, -y         Non-interactive yes to prompts
@@ -30,7 +31,7 @@ USG
 }
 
 # Parse CLI flags
-ref=""; comps=""; do_all=0; do_full=0; migrate=1; yes=0; skip_components=0; telemetry=1
+ref=""; comps=""; do_all=0; do_full=0; do_backup=0; migrate=1; yes=0; skip_components=0; telemetry=1
 [[ "${NO_TELEMETRY:-0}" == "1" ]] && telemetry=0
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -49,6 +50,7 @@ while [[ $# -gt 0 ]]; do
       shift 2
       ;;
     --full) do_full=1; shift ;;
+    --backup) do_backup=1; shift ;;
     --no-migrate) migrate=0; shift ;;
     --no-telemetry) telemetry=0; shift ;;
     --yes|-y) yes=1; shift ;;
@@ -240,6 +242,19 @@ elif [[ -z "$comps" ]] && ((do_all==0)) && ((do_full==0)) && [[ -n "$meta_compon
   comps="$meta_components"
 fi
 
+# Backup (recommended)
+if ((do_backup==1)); then
+  printf '\n'
+  run_cmd "Creating backup" bash "$SCRIPT_DIR/backup.sh"
+elif ((yes!=1)) && [[ -t 0 ]]; then
+  printf '\n'
+  read -r -p "Create a backup before updating? [Y/n]: " ans
+  if [[ -z "${ans:-}" || "$ans" =~ ^[Yy]([Ee][Ss])?$ ]]; then
+    printf '\n'
+    run_cmd "Creating backup" bash "$SCRIPT_DIR/backup.sh"
+  fi
+fi
+
 # Full stack update helper (with downtime)
 full_update() {
   printf '\n'

scripts/update.sh

@@ -25,6 +25,7 @@ Update /dev/push by Git tag; performs rollouts (blue-green rollouts or simple re
   --components <csv>
                     Comma-separated list of services (${VALID_COMPONENTS//|/, })
   --full            Full stack update (down whole stack, then up). Causes downtime
+  --backup          Create a backup before applying the update (recommended)
   --no-migrate      Do not run DB migrations after app update
   --no-telemetry    Do not send telemetry
   --yes, -y         Non-interactive yes to prompts
@@ -35,7 +36,7 @@ USG
 }
 
 # Parse CLI flags
-ref=""; comps=""; do_all=0; do_full=0; migrate=1; yes=0; telemetry=1
+ref=""; comps=""; do_all=0; do_full=0; do_backup=0; migrate=1; yes=0; telemetry=1
 [[ "${NO_TELEMETRY:-0}" == "1" ]] && telemetry=0
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -54,6 +55,7 @@ while [[ $# -gt 0 ]]; do
       shift 2
       ;;
     --full) do_full=1; shift ;;
+    --backup) do_backup=1; shift ;;
     --no-migrate) migrate=0; shift ;;
     --no-telemetry) telemetry=0; shift ;;
     --yes|-y) yes=1; shift ;;
@@ -127,6 +129,9 @@ elif [[ -n "$comps" ]]; then
 elif ((do_full==1)); then
   apply_args+=(--full)
 fi
+if ((do_backup==1)); then
+  apply_args+=(--backup)
+fi
 if ((migrate==0)); then
   apply_args+=(--no-migrate)
 fi