diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da47d69..effc0de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,17 @@ jobs: tags: triangle-cms-frontend:${{ github.sha }} cache-from: type=gha,scope=frontend cache-to: type=gha,mode=max,scope=frontend + # Cheap, unlike the embeddings image, and its build fails if the bundled + # libvips cannot write WebP, which is worth knowing before a deploy. + - name: Build imaging sidecar image + uses: docker/build-push-action@v6 + with: + context: ./imaging + file: ./imaging/Dockerfile + push: false + tags: triangle-cms-imaging:${{ github.sha }} + cache-from: type=gha,scope=imaging + cache-to: type=gha,mode=max,scope=imaging # The deployment scripts are the least reversible code in the repo, so their # test suite runs on every PR. It stubs docker/curl/nginx on PATH and needs no @@ -200,4 +211,5 @@ jobs: # checks that the derivation deploy.sh uses still resolves. run: | CMS_EMBEDDINGS_TAG="$(git rev-parse HEAD:embeddings)" \ + CMS_IMAGING_TAG="$(git rev-parse HEAD:imaging)" \ docker compose -f deploy/compose.cms.yml config >/dev/null diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 557be08..0dbc462 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -91,6 +91,7 @@ jobs: echo "backend=ghcr.io/${repo}-backend" >> "$GITHUB_OUTPUT" echo "frontend=ghcr.io/${repo}-frontend" >> "$GITHUB_OUTPUT" echo "embeddings=ghcr.io/${repo}-embeddings" >> "$GITHUB_OUTPUT" + echo "imaging=ghcr.io/${repo}-imaging" >> "$GITHUB_OUTPUT" - uses: docker/setup-buildx-action@v3 @@ -122,6 +123,21 @@ jobs: echo "value=false" >> "$GITHUB_OUTPUT" fi + # Same content tagging for the imaging sidecar, from imaging/. + - name: Compute the imaging tag + id: imaging_tag + run: echo "value=$(git rev-parse HEAD:imaging)" >> "$GITHUB_OUTPUT" + + - name: Check whether the imaging image already exists + id: imaging_exists + run: | + if docker manifest inspect "${{ steps.image.outputs.imaging }}:${{ steps.imaging_tag.outputs.value }}" >/dev/null 2>&1; then + echo "skipping the imaging build: ${{ steps.imaging_tag.outputs.value }} is already published" + echo "value=true" >> "$GITHUB_OUTPUT" + else + echo "value=false" >> "$GITHUB_OUTPUT" + fi + - name: Build and publish backend uses: docker/build-push-action@v6 with: @@ -161,3 +177,17 @@ jobs: org.opencontainers.image.revision=${{ steps.sha.outputs.value }} cache-from: type=gha,scope=embeddings cache-to: type=gha,mode=max,scope=embeddings + + - name: Build and publish imaging sidecar + if: steps.imaging_exists.outputs.value != 'true' + uses: docker/build-push-action@v6 + with: + context: ./imaging + file: ./imaging/Dockerfile + push: true + tags: ${{ steps.image.outputs.imaging }}:${{ steps.imaging_tag.outputs.value }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ steps.sha.outputs.value }} + cache-from: type=gha,scope=imaging + cache-to: type=gha,mode=max,scope=imaging diff --git a/deploy/compose.cms.yml b/deploy/compose.cms.yml index 1c03dd1..fc235dd 100644 --- a/deploy/compose.cms.yml +++ b/deploy/compose.cms.yml @@ -51,6 +51,11 @@ x-backend-base: &backend-base # is no reason to run one per slot. Leave empty to run lexical-only search: # the backend then skips query embedding and its reconciler exits at start. EMBEDDINGS_URL: ${EMBEDDINGS_URL-http://embeddings:8000} + # Resized image renditions. Shared by both slots like the embeddings + # sidecar; the reconciler that drives it takes a database lock so only one + # slot renders at a time. Leave empty to stop rendering; images already + # rendered keep being served. + IMAGING_URL: ${IMAGING_URL-http://imaging:8000} volumes: # CephFS media tree (host). rw so the upload endpoint can store new files; # host Nginx serves the same tree read-only (that site is in triangle-infrastructure). @@ -118,6 +123,37 @@ services: networks: - triangle_net + # Renders resized WebP copies of library images. Stateless: the backend reads + # the original from the media mount, sends the bytes, and writes what comes + # back, so this container needs no volume and never touches CephFS itself. + imaging: + # Tagged by content, like embeddings: CMS_IMAGING_TAG is the git tree hash + # of imaging/. deploy.sh derives the value. + image: ${CMS_IMAGING_IMAGE:-ghcr.io/drexeltriangle/triangle-cms-imaging}:${CMS_IMAGING_TAG:?CMS_IMAGING_TAG is required} + restart: unless-stopped + stop_grace_period: 10s + # The backfill of the existing library (~13k originals, about 1.5s each) + # runs for hours, so it gets less of Delta's 6 cores than the embeddings + # sidecar does. libvips sizes its thread pool from VIPS_CONCURRENCY, not + # from the cgroup, so the two must match for the same reason + # OMP_NUM_THREADS does above. + cpus: ${IMAGE_CPUS:-2} + # Decoding is the memory peak: a 120MP original (the sidecar's ceiling) + # is ~480MB of pixels. A ceiling here turns a pathological file into one + # OOM-killed render, which the reconciler gives up on after three tries, + # rather than pressure on the backends. + mem_limit: ${IMAGE_MEM_LIMIT:-1536m} + environment: + VIPS_CONCURRENCY: ${IMAGE_CPUS:-2} + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health')\""] + interval: 15s + timeout: 5s + retries: 4 + start_period: 10s + networks: + - triangle_net + backend-blue: <<: *backend-base ports: diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh index 13eb5f9..34d5fbe 100755 --- a/deploy/scripts/common.sh +++ b/deploy/scripts/common.sh @@ -17,6 +17,8 @@ PUBLIC_HEALTH_TIMEOUT="${PUBLIC_HEALTH_TIMEOUT:-30}" # Generous: this covers pulling the image and loading the ONNX model on a host # with no GPU. Exceeding it only costs semantic search, never the deployment. EMBEDDINGS_HEALTH_TIMEOUT="${EMBEDDINGS_HEALTH_TIMEOUT:-240}" +# No model to load: the imaging sidecar is healthy as soon as uvicorn is up. +IMAGING_HEALTH_TIMEOUT="${IMAGING_HEALTH_TIMEOUT:-60}" compose() { docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" "$@" @@ -245,16 +247,19 @@ wait_for_url() { done } -# wait_for_embeddings polls the container's health state rather than an HTTP -# endpoint, because the sidecar is deliberately not published to the host: only -# the backends reach it, over the compose network. Its healthcheck 503s until the -# model has finished loading, so "healthy" here means it can actually answer. -wait_for_embeddings() { - local deadline=$((SECONDS + EMBEDDINGS_HEALTH_TIMEOUT)) +# wait_for_sidecar polls a shared sidecar's container health rather than an +# HTTP endpoint, because the sidecars are deliberately not published to the +# host: only the backends reach them, over the compose network. The embeddings +# healthcheck 503s until the model has finished loading, so "healthy" here means +# it can actually answer. +wait_for_sidecar() { + local service="$1" + local timeout="$2" + local deadline=$((SECONDS + timeout)) local container status="" while true; do - container="$(compose ps -q embeddings 2>/dev/null || true)" + container="$(compose ps -q "${service}" 2>/dev/null || true)" if [[ -n "${container}" ]]; then status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "${container}" 2>/dev/null || true)" if [[ "${status}" == "healthy" ]]; then @@ -262,13 +267,21 @@ wait_for_embeddings() { fi fi if (( SECONDS >= deadline )); then - echo "timed out waiting for the embeddings sidecar (last status: ${status:-unknown})" >&2 + echo "timed out waiting for the ${service} sidecar (last status: ${status:-unknown})" >&2 return 1 fi sleep 3 done } +wait_for_embeddings() { + wait_for_sidecar embeddings "${EMBEDDINGS_HEALTH_TIMEOUT}" +} + +wait_for_imaging() { + wait_for_sidecar imaging "${IMAGING_HEALTH_TIMEOUT}" +} + wait_for_slot() { local slot="$1" validate_slot "${slot}" diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh index d6c8009..710717b 100755 --- a/deploy/scripts/deploy.sh +++ b/deploy/scripts/deploy.sh @@ -41,6 +41,17 @@ fi export CMS_EMBEDDINGS_TAG echo "embeddings image tag: ${CMS_EMBEDDINGS_TAG}" +# The imaging sidecar is tagged the same way, from imaging/, and for the same +# reason: it should only be rebuilt and recreated when it changes. +if imaging_tag="$(git -C "${REPO_DIR}" rev-parse HEAD:imaging 2>/dev/null)"; then + CMS_IMAGING_TAG="${imaging_tag}" +else + echo "warning: could not derive the imaging tag from git; falling back to the commit tag" >&2 + CMS_IMAGING_TAG="${CMS_IMAGE_TAG}" +fi +export CMS_IMAGING_TAG +echo "imaging image tag: ${CMS_IMAGING_TAG}" + require_file "${COMPOSE_FILE}" acquire_deploy_lock deployment_preflight @@ -74,6 +85,20 @@ else echo "warning: could not start the embeddings sidecar; search will serve lexical results" >&2 fi +# Same arrangement for the imaging sidecar, and just as non-fatal: without it +# the backends stop producing resized images, and the site serves the originals +# it served before this sidecar existed. +if ! compose pull imaging; then + echo "warning: could not pull the imaging sidecar; new uploads will not be resized" >&2 +fi +if compose up -d --no-deps imaging; then + if ! wait_for_imaging; then + echo "warning: the imaging sidecar did not become healthy; new uploads will not be resized until it does" >&2 + fi +else + echo "warning: could not start the imaging sidecar; new uploads will not be resized" >&2 +fi + compose pull "backend-${next_slot}" "frontend-${next_slot}" compose up -d --no-deps "backend-${next_slot}" "frontend-${next_slot}" diff --git a/deploy/scripts/deploy_scripts_test.sh b/deploy/scripts/deploy_scripts_test.sh index 0699c53..814af1d 100755 --- a/deploy/scripts/deploy_scripts_test.sh +++ b/deploy/scripts/deploy_scripts_test.sh @@ -58,6 +58,7 @@ make_case() { # the fake compose in these tests. Left at its 240s production default it made # every deploy case sit out the full timeout; the suite took 8 minutes. EMBEDDINGS_HEALTH_TIMEOUT=0 + IMAGING_HEALTH_TIMEOUT=0 DEPLOY_TEST_MODE=1 NGINX_TEST_CMD='exit "${FAKE_NGINX_TEST_STATUS:-0}"' NGINX_RELOAD_CMD='exit "${FAKE_NGINX_RELOAD_STATUS:-0}"' @@ -69,7 +70,7 @@ make_case() { FAIL_PUBLIC=0 export NGINX_ACTIVE_INCLUDE ENV_FILE COMPOSE_FILE DEPLOY_LOCK_FILE PUBLIC_BASE_URL export BACKEND_HEALTH_TIMEOUT FRONTEND_HEALTH_TIMEOUT PUBLIC_HEALTH_TIMEOUT - export EMBEDDINGS_HEALTH_TIMEOUT + export EMBEDDINGS_HEALTH_TIMEOUT IMAGING_HEALTH_TIMEOUT export DEPLOY_TEST_MODE NGINX_TEST_CMD NGINX_RELOAD_CMD NGINX_RELOAD_CHECK_CMD export FAKE_NGINX_TEST_STATUS FAKE_NGINX_RELOAD_STATUS FAKE_NGINX_RELOAD_CHECK_STATUS export FAIL_READINESS FAIL_PUBLIC diff --git a/docker-compose.yml b/docker-compose.yml index ac442de..29aa7d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,19 @@ services: networks: - triangle_net + imaging: + build: + context: ./imaging + dockerfile: Dockerfile + restart: unless-stopped + cpus: ${IMAGE_CPUS:-2} + environment: + VIPS_CONCURRENCY: ${IMAGE_CPUS:-2} + # Stateless and internal-only, like embeddings. Locally the CMS has no + # MEDIA_ROOT, so its reconciler stays idle unless you mount one. + networks: + - triangle_net + cms: build: context: ./server @@ -61,6 +74,7 @@ services: restart: unless-stopped environment: EMBEDDINGS_URL: ${EMBEDDINGS_URL:-http://embeddings:8000} + IMAGING_URL: ${IMAGING_URL:-http://imaging:8000} DB_NAME: ${MARIADB_DATABASE:-triangle} DB_USER: ${MARIADB_USER:-triangle_user} DB_PASSWORD: ${MARIADB_PASSWORD:?MARIADB_PASSWORD is required} diff --git a/imaging/Dockerfile b/imaging/Dockerfile new file mode 100644 index 0000000..9c1182a --- /dev/null +++ b/imaging/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py . + +# Fail the build, not the first render, if the bundled libvips cannot write +# WebP. +RUN python -c "import pyvips; pyvips.Image.black(8, 8).webpsave_buffer()" + +RUN useradd --create-home --uid 10001 imaging +USER imaging + +EXPOSE 8000 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] diff --git a/imaging/app.py b/imaging/app.py new file mode 100644 index 0000000..792842b --- /dev/null +++ b/imaging/app.py @@ -0,0 +1,138 @@ +"""Image rendition sidecar for the CMS. + +The media library stores what photographers upload, which since 2023 is mostly +straight off the camera: 6000px, 5-80MB JPEGs. The public site was putting those +originals into 400px cards. WordPress used to hide this by generating resized +copies on upload; the CMS does not, so this service is that step. + +It is deliberately stateless, like the embedding sidecar. The CMS sends the +original's bytes and gets one encoded rendition back; it owns the files, the +database rows and the decision about what to render. If this service restarts +or is missing entirely, the site keeps serving originals. +""" + +from __future__ import annotations + +import logging +import os +import threading + +import pyvips +from fastapi import FastAPI, HTTPException, Query, Request, Response +from starlette.concurrency import run_in_threadpool + +# The recipe names everything that decides what a rendition looks like: format, +# quality, and the width ladder. The CMS stores it with each rendition and puts +# it in the file path, so changing any of those means changing this string. That +# is what makes the CMS re-render the library rather than keep serving stale +# files, and what keeps Cloudflare's 30-day immutable cache from pinning the old +# bytes under an unchanged URL. +RECIPE = os.getenv("IMAGE_RECIPE", "webp-q80-v1") +QUALITY = int(os.getenv("IMAGE_QUALITY", "80")) + +# Ascending. 480/960 cover phone cards at 1x/2x, 1600 the article lead on a +# laptop, 2400 a full-bleed lead on a large or high-DPI screen. Nothing on the +# site is displayed wider than that. +WIDTHS = [int(w) for w in os.getenv("IMAGE_WIDTHS", "480,960,1600,2400").split(",")] + +# Bounds on what one request can make this process allocate. The upload cap is +# 90MB, so the byte limit only has to clear that. The pixel limit is the real +# guard: a small PNG can declare a huge canvas, and decoding allocates by pixel +# count, not file size. 120MP is above anything a current camera produces. +MAX_BYTES = int(os.getenv("IMAGE_MAX_BYTES", str(100 * 1024 * 1024))) +MAX_PIXELS = int(os.getenv("IMAGE_MAX_PIXELS", str(120_000_000))) + +# libvips already uses every core it is given for a single image, so rendering +# two at once only doubles peak memory. The CMS sends one at a time anyway; this +# holds if something else ever doesn't. +_render_lock = threading.Semaphore(int(os.getenv("IMAGE_MAX_CONCURRENT", "1"))) + +logger = logging.getLogger("imaging") + +app = FastAPI(title="Triangle CMS imaging") + + +@app.get("/health") +def health() -> dict[str, object]: + return { + "status": "ok", + "recipe": RECIPE, + "format": "webp", + "widths": WIDTHS, + "libvips": f"{pyvips.version(0)}.{pyvips.version(1)}.{pyvips.version(2)}", + } + + +class Unprocessable(Exception): + """The input itself is the problem; retrying the same bytes will not help.""" + + +def _render(data: bytes, width: int) -> tuple[bytes, int, int]: + try: + # Header only: new_from_buffer does not decode pixels until asked, so + # this is how the canvas size is checked before anything is allocated. + probe = pyvips.Image.new_from_buffer(data, "") + except pyvips.Error as exc: + raise Unprocessable(f"not a decodable image: {exc}") from exc + if probe.width * probe.height > MAX_PIXELS: + raise Unprocessable(f"{probe.width}x{probe.height} exceeds {MAX_PIXELS} pixels") + + try: + # thumbnail_buffer is the libvips fast path. For JPEG it decodes at a + # reduced scale directly (shrink-on-load), so a 6000px original costs + # about what a 1500px one would. It also applies the EXIF orientation, + # which phone photos depend on, and converts any embedded profile (Adobe + # RGB, CMYK) to sRGB, without which colours shift visibly in browsers. + # + # size="down" never enlarges, so asking for 2400 from a 1200px original + # returns 1200. The CMS reads the width it got back rather than assuming. + image = pyvips.Image.thumbnail_buffer( + data, + width, + height=10_000_000, + size="down", + export_profile="srgb", + ) + # 16-bit PNGs come out of thumbnail as 16-bit; WebP is 8-bit only. + if image.format != "uchar": + image = image.colourspace("srgb") + # keep="none" drops EXIF, XMP and ICC. The originals are served with + # their EXIF intact, GPS included; the renditions should not be. + encoded = image.webpsave_buffer(Q=QUALITY, effort=4, keep="none") + except pyvips.Error as exc: + raise Unprocessable(f"render failed: {exc}") from exc + + return encoded, image.width, image.height + + +@app.post("/render") +async def render(request: Request, width: int = Query(gt=0, le=10_000)) -> Response: + declared = request.headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > MAX_BYTES: + raise HTTPException(status_code=413, detail=f"body exceeds {MAX_BYTES} bytes") + data = await request.body() + if not data: + raise HTTPException(status_code=400, detail="empty body") + if len(data) > MAX_BYTES: + raise HTTPException(status_code=413, detail=f"body exceeds {MAX_BYTES} bytes") + + def work() -> tuple[bytes, int, int]: + with _render_lock: + return _render(data, width) + + try: + encoded, out_width, out_height = await run_in_threadpool(work) + except Unprocessable as exc: + # 422 tells the CMS to record this file as failed rather than retry it + # on every pass forever. + raise HTTPException(status_code=422, detail=str(exc)) from exc + + return Response( + content=encoded, + media_type="image/webp", + headers={ + "X-Image-Width": str(out_width), + "X-Image-Height": str(out_height), + "X-Image-Recipe": RECIPE, + }, + ) diff --git a/imaging/requirements.txt b/imaging/requirements.txt new file mode 100644 index 0000000..1dd7008 --- /dev/null +++ b/imaging/requirements.txt @@ -0,0 +1,8 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +# pyvips-binary ships a current libvips (with libwebp, libjpeg-turbo, lcms) as +# a wheel. Debian's libvips42 is several releases older and splits format +# support across optional packages, which is how a container ends up healthy but +# unable to write WebP. +pyvips==3.0.0 +pyvips-binary==8.18.7 diff --git a/server/docs/docs.go b/server/docs/docs.go index 55d979b..f8b8727 100644 --- a/server/docs/docs.go +++ b/server/docs/docs.go @@ -4629,6 +4629,13 @@ const docTemplate = `{ "description": "FeaturedImageAlt is the article's own description of its featured image.\nEmpty means the public site has nothing to render, which is a defect worth\nsurfacing rather than papering over with the headline: an alt that repeats\nthe adjacent headline tells a screen-reader user nothing new.", "type": "string" }, + "featured_image_variants": { + "description": "FeaturedImageVariants are resized copies of FeaturedImage, narrowest\nfirst, for a srcset. Absent until they have been rendered.", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "id": { "type": "integer" }, @@ -4764,6 +4771,13 @@ const docTemplate = `{ "featured_image": { "type": "string" }, + "featured_image_variants": { + "description": "FeaturedImageVariants are resized copies of FeaturedImage, narrowest\nfirst, for a srcset. Absent until they have been rendered; keep\nFeaturedImage as the src fallback either way.", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "id": { "type": "integer" }, @@ -5536,6 +5550,20 @@ const docTemplate = `{ } } }, + "models.ImageVariant": { + "type": "object", + "properties": { + "height": { + "type": "integer" + }, + "url": { + "type": "string" + }, + "width": { + "type": "integer" + } + } + }, "models.Media": { "type": "object", "properties": { @@ -5576,6 +5604,13 @@ const docTemplate = `{ "url": { "type": "string" }, + "variants": { + "description": "Variants are resized WebP copies of the original, narrowest first. Empty\nuntil the imaging sidecar has rendered this item, and always empty for\nformats it does not render (GIF, SVG).", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "width": { "type": "integer" } diff --git a/server/docs/swagger.json b/server/docs/swagger.json index 73ca8fc..3ddfa12 100644 --- a/server/docs/swagger.json +++ b/server/docs/swagger.json @@ -4626,6 +4626,13 @@ "description": "FeaturedImageAlt is the article's own description of its featured image.\nEmpty means the public site has nothing to render, which is a defect worth\nsurfacing rather than papering over with the headline: an alt that repeats\nthe adjacent headline tells a screen-reader user nothing new.", "type": "string" }, + "featured_image_variants": { + "description": "FeaturedImageVariants are resized copies of FeaturedImage, narrowest\nfirst, for a srcset. Absent until they have been rendered.", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "id": { "type": "integer" }, @@ -4761,6 +4768,13 @@ "featured_image": { "type": "string" }, + "featured_image_variants": { + "description": "FeaturedImageVariants are resized copies of FeaturedImage, narrowest\nfirst, for a srcset. Absent until they have been rendered; keep\nFeaturedImage as the src fallback either way.", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "id": { "type": "integer" }, @@ -5533,6 +5547,20 @@ } } }, + "models.ImageVariant": { + "type": "object", + "properties": { + "height": { + "type": "integer" + }, + "url": { + "type": "string" + }, + "width": { + "type": "integer" + } + } + }, "models.Media": { "type": "object", "properties": { @@ -5573,6 +5601,13 @@ "url": { "type": "string" }, + "variants": { + "description": "Variants are resized WebP copies of the original, narrowest first. Empty\nuntil the imaging sidecar has rendered this item, and always empty for\nformats it does not render (GIF, SVG).", + "type": "array", + "items": { + "$ref": "#/definitions/models.ImageVariant" + } + }, "width": { "type": "integer" } diff --git a/server/docs/swagger.yaml b/server/docs/swagger.yaml index b9b3e10..804b5bd 100644 --- a/server/docs/swagger.yaml +++ b/server/docs/swagger.yaml @@ -197,6 +197,13 @@ definitions: surfacing rather than papering over with the headline: an alt that repeats the adjacent headline tells a screen-reader user nothing new. type: string + featured_image_variants: + description: |- + FeaturedImageVariants are resized copies of FeaturedImage, narrowest + first, for a srcset. Absent until they have been rendered. + items: + $ref: '#/definitions/models.ImageVariant' + type: array id: type: integer is_featured: @@ -293,6 +300,14 @@ definitions: type: string featured_image: type: string + featured_image_variants: + description: |- + FeaturedImageVariants are resized copies of FeaturedImage, narrowest + first, for a srcset. Absent until they have been rendered; keep + FeaturedImage as the src fallback either way. + items: + $ref: '#/definitions/models.ImageVariant' + type: array id: type: integer is_featured: @@ -798,6 +813,15 @@ definitions: $ref: '#/definitions/models.ArticleListItem' type: array type: object + models.ImageVariant: + properties: + height: + type: integer + url: + type: string + width: + type: integer + type: object models.Media: properties: alt_text: @@ -828,6 +852,14 @@ definitions: type: string url: type: string + variants: + description: |- + Variants are resized WebP copies of the original, narrowest first. Empty + until the imaging sidecar has rendered this item, and always empty for + formats it does not render (GIF, SVG). + items: + $ref: '#/definitions/models.ImageVariant' + type: array width: type: integer type: object diff --git a/server/internal/database/media_renditions.go b/server/internal/database/media_renditions.go new file mode 100644 index 0000000..3e8c985 --- /dev/null +++ b/server/internal/database/media_renditions.go @@ -0,0 +1,234 @@ +package database + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" +) + +// Rendition statuses. A failed row is not retried until the recipe changes: the +// sidecar only reports a failure as permanent when the bytes themselves are the +// problem, and re-sending them every pass would just repeat it. +const ( + RenditionStatusOK = "ok" + RenditionStatusFailed = "failed" +) + +// RenditionVariant is one rendered file. Path is MEDIA_ROOT-relative, the same +// shape as media.path, so URLs come from it the same way. +type RenditionVariant struct { + Path string `json:"path"` + Width int `json:"width"` + Height int `json:"height"` + SizeBytes int64 `json:"size_bytes"` +} + +// EnsureMediaRenditionsTable creates the table recording which resized copies +// exist for each media row. +// +// One row per media item rather than one per variant, so a re-render replaces +// the whole set in a single upsert and a reader never sees half of the old +// ladder mixed with half of the new one. +// +// There is deliberately no foreign key to media. A cascade would drop the row +// the moment the media row goes, and with it the only record of which variant +// files to delete. The reconciler removes orphans itself, files first. +func EnsureMediaRenditionsTable(ctx context.Context, conn *sql.DB) error { + _, err := conn.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS media_renditions ( + media_id BIGINT NOT NULL PRIMARY KEY, + recipe VARCHAR(64) NOT NULL, + status VARCHAR(16) NOT NULL, + variants LONGTEXT NULL, + error VARCHAR(512) NULL, + updated_at DATETIME NOT NULL, + INDEX idx_media_renditions_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + `) + return err +} + +// RenditionSource is a media item that needs rendering, plus whatever variants +// it currently has so the caller can delete them once they are replaced. +type RenditionSource struct { + MediaID int64 + Path string + Previous []RenditionVariant +} + +// renderableMimeTypes are the formats worth rendering. GIF is left out on +// purpose: most of the GIFs in the library are animated, and a WebP rendition +// of the first frame would silently replace an animation with a still. +var renderableMimeTypes = []any{"image/jpeg", "image/png", "image/webp"} + +// MediaNeedingRenditions returns up to limit media items with no rendition for +// recipe, newest first so a backfill reaches the images readers are looking at +// before the 2011 archive. +func MediaNeedingRenditions(ctx context.Context, conn *sql.DB, recipe string, limit int) ([]RenditionSource, error) { + if limit <= 0 { + return nil, fmt.Errorf("limit must be greater than 0") + } + + args := append(append([]any{}, renderableMimeTypes...), recipe) + rows, err := conn.QueryContext(ctx, ` + SELECT m.id, m.path, r.variants + FROM media AS m + LEFT JOIN media_renditions AS r ON r.media_id = m.id + WHERE m.mime_type IN (?, ?, ?) + AND (r.media_id IS NULL OR r.recipe <> ?) + ORDER BY m.created_at DESC, m.id DESC + LIMIT `+strconv.Itoa(limit), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var sources []RenditionSource + for rows.Next() { + var source RenditionSource + var variants sql.NullString + if err := rows.Scan(&source.MediaID, &source.Path, &variants); err != nil { + return nil, err + } + if source.Previous, err = decodeVariants(variants); err != nil { + return nil, fmt.Errorf("media %d: %w", source.MediaID, err) + } + sources = append(sources, source) + } + return sources, rows.Err() +} + +// SaveRenditions records a successful render, replacing any earlier set. +func SaveRenditions(ctx context.Context, conn *sql.DB, mediaID int64, recipe string, variants []RenditionVariant) error { + encoded, err := json.Marshal(variants) + if err != nil { + return err + } + return upsertRendition(ctx, conn, mediaID, recipe, RenditionStatusOK, sql.NullString{String: string(encoded), Valid: true}, sql.NullString{}) +} + +// SaveRenditionFailure records that mediaID cannot be rendered under recipe. +// Any variants from an earlier recipe stay on disk and in use: they are still +// valid images, just made by an older recipe, so they are kept in the row. +func SaveRenditionFailure(ctx context.Context, conn *sql.DB, mediaID int64, recipe string, previous []RenditionVariant, reason string) error { + if len(reason) > 512 { + reason = reason[:512] + } + variants := sql.NullString{} + if len(previous) > 0 { + encoded, err := json.Marshal(previous) + if err != nil { + return err + } + variants = sql.NullString{String: string(encoded), Valid: true} + } + return upsertRendition(ctx, conn, mediaID, recipe, RenditionStatusFailed, variants, sql.NullString{String: reason, Valid: true}) +} + +func upsertRendition(ctx context.Context, conn *sql.DB, mediaID int64, recipe, status string, variants, reason sql.NullString) error { + _, err := conn.ExecContext(ctx, ` + INSERT INTO media_renditions (media_id, recipe, status, variants, error, updated_at) + VALUES (?, ?, ?, ?, ?, UTC_TIMESTAMP()) + ON DUPLICATE KEY UPDATE + recipe = VALUES(recipe), + status = VALUES(status), + variants = VALUES(variants), + error = VALUES(error), + updated_at = UTC_TIMESTAMP() + `, mediaID, recipe, status, variants, reason) + return err +} + +// OrphanedRendition is a rendition whose media row no longer exists. +type OrphanedRendition struct { + MediaID int64 + Variants []RenditionVariant +} + +// OrphanedRenditions returns up to limit renditions left behind by deleted +// media, so their files can be removed before their rows. +func OrphanedRenditions(ctx context.Context, conn *sql.DB, limit int) ([]OrphanedRendition, error) { + if limit <= 0 { + return nil, fmt.Errorf("limit must be greater than 0") + } + + rows, err := conn.QueryContext(ctx, ` + SELECT r.media_id, r.variants + FROM media_renditions AS r + LEFT JOIN media AS m ON m.id = r.media_id + WHERE m.id IS NULL + LIMIT `+strconv.Itoa(limit)) + if err != nil { + return nil, err + } + defer rows.Close() + + var orphans []OrphanedRendition + for rows.Next() { + var orphan OrphanedRendition + var variants sql.NullString + if err := rows.Scan(&orphan.MediaID, &variants); err != nil { + return nil, err + } + if orphan.Variants, err = decodeVariants(variants); err != nil { + return nil, fmt.Errorf("media %d: %w", orphan.MediaID, err) + } + orphans = append(orphans, orphan) + } + return orphans, rows.Err() +} + +// DeleteRendition removes one rendition row. Callers delete its files first. +func DeleteRendition(ctx context.Context, conn *sql.DB, mediaID int64) error { + _, err := conn.ExecContext(ctx, `DELETE FROM media_renditions WHERE media_id = ?`, mediaID) + return err +} + +// RenditionsByMediaPath returns every usable variant set keyed by its +// original's media path. It is the whole table in one query (tens of thousands +// of short rows), which the public API holds in memory so that listing a page +// of articles costs no extra database round trip. +func RenditionsByMediaPath(ctx context.Context, conn *sql.DB) (map[string][]RenditionVariant, error) { + rows, err := conn.QueryContext(ctx, ` + SELECT m.path, r.variants + FROM media_renditions AS r + JOIN media AS m ON m.id = r.media_id + WHERE r.variants IS NOT NULL AND r.variants <> '' + `) + if err != nil { + return nil, err + } + defer rows.Close() + + byPath := make(map[string][]RenditionVariant) + for rows.Next() { + var path string + var variants sql.NullString + if err := rows.Scan(&path, &variants); err != nil { + return nil, err + } + decoded, err := decodeVariants(variants) + if err != nil { + // One corrupt row should not take every other image's variants out + // of the API with it. + continue + } + if len(decoded) > 0 { + byPath[path] = decoded + } + } + return byPath, rows.Err() +} + +func decodeVariants(raw sql.NullString) ([]RenditionVariant, error) { + if !raw.Valid || raw.String == "" { + return nil, nil + } + var variants []RenditionVariant + if err := json.Unmarshal([]byte(raw.String), &variants); err != nil { + return nil, fmt.Errorf("decode rendition variants: %w", err) + } + return variants, nil +} diff --git a/server/internal/database/media_renditions_integration_test.go b/server/internal/database/media_renditions_integration_test.go new file mode 100644 index 0000000..c55fd5c --- /dev/null +++ b/server/internal/database/media_renditions_integration_test.go @@ -0,0 +1,145 @@ +package database + +import ( + "context" + "database/sql" + "os" + "testing" + + _ "github.com/go-sql-driver/mysql" +) + +// CMS_TEST_DSN='user:pw@tcp(127.0.0.1:3306)/cms_test?parseTime=true&multiStatements=true' go test ./internal/database/ -run MediaRenditions -v +func mediaRenditionsTestDB(t *testing.T) *sql.DB { + t.Helper() + + dsn := os.Getenv("CMS_TEST_DSN") + if dsn == "" { + t.Skip("CMS_TEST_DSN not set; skipping media renditions integration test") + } + + conn, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open test database: %v", err) + } + conn.SetMaxOpenConns(1) + if err := conn.Ping(); err != nil { + t.Fatalf("ping test database: %v", err) + } + + ctx := context.Background() + var acquired sql.NullInt64 + if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, 60)", "cms_integration_test_shared_tables").Scan(&acquired); err != nil { + t.Fatalf("acquire test lock: %v", err) + } + if !acquired.Valid || acquired.Int64 != 1 { + t.Fatal("timed out waiting for the media renditions test lock") + } + t.Cleanup(func() { + _, _ = conn.ExecContext(context.Background(), "SELECT RELEASE_LOCK(?)", "cms_integration_test_shared_tables") + conn.Close() + }) + + for _, table := range []string{"media_renditions", "media"} { + if _, err := conn.ExecContext(ctx, "DROP TABLE IF EXISTS "+table); err != nil { + t.Fatalf("drop %s table: %v", table, err) + } + } + if err := EnsureMediaTable(ctx, conn); err != nil { + t.Fatalf("create media table: %v", err) + } + if err := EnsureMediaRenditionsTable(ctx, conn); err != nil { + t.Fatalf("create media renditions table: %v", err) + } + // Idempotent, since it runs on every boot. + if err := EnsureMediaRenditionsTable(ctx, conn); err != nil { + t.Fatalf("re-run media renditions migration: %v", err) + } + return conn +} + +func insertTestMedia(t *testing.T, conn *sql.DB, path, mime, createdAt string) int64 { + t.Helper() + result, err := conn.Exec(` + INSERT INTO media (path, file_name, mime_type, size_bytes, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?) + `, path, path, mime, createdAt, createdAt) + if err != nil { + t.Fatalf("insert media %s: %v", path, err) + } + id, _ := result.LastInsertId() + return id +} + +func TestMediaRenditionsLifecycle(t *testing.T) { + conn := mediaRenditionsTestDB(t) + ctx := context.Background() + + old := insertTestMedia(t, conn, "wp-content/uploads/2012/01/old.jpg", "image/jpeg", "2012-01-01 00:00:00") + recent := insertTestMedia(t, conn, "wp-content/uploads/2026/08/new.png", "image/png", "2026-08-01 00:00:00") + insertTestMedia(t, conn, "wp-content/uploads/2026/08/anim.gif", "image/gif", "2026-08-02 00:00:00") + insertTestMedia(t, conn, "wp-content/uploads/2026/08/logo.svg", "image/svg+xml", "2026-08-03 00:00:00") + + // Newest first, and only formats the sidecar renders. + sources, err := MediaNeedingRenditions(ctx, conn, "r1", 10) + if err != nil { + t.Fatalf("needing renditions: %v", err) + } + if len(sources) != 2 || sources[0].MediaID != recent || sources[1].MediaID != old { + t.Fatalf("unexpected sources %+v", sources) + } + + v1 := []RenditionVariant{{Path: "wp-content/variants/r1/2026/08/new.png.480w.webp", Width: 480, Height: 240, SizeBytes: 10}} + if err := SaveRenditions(ctx, conn, recent, "r1", v1); err != nil { + t.Fatalf("save renditions: %v", err) + } + if err := SaveRenditionFailure(ctx, conn, old, "r1", nil, "not a decodable image"); err != nil { + t.Fatalf("save failure: %v", err) + } + + // Both are settled for r1, the failure included: it is not retried. + if sources, err = MediaNeedingRenditions(ctx, conn, "r1", 10); err != nil || len(sources) != 0 { + t.Fatalf("expected nothing left for r1, got %+v (%v)", sources, err) + } + + // A new recipe re-queues everything, carrying the old variants along so the + // caller can delete them once replaced. + sources, err = MediaNeedingRenditions(ctx, conn, "r2", 10) + if err != nil || len(sources) != 2 { + t.Fatalf("expected both re-queued for r2, got %+v (%v)", sources, err) + } + if len(sources[0].Previous) != 1 || sources[0].Previous[0].Path != v1[0].Path { + t.Fatalf("previous variants not carried: %+v", sources[0].Previous) + } + + byPath, err := RenditionsByMediaPath(ctx, conn) + if err != nil { + t.Fatalf("renditions by path: %v", err) + } + if len(byPath) != 1 || len(byPath["wp-content/uploads/2026/08/new.png"]) != 1 { + t.Fatalf("unexpected index contents %+v", byPath) + } + + // A failed re-render under a new recipe keeps serving the old variants. + if err := SaveRenditionFailure(ctx, conn, recent, "r2", v1, "timeout"); err != nil { + t.Fatalf("save failure with previous: %v", err) + } + if byPath, _ = RenditionsByMediaPath(ctx, conn); len(byPath["wp-content/uploads/2026/08/new.png"]) != 1 { + t.Fatalf("old variants dropped after a failed re-render: %+v", byPath) + } + + // Deleting the media row leaves an orphan whose files can still be found. + if _, err := conn.Exec("DELETE FROM media WHERE id = ?", recent); err != nil { + t.Fatal(err) + } + orphans, err := OrphanedRenditions(ctx, conn, 10) + if err != nil || len(orphans) != 1 || orphans[0].MediaID != recent || len(orphans[0].Variants) != 1 { + t.Fatalf("unexpected orphans %+v (%v)", orphans, err) + } + if err := DeleteRendition(ctx, conn, recent); err != nil { + t.Fatal(err) + } + if orphans, _ = OrphanedRenditions(ctx, conn, 10); len(orphans) != 0 { + t.Fatalf("orphan survived deletion: %+v", orphans) + } +} diff --git a/server/internal/handlers/handlers.go b/server/internal/handlers/handlers.go index 0a02397..a4e43d3 100644 --- a/server/internal/handlers/handlers.go +++ b/server/internal/handlers/handlers.go @@ -710,17 +710,18 @@ func articleListItems(articles []models.Article, excerptWords int, preferSlugs . } item := models.ArticleListItem{ - Title: article.Title, - ID: article.ID, - Authors: authors, - Categories: categories, - Excerpt: truncateWords(article.Excerpt, excerptWords), - Slug: article.Slug, - Status: article.Status, - CommentStatus: article.CommentStatus, - FeaturedImage: article.PhotoURL, - IsFeatured: article.IsFeatured, - BreakingNews: article.BreakingNews, + Title: article.Title, + ID: article.ID, + Authors: authors, + Categories: categories, + Excerpt: truncateWords(article.Excerpt, excerptWords), + Slug: article.Slug, + Status: article.Status, + CommentStatus: article.CommentStatus, + FeaturedImage: article.PhotoURL, + FeaturedImageVariants: imageVariants.ForURL(article.PhotoURL), + IsFeatured: article.IsFeatured, + BreakingNews: article.BreakingNews, } item.PublishedDate = article.PublishedAt item.CreationDate = article.CreatedAt @@ -1860,17 +1861,18 @@ func GetSearch(conn *sql.DB, embedder QueryEmbedder) http.HandlerFunc { } item := models.ArticleListItem{ - Title: article.Title, - ID: article.ID, - Authors: authors, - Categories: categories, - Excerpt: article.Excerpt, - Slug: article.Slug, - Status: article.Status, - CommentStatus: article.CommentStatus, - FeaturedImage: article.PhotoURL, - IsFeatured: article.IsFeatured, - BreakingNews: article.BreakingNews, + Title: article.Title, + ID: article.ID, + Authors: authors, + Categories: categories, + Excerpt: article.Excerpt, + Slug: article.Slug, + Status: article.Status, + CommentStatus: article.CommentStatus, + FeaturedImage: article.PhotoURL, + FeaturedImageVariants: imageVariants.ForURL(article.PhotoURL), + IsFeatured: article.IsFeatured, + BreakingNews: article.BreakingNews, } item.PublishedDate = article.PublishedAt resp = append(resp, item) @@ -2027,36 +2029,38 @@ func GetArticle(conn *sql.DB) http.HandlerFunc { } relatedItem := models.ArticleListItem{ - Title: relatedArticle.Title, - ID: relatedArticle.ID, - Authors: relatedAuthors, - Categories: relatedCategories, - Excerpt: relatedArticle.Excerpt, - Slug: relatedArticle.Slug, - Status: relatedArticle.Status, - CommentStatus: relatedArticle.CommentStatus, - FeaturedImage: relatedArticle.PhotoURL, - IsFeatured: relatedArticle.IsFeatured, - BreakingNews: relatedArticle.BreakingNews, + Title: relatedArticle.Title, + ID: relatedArticle.ID, + Authors: relatedAuthors, + Categories: relatedCategories, + Excerpt: relatedArticle.Excerpt, + Slug: relatedArticle.Slug, + Status: relatedArticle.Status, + CommentStatus: relatedArticle.CommentStatus, + FeaturedImage: relatedArticle.PhotoURL, + FeaturedImageVariants: imageVariants.ForURL(relatedArticle.PhotoURL), + IsFeatured: relatedArticle.IsFeatured, + BreakingNews: relatedArticle.BreakingNews, } relatedItem.PublishedDate = relatedArticle.PublishedAt related = append(related, relatedItem) } resp := models.ArticleDetailResponse{ - ID: a.ID, - Title: a.Title, - Slug: a.Slug, - Content: a.Content, - Excerpt: a.Excerpt, - Categories: categories, - CommentStatus: a.CommentStatus, - IsFeatured: a.IsFeatured, - BreakingNews: a.BreakingNews, - Status: a.Status, - FeaturedImage: a.PhotoURL, - FeaturedImageAlt: a.PhotoAlt, - Authors: authors, + ID: a.ID, + Title: a.Title, + Slug: a.Slug, + Content: a.Content, + Excerpt: a.Excerpt, + Categories: categories, + CommentStatus: a.CommentStatus, + IsFeatured: a.IsFeatured, + BreakingNews: a.BreakingNews, + Status: a.Status, + FeaturedImage: a.PhotoURL, + FeaturedImageAlt: a.PhotoAlt, + FeaturedImageVariants: imageVariants.ForURL(a.PhotoURL), + Authors: authors, SEO: models.SEOResponse{ SEOTitle: a.SEOTitle, MetaDescription: a.MetaDescription, diff --git a/server/internal/handlers/image_variants.go b/server/internal/handlers/image_variants.go new file mode 100644 index 0000000..39dba13 --- /dev/null +++ b/server/internal/handlers/image_variants.go @@ -0,0 +1,43 @@ +package handlers + +import "server/internal/models" + +// ImageVariantLookup finds the resized renditions of a library image. It is +// satisfied by *imaging.Index; an interface so this package does not depend on +// how renditions are made. +type ImageVariantLookup interface { + ForURL(imageURL string) []models.ImageVariant + ForPath(mediaPath string) []models.ImageVariant +} + +// imageVariants is package state rather than a handler argument because +// article list items are assembled in helpers several calls away from any +// handler constructor. It is set once at startup, before the server listens. +var imageVariants variantLookup + +// SetImageVariants installs the rendition lookup. Without it, responses simply +// carry no variants and clients fall back to the original image. +func SetImageVariants(lookup ImageVariantLookup) { + imageVariants = variantLookup{lookup: lookup} +} + +// variantLookup makes an unset lookup answer "no variants" instead of +// panicking, which is what every test and every deployment without the +// sidecar relies on. +type variantLookup struct { + lookup ImageVariantLookup +} + +func (v variantLookup) ForURL(imageURL string) []models.ImageVariant { + if v.lookup == nil { + return nil + } + return v.lookup.ForURL(imageURL) +} + +func (v variantLookup) ForPath(mediaPath string) []models.ImageVariant { + if v.lookup == nil { + return nil + } + return v.lookup.ForPath(mediaPath) +} diff --git a/server/internal/handlers/media.go b/server/internal/handlers/media.go index bb1e53a..d72e182 100644 --- a/server/internal/handlers/media.go +++ b/server/internal/handlers/media.go @@ -691,6 +691,7 @@ func randomHex(n int) (string, error) { // response rather than persisted. func withMediaURL(item models.Media) models.Media { item.URL = uploadURL(item.Path) + item.Variants = imageVariants.ForPath(item.Path) return item } diff --git a/server/internal/imaging/client.go b/server/internal/imaging/client.go new file mode 100644 index 0000000..b3ca53a --- /dev/null +++ b/server/internal/imaging/client.go @@ -0,0 +1,157 @@ +// Package imaging renders resized copies of library images through the imaging +// sidecar and serves the result to the public API. +// +// The CMS stores originals exactly as uploaded, which for photo desk work means +// 5-80MB camera JPEGs. WordPress used to generate smaller copies on upload; the +// CMS did not, so the public site served the originals into 400px cards. This +// package restores that step, off the request path: a reconciler renders the +// library in the background, and an in-memory index lets article responses +// advertise whatever has been rendered so far. +package imaging + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// ErrDisabled is returned when no sidecar is configured. +var ErrDisabled = errors.New("imaging: no sidecar configured") + +// UnprocessableError means the sidecar rejected the image itself: undecodable, +// too many pixels, too large. Sending the same bytes again will fail the same +// way, so the reconciler records it instead of retrying. +type UnprocessableError struct { + Detail string +} + +func (e *UnprocessableError) Error() string { + return "imaging: sidecar rejected the image: " + e.Detail +} + +type Client struct { + baseURL string + http *http.Client +} + +// New returns a client for baseURL. An empty baseURL yields a disabled client, +// which is how a deployment without the sidecar keeps serving originals. +func New(baseURL string, timeout time.Duration) *Client { + return &Client{ + baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), + http: &http.Client{Timeout: timeout}, + } +} + +// Enabled reports whether a sidecar is configured. +func (c *Client) Enabled() bool { return c != nil && c.baseURL != "" } + +// Recipe is what the sidecar will produce: its name, and the widths it renders, +// ascending. The name changes whenever the output would. +type Recipe struct { + Name string `json:"recipe"` + Format string `json:"format"` + Widths []int `json:"widths"` +} + +// Recipe asks the sidecar what it is currently configured to render. +func (c *Client) Recipe(ctx context.Context) (Recipe, error) { + if !c.Enabled() { + return Recipe{}, ErrDisabled + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health", nil) + if err != nil { + return Recipe{}, err + } + + resp, err := c.http.Do(req) + if err != nil { + return Recipe{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return Recipe{}, fmt.Errorf("imaging: sidecar health returned %s", resp.Status) + } + + var recipe Recipe + if err := json.NewDecoder(resp.Body).Decode(&recipe); err != nil { + return Recipe{}, err + } + return recipe, nil +} + +// Rendered is one encoded rendition and the dimensions it actually came out at. +// The sidecar never enlarges, so Width can be smaller than what was asked for. +type Rendered struct { + Data []byte + Width int + Height int +} + +// Render resizes src to fit width and returns the encoded result. +func (c *Client) Render(ctx context.Context, src []byte, width int) (Rendered, error) { + if !c.Enabled() { + return Rendered{}, ErrDisabled + } + + endpoint := c.baseURL + "/render?width=" + strconv.Itoa(width) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(src)) + if err != nil { + return Rendered{}, err + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := c.http.Do(req) + if err != nil { + return Rendered{}, err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusOK: + case resp.StatusCode == http.StatusUnprocessableEntity, + resp.StatusCode == http.StatusRequestEntityTooLarge: + return Rendered{}, &UnprocessableError{Detail: sidecarDetail(resp.Body)} + default: + return Rendered{}, fmt.Errorf("imaging: sidecar returned %s: %s", resp.Status, sidecarDetail(resp.Body)) + } + + out := Rendered{} + if out.Width, err = strconv.Atoi(resp.Header.Get("X-Image-Width")); err != nil || out.Width <= 0 { + return Rendered{}, fmt.Errorf("imaging: sidecar sent no usable X-Image-Width") + } + if out.Height, err = strconv.Atoi(resp.Header.Get("X-Image-Height")); err != nil || out.Height <= 0 { + return Rendered{}, fmt.Errorf("imaging: sidecar sent no usable X-Image-Height") + } + if out.Data, err = io.ReadAll(resp.Body); err != nil { + return Rendered{}, err + } + if len(out.Data) == 0 { + return Rendered{}, fmt.Errorf("imaging: sidecar returned an empty rendition") + } + return out, nil +} + +// sidecarDetail pulls FastAPI's {"detail": ...} out of an error body, falling +// back to the raw text. +func sidecarDetail(body io.Reader) string { + raw, _ := io.ReadAll(io.LimitReader(body, 512)) + var decoded struct { + Detail any `json:"detail"` + } + if json.Unmarshal(raw, &decoded) == nil && decoded.Detail != nil { + if s, ok := decoded.Detail.(string); ok { + return s + } + } + return strings.TrimSpace(string(raw)) +} diff --git a/server/internal/imaging/imaging_test.go b/server/internal/imaging/imaging_test.go new file mode 100644 index 0000000..bf21ce1 --- /dev/null +++ b/server/internal/imaging/imaging_test.go @@ -0,0 +1,250 @@ +package imaging + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + db "server/internal/database" + "server/internal/models" +) + +// stubSidecar renders by echoing the width it would produce: never wider than +// sourceWidth, the way the real sidecar's size="down" behaves. +func stubSidecar(t *testing.T, sourceWidth int, status int) *httptest.Server { + t.Helper() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/health": + _ = json.NewEncoder(w).Encode(Recipe{Name: "webp-test-v1", Format: "webp", Widths: []int{480, 960, 1600, 2400}}) + case "/render": + if _, err := io.ReadAll(r.Body); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + if status != http.StatusOK { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"detail":"stub failure"}`)) + return + } + width, _ := strconv.Atoi(r.URL.Query().Get("width")) + if width > sourceWidth { + width = sourceWidth + } + w.Header().Set("X-Image-Width", strconv.Itoa(width)) + w.Header().Set("X-Image-Height", strconv.Itoa(width/2)) + _, _ = w.Write([]byte("webp:" + strconv.Itoa(width))) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + return server +} + +func TestClientDisabledWithoutURL(t *testing.T) { + client := New("", time.Second) + if client.Enabled() { + t.Fatal("expected an empty URL to disable the client") + } + if _, err := client.Recipe(context.Background()); !errors.Is(err, ErrDisabled) { + t.Fatalf("expected ErrDisabled, got %v", err) + } + if _, err := client.Render(context.Background(), []byte("x"), 480); !errors.Is(err, ErrDisabled) { + t.Fatalf("expected ErrDisabled, got %v", err) + } +} + +func TestClientRenderReportsActualDimensions(t *testing.T) { + server := stubSidecar(t, 1000, http.StatusOK) + client := New(server.URL, time.Second) + + got, err := client.Render(context.Background(), []byte("original"), 1600) + if err != nil { + t.Fatalf("render: %v", err) + } + if got.Width != 1000 || got.Height != 500 || string(got.Data) != "webp:1000" { + t.Fatalf("unexpected rendition %+v (%q)", got, got.Data) + } +} + +func TestClientRenderClassifiesFailures(t *testing.T) { + for _, tc := range []struct { + status int + unprocessable bool + }{ + {http.StatusUnprocessableEntity, true}, + {http.StatusRequestEntityTooLarge, true}, + {http.StatusInternalServerError, false}, + {http.StatusServiceUnavailable, false}, + } { + server := stubSidecar(t, 1000, tc.status) + _, err := New(server.URL, time.Second).Render(context.Background(), []byte("x"), 480) + var unprocessable *UnprocessableError + if got := errors.As(err, &unprocessable); got != tc.unprocessable { + t.Errorf("status %d: unprocessable = %v, want %v (err %v)", tc.status, got, tc.unprocessable, err) + } + if tc.unprocessable && unprocessable.Detail != "stub failure" { + t.Errorf("status %d: detail = %q, want the sidecar's detail", tc.status, unprocessable.Detail) + } + } +} + +func TestVariantPath(t *testing.T) { + for _, tc := range []struct{ media, want string }{ + {"wp-content/uploads/2026/08/staff.jpg", "wp-content/variants/r1/2026/08/staff.jpg.960w.webp"}, + {"wp-content/uploads/2026/08/staff.png", "wp-content/variants/r1/2026/08/staff.png.960w.webp"}, + {"wp-content/uploads/../../../etc/passwd", "wp-content/variants/r1/etc/passwd.960w.webp"}, + } { + if got := variantPath("r1", tc.media, 960); got != tc.want { + t.Errorf("variantPath(%q) = %q, want %q", tc.media, got, tc.want) + } + } +} + +func TestValidateRecipe(t *testing.T) { + ok := Recipe{Name: "webp-q80-v1", Format: "webp", Widths: []int{480, 960}} + if err := validateRecipe(ok); err != nil { + t.Fatalf("valid recipe rejected: %v", err) + } + for name, bad := range map[string]Recipe{ + "traversal": {Name: "../x", Format: "webp", Widths: []int{480}}, + "slash": {Name: "a/b", Format: "webp", Widths: []int{480}}, + "empty": {Name: "", Format: "webp", Widths: []int{480}}, + "format": {Name: "r1", Format: "avif", Widths: []int{480}}, + "no widths": {Name: "r1", Format: "webp"}, + "descending": {Name: "r1", Format: "webp", Widths: []int{960, 480}}, + } { + if err := validateRecipe(bad); err == nil { + t.Errorf("%s: expected rejection", name) + } + } +} + +func TestRenderStopsAtOriginalWidthAndWritesReadableFiles(t *testing.T) { + root := t.TempDir() + mediaPath := "wp-content/uploads/2026/08/staff.jpg" + if err := os.MkdirAll(filepath.Join(root, "wp-content/uploads/2026/08"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, mediaPath), []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + + server := stubSidecar(t, 1000, http.StatusOK) + client := New(server.URL, time.Second) + recipe, err := client.Recipe(context.Background()) + if err != nil { + t.Fatal(err) + } + r := NewReconciler(nil, client, root, nil) + + variants, err := r.render(context.Background(), recipe, db.RenditionSource{MediaID: 1, Path: mediaPath}) + if err != nil { + t.Fatalf("render: %v", err) + } + + // 480 and 960 fit; 1600 comes back at the original's 1000; 2400 is never + // asked for, because it could only produce the same 1000px image again. + wantWidths := []int{480, 960, 1000} + if len(variants) != len(wantWidths) { + t.Fatalf("got %d variants %+v, want widths %v", len(variants), variants, wantWidths) + } + for i, variant := range variants { + if variant.Width != wantWidths[i] { + t.Errorf("variant %d width = %d, want %d", i, variant.Width, wantWidths[i]) + } + info, err := os.Stat(filepath.Join(root, variant.Path)) + if err != nil { + t.Fatalf("variant %s not written: %v", variant.Path, err) + } + // Nginx runs as a different user; a 0600 file is a 403. + if info.Mode().Perm() != 0o644 { + t.Errorf("variant %s mode = %v, want 0644", variant.Path, info.Mode().Perm()) + } + if info.Size() != variant.SizeBytes { + t.Errorf("variant %s size = %d, recorded %d", variant.Path, info.Size(), variant.SizeBytes) + } + } + + leftovers, _ := filepath.Glob(filepath.Join(root, "wp-content/variants/webp-test-v1/2026/08/.rendition-*")) + if len(leftovers) > 0 { + t.Fatalf("temp files left behind: %v", leftovers) + } +} + +func TestRenderMissingOriginalIsPermanent(t *testing.T) { + server := stubSidecar(t, 1000, http.StatusOK) + client := New(server.URL, time.Second) + recipe, _ := client.Recipe(context.Background()) + r := NewReconciler(nil, client, t.TempDir(), nil) + + _, err := r.render(context.Background(), recipe, db.RenditionSource{MediaID: 1, Path: "wp-content/uploads/gone.jpg"}) + if !errors.Is(err, errUnusableSource) { + t.Fatalf("expected errUnusableSource, got %v", err) + } +} + +func TestRemoveFilesStaysInsideVariantsTree(t *testing.T) { + root := t.TempDir() + original := filepath.Join(root, "wp-content/uploads/keep.jpg") + variant := filepath.Join(root, "wp-content/variants/r0/keep.jpg.480w.webp") + for _, p := range []string{original, variant} { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + r := NewReconciler(nil, New("", time.Second), root, nil) + r.removeFiles([]db.RenditionVariant{ + {Path: "wp-content/uploads/keep.jpg"}, + {Path: "wp-content/variants/../uploads/keep.jpg"}, + {Path: "wp-content/variants/r0/keep.jpg.480w.webp"}, + }, nil) + + if _, err := os.Stat(original); err != nil { + t.Fatalf("an original was deleted: %v", err) + } + if _, err := os.Stat(variant); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("the stale variant was not deleted: %v", err) + } +} + +func TestIndexForURL(t *testing.T) { + index := NewIndex(nil, "https://delta.example") + index.byPath = map[string][]models.ImageVariant{ + "wp-content/uploads/2026/08/staff photo.jpg": {{URL: "u", Width: 480, Height: 240}}, + } + + for _, imageURL := range []string{ + "https://delta.example/wp-content/uploads/2026/08/staff%20photo.jpg", + "https://www.thetriangle.org/wp-content/uploads/2026/08/staff%20photo.jpg?ver=2", + "/wp-content/uploads/2026/08/staff%20photo.jpg", + } { + if got := index.ForURL(imageURL); len(got) != 1 { + t.Errorf("ForURL(%q) = %v, want the stored variant", imageURL, got) + } + } + for _, imageURL := range []string{"", "https://elsewhere.example/photo.jpg", "https://delta.example/wp-content/uploads/other.jpg"} { + if got := index.ForURL(imageURL); got != nil { + t.Errorf("ForURL(%q) = %v, want nil", imageURL, got) + } + } + + var unset *Index + if got := unset.ForURL("https://delta.example/wp-content/uploads/x.jpg"); got != nil { + t.Errorf("nil index returned %v", got) + } +} diff --git a/server/internal/imaging/index.go b/server/internal/imaging/index.go new file mode 100644 index 0000000..927989e --- /dev/null +++ b/server/internal/imaging/index.go @@ -0,0 +1,134 @@ +package imaging + +import ( + "context" + "database/sql" + "log/slog" + "net/url" + "strings" + "sync" + "time" + + db "server/internal/database" + "server/internal/models" +) + +// Index answers "which renditions does this image have?" from memory. +// +// Article lists are the hot path of the public API: the homepage alone builds +// several of them. A query per list, or worse per article, to attach variants +// would add round trips to every one. The whole rendition table is a few MB, so +// each backend keeps a copy and reloads it on a timer. A rendition becomes +// visible within one Interval of being written, which is far shorter than the +// public cache in front of the API anyway. +type Index struct { + conn *sql.DB + baseURL string + + // Interval between reloads. + Interval time.Duration + + mu sync.RWMutex + byPath map[string][]models.ImageVariant +} + +// NewIndex returns an empty index. baseURL is MEDIA_BASE_URL, so rendition URLs +// are built exactly the way original URLs are. +func NewIndex(conn *sql.DB, baseURL string) *Index { + return &Index{ + conn: conn, + baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), + Interval: time.Minute, + } +} + +// Run reloads the index until ctx is cancelled. A failed reload keeps serving +// the previous copy: stale variants still exist on disk, so they are still +// correct to advertise. +func (i *Index) Run(ctx context.Context) { + for { + if err := i.Refresh(ctx); err != nil && ctx.Err() == nil { + slog.Warn("could not reload image renditions; serving the previous copy", "error", err) + } + select { + case <-ctx.Done(): + return + case <-time.After(i.Interval): + } + } +} + +// Refresh reloads the index from the database. +func (i *Index) Refresh(ctx context.Context) error { + if i.conn == nil { + return nil + } + rows, err := db.RenditionsByMediaPath(ctx, i.conn) + if err != nil { + return err + } + + byPath := make(map[string][]models.ImageVariant, len(rows)) + for mediaPath, variants := range rows { + out := make([]models.ImageVariant, 0, len(variants)) + for _, variant := range variants { + out = append(out, models.ImageVariant{ + URL: i.url(variant.Path), + Width: variant.Width, + Height: variant.Height, + }) + } + byPath[mediaPath] = out + } + + i.mu.Lock() + i.byPath = byPath + i.mu.Unlock() + return nil +} + +func (i *Index) url(relPath string) string { + if i.baseURL != "" { + return i.baseURL + "/" + relPath + } + return "/" + relPath +} + +// ForPath returns the renditions of the media item at a MEDIA_ROOT-relative +// path, narrowest first, or nil. The slice is shared: callers must not modify +// it. +func (i *Index) ForPath(mediaPath string) []models.ImageVariant { + if i == nil { + return nil + } + i.mu.RLock() + defer i.mu.RUnlock() + return i.byPath[mediaPath] +} + +// ForURL returns the renditions of the image an article's featured_image URL +// points at, or nil. Article image URLs are stored absolute and have carried +// more than one host over the years (the WordPress origin, then Delta), so the +// match is on the wp-content path alone. +func (i *Index) ForURL(imageURL string) []models.ImageVariant { + if i == nil || imageURL == "" { + return nil + } + mediaPath, ok := mediaPathFromURL(imageURL) + if !ok { + return nil + } + return i.ForPath(mediaPath) +} + +func mediaPathFromURL(imageURL string) (string, bool) { + parsed, err := url.Parse(strings.TrimSpace(imageURL)) + if err != nil { + return "", false + } + at := strings.Index(parsed.Path, "/wp-content/") + if at < 0 { + return "", false + } + return parsed.Path[at+1:], true +} diff --git a/server/internal/imaging/reconciler.go b/server/internal/imaging/reconciler.go new file mode 100644 index 0000000..5bbdb84 --- /dev/null +++ b/server/internal/imaging/reconciler.go @@ -0,0 +1,393 @@ +package imaging + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + db "server/internal/database" +) + +// VariantsDir is where renditions live, MEDIA_ROOT-relative. It sits beside +// wp-content/uploads rather than inside it for two reasons: the media indexer +// walks uploads/ and would adopt every rendition as a new library item, and +// Nginx already serves all of /wp-content/, so no new location is needed. +const VariantsDir = "wp-content/variants" + +const uploadsPrefix = "wp-content/uploads/" + +// recipePattern bounds what a sidecar-reported recipe may look like, because it +// becomes a directory name under MEDIA_ROOT. +var recipePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{0,63}$`) + +// Reconciler keeps media_renditions in step with the media library. +// +// A background loop for the same reasons as the embedding reconciler: uploads, +// sideloads and the indexer's adoption of migrated files are three ways into +// the library, rendering on the upload path would make an editor wait on (and +// fail with) the sidecar, and only a loop converges the 13,000 originals that +// were already there. The first pass after deploy is the backfill. +type Reconciler struct { + conn *sql.DB + client *Client + root string + + // checkStorage guards against writing into the empty directory Docker + // creates when CephFS is not mounted. Renditions written there would be + // recorded as done and then vanish when the mount returns. + checkStorage func() error + + // Interval between passes once the library has converged. Short, unlike the + // embedding reconciler's, because a fresh upload is usually about to go out + // as a featured image and an idle pass is one cheap query. + Interval time.Duration + + // BatchSize is how many images one pass renders before releasing the lock. + BatchSize int + + // MaxSourceBytes skips originals the sidecar would refuse anyway, without + // reading them into memory first. + MaxSourceBytes int64 + + // MaxAttempts is how many passes may fail on the same image, with the + // sidecar otherwise healthy, before it is recorded as failed. Without a cap + // one image that crashes the sidecar would stall the queue behind it. + MaxAttempts int + + attempts map[int64]int +} + +func NewReconciler(conn *sql.DB, client *Client, root string, checkStorage func() error) *Reconciler { + return &Reconciler{ + conn: conn, + client: client, + root: strings.TrimRight(strings.TrimSpace(root), "/"), + checkStorage: checkStorage, + Interval: time.Minute, + BatchSize: 8, + MaxSourceBytes: 100 << 20, + MaxAttempts: 3, + attempts: make(map[int64]int), + } +} + +// Run blocks until ctx is cancelled. With no sidecar or no media root it logs +// once and returns, and the site keeps serving originals. +func (r *Reconciler) Run(ctx context.Context) { + if !r.client.Enabled() { + slog.Info("image reconciler disabled; no sidecar configured") + return + } + if r.root == "" { + slog.Info("image reconciler disabled; MEDIA_ROOT is not set") + return + } + + for { + worked, err := r.pass(ctx) + if ctx.Err() != nil { + return + } + if err != nil { + slog.Warn("image reconciler pass failed", "error", err) + } + + delay := r.Interval + if worked && err == nil { + delay = time.Second + } + select { + case <-ctx.Done(): + return + case <-time.After(delay): + } + } +} + +// reconcilerLockName serializes reconcilers across the blue and green slots, +// which are both live during a deploy. +const reconcilerLockName = "cms_image_reconciler" + +func (r *Reconciler) pass(ctx context.Context) (bool, error) { + lockConn, err := r.conn.Conn(ctx) + if err != nil { + return false, err + } + defer lockConn.Close() + + var acquired sql.NullInt64 + if err := lockConn.QueryRowContext(ctx, "SELECT GET_LOCK(?, 0)", reconcilerLockName).Scan(&acquired); err != nil { + return false, err + } + if !acquired.Valid || acquired.Int64 != 1 { + return false, nil + } + defer func() { + _, _ = lockConn.ExecContext(context.Background(), "SELECT RELEASE_LOCK(?)", reconcilerLockName) + }() + + if r.checkStorage != nil { + if err := r.checkStorage(); err != nil { + return false, err + } + } + + recipe, err := r.client.Recipe(ctx) + if err != nil { + return false, err + } + if err := validateRecipe(recipe); err != nil { + return false, err + } + + orphans, err := db.OrphanedRenditions(ctx, r.conn, 100) + if err != nil { + return false, err + } + for _, orphan := range orphans { + r.removeFiles(orphan.Variants, nil) + if err := db.DeleteRendition(ctx, r.conn, orphan.MediaID); err != nil { + return false, err + } + } + if len(orphans) > 0 { + slog.Info("removed renditions of deleted media", "count", len(orphans)) + } + + sources, err := db.MediaNeedingRenditions(ctx, r.conn, recipe.Name, r.BatchSize) + if err != nil { + return false, err + } + + rendered, failed := 0, 0 + for _, source := range sources { + variants, err := r.render(ctx, recipe, source) + var unprocessable *UnprocessableError + switch { + case err == nil: + case errors.As(err, &unprocessable), errors.Is(err, errUnusableSource): + if err := r.fail(ctx, recipe, source, err); err != nil { + return rendered > 0, err + } + failed++ + continue + case ctx.Err() != nil: + return rendered > 0, ctx.Err() + default: + // The sidecar answered /health at the top of this pass, so a + // failure now is more likely this image than an outage. Count it, + // and give up on the image once it has used its attempts. + r.attempts[source.MediaID]++ + if r.attempts[source.MediaID] < r.MaxAttempts { + return rendered > 0, fmt.Errorf("media %d (%s): %w", source.MediaID, source.Path, err) + } + if err := r.fail(ctx, recipe, source, err); err != nil { + return rendered > 0, err + } + failed++ + continue + } + + if err := db.SaveRenditions(ctx, r.conn, source.MediaID, recipe.Name, variants); err != nil { + return rendered > 0, err + } + delete(r.attempts, source.MediaID) + r.removeFiles(source.Previous, variants) + rendered++ + } + + if rendered+failed > 0 { + slog.Info("rendered media", "count", rendered, "failed", failed, "recipe", recipe.Name) + } + return len(sources) > 0, nil +} + +func validateRecipe(recipe Recipe) error { + if !recipePattern.MatchString(recipe.Name) { + return fmt.Errorf("imaging: sidecar recipe %q is not a safe directory name", recipe.Name) + } + if recipe.Format != "webp" { + return fmt.Errorf("imaging: sidecar renders %q; only webp is supported", recipe.Format) + } + if len(recipe.Widths) == 0 { + return fmt.Errorf("imaging: sidecar recipe %q has no widths", recipe.Name) + } + for i, width := range recipe.Widths { + if width <= 0 || (i > 0 && width <= recipe.Widths[i-1]) { + return fmt.Errorf("imaging: sidecar widths %v must be positive and ascending", recipe.Widths) + } + } + return nil +} + +// errUnusableSource marks a media row whose original cannot be sent at all. +var errUnusableSource = errors.New("unusable source") + +// render produces the full width ladder for one original and writes it to disk. +// Every width is rendered before anything is written, so a failure partway +// leaves no files that the database does not know about. +func (r *Reconciler) render(ctx context.Context, recipe Recipe, source db.RenditionSource) ([]db.RenditionVariant, error) { + abs, ok := resolveWithin(r.root, source.Path) + if !ok { + return nil, fmt.Errorf("%w: path escapes the media root", errUnusableSource) + } + info, err := os.Stat(abs) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: original is missing from the media root", errUnusableSource) + } + if err != nil { + return nil, err + } + if info.Size() > r.MaxSourceBytes { + return nil, fmt.Errorf("%w: original is %d bytes, over the %d limit", errUnusableSource, info.Size(), r.MaxSourceBytes) + } + src, err := os.ReadFile(abs) + if err != nil { + return nil, err + } + + type pending struct { + variant db.RenditionVariant + data []byte + } + var out []pending + for _, width := range recipe.Widths { + result, err := r.client.Render(ctx, src, width) + if err != nil { + return nil, err + } + // The sidecar never enlarges. Once it returns less than was asked for, + // that is the original's own width, and every wider step would be the + // same image again. + if len(out) == 0 || result.Width > out[len(out)-1].variant.Width { + out = append(out, pending{ + variant: db.RenditionVariant{ + Path: variantPath(recipe.Name, source.Path, result.Width), + Width: result.Width, + Height: result.Height, + SizeBytes: int64(len(result.Data)), + }, + data: result.Data, + }) + } + if result.Width < width { + break + } + } + + variants := make([]db.RenditionVariant, 0, len(out)) + for _, item := range out { + if err := r.writeFile(item.variant.Path, item.data); err != nil { + return nil, err + } + variants = append(variants, item.variant) + } + return variants, nil +} + +func (r *Reconciler) fail(ctx context.Context, recipe Recipe, source db.RenditionSource, cause error) error { + delete(r.attempts, source.MediaID) + slog.Warn("could not render media; it will keep serving the original", + "media_id", source.MediaID, "path", source.Path, "error", cause) + return db.SaveRenditionFailure(ctx, r.conn, source.MediaID, recipe.Name, source.Previous, cause.Error()) +} + +// variantPath maps an original to one of its renditions: +// +// wp-content/uploads/2026/08/staff.jpg -> wp-content/variants//2026/08/staff.jpg.960w.webp +// +// The recipe is in the path because Cloudflare caches these for 30 days as +// immutable. A recipe change has to produce new URLs, not new bytes at old ones. +// The original's extension stays in the name so staff.jpg and staff.png, which +// both exist in the migrated corpus, cannot collide. +func variantPath(recipe, mediaPath string, width int) string { + rel := strings.TrimPrefix(path.Clean("/"+mediaPath), "/") + rel = strings.TrimPrefix(rel, uploadsPrefix) + return path.Join(VariantsDir, recipe, rel) + "." + strconv.Itoa(width) + "w.webp" +} + +// writeFile stores one rendition atomically: a partially written file is never +// visible to Nginx, and a rerun overwrites a leftover from an interrupted pass. +func (r *Reconciler) writeFile(relPath string, data []byte) error { + abs, ok := resolveWithin(r.root, relPath) + if !ok { + return fmt.Errorf("rendition path %q escapes the media root", relPath) + } + dir := filepath.Dir(abs) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + + tmp, err := os.CreateTemp(dir, ".rendition-*") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer func() { + _ = tmp.Close() + _ = os.Remove(tmpPath) + }() + + // CreateTemp makes 0600 files, and Nginx runs as another user. + if err := tmp.Chmod(0o644); err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, abs) +} + +// removeFiles deletes the variants in old that are not also in keep. Failures +// are logged, not returned: a leftover file costs some disk, nothing more. +func (r *Reconciler) removeFiles(old, keep []db.RenditionVariant) { + kept := make(map[string]struct{}, len(keep)) + for _, variant := range keep { + kept[variant.Path] = struct{}{} + } + for _, variant := range old { + if _, ok := kept[variant.Path]; ok { + continue + } + // Only ever delete inside the variants tree, whatever a row says. The + // check is on the cleaned path: "wp-content/variants/../uploads/x.jpg" + // starts with the right prefix and names an original. + cleaned := strings.TrimPrefix(path.Clean("/"+variant.Path), "/") + if !strings.HasPrefix(cleaned, VariantsDir+"/") { + continue + } + abs, ok := resolveWithin(r.root, cleaned) + if !ok { + continue + } + if err := os.Remove(abs); err != nil && !errors.Is(err, os.ErrNotExist) { + slog.Warn("could not remove rendition", "path", variant.Path, "error", err) + } + } +} + +// resolveWithin joins a MEDIA_ROOT-relative path onto root, refusing anything +// that would land outside it. +func resolveWithin(root, relPath string) (string, bool) { + cleaned := path.Clean("/" + strings.TrimSpace(relPath)) + if cleaned == "/" { + return "", false + } + abs := filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(cleaned, "/"))) + if !strings.HasPrefix(abs, filepath.Clean(root)+string(os.PathSeparator)) { + return "", false + } + return abs, true +} diff --git a/server/internal/models/api_responses.go b/server/internal/models/api_responses.go index 2e7749f..7dd773b 100644 --- a/server/internal/models/api_responses.go +++ b/server/internal/models/api_responses.go @@ -45,9 +45,13 @@ type ArticleListItem struct { Status ArticleStatus `json:"status"` CommentStatus string `json:"comment_status"` FeaturedImage string `json:"featured_image"` - IsFeatured bool `json:"is_featured"` - BreakingNews bool `json:"breaking_news"` - PublishedDate *time.Time `json:"published_date,omitempty"` + // FeaturedImageVariants are resized copies of FeaturedImage, narrowest + // first, for a srcset. Absent until they have been rendered; keep + // FeaturedImage as the src fallback either way. + FeaturedImageVariants []ImageVariant `json:"featured_image_variants,omitempty"` + IsFeatured bool `json:"is_featured"` + BreakingNews bool `json:"breaking_news"` + PublishedDate *time.Time `json:"published_date,omitempty"` // Drafts have no published_date; the CMS listing falls back to this so an // unpublished row still shows a date. CreationDate *time.Time `json:"creation_date,omitempty"` @@ -135,11 +139,14 @@ type ArticleDetailResponse struct { // Empty means the public site has nothing to render, which is a defect worth // surfacing rather than papering over with the headline: an alt that repeats // the adjacent headline tells a screen-reader user nothing new. - FeaturedImageAlt string `json:"featured_image_alt"` - Authors []AuthorSummary `json:"authors"` - SEO SEOResponse `json:"seo"` - Related []ArticleListItem `json:"related"` - PublishedDate *time.Time `json:"published_date,omitempty"` + FeaturedImageAlt string `json:"featured_image_alt"` + // FeaturedImageVariants are resized copies of FeaturedImage, narrowest + // first, for a srcset. Absent until they have been rendered. + FeaturedImageVariants []ImageVariant `json:"featured_image_variants,omitempty"` + Authors []AuthorSummary `json:"authors"` + SEO SEOResponse `json:"seo"` + Related []ArticleListItem `json:"related"` + PublishedDate *time.Time `json:"published_date,omitempty"` // ModifiedDate is the article's last-edited time (the `mod_date` column the // public sitemap already reports as ). Exposed so the public site's // NewsArticle dateModified agrees with the sitemap instead of silently diff --git a/server/internal/models/types.go b/server/internal/models/types.go index 752e0f2..47cb916 100644 --- a/server/internal/models/types.go +++ b/server/internal/models/types.go @@ -224,9 +224,22 @@ type Media struct { // InGallery is the editor's "show this on the public photo gallery" flag. // It is off by default: the library is every file on the media mount, house // ads and comics included, and the gallery is a curated selection of it. - InGallery bool `json:"in_gallery"` - CreatedAt *time.Time `json:"created_at,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` + InGallery bool `json:"in_gallery"` + // Variants are resized WebP copies of the original, narrowest first. Empty + // until the imaging sidecar has rendered this item, and always empty for + // formats it does not render (GIF, SVG). + Variants []ImageVariant `json:"variants,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` +} + +// ImageVariant is one resized rendition of a library image. A client builds a +// srcset from these (" w", ...) and keeps the original as the src +// fallback. +type ImageVariant struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` } type MediaOverview struct { diff --git a/server/main.go b/server/main.go index 1c4370c..065d1ff 100644 --- a/server/main.go +++ b/server/main.go @@ -16,6 +16,7 @@ import ( "server/internal/database" "server/internal/embeddings" "server/internal/handlers" + "server/internal/imaging" "server/internal/middleware" "server/internal/routes" "server/internal/slack" @@ -169,6 +170,10 @@ func main() { slog.Error("failed to create media table", "error", err) os.Exit(1) } + // Non-fatal: without it the site serves original images, as it always has. + if err := database.EnsureMediaRenditionsTable(context.Background(), db); err != nil { + slog.Error("failed to create media renditions table; images are served unresized", "error", err) + } // Surface a missing media mount at boot rather than on the first failed // upload. Deliberately not fatal: the rest of the CMS works fine without the @@ -444,6 +449,9 @@ func run(deps runDeps, conn *sql.DB) error { // public search path, where waiting is worse than a slightly worse ranking. embedder := embeddings.New(os.Getenv("EMBEDDINGS_URL"), 2*time.Second) + variantIndex := imaging.NewIndex(conn, os.Getenv("MEDIA_BASE_URL")) + handlers.SetImageVariants(variantIndex) + mux := http.NewServeMux() routes.Register(mux, conn, deps.oidcVerifier, deps.oidcCfg, deps.spamChecker, deps.slackNotifier, embedder) server := deps.newServer(cert, mux, slog.Default()) @@ -457,6 +465,14 @@ func run(deps runDeps, conn *sql.DB) error { // a query, and unlike search it has no reason to give up quickly. go embeddings.NewReconciler(conn, embeddings.New(os.Getenv("EMBEDDINGS_URL"), 2*time.Minute)).Run(schedulerCtx) + // Resized copies of library images. The reconciler renders them in the + // background through the imaging sidecar; the index tells article responses + // which ones exist. Its long timeout covers one large original, not a batch. + // An unset IMAGING_URL disables rendering, and responses carry whatever + // renditions were made before, which is none on a fresh install. + go variantIndex.Run(schedulerCtx) + go imaging.NewReconciler(conn, imaging.New(os.Getenv("IMAGING_URL"), 2*time.Minute), os.Getenv("MEDIA_ROOT"), handlers.CheckMediaStorage).Run(schedulerCtx) + serverErr := make(chan error, 1) go func() { var err error