Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@ NEXT_PUBLIC_SITE_URL=http://localhost:3000
NEXT_PUBLIC_DATASETS_API_URL=http://localhost:3000
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=GTM-XXXXXXX
DEPLOYMENT_VERSION=development

# Production container limits and SSH destination are managed by deployment
# Compose and GitHub production-environment settings, not public app config.
10 changes: 5 additions & 5 deletions .github/workflows/deploy-production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ jobs:
echo "runtime-image=${IMAGE}@${DIGEST}" >> "$GITHUB_OUTPUT"

deploy:
name: Deploy to syr-prod
name: Deploy to configured production host
needs:
- deploy-policy
- build
Expand Down Expand Up @@ -253,8 +253,8 @@ jobs:
DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}
DEPLOY_SSH_KNOWN_HOSTS: ${{ secrets.DEPLOY_SSH_KNOWN_HOSTS }}
run: |
test "$DEPLOY_HOST" = "syr-prod"
test "$DEPLOY_USER" = "mustafa"
[[ "$DEPLOY_HOST" =~ ^[a-zA-Z0-9][a-zA-Z0-9.-]*$ ]]
[[ "$DEPLOY_USER" =~ ^[a-z_][a-z0-9_-]*$ ]]
test "${CONFIGURED_DEPLOY_ROOT:-$DEPLOY_ROOT}" = "$DEPLOY_ROOT"
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
test -n "$DEPLOY_SSH_PRIVATE_KEY"
Expand All @@ -267,7 +267,7 @@ jobs:
oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
audience: ${{ secrets.TS_AUDIENCE }}
tags: tag:ci
ping: syr-prod
ping: ${{ vars.DEPLOY_HOST }}

- name: Install SSH credentials
shell: bash
Expand All @@ -293,7 +293,7 @@ jobs:
-o IdentitiesOnly=yes \
-o StrictHostKeyChecking=yes \
"${DEPLOY_USER}@${DEPLOY_HOST}" \
'sudo -n /usr/bin/install -d -m 0750 -o mustafa -g mustafa /opt/syr/apps/opensyria /opt/syr/apps/opensyria/production /opt/syr/apps/opensyria/production/website /opt/syr/apps/opensyria/production/datasets-api'
'root=/opt/syr/apps/opensyria/production/website; test -d "$root" && test ! -L "$root" && test -w "$root"'
ssh -i ~/.ssh/opensyria_deploy \
-o BatchMode=yes \
-o IdentitiesOnly=yes \
Expand Down
18 changes: 17 additions & 1 deletion devops/production/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ drains old workers. Shared nginx changes wait on the cross-application lock
instead of failing when another OpenSyria rollout is finishing. The previous
slot is retained.

`finalize` rechecks the public route, drains existing requests, stops the
`finalize` rechecks the private ingress route, drains existing requests, stops the
previous slot, and records the new active state.

`rollback` restores the backed-up nginx include and previous slot. If no prior
Expand All @@ -96,3 +96,19 @@ bin/release.sh status
```

See `docs/deployment.md` for GitHub configuration and Cloudflare cutover.

## Restricted production host deployment

The production GitHub environment selects `DEPLOY_HOST`, `DEPLOY_USER`, the SSH
key and its pinned known-hosts entry. The host must be provisioned in advance;
CI only verifies the application directory and cannot create directories with
unrestricted sudo. The deployment identity must have only the fixed Docker
operations for this application. Keep automatic deployment paused while moving
data and use `VERIFY_PUBLIC_DEPLOYMENT=false` for the private cutover checks.
Set it back to `true` when the public route points to the prepared destination.

The long-running application has a 1 CPU burst ceiling and 512 MiB memory/swap
ceiling, with Node heap capped at 320 MiB. These limits apply to each blue/green
slot; allow temporary overlap during a rollout.

The host-side switch verifies the private ingress at `127.0.0.1:18080` using the production Host header. GitHub separately verifies the public HTTPS route; an old DNS destination therefore cannot accidentally validate or block a private candidate.
32 changes: 17 additions & 15 deletions devops/production/bin/release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ ACTIVE_VERSION_FILE="${STATE_DIR}/active-version"
PENDING_FILE="${STATE_DIR}/pending.env"
PREVIOUS_UPSTREAM_FILE="${STATE_DIR}/previous-upstream.conf"
DEPLOY_LOCK_FILE="${ROOT_DIR}/.deploy.lock"
NGINX_DEPLOY_LOCK_FILE="/opt/syr/services/staging/.nginx-deploy.lock"
NGINX_ACTIVE_INCLUDE="/opt/syr/services/staging/infrastructure/nginx/conf.d/includes/opensyria-production-website-active.conf"
NGINX_DEPLOY_LOCK_FILE="/opt/syr/services/staging/infrastructure/nginx/conf.d/includes/opensyria/.deploy.lock"
NGINX_ACTIVE_INCLUDE="/opt/syr/services/staging/infrastructure/nginx/conf.d/includes/opensyria/opensyria-production-website-active.conf"
NGINX_CONTAINER="infra-nginx"
PUBLIC_HOST="opensyria.org"
PUBLIC_URL="https://${PUBLIC_HOST}"
PRIVATE_URL="http://127.0.0.1:18080"
EDGE_NETWORK="syr-staging-edge"
COMPOSE_PROJECT="opensyria-production-website"
COMPOSE_PS_FORMAT='table {{.Name}}\t{{.Image}}\t{{.State}}\t{{.Health}}'
Expand All @@ -50,7 +50,7 @@ readonly COMPOSE_ENV_FILE RUNTIME_ENV_FILE RUNTIME_ENV_VALIDATOR
readonly INFISICAL_CONFIG_FILE STATE_DIR ACTIVE_COLOR_FILE
readonly ACTIVE_VERSION_FILE PENDING_FILE PREVIOUS_UPSTREAM_FILE DEPLOY_LOCK_FILE
readonly NGINX_DEPLOY_LOCK_FILE NGINX_ACTIVE_INCLUDE NGINX_CONTAINER PUBLIC_HOST
readonly PUBLIC_URL EDGE_NETWORK COMPOSE_PROJECT COMPOSE_PS_FORMAT
readonly PRIVATE_URL EDGE_NETWORK COMPOSE_PROJECT COMPOSE_PS_FORMAT
readonly MAX_HOMEPAGE_HEADER_BYTES NGINX_LOCK_TIMEOUT_SECONDS
readonly NGINX_ROUTE_TIMEOUT_SECONDS

Expand Down Expand Up @@ -355,7 +355,7 @@ wait_for_service_health() {
done
}

probe_public_route() {
probe_private_route() {
local expected_version="$1"
local enforce_header_budget="$2"
local health_body health_code homepage_headers homepage_code
Expand All @@ -370,11 +370,12 @@ probe_public_route() {
--max-time 10 \
--header 'Cache-Control: no-cache' \
--header 'Pragma: no-cache' \
--header "Host: ${PUBLIC_HOST}" \
--get \
--data-urlencode "deployment_probe=${expected_version}" \
--output "${health_body}" \
--write-out '%{http_code}' \
"${PUBLIC_URL}/health" 2>/dev/null || true
"${PRIVATE_URL}/health" 2>/dev/null || true
)"
if [[ "${health_code}" != "200" ]] \
|| ! grep -Fq "\"version\":\"${expected_version}\"" "${health_body}"; then
Expand All @@ -388,12 +389,13 @@ probe_public_route() {
--max-time 15 \
--header 'Cache-Control: no-cache' \
--header 'Pragma: no-cache' \
--header "Host: ${PUBLIC_HOST}" \
--get \
--data-urlencode "deployment_probe=${expected_version}" \
--dump-header "${homepage_headers}" \
--output /dev/null \
--write-out '%{http_code}' \
"${PUBLIC_URL}/" 2>/dev/null || true
"${PRIVATE_URL}/" 2>/dev/null || true
)"
if [[ "${homepage_code}" != "200" ]]; then
rm -f -- "${health_body}" "${homepage_headers}"
Expand Down Expand Up @@ -426,14 +428,14 @@ probe_public_route() {
rm -f -- "${health_body}" "${homepage_headers}"
}

verify_public_route() {
verify_private_route() {
local expected_version="$1"
local enforce_header_budget="${2:-true}"
local started_at now

started_at="$(date +%s)"
while true; do
if probe_public_route "${expected_version}" "${enforce_header_budget}"; then
if probe_private_route "${expected_version}" "${enforce_header_budget}"; then
return 0
fi

Expand All @@ -449,7 +451,7 @@ verify_public_route() {
verify_previous_public_route() {
[[ "${HAS_ROLLBACK}" == "true" ]] || return 1
[[ "${PREVIOUS_VERSION}" =~ ^[0-9a-f]{40}$ ]] || return 1
verify_public_route "${PREVIOUS_VERSION}" false
verify_private_route "${PREVIOUS_VERSION}" false
}

restore_previous_route() {
Expand All @@ -462,7 +464,7 @@ restore_previous_route() {
echo "Previous website route could not be verified; restoring the healthy candidate." >&2
if (write_nginx_upstream "${TARGET_COLOR}") \
&& reload_nginx \
&& verify_public_route "${DEPLOYMENT_VERSION}"; then
&& verify_private_route "${DEPLOYMENT_VERSION}"; then
echo "Restored and verified the candidate website route." >&2
else
echo "Candidate remains running, but automatic route recovery could not be verified." >&2
Expand Down Expand Up @@ -787,7 +789,7 @@ prepare_release() {
"WEBSITE_${routed_color^^}_VERSION"
)"
if service_is_healthy "${routed_color}" \
&& verify_public_route "${routed_version}" false; then
&& verify_private_route "${routed_version}" false; then
CURRENT_COLOR="${routed_color}"
HAS_ROLLBACK="true"
PREVIOUS_VERSION="${routed_version}"
Expand Down Expand Up @@ -827,7 +829,7 @@ switch_release() {
write_pending_state switching
write_nginx_upstream "${TARGET_COLOR}"
write_pending_state switched
if ! reload_nginx || ! verify_public_route "${DEPLOYMENT_VERSION}"; then
if ! reload_nginx || ! verify_private_route "${DEPLOYMENT_VERSION}"; then
echo "Website cutover verification failed." >&2
if [[ "${HAS_ROLLBACK}" == "true" ]] && restore_previous_route; then
write_pending_state prepared
Expand All @@ -848,12 +850,12 @@ finalize_release() {
|| fail "Shared nginx is no longer routed to ${TARGET_COLOR}"

wait_for_service_health "${TARGET_COLOR}"
verify_public_route "${DEPLOYMENT_VERSION}"
verify_private_route "${DEPLOYMENT_VERSION}"
sleep "${DRAIN_SECONDS}"
[[ "$(current_upstream_color)" == "${TARGET_COLOR}" ]] \
|| fail "Shared nginx changed during the drain period"
wait_for_service_health "${TARGET_COLOR}"
verify_public_route "${DEPLOYMENT_VERSION}"
verify_private_route "${DEPLOYMENT_VERSION}"

if [[ "${HAS_ROLLBACK}" == "true" && "${CURRENT_COLOR}" != "${TARGET_COLOR}" ]]; then
compose stop "$(service_for_color "${CURRENT_COLOR}")"
Expand Down
10 changes: 10 additions & 0 deletions devops/production/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ x-website: &website
init: true
user: "1001:1001"
read_only: true
cpus: "1.0"
mem_limit: 512m
memswap_limit: 512m
pids_limit: 128
logging:
driver: json-file
options: {max-size: 10m, max-file: "3"}
cap_drop:
- ALL
security_opt:
Expand All @@ -15,6 +22,7 @@ x-website: &website
DEPLOYMENT_VERSION: ${DEPLOYMENT_VERSION:?DEPLOYMENT_VERSION is required}
HOSTNAME: 0.0.0.0
NEXT_TELEMETRY_DISABLED: "1"
NODE_OPTIONS: --max-old-space-size=320
NODE_ENV: production
PORT: "3000"
expose:
Expand Down Expand Up @@ -43,6 +51,7 @@ services:
DEPLOYMENT_VERSION: ${WEBSITE_BLUE_VERSION:?WEBSITE_BLUE_VERSION is required}
HOSTNAME: 0.0.0.0
NEXT_TELEMETRY_DISABLED: "1"
NODE_OPTIONS: --max-old-space-size=320
NODE_ENV: production
PORT: "3000"
labels:
Expand All @@ -61,6 +70,7 @@ services:
DEPLOYMENT_VERSION: ${WEBSITE_GREEN_VERSION:?WEBSITE_GREEN_VERSION is required}
HOSTNAME: 0.0.0.0
NEXT_TELEMETRY_DISABLED: "1"
NODE_OPTIONS: --max-old-space-size=320
NODE_ENV: production
PORT: "3000"
labels:
Expand Down
24 changes: 20 additions & 4 deletions docs/deployment.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Production Deployment

The website runs on `syr-prod` as a standalone Next.js container. GitHub Actions
The website runs on the configured production host as a standalone Next.js container. GitHub Actions
builds one `linux/amd64` image, pushes it to GHCR, and deploys its immutable
digest. The production host never installs pnpm dependencies or builds source.

Expand Down Expand Up @@ -68,8 +68,8 @@ NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID
VERIFY_PUBLIC_DEPLOYMENT
```

The deployment values are pinned to `syr-prod`, `mustafa`, and
`/opt/syr/apps/opensyria/production/website`. Production build configuration
The host and dedicated SSH account are selected by `DEPLOY_HOST` and
`DEPLOY_USER`; the root remains pinned to `/opt/syr/apps/opensyria/production/website`. Production build configuration
must use the following exact public values:

```dotenv
Expand Down Expand Up @@ -111,7 +111,7 @@ chmod 600 .infisical.env
Configure immutable OpenSyria project ID
`5922e0e7-9672-4195-a61f-90db3eb60ce5`, the `production` environment, and the
`/website` secret path. The Universal Auth credentials remain only on
`syr-prod`; they are not GitHub secrets.
the production host; they are not GitHub secrets.

During `prepare`, the host runs:

Expand Down Expand Up @@ -216,3 +216,19 @@ After cutover:
- The two slots do not publish host ports.
- A shared cache is unnecessary because only one website slot receives new
traffic at a time.

## Restricted production host deployment

The production GitHub environment selects `DEPLOY_HOST`, `DEPLOY_USER`, the SSH
key and its pinned known-hosts entry. The host must be provisioned in advance;
CI only verifies the application directory and cannot create directories with
unrestricted sudo. The deployment identity must have only the fixed Docker
operations for this application. Keep automatic deployment paused while moving
data and use `VERIFY_PUBLIC_DEPLOYMENT=false` for the private cutover checks.
Set it back to `true` when the public route points to the prepared destination.

The long-running application has a 1 CPU burst ceiling and 512 MiB memory/swap
ceiling, with Node heap capped at 320 MiB. These limits apply to each blue/green
slot; allow temporary overlap during a rollout.

The host-side switch verifies the private ingress at `127.0.0.1:18080` using the production Host header. GitHub separately verifies the public HTTPS route; an old DNS destination therefore cannot accidentally validate or block a private candidate.
3 changes: 0 additions & 3 deletions next.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ const nextConfig: NextConfig = {
},
],
},
experimental: {
rootParams: true,
},
}

export default withNextIntl(nextConfig)
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"lucide-react": "^1.23.0",
"next": "16.2.11",
"next": "16.3.3",
"next-intl": "4.13.4",
"next-themes": "^0.4.6",
"react": "19.2.8",
Expand Down
Loading