From 8a3f144a437c22956ee993b115a6896352077ce2 Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Fri, 6 Nov 2026 16:12:26 -0600 Subject: [PATCH 1/4] fix: run Redis memory-only to match the no-persistence spec SPECIFICATION.md 6.2 forbids persistent storage, but the default compose stack snapshotted Redis to a named volume, so ciphertext, verification hashes and delete tokens could stay on disk after a burn. Disable RDB and AOF and drop the volume in both compose files and the `pnpm redis` script. The image snapshots by default, so removing `--save` alone would not be enough. Document that a Redis restart now drops unexpired secrets and queued webhooks, and how to remove an old volume. --- README.md | 12 ++++++++++++ docker-compose.yaml | 14 +++++++------- package.json | 2 +- packages/server/docker-compose.yaml | 13 ++++++++----- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5e4ee7a..03ddabb 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,18 @@ It runs with `NODE_ENV=production` and requires `REDIS_URL` and `ENCRYPTION_KEY` The web client image is not published to GHCR yet; build it with `docker compose` as above. +#### Redis is memory-only + +The compose file runs Redis with RDB and AOF disabled and no data volume, because the [specification](./SPECIFICATION.md) forbids persistent storage: a snapshot would keep ciphertext, verification hashes and delete tokens on disk after a secret is burned. As a result, **any Redis restart, including recreating the container during an upgrade, drops all unexpired secrets and queued webhooks**. Their links stop working, and undelivered webhooks are not sent. Rate-limit counters reset too. + +If you deployed before persistence was disabled, your old volume still holds a snapshot and is no longer used. Delete it (`docker volume ls` shows the name, usually `_redis_data`): + +```bash +docker volume rm _redis_data +``` + +If you run Redis any other way (the GHCR image with your own Redis, a managed service, a hosting platform), set `save ""` and `appendonly no` and don't attach a volume yourself, since Redis snapshots by default and these settings aren't applied by the server. + ### Railway [![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/Pmkrsc?referralCode=ToZEjF&utm_medium=integration&utm_source=template&utm_campaign=generic) diff --git a/docker-compose.yaml b/docker-compose.yaml index ad63391..ca174f8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,6 +5,10 @@ services: # TTL'd and must never be silently dropped, so use noeviction: once the cap # is hit, new writes are rejected (surfacing pressure) rather than evicting # live secrets. Tune REDIS_MAXMEMORY to the host. + # + # Memory only (SPECIFICATION.md 6.2): a snapshot or AOF would keep ciphertext, + # verification hashes and delete tokens on disk after a burn. A Redis restart + # therefore drops unexpired secrets and queued webhooks. command: - redis-server - --maxmemory @@ -12,12 +16,11 @@ services: - --maxmemory-policy - noeviction - --save - - '900' - - '1' + - '' + - --appendonly + - 'no' - --loglevel - warning - volumes: - - redis_data:/data server: build: @@ -43,6 +46,3 @@ services: - '${WEB_PORT:-80}:80' depends_on: - server - -volumes: - redis_data: diff --git a/package.json b/package.json index 938f82e..b1c510c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "format": "prettier --write .", "format:check": "prettier --check .", "test:e2e": "pnpm --filter @crypt.fyi/web test:e2e", - "redis": "docker run --rm -p 6379:6379 -v ./redis-data:/data redis:latest redis-server --save 1 1 --loglevel notice", + "redis": "docker run --rm -p 6379:6379 redis:latest redis-server --save \"\" --appendonly no --loglevel notice", "changeset": "changeset", "version-packages": "changeset version", "release": "node ./scripts/release.mjs", diff --git a/packages/server/docker-compose.yaml b/packages/server/docker-compose.yaml index e3ffaea..9ca0534 100644 --- a/packages/server/docker-compose.yaml +++ b/packages/server/docker-compose.yaml @@ -1,10 +1,16 @@ services: redis: image: redis:latest + # Memory only, matching the root docker-compose.yaml. The image snapshots by + # default, so persistence has to be turned off explicitly. + command: + - redis-server + - --save + - '' + - --appendonly + - 'no' ports: - '6379:6379' - volumes: - - redis_data:/data jaeger: image: jaegertracing/all-in-one:latest environment: @@ -12,6 +18,3 @@ services: ports: - 16686:16686 - 4318:4318 - -volumes: - redis_data: From a31d87abbd328db78b62082407cd07e7e2092100 Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Fri, 6 Nov 2026 16:28:17 -0600 Subject: [PATCH 2/4] fix: persist Redis with a synced AOF instead of snapshots Memory-only Redis met the spec but lost every unexpired secret on any restart. Snapshots kept data across clean restarts, but a crash could bring back a secret burned since the last snapshot. Use an AOF synced before each write is acknowledged, so secrets survive restarts and an acknowledged burn survives a crash. Update the spec to allow this and forbid recovering from snapshots, and correct the security copy that claimed no persistent storage. Document the upgrade steps, because Redis ignores an existing dump.rdb once AOF is on. Dev setups stay memory-only. --- .changeset/redis-aof-privacy-copy.md | 5 ++++ README.md | 37 ++++++++++++++++++++++++---- SPECIFICATION.md | 23 +++++++++++++---- docker-compose.yaml | 19 +++++++++++--- packages/core/src/i18n/locales/de.ts | 3 ++- packages/core/src/i18n/locales/en.ts | 3 ++- packages/core/src/i18n/locales/es.ts | 3 ++- packages/core/src/i18n/locales/fr.ts | 3 ++- packages/core/src/i18n/locales/zh.ts | 3 ++- packages/server/docker-compose.yaml | 4 +-- 10 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 .changeset/redis-aof-privacy-copy.md diff --git a/.changeset/redis-aof-privacy-copy.md b/.changeset/redis-aof-privacy-copy.md new file mode 100644 index 0000000..3cd3064 --- /dev/null +++ b/.changeset/redis-aof-privacy-copy.md @@ -0,0 +1,5 @@ +--- +'@crypt.fyi/core': patch +--- + +Correct the security copy now that Redis persists data with an AOF: secrets reach disk only in encrypted form, and deleted entries can remain until the log is next compacted. diff --git a/README.md b/README.md index 03ddabb..4926e33 100644 --- a/README.md +++ b/README.md @@ -96,17 +96,44 @@ It runs with `NODE_ENV=production` and requires `REDIS_URL` and `ENCRYPTION_KEY` The web client image is not published to GHCR yet; build it with `docker compose` as above. -#### Redis is memory-only +#### Redis persistence -The compose file runs Redis with RDB and AOF disabled and no data volume, because the [specification](./SPECIFICATION.md) forbids persistent storage: a snapshot would keep ciphertext, verification hashes and delete tokens on disk after a secret is burned. As a result, **any Redis restart, including recreating the container during an upgrade, drops all unexpired secrets and queued webhooks**. Their links stop working, and undelivered webhooks are not sent. Rate-limit counters reset too. +The compose file stores Redis data in the `redis_data` volume using an append-only file (AOF) that is synced to disk before each write is acknowledged, so secrets and queued webhooks survive restarts and crashes. A secret that was burned stays burned, even after a crash. RDB snapshots stay off because restoring an older snapshot could bring a burned secret back. See the [specification](./SPECIFICATION.md) for the requirements. -If you deployed before persistence was disabled, your old volume still holds a snapshot and is no longer used. Delete it (`docker volume ls` shows the name, usually `_redis_data`): +What this means for operators: + +- **The disk holds sensitive data.** The volume contains ciphertext, verification hashes and delete tokens, but never plaintext or decryption keys. A deleted secret's ciphertext can remain in the AOF until Redis next compacts it, so encrypt the disk. +- **Don't back up the volume.** Restoring a backup can bring back burned secrets. +- **Writes are slower** than without persistence, because each one waits for the disk. +- **`docker compose down -v` deletes the volume** and every unexpired secret with it. + +To run without persistence, set `--appendonly no` and remove the volume. A Redis restart then drops all unexpired secrets and queued webhooks. If you run Redis any other way (the GHCR image with your own Redis, a managed service, a hosting platform), apply these settings yourself: by default Redis takes snapshots and doesn't use AOF. + +##### Upgrading from RDB snapshots + +Deployments from before this change store data in a `dump.rdb` snapshot, which Redis ignores once AOF is on. **Upgrading without these steps loses all unexpired secrets**, and the old snapshot stays on disk. Before pulling the new compose file, turn on AOF in the running Redis: + +```bash +docker compose exec redis redis-cli config set appendonly yes +``` + +Wait until this shows `aof_rewrite_in_progress:0` and `aof_last_bgrewrite_status:ok`: ```bash -docker volume rm _redis_data +docker compose exec redis redis-cli info persistence ``` -If you run Redis any other way (the GHCR image with your own Redis, a managed service, a hosting platform), set `save ""` and `appendonly no` and don't attach a volume yourself, since Redis snapshots by default and these settings aren't applied by the server. +Then pull the new compose file and restart: + +```bash +docker compose up -d +``` + +Once you've confirmed existing secrets still open, delete the old snapshot: + +```bash +docker compose exec redis rm /data/dump.rdb +``` ### Railway diff --git a/SPECIFICATION.md b/SPECIFICATION.md index 3ea2567..a4af4f7 100644 --- a/SPECIFICATION.md +++ b/SPECIFICATION.md @@ -63,7 +63,7 @@ Abstract o Web Client (Browser-based interface) o Web Server (Static file server) o API Server - o Data Store (Redis for ephemeral storage) + o Data Store (Redis, holding entries until they expire or are burned) ### 2.2. Component Interaction Flow @@ -273,9 +273,21 @@ Abstract - Burn after reading MUST use atomic Redis operations - Read count limits MUST NOT exceed 10 reads - Failed attempt limits MUST be between 1-10 attempts before burning - - Implementation MUST NOT use persistent storage - ephemeral Redis - storage only - - Data MUST be securely deleted upon expiration or burning + - Implementations MAY keep data in memory only, in which case a data + store restart loses all unexpired entries + - Implementations that persist data MUST use an append-only log that is + synced to disk before each write is acknowledged (in Redis, + `appendonly yes` and `appendfsync always`), so that an acknowledged + burn or delete survives a crash + - Implementations MUST NOT recover from periodic snapshots (Redis RDB save + points), because restoring one can bring back a burned entry + - Backups of persisted data SHOULD NOT be taken, for the same reason + - Persisted data MUST NOT include plaintext secrets, encryption keys, or + passwords + - Data MUST be deleted upon expiration or burning. A deleted entry can + remain in an append-only log until the log is next compacted, so + implementations SHOULD compact it regularly and SHOULD encrypt the + underlying disk ### 6.3. Access Control @@ -315,7 +327,8 @@ Abstract #### 7.1.1. Redis Configuration - - Implementations MUST use ephemeral storage (Redis recommended) + - Implementations MUST meet the persistence requirements in Section 6.2 + (Redis recommended) - TTL per entry MUST be configurable (1 second to 7 days) - Automatic expiration and cleanup MUST be implemented - Atomic operations MUST be used for critical functions: diff --git a/docker-compose.yaml b/docker-compose.yaml index ca174f8..dafae4e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,9 +6,11 @@ services: # is hit, new writes are rejected (surfacing pressure) rather than evicting # live secrets. Tune REDIS_MAXMEMORY to the host. # - # Memory only (SPECIFICATION.md 6.2): a snapshot or AOF would keep ciphertext, - # verification hashes and delete tokens on disk after a burn. A Redis restart - # therefore drops unexpired secrets and queued webhooks. + # Secrets survive restarts through an AOF synced before each write is + # acknowledged (SPECIFICATION.md 6.2), so a burn can't be undone by a crash. + # RDB save points stay off because restoring an older snapshot could bring a + # burned secret back. The low rewrite threshold limits how long deleted + # entries remain in the log on small instances. command: - redis-server - --maxmemory @@ -18,9 +20,15 @@ services: - --save - '' - --appendonly - - 'no' + - 'yes' + - --appendfsync + - always + - --auto-aof-rewrite-min-size + - 1mb - --loglevel - warning + volumes: + - redis_data:/data server: build: @@ -46,3 +54,6 @@ services: - '${WEB_PORT:-80}:80' depends_on: - server + +volumes: + redis_data: diff --git a/packages/core/src/i18n/locales/de.ts b/packages/core/src/i18n/locales/de.ts index 87f2646..5c9cb89 100644 --- a/packages/core/src/i18n/locales/de.ts +++ b/packages/core/src/i18n/locales/de.ts @@ -395,7 +395,8 @@ export const de: TranslationKeys = { tls: 'TLS-Verschlüsselung für alle API-Kommunikation', expiration: 'Automatischer Datenablauf mit konfigurierbarer Lebensdauer (TTL)', deletion: 'Sichere Datenlöschung nach Zugriff oder Ablauf', - storage: 'Keine dauerhafte Speicherung sensibler Informationen', + storage: + 'Geheimnisse gelangen nur verschlüsselt auf die Festplatte, und gelöschte Einträge können dort bis zur nächsten Log-Komprimierung verbleiben; Klartext und Schlüssel werden nie gespeichert', }, }, thirdParty: { diff --git a/packages/core/src/i18n/locales/en.ts b/packages/core/src/i18n/locales/en.ts index d631ceb..cce091f 100644 --- a/packages/core/src/i18n/locales/en.ts +++ b/packages/core/src/i18n/locales/en.ts @@ -387,7 +387,8 @@ export const en: TranslationKeys = { tls: 'TLS encryption for all API communications', expiration: 'Automatic data expiration with configurable Time-To-Live (TTL)', deletion: 'Secure deletion of data after access or expiration', - storage: 'No persistent storage of sensitive information', + storage: + 'Secrets reach disk only in encrypted form and deleted entries can remain there until the log is next compacted; plaintext and keys are never stored', }, }, thirdParty: { diff --git a/packages/core/src/i18n/locales/es.ts b/packages/core/src/i18n/locales/es.ts index 5b908dc..0e8d285 100644 --- a/packages/core/src/i18n/locales/es.ts +++ b/packages/core/src/i18n/locales/es.ts @@ -393,7 +393,8 @@ export const es: TranslationKeys = { tls: 'Cifrado TLS para todas las comunicaciones API', expiration: 'Vencimiento automático de datos con tiempo de vida configurable (TTL)', deletion: 'Eliminación segura de datos después del acceso o vencimiento', - storage: 'Sin almacenamiento persistente de información sensible', + storage: + 'Los secretos solo llegan al disco cifrados y las entradas eliminadas pueden permanecer allí hasta la siguiente compactación del registro; el texto plano y las claves nunca se almacenan', }, }, thirdParty: { diff --git a/packages/core/src/i18n/locales/fr.ts b/packages/core/src/i18n/locales/fr.ts index f75b298..3091dc9 100644 --- a/packages/core/src/i18n/locales/fr.ts +++ b/packages/core/src/i18n/locales/fr.ts @@ -394,7 +394,8 @@ export const fr: TranslationKeys = { tls: 'Chiffrement TLS pour toutes les communications API', expiration: 'Expiration automatique des données avec durée de vie configurable (TTL)', deletion: 'Suppression sécurisée des données après accès ou expiration', - storage: "Pas de stockage persistant d'informations sensibles", + storage: + "Les secrets n'atteignent le disque que chiffrés et les entrées supprimées peuvent y rester jusqu'au prochain compactage du journal ; le texte en clair et les clés ne sont jamais stockés", }, }, thirdParty: { diff --git a/packages/core/src/i18n/locales/zh.ts b/packages/core/src/i18n/locales/zh.ts index a9d6cfe..35af59c 100644 --- a/packages/core/src/i18n/locales/zh.ts +++ b/packages/core/src/i18n/locales/zh.ts @@ -371,7 +371,8 @@ export const zh: TranslationKeys = { tls: '所有 API 通信都使用 TLS 加密', expiration: '具有可配置生存时间 (TTL) 的自动数据过期', deletion: '访问或过期后安全删除数据', - storage: '不持久存储敏感信息', + storage: + '秘密内容仅以加密形式写入磁盘,已删除的条目可能保留到下一次日志压缩;绝不存储明文和密钥', }, }, thirdParty: { diff --git a/packages/server/docker-compose.yaml b/packages/server/docker-compose.yaml index 9ca0534..cd87c4e 100644 --- a/packages/server/docker-compose.yaml +++ b/packages/server/docker-compose.yaml @@ -1,8 +1,8 @@ services: redis: image: redis:latest - # Memory only, matching the root docker-compose.yaml. The image snapshots by - # default, so persistence has to be turned off explicitly. + # Memory only: dev data is disposable. The image snapshots by default, so + # persistence has to be turned off explicitly. command: - redis-server - --save From b02b4802cd37d0314c49fb09c6d8a76ae1fa932c Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Fri, 6 Nov 2026 16:33:44 -0600 Subject: [PATCH 3/4] docs: describe protected hashes and tokens on the Redis volume --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d4fe40..3ff5629 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ The compose file stores Redis data in the `redis_data` volume using an append-on What this means for operators: -- **The disk holds sensitive data.** The volume contains ciphertext, verification hashes and delete tokens, but never plaintext or decryption keys. A deleted secret's ciphertext can remain in the AOF until Redis next compacts it, so encrypt the disk. +- **The disk holds sensitive data.** The volume contains ciphertext, keyed digests of verification hashes and encrypted delete tokens, but never plaintext or decryption keys. The digests and tokens rely on `ENCRYPTION_KEY`, so keep that key off this disk where you can. A deleted secret's ciphertext can remain in the AOF until Redis next compacts it, so encrypt the disk. - **Don't back up the volume.** Restoring a backup can bring back burned secrets. - **Writes are slower** than without persistence, because each one waits for the disk. - **`docker compose down -v` deletes the volume** and every unexpired secret with it. From c46edcb9e08b118f3618ef3a6171930e79cafe80 Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Fri, 6 Nov 2026 16:44:39 -0600 Subject: [PATCH 4/4] docs(spec): separate the no-release and purge guarantees for deletion "No persistent storage" conflated two guarantees. A deleted entry must never be released, with no races and across crash recovery. It must also be purged as soon as the backing store can manage at scale, and data still on disk is acceptable until then. Keep the requirements store-agnostic so other adapters can be added, and move the Redis settings under the adapter requirements. --- README.md | 2 +- SPECIFICATION.md | 54 ++++++++++++++++++++++++++++++------------------ 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3ff5629..9652ef7 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ The compose file stores Redis data in the `redis_data` volume using an append-on What this means for operators: -- **The disk holds sensitive data.** The volume contains ciphertext, keyed digests of verification hashes and encrypted delete tokens, but never plaintext or decryption keys. The digests and tokens rely on `ENCRYPTION_KEY`, so keep that key off this disk where you can. A deleted secret's ciphertext can remain in the AOF until Redis next compacts it, so encrypt the disk. +- **The disk holds sensitive data.** The volume contains ciphertext, keyed digests of verification hashes and encrypted delete tokens, but never plaintext or decryption keys. The digests and tokens rely on `ENCRYPTION_KEY`, so keep that key off this disk where you can. Once a secret is deleted, the server never returns it, even if its ciphertext remains in the AOF until Redis next compacts it. Encrypt the disk. - **Don't back up the volume.** Restoring a backup can bring back burned secrets. - **Writes are slower** than without persistence, because each one waits for the disk. - **`docker compose down -v` deletes the volume** and every unexpired secret with it. diff --git a/SPECIFICATION.md b/SPECIFICATION.md index 27776b3..f054cab 100644 --- a/SPECIFICATION.md +++ b/SPECIFICATION.md @@ -63,7 +63,7 @@ Abstract o Web Client (Browser-based interface) o Web Server (Static file server) o API Server - o Data Store (Redis, holding entries until they expire or are burned) + o Data Store (currently Redis, holding entries until they are deleted) ### 2.2. Component Interaction Flow @@ -272,24 +272,31 @@ Abstract ### 6.2. Data Security - Data MUST have automatic expiration (TTL: 1 second to 7 days) - - Burn after reading MUST use atomic Redis operations + - Burn after reading MUST use atomic data store operations - Read count limits MUST NOT exceed 10 reads - Failed attempt limits MUST be between 1-10 attempts before burning - - Implementations MAY keep data in memory only, in which case a data - store restart loses all unexpired entries - - Implementations that persist data MUST use an append-only log that is - synced to disk before each write is acknowledged (in Redis, - `appendonly yes` and `appendfsync always`), so that an acknowledged - burn or delete survives a crash - - Implementations MUST NOT recover from periodic snapshots (Redis RDB save - points), because restoring one can bring back a burned entry - - Backups of persisted data SHOULD NOT be taken, for the same reason + - Implementations MAY persist data to disk or keep it in memory only. A + memory-only data store loses all unexpired entries when it restarts - Persisted data MUST NOT include plaintext secrets, encryption keys, or passwords - - Data MUST be deleted upon expiration or burning. A deleted entry can - remain in an append-only log until the log is next compacted, so - implementations SHOULD compact it regularly and SHOULD encrypt the - underlying disk + - An entry is deleted when it expires, is burned, exhausts its failed + attempts, or is removed with its delete token. Deletion carries two + separate guarantees: + - Deleted entries MUST NOT be released. Deciding to release an entry + and deleting it MUST be a single atomic operation, so that concurrent + requests cannot race, and once an entry is deleted the server MUST NOT + release it through the API. This MUST also hold after a data store + restart or crash: an acknowledged deletion MUST be durable, and + recovery MUST NOT restore a deleted entry. Backups of persisted data + SHOULD NOT be taken, because restoring one can bring back deleted + entries + - Deleted entries MUST be purged. The data store MUST release any + snapshot, memory, or other reference to a deleted entry as soon as is + possible at scale. Data that remains on disk but cannot be released + through the API is acceptable until the data store is able to purge + it, which SHOULD happen as soon as it can. Implementations SHOULD + encrypt the underlying disk, because purging does not guarantee that + the physical blocks are erased - The verification hash MUST NOT be stored in plaintext; servers MUST store a keyed digest (HMAC-SHA-256) bound to the vault ID, so a datastore dump alone cannot be replayed to read or burn an entry @@ -333,10 +340,16 @@ Abstract ### 7.1. Storage Requirements -#### 7.1.1. Redis Configuration +#### 7.1.1. Data Store Configuration - - Implementations MUST meet the persistence requirements in Section 6.2 - (Redis recommended) + - Implementations MUST meet the requirements in Section 6.2 through a data + store adapter. Redis is the current adapter; others, such as PostgreSQL + or SQLite, MAY be added if they meet the same requirements + - The Redis adapter, when persisting data, MUST use `appendonly yes` and + `appendfsync always` so that a deletion is on disk before it is + acknowledged. RDB save points MUST be disabled, because restoring an + older snapshot can bring back a deleted entry. Deleted data is purged + when Redis rewrites the append-only file - TTL per entry MUST be configurable (1 second to 7 days) - Automatic expiration and cleanup MUST be implemented - Atomic operations MUST be used for critical functions: @@ -348,8 +361,9 @@ Abstract - Implementations MUST support concurrent access - Data consistency MUST be maintained using atomic transactions - - Atomic operations SHOULD use Lua scripts to prevent race conditions - - Concurrent read/delete operations MUST be handled properly + - Atomic operations SHOULD use the data store's native mechanism to prevent + race conditions (Lua scripts in Redis) + - Concurrent read/delete operations MUST meet the guarantees in Section 6.2 #### 7.1.3. Performance Requirements