diff --git a/.changeset/config.json b/.changeset/config.json index 05ae9beb..e9557e59 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -7,5 +7,8 @@ "access": "public", "baseBranch": "master", "updateInternalDependencies": "patch", - "ignore": ["example", "example-modules"] + "ignore": ["example", "example-modules"], + "snapshot": { + "useCalculatedVersion": true + } } diff --git a/.changeset/persistent-cache-debug-log.md b/.changeset/persistent-cache-debug-log.md new file mode 100644 index 00000000..baa22e5e --- /dev/null +++ b/.changeset/persistent-cache-debug-log.md @@ -0,0 +1,9 @@ +--- +'arui-scripts': minor +--- + +**Что изменилось** +Добавлена диагностика persistent cache для проверки локально и в Jenkins: настройки каждого компилятора, подтвержденное восстановление snapshot, инвалидация, счетчики сжатия и завершение закрытия компилятора. Значения переменных окружения не выводятся. + +**Что делать потребителю** +При включенном `persistentCache` задайте `ARUI_SCRIPTS_CACHE_DEBUG=1` на время проверки. Сравните холодную и повторную сборки по событиям с префиксом `[arui-scripts cache]`. Без переменной дополнительный лог выключен. diff --git a/.changeset/persistent-compression-results.md b/.changeset/persistent-compression-results.md new file mode 100644 index 00000000..9cce3d21 --- /dev/null +++ b/.changeset/persistent-compression-results.md @@ -0,0 +1,9 @@ +--- +'arui-scripts': minor +--- + +**Что изменилось** +При включенном `persistentCache` результаты gzip, Brotli и DCB сохраняются между сборками. Изменение исходных данных, параметров сжатия или словаря вызывает пересчет; поврежденные записи восстанавливаются автоматически. + +**Что делать потребителю** +Включите `persistentCache: true`. Для сохранения только кеша компиляции задайте `persistentCache: { compression: false }`. В Jenkins сохраняйте и восстанавливайте каталог `.cache/arui-scripts/rspack` целиком. diff --git a/.changeset/persistent-rspack-cache.md b/.changeset/persistent-rspack-cache.md new file mode 100644 index 00000000..4aac33c5 --- /dev/null +++ b/.changeset/persistent-rspack-cache.md @@ -0,0 +1,9 @@ +--- +'arui-scripts': minor +--- + +**Что изменилось** +Добавлен постоянный кеш Rspack для dev и production-сборок с настройкой `persistentCache`, командами `cache:info` и `cache:clear` и сохранением данных при штатном завершении компиляторов. Добавлен пример использования в Jenkins. + +**Что делать потребителю** +Для включения добавьте `persistentCache: true` в конфигурацию arui-scripts и `.cache/arui-scripts/` в `.gitignore` и `.prettierignore` приложения. В Jenkins настройте восстановление и сохранение этого каталога между сборками. По умолчанию постоянный кеш выключен. diff --git a/.gitignore b/.gitignore index 4b25c56c..dfb9386e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tsconfig.tsbuildinfo build .DS_Store .vscode/ +**/.cache/arui-scripts/ diff --git a/packages/arui-scripts/.gitignore b/packages/arui-scripts/.gitignore index a9c5d479..1401140a 100644 --- a/packages/arui-scripts/.gitignore +++ b/packages/arui-scripts/.gitignore @@ -1,2 +1,3 @@ build +!src/commands/build/ *.tsbuildinfo diff --git a/packages/arui-scripts/.prettierignore b/packages/arui-scripts/.prettierignore index 0d6a1e94..79dff6bd 100644 --- a/packages/arui-scripts/.prettierignore +++ b/packages/arui-scripts/.prettierignore @@ -1,2 +1,3 @@ .turbo +.cache/arui-scripts/ build diff --git a/packages/arui-scripts/README.md b/packages/arui-scripts/README.md index 854562dd..d29014a8 100644 --- a/packages/arui-scripts/README.md +++ b/packages/arui-scripts/README.md @@ -19,6 +19,9 @@ yarn start CLI `arui-scripts` поддерживает глобальные флаги `--help`, `--version` и справку по командам `arui-scripts <команда> --help`. Подробнее в [CLI](docs/cli.md). +Для повторных запусков dev и production-сборок можно включить [постоянный кеш Rspack](docs/persistent-cache.md) +настройкой `persistentCache: true`. По умолчанию он выключен. В документации есть пример подключения к Jenkins. + Команду `bundle-diff` можно запускать локально или в CI/CD. Для анализа изменений клиентских бандлов необходимы snapshot двух production сборок: diff --git a/packages/arui-scripts/docs/cli.md b/packages/arui-scripts/docs/cli.md index 38d12586..75f532f8 100644 --- a/packages/arui-scripts/docs/cli.md +++ b/packages/arui-scripts/docs/cli.md @@ -20,6 +20,9 @@ npx create-arui-scripts-app my-app Полный список и описание — в [docs/commands.md](commands.md): `start`, `start:prod`, `build`, `docker-build`, `docker-build:compiled`, `test`, `test:vitest`, -`archive-build`, `bundle-analyze`, `bundle-diff`, `ensure-yarn`, `changelog`. +`archive-build`, `bundle-analyze`, `bundle-diff`, `cache:info`, `cache:clear`, `ensure-yarn`, `changelog`. + +Постоянный кеш включается настройкой `persistentCache`. Команды `cache:info --json` и `cache:clear` +позволяют проверить его состояние и очистить данные. Подробнее — в [документации кеша](persistent-cache.md). При production-сборке можно включить [проверку размера начального JS и CSS](settings.md#buildsizebudgets). Лимиты задаются через `buildSizeBudgets`; их превышение завершает сборку с ненулевым кодом. diff --git a/packages/arui-scripts/docs/commands.md b/packages/arui-scripts/docs/commands.md index f0b1d335..54349948 100644 --- a/packages/arui-scripts/docs/commands.md +++ b/packages/arui-scripts/docs/commands.md @@ -41,6 +41,26 @@ arui-scripts start:prod arui-scripts build ``` +## cache:info + +Показывает расположение, размер и области постоянного кеша Rspack. Флаг `--json` включает JSON-вывод. +Команда не запускает компиляцию; для определения настроек она загружает конфигурацию и overrides проекта. + +```bash +arui-scripts cache:info --json +``` + +## cache:clear + +Удаляет только области постоянного кеша, созданные arui-scripts. Сначала остановите dev/build: +при активной записи команда завершится ошибкой. Результат сборки и остальные кеши сохраняются. + +```bash +arui-scripts cache:clear +``` + +Подробнее о настройках, ограничениях и Jenkins — в [документации кеша](persistent-cache.md). + ## test Конфигурация включает в себя: diff --git a/packages/arui-scripts/docs/persistent-cache.md b/packages/arui-scripts/docs/persistent-cache.md new file mode 100644 index 00000000..39b72575 --- /dev/null +++ b/packages/arui-scripts/docs/persistent-cache.md @@ -0,0 +1,214 @@ +# Постоянный кэш Rspack + +Кэш сохраняет промежуточную работу на диске и использует ее при следующем запуске `start`, `start:prod` или `build`. Это особенно полезно при повторных запусках dev и восстановлении кэша в CI. Итоговая сборка всегда создается заново; кэш не заменяет `.build` и проверку типов. + +По умолчанию функция выключена: dev использует прежний кэш в памяти, production собирается без постоянного кэша. + +```ts +// arui-scripts.config.ts +import type { PackageSettings } from 'arui-scripts'; + +export default { + persistentCache: true, +} satisfies PackageSettings; +``` + +По умолчанию данные хранятся в `/.cache/arui-scripts/rspack`. Добавьте `.cache/arui-scripts/` в `.gitignore` приложения +и в `.prettierignore`, если форматтер обходит все JSON-файлы проекта. Кеш не попадает в очищаемую папку сборки. +В dev его изменения игнорируются watcher. + +При упаковке приложения в Docker добавьте в `.dockerignore` контекста: + +```text +.cache/arui-scripts/ +``` + +Штатный SSR-шаблон копирует проект через `ADD . /src`; `.gitignore` не исключает файлы из Docker-контекста. +Для нестандартного `persistentCache.directory` исключите его фактический путь относительно контекста, +если каталог находится внутри него. Кеш нужно сохранять средствами Jenkins отдельно от runtime-образа. + +## Настройки + +Вместо `true` можно передать объект: + +```ts +persistentCache: { + modes: ['dev', 'production'], + directory: '.cache/arui-scripts/rspack', + version: 'my-pipeline-v1', + buildDependencies: ['./build-values.json'], + env: ['MY_COMPILE_FLAG'], + portable: false, + readonly: false, + compression: true, +} +``` + +| Поле | Значение и назначение | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `modes` | Непустой массив `dev` / `production`. По умолчанию оба. | +| `directory` | Путь от корня приложения или абсолютный путь к отдельному каталогу кэша. Нельзя использовать корень диска, каталог проекта, `node_modules`, buildPath или пути через symlink. | +| `version` | Добавка к автоматической совместимости. Измените ее, если изменилось поведение внешнего инструмента, которое невозможно отследить файлом или переменной. | +| `buildDependencies` | Дополнительные существующие файлы/каталоги, которые читает ваш override/plugin. Неверный путь вызывает ошибку настройки. | +| `env` | Имена дополнительных переменных, влияющих на компиляцию. Значения учитываются в хэше и не печатаются в `cache:info`. Не перечисляйте здесь обычные runtime-переменные. | +| `portable` | Переносимый формат Rspack. В версии 2.2.5 при смене абсолютного пути проекта первая сборка пересчитывается, чтобы избежать устаревших путей модулей и source maps. | +| `readonly` | По умолчанию `false`. Читать имеющийся кеш компиляции и сжатия, не сохранять изменения и не удалять старые записи. Отсутствующие результаты пересчитываются. | +| `compression` | По умолчанию `true` при включенном `persistentCache`. Сохранять результаты gzip, Brotli и DCB между сборками. `false` оставляет только native-кеш компиляции. | + +Автоматически учитываются настройки arui, окончательная конфигурация обработки и выходов, Node/ОС/архитектура, app/package/lock-файлы в родительских workspace, конфиги TS с `extends`, browserslist, presets/overrides и их обычные импортируемые зависимости, theme и словари сжатия. Изменения установленного кода arui тоже инвалидируют кэш. Локальные linked-пакеты проверяются как изменяемые файлы. + +Произвольный plugin может читать файлы динамически или захватывать переменные из окружения. Его автор должен перечислить такие входы через `buildDependencies`, `env` или `version`. Экземпляры plugin не сериализуются. Если остальные настройки невозможно безопасно fingerprint-ить, arui выводит предупреждение и собирает с прежней стратегией кэша. + +Client/server, режимы, имена, входы/выходы и профили обработки изолированы. Порядок конфигураций в массиве не влияет на область хранения. Низкоуровневое переопределение `rspackConfig.cache` имеет приоритет: такой кэш помечен `custom`, его путь и очисткой управляет автор override. + +В dev для clientOnly runtime `env-config.json` генерируется заново при новом процессе с текущим окружением. Production сохраняет прежний контракт подстановки окружения при запуске приложения. Runtime-значения не включаются в compile-time fingerprint автоматически. + +## Кеш сжатия + +При `persistentCache: true` production-сборка сохраняет готовые байты `.gz`, `.br` и `.dcb` +в подкаталоге `compression-v1` каждой области компилятора. Уровни сжатия, правила выбора файлов, +пороги и имена выходов остаются прежними. В dev стандартная конфигурация не сжимает ассеты. + +Ключ учитывает исходные байты, алгоритм, параметры сжатия, содержимое DCB-словаря, версии +arui/Node/библиотеки сжатия и `persistentCache.version`. Изменение входа приводит к пересчету; +смена только имени файла позволяет повторно использовать те же сжатые байты. +Пользовательские callback-алгоритмы `CompressionPlugin` не используют дисковый кеш: +захваченное ими состояние невозможно надежно определить автоматически. + +Записи сохраняются атомарно. При чтении проверяются ключ и SHA-256 результата; +поврежденная или недоступная запись пересчитывается с предупреждением. Ошибка самого +компрессора по-прежнему завершает сборку с ошибкой. При первом обращении в процессе удаляются +записи сжатия, записанные более семи дней назад; `readonly` отключает запись и эту очистку. + +`cache:info` включает размер сжатых данных, `cache:clear` удаляет их вместе с native-кешем. +В Jenkins достаточно сохранять существующий каталог `.cache/arui-scripts/rspack` целиком. +Для сравнения с native-кешем без сохранения результатов сжатия используйте +`persistentCache: { compression: false }`. + +## Диагностика и очистка + +```sh +yarn arui-scripts cache:info +yarn arui-scripts cache:info --json +yarn arui-scripts cache:clear +``` + +Info показывает путь, режимы, размер сохраненных данных, namespaces, fingerprint и политику текущих конфигураций (`arui` / `custom` / `legacy`). Ни содержимое настроек, ни значения env не выводятся. + +Clear удаляет данные только из каталога с marker владения arui-scripts. Сохраняет build output, `node_modules` и другие кэши. Отсутствующий кэш считается уже очищенным. Clear откажется работать при активной сборке, symlink внутри каталога или отсутствии marker. Сначала остановите dev/build. Если другой процесс уже пишет в ту же область, новая сборка выдаст предупреждение и продолжит без постоянного кэша. + +При недоступном для записи каталоге сборка продолжится с предупреждением. Поврежденные native-данные Rspack восстанавливает пересборкой. Ошибки кода и проверки типов продолжают завершать сборку с ошибкой. + +### Отладочный лог для беты и Jenkins + +Включите `persistentCache: true` в приложении и добавьте переменную окружения только на время проверки: + +```sh +ARUI_SCRIPTS_CACHE_DEBUG=1 yarn arui-scripts build +``` + +В Jenkins можно обернуть существующую команду в `withEnv(['ARUI_SCRIPTS_CACHE_DEBUG=1'])`. +Поддерживается также значение `true`; по умолчанию диагностика выключена. Переключение этой переменной +не меняет fingerprint. Лог пишется в stderr строками с префиксом `[arui-scripts cache] ` и JSON после него. +В каждой записи есть `schemaVersion: 1`, PID, имя компилятора, target, режим и namespace, если он известен. +Лог не содержит значений env, содержимого настроек, исходников или сырых сообщений native-кеша. + +Основные события: + +| `event` | Что проверять | +| --- | --- | +| `configured` | `cacheType: persistent`, путь `location`, `namespace`, `fingerprint`, `versionHash`, версии arui/Rspack/Node, `readonly`, `portable` и включение кеша сжатия. `reason` объясняет выключение, override или fallback. | +| `compilation` | `durationMs`, результат сборки, `native.snapshot`, число `native.errors`, число проверенных build dependencies и `compression.hits/misses/writes`. | +| `close` | `result: completed` означает завершение `compiler.close()` и освобождение записи в кеш. `persistence` различает `read-write`, `readonly` и выключенный кеш. | +| `diagnostic-error` | Статистику прочитать не удалось; диагностика не меняет результат сборки. | + +`native.snapshot` принимает следующие значения: + +- `hit`: Rspack явно подтвердил восстановление snapshot без измененных зависимостей. +- `invalidated`: Rspack сообщил об изменении snapshot, build dependencies или версии кеша. +- `cold`: snapshot прочитан, но подтверждения восстановления нет; это ожидаемо при первом запуске. +- `unavailable`: чтение/декодирование native-кеша завершилось ошибкой без подтвержденного восстановления. +- `unknown`: версия Rspack не дала достаточных данных; такой результат не считается попаданием. +- `in-process`: повторная компиляция в том же dev-процессе; это не доказательство восстановления с диска. +- `disabled`: текущий компилятор не использует постоянный native-кеш. + +Для проверки Jenkins выполните два последовательных job с одинаковым checkout, Node, lock-файлом, +конфигурацией и исходниками. В первом ожидаются `cold`, промахи и записи сжатия; после завершения команды +сохраните каталог кеша. Во втором восстановите его до сборки: должны совпасть namespace/fingerprint/versionHash, +появиться `native.snapshot: hit` с `errors: 0`, а у компиляторов, использующих сжатие, — ненулевые hits и нулевые misses. +Если соответствующих ассетов нет, нулевые счетчики сжатия нормальны. + +Затем измените исходник: `invalidated` и пересчет затронутых результатов ожидаемы. Смена только DCB-словаря +должна пересчитать DCB, сохранив попадания gzip/Brotli для неизменных входов. Для проверки результата сравните +артефакты с отдельной сборкой `persistentCache: false`; сами счетчики не доказывают равенство выходов. + +`close: completed` подтверждает штатное закрытие компилятора, а успешный `hit` следующего процесса — +фактическое переиспользование сохраненных данных. Окончание сборки само по себе не считается попаданием в кеш. +Сообщение Rspack `make persistent cache recovery succeeded` встречается и на холодном запуске, +поэтому по нему одному диагностика не ставит `hit`. При смене пути checkout см. ограничение portable ниже. + +## Jenkins: восстановить → собрать → сохранить + +1. Установить зависимости согласно lock-файлу. +2. Восстановить `.cache/arui-scripts/rspack` средствами CI. +3. Выполнить обычную команду сборки. +4. После завершения сохранить тот же каталог в cache-хранилище CI. +5. `.build` сохранить отдельно как артефакт результата. + +В cache key CI включите проект, ОС/архитектуру, Node, версию arui и hash lock-файла. Для ускорения на Rspack 2.2.5 сохраняйте одинаковый абсолютный путь checkout. При смене пути первая сборка пересчитывается, включая режим `portable: true`: проверка обнаружила старые пути модулей/source maps в native portable-восстановлении. Следующие сборки в новой папке используют кэш. Разные проекты не должны делить один каталог. Не восстанавливайте и не сохраняйте его во время работы компилятора. + +Arui сохраняет кэш при штатном завершении: build ждет `compiler.close()`, dev сначала останавливает сервер/watch, затем закрывает compiler. Parent-процесс ждет остановки дочерних компиляторов. Для dev установлен предел 10 секунд; повторный сигнал завершает процесс принудительно. После SIGKILL/падения ОС сохранение не гарантируется. + +### Пример для Job Cacher + +Следующий фрагмент предназначен для Linux-агента и требует установленного +[Job Cacher](https://plugins.jenkins.io/jobcacher/). Наличие плагина в вашей Jenkins-инсталляции нужно проверить. +Если кешированием управляет Jenkins Shared Library, используйте ту же последовательность в ней. + +```groovy +stage('Build') { + steps { + sh 'yarn install --immutable' + script { + def platform = sh(returnStdout: true, + script: 'node -p "process.platform + \'-\' + process.arch"').trim() + def nodeVersion = sh(returnStdout: true, script: 'node --version').trim() + def aruiVersion = sh(returnStdout: true, script: 'yarn arui-scripts --version').trim() + def lockHash = sh(returnStdout: true, script: 'sha256sum yarn.lock | cut -d " " -f 1').trim() + + cache(maxCacheSize: 1024, caches: [arbitraryFileCache( + path: '.cache/arui-scripts/rspack', + cacheName: "rspack-${platform}-${nodeVersion}-${aruiVersion}-${lockHash}", + includes: '**/*', + excludes: '**/.writer.json,**/.operation.lock', + useDefaultExcludes: false + )]) { + sh 'yarn arui-scripts build' + } + } + } +} +``` + +В конфигурации приложения должно быть `persistentCache: true` (либо объект с `modes: ['production']`). +Job Cacher восстанавливает каталог перед блоком и сохраняет после его выполнения; настройка `cacheValidityDecidingFile` +здесь не используется, чтобы обновленная компиляция сохранялась и при неизменном lock-файле. +Плагин изолирует кеши по job; при собственном хранилище добавьте в ключ также идентификатор проекта/job. +Не давайте параллельным сборкам один живой каталог: каждая сборка работает со своей восстановленной копией. +Каталог `.build` публикуется отдельно средствами существующего pipeline. + +Диагностические сборки `bundle-analyze` и сборки с `ARUI_SCRIPTS_RSDOCTOR_OUTPUT` выполняются без постоянного кеша, +чтобы профилировщик собирал свежие данные обо всем графе модулей. + +## Проверка разработчиком arui-scripts + +После сборки пакетов: + +```sh +node packages/arui-scripts/test/persistent-cache.cjs +node packages/arui-scripts/test/persistent-cache-lifecycle.cjs +``` + +Каждый build fixture запускается отдельным процессом. Проверяется подтвержденное native-восстановление, совпадение ассетов со сборкой без кэша, инвалидация входов и перенос checkout. Время печатается для диагностики: небольшой fixture не является прогнозом ускорения большого приложения. + +Lifecycle-проверка запускает настоящий dev-сервер и серверный watcher, отправляет SIGTERM родительскому процессу и проверяет завершение дочерних процессов и сохранение кэша. Ей требуется доступ к локальному порту. diff --git a/packages/arui-scripts/docs/settings.md b/packages/arui-scripts/docs/settings.md index f3e89f91..0fff108f 100644 --- a/packages/arui-scripts/docs/settings.md +++ b/packages/arui-scripts/docs/settings.md @@ -270,6 +270,23 @@ const packageSettings = { }; ``` +#### persistentCache + +Сохраняет кеш компиляции Rspack между запусками `start`, `start:prod` и `build`. +По умолчанию `false`: dev использует кеш в памяти, production собирается без постоянного кеша. + +```ts +const packageSettings = { + persistentCache: true, +}; +``` + +Объект позволяет выбрать режимы (`modes`), каталог (`directory`), дополнительную версию (`version`), +файлы (`buildDependencies`) и переменные окружения (`env`), влияющие на сборку. +По умолчанию включаются оба режима, данные хранятся в `.cache/arui-scripts/rspack` вне каталога сборки. + +Полный список настроек, правила инвалидации и пример Jenkins — в [документации кеша](persistent-cache.md). + #### installServerSourceMaps Добавлять ли в серверную сборку пакет source-map-support. По умолчанию `false`. diff --git a/packages/arui-scripts/package.json b/packages/arui-scripts/package.json index 9b0cea51..d0b0d6d6 100644 --- a/packages/arui-scripts/package.json +++ b/packages/arui-scripts/package.json @@ -162,6 +162,7 @@ "build": "sh bin/build.sh", "cli": "node build/bin/index.js", "test": "jest", + "test:persistent-cache": "node test/persistent-cache.cjs && node test/persistent-cache-lifecycle.cjs", "test:vitest": "vitest", "lint:scripts": "arui-presets-lint scripts", "format": "arui-presets-lint format", diff --git a/packages/arui-scripts/src/bin/commands-registry.ts b/packages/arui-scripts/src/bin/commands-registry.ts index 86f645fc..fe019ad0 100644 --- a/packages/arui-scripts/src/bin/commands-registry.ts +++ b/packages/arui-scripts/src/bin/commands-registry.ts @@ -16,6 +16,19 @@ export type CliCommand = { }; export const commands: CliCommand[] = [ + { + name: 'cache:info', + description: 'Показать состояние постоянного кеша Rspack', + options: [{ flags: '--json', description: 'Вывести данные в JSON' }], + // eslint-disable-next-line @typescript-eslint/no-var-requires + load: (options) => require('../commands/cache').cacheInfo(Boolean(options.json)), + }, + { + name: 'cache:clear', + description: 'Очистить постоянный кеш Rspack после остановки сборки', + // eslint-disable-next-line @typescript-eslint/no-var-requires + load: () => require('../commands/cache').cacheClear(), + }, { name: 'start', description: 'Dev сервер для клиента и серверный код в watch режиме', diff --git a/packages/arui-scripts/src/commands/build/__tests__/build-wrapper.tests.ts b/packages/arui-scripts/src/commands/build/__tests__/build-wrapper.tests.ts new file mode 100644 index 00000000..50a88f85 --- /dev/null +++ b/packages/arui-scripts/src/commands/build/__tests__/build-wrapper.tests.ts @@ -0,0 +1,72 @@ +import build from '../build-wrapper'; + +const mockRun = jest.fn(); +const mockClose = jest.fn(); + +jest.mock('../../util/create-compiler', () => ({ + createCompiler: () => ({ run: mockRun, close: mockClose }), +})); + +const stats = { toJson: jest.fn(() => ({ errors: [], warnings: [] })) }; + +beforeEach(() => { + jest.clearAllMocks(); + mockRun.mockImplementation((callback) => callback(null, stats)); + mockClose.mockImplementation((callback) => callback()); + stats.toJson.mockReturnValue({ errors: [], warnings: [] }); +}); + +it('consumes native statistics before closing and waits for cache flush', async () => { + const order: string[] = []; + let finishClose: () => void = () => {}; + + mockClose.mockImplementation((callback) => { + order.push('close'); + finishClose = callback; + }); + const pending = build({}, async (result) => { + expect(result.stats).toBe(stats); + order.push('statistics'); + }); + + await new Promise(setImmediate); + expect(order).toEqual(['statistics', 'close']); + finishClose(); + await expect(pending).resolves.toEqual({ warnings: [] }); +}); + +it('closes the compiler after a native compilation error', async () => { + mockRun.mockImplementation((callback) => callback(new Error('compile failed'))); + await expect(build({})).rejects.toThrow('compile failed'); + expect(mockClose).toHaveBeenCalledTimes(1); +}); + +it('closes the compiler when output validation rejects', async () => { + await expect( + build({}, async () => { + throw new Error('budget exceeded'); + }), + ).rejects.toThrow('budget exceeded'); + expect(mockClose).toHaveBeenCalledTimes(1); +}); + +it('rejects an unsuccessful cache flush', async () => { + mockClose.mockImplementation((callback) => callback(new Error('flush failed'))); + await expect(build({})).rejects.toThrow('flush failed'); +}); + +it('preserves the original error when closing also fails', async () => { + const log = jest.spyOn(console, 'error').mockImplementation(() => {}); + + mockRun.mockImplementation((callback) => callback(new Error('compile failed'))); + mockClose.mockImplementation((callback) => callback(new Error('flush failed'))); + await expect(build({})).rejects.toThrow('compile failed'); + expect(log).toHaveBeenCalled(); + log.mockRestore(); +}); + +it('closes the compiler when statistics are unavailable', async () => { + mockRun.mockImplementation((callback) => callback(null, undefined)); + await expect(build({})).rejects.toThrow('statistics'); + expect(mockClose).toHaveBeenCalledTimes(1); +}); diff --git a/packages/arui-scripts/src/commands/build/__tests__/client.tests.ts b/packages/arui-scripts/src/commands/build/__tests__/client.tests.ts index 3333f01b..2d6c351a 100644 --- a/packages/arui-scripts/src/commands/build/__tests__/client.tests.ts +++ b/packages/arui-scripts/src/commands/build/__tests__/client.tests.ts @@ -39,7 +39,11 @@ describe('build client', () => { jest.clearAllMocks(); jest.spyOn(console, 'log').mockImplementation(() => undefined); exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never); - mockBuild.mockResolvedValue({ stats, warnings: [] }); + mockBuild.mockImplementation(async (_config, beforeClose) => { + await beforeClose({ stats, warnings: [] }); + + return { warnings: [] }; + }); }); afterEach(() => { diff --git a/packages/arui-scripts/src/commands/build/build-wrapper.ts b/packages/arui-scripts/src/commands/build/build-wrapper.ts index cf66e5da..07a4aebe 100644 --- a/packages/arui-scripts/src/commands/build/build-wrapper.ts +++ b/packages/arui-scripts/src/commands/build/build-wrapper.ts @@ -1,55 +1,81 @@ -import { type Configuration, type MultiStats, rspack, type Stats } from '@rspack/core'; +import { type Configuration, type MultiStats, type Stats } from '@rspack/core'; import chalk from 'chalk'; +import { createCompiler } from '../util/create-compiler'; import { formatRspackMessages } from '../util/format-rspack-messages'; +import { closeCompiler } from '../util/graceful-shutdown'; -type BuildResult = { +type CompilationResult = { stats: Stats | MultiStats; warnings: string[]; - previousFileSizes: unknown; }; -function build(config: Configuration | Configuration[], previousFileSizes?: unknown) { - const compiler = rspack(config); - - return new Promise((resolve, reject) => { - compiler.run((err, stats) => { - if (err) { - return reject(err); - } - const messages = formatRspackMessages(stats?.toJson({})); - - if (messages.errors.length) { - // Only keep the first error. Others are often indicative - // of the same problem, but confuse the reader with noise. - if (messages.errors.length > 1) { - messages.errors.length = 1; - } +async function build( + config: Configuration | Configuration[], + beforeClose?: (result: CompilationResult) => void | Promise, +) { + const compiler = createCompiler(config); + let failed = false; + let failure: unknown; + let warnings: string[] = []; + + try { + const result = await new Promise((resolve, reject) => { + compiler.run((err, stats) => { + try { + if (err) return reject(err); + if (!stats) + return reject(new Error('Compiler did not return build statistics')); + const messages = formatRspackMessages(stats.toJson({})); + + if (messages.errors.length) { + return reject(new Error(messages.errors[0])); + } + if ( + process.env.CI && + process.env.CI.toLowerCase() !== 'false' && + messages.warnings.length + ) { + console.log( + chalk.yellow( + '\nTreating warnings as errors because process.env.CI = true.\n' + + 'Most CI servers set it automatically.\n', + ), + ); - return reject(new Error(messages.errors.join('\n\n'))); - } - if ( - process.env.CI && - process.env.CI.toLowerCase() !== 'false' && - messages.warnings.length - ) { - console.log( - chalk.yellow( - '\nTreating warnings as errors because process.env.CI = true.\n' + - 'Most CI servers set it automatically.\n', - ), - ); - - return reject(new Error(messages.warnings.join('\n\n'))); - } - - return resolve({ - stats: stats as Stats | MultiStats, - warnings: messages.warnings, - previousFileSizes, + return reject(new Error(messages.warnings.join('\n\n'))); + } + + return resolve({ stats, warnings: messages.warnings }); + } catch (error) { + return reject(error); + } }); }); - }); + + // Native Stats/Compilation become invalid after close(). Consume them before flushing. + await beforeClose?.(result); + + warnings = result.warnings; + } catch (error) { + failed = true; + failure = error; + } + + try { + await closeCompiler(compiler); + } catch (error) { + if (failed) { + console.error('Failed to close Rspack compiler:', error); + } else { + failed = true; + failure = error; + } + } + + if (failed) throw failure; + + return { warnings }; } export default build; diff --git a/packages/arui-scripts/src/commands/build/client.ts b/packages/arui-scripts/src/commands/build/client.ts index 4a011923..f7c2fe5f 100644 --- a/packages/arui-scripts/src/commands/build/client.ts +++ b/packages/arui-scripts/src/commands/build/client.ts @@ -42,32 +42,35 @@ async function main() { process.env.ARUI_SCRIPTS_RSDOCTOR_OUTPUT, configs.appSrc, ); - const { stats, warnings } = await build(rspackClientConfig); - if (warnings.length) { - console.log(chalk.yellow('Client compiled with warnings.\n')); - console.log(warnings.join('\n\n')); - console.log( - `Search for the ${chalk.underline( - chalk.yellow('keywords'), - )} to learn more about each warning.`, - ); - console.log( - `To ignore, add ${chalk.cyan('// eslint-disable-next-line')} to the line before.`, - ); - } else { - console.log(chalk.green('Client compiled successfully.\n')); - } + await build(rspackClientConfig, async ({ stats, warnings }) => { + if (warnings.length) { + console.log(chalk.yellow('Client compiled with warnings.\n')); + console.log(warnings.join('\n\n')); + console.log( + `Search for the ${chalk.underline( + chalk.yellow('keywords'), + )} to learn more about each warning.`, + ); + console.log( + `To ignore, add ${chalk.cyan( + '// eslint-disable-next-line', + )} to the line before.`, + ); + } else { + console.log(chalk.green('Client compiled successfully.\n')); + } - if (Array.isArray(rspackClientConfig)) { - for (const [index, conf] of rspackClientConfig.entries()) { - // measure builds sequentially to bound gzip memory use - // eslint-disable-next-line no-await-in-loop - await printOutputSizes(conf, (stats as MultiStats).stats[index]); + if (Array.isArray(rspackClientConfig)) { + for (const [index, conf] of rspackClientConfig.entries()) { + // measure builds sequentially to bound gzip memory use + // eslint-disable-next-line no-await-in-loop + await printOutputSizes(conf, (stats as MultiStats).stats[index]); + } + } else { + await printOutputSizes(rspackClientConfig, stats as Stats); } - } else { - await printOutputSizes(rspackClientConfig, stats as Stats); - } + }); if (snapshot) { await writeRsdoctorSnapshot(snapshot); diff --git a/packages/arui-scripts/src/commands/bundle-analyze/index.ts b/packages/arui-scripts/src/commands/bundle-analyze/index.ts index 8cfa24ab..69a9a7f3 100644 --- a/packages/arui-scripts/src/commands/bundle-analyze/index.ts +++ b/packages/arui-scripts/src/commands/bundle-analyze/index.ts @@ -39,6 +39,8 @@ const bundleAnalyzerStatsOptions: BundleAnalyzerStatsOptions = { /* eslint-disable no-param-reassign */ const promises = clientRspackConfigs.map(async (rspackConfig, i) => { + // Profiling builds use temporary outputs and must collect a fresh module graph. + rspackConfig.cache = false; const tmpDir = await makeTmpDir(i.toString()); const rspackStatsOptions: RspackOptionsNormalized['stats'] = { ...(typeof rspackConfig.stats === 'object' ? rspackConfig.stats : {}), diff --git a/packages/arui-scripts/src/commands/cache.ts b/packages/arui-scripts/src/commands/cache.ts new file mode 100644 index 00000000..529e25ee --- /dev/null +++ b/packages/arui-scripts/src/commands/cache.ts @@ -0,0 +1,76 @@ +/* eslint-disable global-require, @typescript-eslint/no-var-requires -- CLI loads optional configs only for info. */ +import { configs } from '../configs/app-configs'; +import { cacheDirectory, cacheSettings } from '../configs/cache/settings'; +import { clearCache, inspectCache } from '../configs/cache/storage'; +import { describeCompilerCaches } from '../configs/persistent-cache'; + +function configuredCaches(json: boolean) { + const { log } = console; + + // Overrides may log while loading. Keep machine-readable stdout valid JSON. + if (json) console.log = (...args: unknown[]) => console.error(...args); + + try { + // Resolve overrides for diagnostics only; no compiler is created and no cache is written. + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + const dev = require('../configs/rspack.client.dev').config; + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + const prod = require('../configs/rspack.client.prod').rspackClientConfig; + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + const serverDev = configs.clientOnly + ? [] + : require('../configs/rspack.server.dev').rspackServerConfig; + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + const serverProd = configs.clientOnly + ? [] + : require('../configs/rspack.server.prod').rspackServerConfig; + + return [dev, prod, serverDev, serverProd].flatMap(describeCompilerCaches); + } finally { + console.log = log; + } +} + +export function cacheInfo(json: boolean): void { + const directory = cacheDirectory(configs); + const settings = cacheSettings(configs); + const configured = configuredCaches(json); + const info = { + directory, + enabled: !!settings, + modes: settings ? settings.modes || ['dev', 'production'] : [], + configured, + portableCrossPathReuse: false, + ...inspectCache(directory), + }; + + if (json) console.log(JSON.stringify(info, null, 2)); + else { + console.log( + `Rspack cache: ${directory}\nEnabled: ${info.enabled}\nModes: ${ + info.modes.join(', ') || 'none' + }\nSize: ${info.bytes} bytes`, + ); + info.namespaces.forEach((namespace) => + console.log( + ` ${namespace.name}: ${ + namespace.dependencies + } dependencies, fingerprint ${namespace.fingerprint.slice(0, 12)}`, + ), + ); + configured + .filter((cache) => cache.policy === 'custom') + .forEach((cache) => + console.log(` ${cache.name}/${cache.mode}: custom cache override (not managed)`), + ); + if (!info.exists) console.log('Cache does not exist yet.'); + console.log('Raw Rspack cache overrides are custom and are not managed by cache:clear.'); + } +} + +export function cacheClear(): void { + const directory = cacheDirectory(configs); + + clearCache(directory); + console.log(`Rspack cache cleared: ${directory}`); +} diff --git a/packages/arui-scripts/src/commands/util/__tests__/cache-diagnostics.tests.ts b/packages/arui-scripts/src/commands/util/__tests__/cache-diagnostics.tests.ts new file mode 100644 index 00000000..b7979b7e --- /dev/null +++ b/packages/arui-scripts/src/commands/util/__tests__/cache-diagnostics.tests.ts @@ -0,0 +1,185 @@ +import { type Compiler, type Configuration } from '@rspack/core'; + +import { describeCompilerCaches } from '../../../configs/persistent-cache'; +import { getCompressionCache } from '../../../plugins/compression-cache'; +import { attachCacheDiagnostics, summarizeNativeCache } from '../cache-diagnostics'; + +jest.mock('../../../configs/persistent-cache', () => ({ describeCompilerCaches: jest.fn() })); +jest.mock('../../../plugins/compression-cache', () => ({ getCompressionCache: jest.fn() })); + +const unchanged = 'snapshot restored with no changed dependencies'; +const readSnapshot = 'read snapshot from persistent cache: 0.3 ms'; + +it.each([ + [['make persistent cache recovery succeeded'], 'unknown', 'no-native-evidence'], + [[unchanged], 'hit', 'snapshot-unchanged'], + [[readSnapshot], 'cold', 'no-restored-snapshot'], + [ + ['snapshot restored with detected changed dependencies: secret file'], + 'invalidated', + 'dependencies-changed', + ], + [['persistent cache version does not match'], 'invalidated', 'version-changed'], + [['build dependencies are invalid'], 'invalidated', 'build-dependencies-changed'], + [['meta persistent cache recovery failed: secret error'], 'unavailable', 'native-cache-error'], +])( + 'classifies native evidence without treating generic recovery as a hit: %j', + (messages, snapshot, reason) => { + expect(summarizeNativeCache(messages as string[])).toMatchObject({ snapshot, reason }); + }, +); + +it('preserves error evidence even when part of the cache recovered', () => { + expect( + summarizeNativeCache([ + unchanged, + 'source map persistent cache decode failed: secret', + 'build dependencies are valid (42 tracked)', + ]), + ).toEqual({ + snapshot: 'hit', + reason: 'snapshot-unchanged', + errors: 1, + trackedBuildDependencies: 42, + }); +}); + +describe('diagnostic lifecycle', () => { + const originalFlag = process.env.ARUI_SCRIPTS_CACHE_DEBUG; + let output: jest.SpyInstance; + + beforeEach(() => { + process.env.ARUI_SCRIPTS_CACHE_DEBUG = '1'; + output = jest.spyOn(console, 'error').mockImplementation(() => {}); + jest.mocked(describeCompilerCaches).mockReturnValue([ + { + name: 'main', + mode: 'production', + policy: 'arui', + target: 'client', + namespace: 'client-main-test', + fingerprint: 'public-hash', + reason: undefined, + }, + ]); + jest.mocked(getCompressionCache).mockReturnValue(undefined); + }); + afterEach(() => { + if (originalFlag === undefined) delete process.env.ARUI_SCRIPTS_CACHE_DEBUG; + else process.env.ARUI_SCRIPTS_CACHE_DEBUG = originalFlag; + output.mockRestore(); + jest.clearAllMocks(); + }); + + function harness() { + let done: (stats: unknown) => void = () => {}; + let failed: () => void = () => {}; + const compiler = { + options: { + cache: { + type: 'persistent', + version: 'secret-version', + storage: { type: 'filesystem', location: '/tmp/cache' }, + }, + }, + hooks: { + done: { + tap: (_name: string, callback: typeof done) => { + done = callback; + }, + }, + failed: { + tap: (_name: string, callback: typeof failed) => { + failed = callback; + }, + }, + }, + } as unknown as Compiler; + const stats = { + toJson: jest.fn(() => ({ + logging: { + 'rspack.persistentCache': { + entries: [{ message: unchanged }, { message: 'secret-native-payload' }], + }, + }, + })), + hasErrors: () => false, + startTime: 10, + endTime: 30, + }; + const close = attachCacheDiagnostics(compiler, {} as Configuration); + const records = () => + output.mock.calls.map(([line]) => + JSON.parse(line.slice('[arui-scripts cache] '.length)), + ); + + return { complete: () => done(stats), fail: () => failed(), close, records, stats }; + } + + it('is silent and does not read stats when disabled', () => { + process.env.ARUI_SCRIPTS_CACHE_DEBUG = '0'; + const fixture = harness(); + + fixture.complete(); + expect(fixture.close).toBeUndefined(); + expect(fixture.stats.toJson).not.toHaveBeenCalled(); + expect(describeCompilerCaches).not.toHaveBeenCalled(); + expect(output).not.toHaveBeenCalled(); + }); + + it('reports reuse, counters and close without leaking native messages or raw versions', () => { + jest.mocked(getCompressionCache).mockReturnValue({ + stats: { hits: 7, misses: 0, writes: 0 }, + } as ReturnType); + const fixture = harness(); + + fixture.complete(); + fixture.close?.(undefined, 4); + fixture.close?.(undefined, 1); + const records = fixture.records(); + + expect(records.map((record) => record.event)).toEqual([ + 'configured', + 'compilation', + 'close', + ]); + expect(records[0]).toMatchObject({ + cacheType: 'persistent', + location: '/tmp/cache', + fingerprint: 'public-hash', + compression: true, + }); + expect(records[1]).toMatchObject({ + durationMs: 20, + native: { snapshot: 'hit' }, + compression: { enabled: true, hits: 7, misses: 0, writes: 0 }, + }); + expect(records[2]).toMatchObject({ + result: 'completed', + persistence: 'read-write', + durationMs: 4, + }); + expect(JSON.stringify(records)).not.toContain('secret'); + }); + + it('distinguishes subsequent in-process rebuilds from disk restoration', () => { + const fixture = harness(); + + fixture.complete(); + fixture.complete(); + expect(fixture.records()[2]).toMatchObject({ run: 2, native: { snapshot: 'in-process' } }); + expect(fixture.stats.toJson).toHaveBeenCalledTimes(1); + }); + + it('reports a failed compilation and failed close', () => { + const fixture = harness(); + + fixture.fail(); + fixture.close?.(new Error('secret-error'), 2); + expect(fixture.records().slice(1)).toMatchObject([ + { event: 'compilation', result: 'failed' }, + { event: 'close', result: 'failed' }, + ]); + expect(JSON.stringify(fixture.records())).not.toContain('secret'); + }); +}); diff --git a/packages/arui-scripts/src/commands/util/__tests__/graceful-shutdown.tests.ts b/packages/arui-scripts/src/commands/util/__tests__/graceful-shutdown.tests.ts new file mode 100644 index 00000000..49a2a497 --- /dev/null +++ b/packages/arui-scripts/src/commands/util/__tests__/graceful-shutdown.tests.ts @@ -0,0 +1,52 @@ +import { registerShutdown, SHUTDOWN_TIMEOUT } from '../graceful-shutdown'; + +let dispose: () => void; +let exit: jest.SpyInstance; + +beforeEach(() => { + jest.useFakeTimers({ doNotFake: ['performance'] }); + exit = jest.spyOn(process, 'exit').mockImplementation((() => {}) as never); + jest.spyOn(console, 'error').mockImplementation(() => {}); +}); +afterEach(() => { + dispose(); + jest.useRealTimers(); + jest.restoreAllMocks(); +}); + +it('waits for the compiler to flush before exit', async () => { + let finish: () => void = () => {}; + + dispose = registerShutdown( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + process.emit('SIGTERM'); + expect(exit).not.toHaveBeenCalled(); + finish(); + await Promise.resolve(); + expect(exit).toHaveBeenCalledWith(0); +}); + +it('preserves close failure', async () => { + dispose = registerShutdown(() => Promise.reject(new Error('disk flush'))); + process.emit('SIGINT'); + await Promise.resolve(); + expect(exit).toHaveBeenCalledWith(1); +}); + +it('bounds shutdown to ten seconds', () => { + dispose = registerShutdown(() => new Promise(() => {})); + process.emit('SIGTERM'); + jest.advanceTimersByTime(SHUTDOWN_TIMEOUT); + expect(exit).toHaveBeenCalledWith(1); +}); + +it('allows a second signal to force exit', () => { + dispose = registerShutdown(() => new Promise(() => {})); + process.emit('SIGINT'); + process.emit('SIGTERM'); + expect(exit).toHaveBeenCalledWith(1); +}); diff --git a/packages/arui-scripts/src/commands/util/__tests__/rsdoctor-snapshot.tests.ts b/packages/arui-scripts/src/commands/util/__tests__/rsdoctor-snapshot.tests.ts index bc4ff1cb..86064bb6 100644 --- a/packages/arui-scripts/src/commands/util/__tests__/rsdoctor-snapshot.tests.ts +++ b/packages/arui-scripts/src/commands/util/__tests__/rsdoctor-snapshot.tests.ts @@ -68,9 +68,12 @@ function compile(config: Configuration): Promise { return new Promise((resolve, reject) => { compiler.run((error, stats) => { - compiler.close(() => { - if (error || stats?.hasErrors()) { - reject(error ?? new Error(stats?.toString())); + // Native statistics must be read before closing the compiler. + const compilationError = stats?.hasErrors() ? new Error(stats.toString()) : undefined; + + compiler.close((closeError) => { + if (error || compilationError || closeError) { + reject(error ?? compilationError ?? closeError); } else { resolve(); } diff --git a/packages/arui-scripts/src/commands/util/__tests__/run-compilers.tests.ts b/packages/arui-scripts/src/commands/util/__tests__/run-compilers.tests.ts index 12535f3e..099e3e21 100644 --- a/packages/arui-scripts/src/commands/util/__tests__/run-compilers.tests.ts +++ b/packages/arui-scripts/src/commands/util/__tests__/run-compilers.tests.ts @@ -18,14 +18,27 @@ function createFakeProcess(): FakeProcess { describe('runCompilers', () => { const spawnMock = spawn as jest.Mock; let exitSpy: jest.SpyInstance; + let signalListeners: Map<'SIGINT' | 'SIGTERM', NodeJS.SignalsListener[]>; beforeEach(() => { + jest.useFakeTimers(); + signalListeners = new Map([ + ['SIGINT', process.listeners('SIGINT')], + ['SIGTERM', process.listeners('SIGTERM')], + ]); spawnMock.mockReset(); exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never); jest.spyOn(console, 'error').mockImplementation(() => undefined); }); afterEach(() => { + signalListeners.forEach((listeners, signal) => { + process.listeners(signal).forEach((listener) => { + if (!listeners.includes(listener)) process.removeListener(signal, listener); + }); + }); + jest.clearAllTimers(); + jest.useRealTimers(); jest.restoreAllMocks(); }); @@ -56,6 +69,9 @@ describe('runCompilers', () => { expect(processes[0].kill).toHaveBeenCalled(); expect(processes[1].kill).toHaveBeenCalled(); + expect(exitSpy).not.toHaveBeenCalled(); + processes[0].emit('close', -2); + processes[1].emit('close', null, 'SIGTERM'); expect(exitSpy).toHaveBeenCalledWith(1); }); @@ -80,7 +96,6 @@ describe('runCompilers', () => { runCompilers(['client.js']); compiler.emit('close', null, 'SIGABRT'); - expect(compiler.kill).toHaveBeenCalled(); expect(exitSpy).toHaveBeenCalledWith(1); expect(exitSpy).not.toHaveBeenCalledWith(0); expect(exitSpy).not.toHaveBeenCalledWith(null); @@ -97,4 +112,18 @@ describe('runCompilers', () => { expect(compiler.kill).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); }); + + it('waits for all children to flush their caches on SIGTERM', () => { + const processes = [createFakeProcess(), createFakeProcess()]; + + spawnMock.mockImplementation(() => processes[spawnMock.mock.calls.length - 1]); + runCompilers(['client.js', 'server.js']); + process.emit('SIGTERM'); + processes.forEach((child) => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); + expect(exitSpy).not.toHaveBeenCalled(); + processes[0].emit('close', 0); + expect(exitSpy).not.toHaveBeenCalled(); + processes[1].emit('close', 0); + expect(exitSpy).toHaveBeenCalledWith(0); + }); }); diff --git a/packages/arui-scripts/src/commands/util/cache-diagnostics.ts b/packages/arui-scripts/src/commands/util/cache-diagnostics.ts new file mode 100644 index 00000000..b224b7c9 --- /dev/null +++ b/packages/arui-scripts/src/commands/util/cache-diagnostics.ts @@ -0,0 +1,182 @@ +import crypto from 'crypto'; + +import { type Compiler, type Configuration, type MultiCompiler, rspackVersion } from '@rspack/core'; + +import { describeCompilerCaches } from '../../configs/persistent-cache'; +import { getCompressionCache } from '../../plugins/compression-cache'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires, global-require +const { version } = require('../../../package.json') as { version: string }; + +const PREFIX = '[arui-scripts cache] '; + +/** Only explicit snapshot evidence counts as disk reuse; generic recovery also occurs cold. */ +export function summarizeNativeCache(messages: string[]) { + let snapshot = 'unknown'; + let reason = 'no-native-evidence'; + + if ( + messages.some((message) => + message.includes('snapshot restored with detected changed dependencies'), + ) + ) { + snapshot = 'invalidated'; + reason = 'dependencies-changed'; + } else if ( + messages.some((message) => + message.includes('snapshot restored with no changed dependencies'), + ) + ) { + snapshot = 'hit'; + reason = 'snapshot-unchanged'; + } else if ( + messages.some((message) => /version.*(does not match|doesn't match)/i.test(message)) + ) { + snapshot = 'invalidated'; + reason = 'version-changed'; + } else if (messages.some((message) => /build dependencies.*(invalid|changed)/i.test(message))) { + snapshot = 'invalidated'; + reason = 'build-dependencies-changed'; + } else if ( + messages.some((message) => message.includes('read snapshot from persistent cache')) + ) { + snapshot = 'cold'; + reason = 'no-restored-snapshot'; + } + const errors = messages.filter((message) => + /(?:decode|recovery|save|reset) failed/i.test(message), + ).length; + + if (errors && snapshot !== 'hit' && snapshot !== 'invalidated') { + snapshot = 'unavailable'; + reason = 'native-cache-error'; + } + const tracked = messages.join('\n').match(/build dependencies are valid \((\d+) tracked\)/); + + return { + snapshot, + reason, + errors, + trackedBuildDependencies: tracked ? Number(tracked[1]) : undefined, + }; +} + +/** Debug output contains fixed labels, hashes and counters, never raw native logs or env values. */ +export function attachCacheDiagnostics( + compiler: Compiler | MultiCompiler, + configuration: Configuration | Configuration[], +) { + if (!['1', 'true'].includes(process.env.ARUI_SCRIPTS_CACHE_DEBUG?.toLowerCase() || '')) + return undefined; + + const children = 'compilers' in compiler ? compiler.compilers : [compiler]; + const descriptions = describeCompilerCaches(configuration); + const reporters = children.map((child, index) => { + const description = descriptions[index]; + const { cache } = child.options; + const persistent = + typeof cache === 'object' && cache.type === 'persistent' ? cache : undefined; + const common = { + schemaVersion: 1, + pid: process.pid, + compiler: description.name, + target: description.target, + mode: description.mode, + namespace: description.namespace, + }; + const write = (event: string, details: object) => { + console.error(`${PREFIX}${JSON.stringify({ ...common, event, ...details })}`); + }; + let run = 0; + let closed = false; + let cacheType = cache ? 'memory' : 'disabled'; + let persistence = 'disabled'; + + if (persistent) { + cacheType = 'persistent'; + persistence = persistent.readonly ? 'readonly' : 'read-write'; + } + + write('configured', { + policy: description.policy, + reason: description.reason, + cacheType, + location: + persistent?.storage?.type === 'filesystem' + ? persistent.storage.location + : undefined, + fingerprint: description.fingerprint, + versionHash: persistent + ? crypto + .createHash('sha256') + .update(persistent.version || '') + .digest('hex') + : undefined, + readonly: persistent?.readonly || false, + portable: persistent?.portable || false, + compression: !!getCompressionCache(child), + toolchain: { + arui: version, + rspack: rspackVersion, + node: process.versions.node, + platform: process.platform, + arch: process.arch, + }, + }); + child.hooks.done.tap('AruiCacheDiagnostics', (stats) => { + run += 1; + try { + const snapshot = + persistent && run === 1 + ? stats.toJson({ + all: false, + logging: 'verbose', + loggingDebug: /^rspack\.persistentCache$/, + }) + : undefined; + const messages = + snapshot?.logging?.['rspack.persistentCache']?.entries?.map( + (entry) => entry.message || '', + ) || []; + const compression = getCompressionCache(child); + let native: object = { snapshot: 'disabled' }; + + if (persistent) { + native = + run === 1 ? summarizeNativeCache(messages) : { snapshot: 'in-process' }; + } + + write('compilation', { + run, + result: stats.hasErrors() ? 'errors' : 'success', + durationMs: + stats.endTime !== undefined && stats.startTime !== undefined + ? stats.endTime - stats.startTime + : undefined, + native, + compression: compression + ? { enabled: true, ...compression.stats } + : { enabled: false }, + }); + } catch { + write('diagnostic-error', { run, reason: 'stats-unavailable' }); + } + }); + child.hooks.failed.tap('AruiCacheDiagnostics', () => + write('compilation', { run: run + 1, result: 'failed' }), + ); + + return (error: unknown, durationMs: number) => { + if (closed) return; + write('close', { + result: error ? 'failed' : 'completed', + durationMs, + persistence, + }); + closed = !error; + }; + }); + + return (error: unknown, durationMs: number) => + reporters.forEach((report) => report(error, durationMs)); +} diff --git a/packages/arui-scripts/src/commands/util/create-compiler.ts b/packages/arui-scripts/src/commands/util/create-compiler.ts new file mode 100644 index 00000000..dd1fdc68 --- /dev/null +++ b/packages/arui-scripts/src/commands/util/create-compiler.ts @@ -0,0 +1,46 @@ +import { type Compiler, type Configuration, type MultiCompiler, rspack } from '@rspack/core'; + +import { acquireCompilerCaches } from '../../configs/persistent-cache'; + +import { attachCacheDiagnostics } from './cache-diagnostics'; + +/** Keep cache ownership until native close completes, including the error path. */ +export function createCompiler(configuration: Configuration): Compiler; +export function createCompiler(configuration: Configuration[]): MultiCompiler; +export function createCompiler( + configuration: Configuration | Configuration[], +): Compiler | MultiCompiler; +export function createCompiler(configuration: Configuration | Configuration[]) { + const releases = acquireCompilerCaches(configuration); + const release = () => { + releases.splice(0).forEach((callback) => callback()); + }; + + try { + const compiler = rspack(configuration); + const reportClose = attachCacheDiagnostics(compiler, configuration); + const close = compiler.close.bind(compiler); + + compiler.close = (callback: (error?: Error | null) => void) => { + const started = performance.now(); + + close((error) => { + try { + release(); + } catch (releaseError) { + reportClose?.(error || releaseError, Math.round(performance.now() - started)); + callback(error || (releaseError as Error)); + + return; + } + reportClose?.(error, Math.round(performance.now() - started)); + callback(error); + }); + }; + + return compiler; + } catch (error) { + release(); + throw error; + } +} diff --git a/packages/arui-scripts/src/commands/util/graceful-shutdown.ts b/packages/arui-scripts/src/commands/util/graceful-shutdown.ts new file mode 100644 index 00000000..f62dae44 --- /dev/null +++ b/packages/arui-scripts/src/commands/util/graceful-shutdown.ts @@ -0,0 +1,55 @@ +export const SHUTDOWN_TIMEOUT = 10000; + +/** A second signal forces termination; the first allows native cache writes to finish. */ +export function registerShutdown(stop: () => Promise): () => void { + let stopping = false; + let timer: NodeJS.Timeout | undefined; + const dispose = () => { + process.removeListener('SIGINT', onSignal); + process.removeListener('SIGTERM', onSignal); + if (timer) clearTimeout(timer); + }; + + function onSignal() { + if (stopping) { + console.error('Shutdown interrupted by a second signal; cache may not be saved.'); + dispose(); + process.exit(1); + + return; + } + stopping = true; + timer = setTimeout(() => { + console.error('Compiler shutdown timed out after 10 seconds; cache may not be saved.'); + dispose(); + process.exit(1); + }, SHUTDOWN_TIMEOUT); + stop().then( + () => { + dispose(); + process.exit(0); + }, + (error) => { + console.error('Compiler shutdown failed:', error); + dispose(); + process.exit(1); + }, + ); + } + + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + + return dispose; +} + +export function closeCompiler(compiler: { + close: (callback: (error?: Error | null) => void) => void; +}): Promise { + return new Promise((resolve, reject) => { + compiler.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); +} diff --git a/packages/arui-scripts/src/commands/util/rsdoctor-snapshot.ts b/packages/arui-scripts/src/commands/util/rsdoctor-snapshot.ts index a67ea311..2d63392f 100644 --- a/packages/arui-scripts/src/commands/util/rsdoctor-snapshot.ts +++ b/packages/arui-scripts/src/commands/util/rsdoctor-snapshot.ts @@ -111,6 +111,9 @@ export async function enableRsdoctor( const { RsdoctorRspackPlugin } = await import('@rsdoctor/rspack-plugin'); const bundles = configurations.map((config, index) => { + // Rsdoctor needs fresh attribution data even when application compilation is cached. + // eslint-disable-next-line no-param-reassign + config.cache = false; const reportDir = path.join(directory, `client-${index}`); // eslint-disable-next-line no-param-reassign diff --git a/packages/arui-scripts/src/commands/util/run-client-dev-server.ts b/packages/arui-scripts/src/commands/util/run-client-dev-server.ts index ea7e4f46..d4b734e6 100644 --- a/packages/arui-scripts/src/commands/util/run-client-dev-server.ts +++ b/packages/arui-scripts/src/commands/util/run-client-dev-server.ts @@ -1,13 +1,29 @@ import { types } from 'util'; -import { type Configuration, rspack, type Stats } from '@rspack/core'; +import { type Configuration, type Stats } from '@rspack/core'; import { RspackDevServer } from '@rspack/dev-server'; import { devServerConfig } from '../../configs/dev-server'; import { printCompilerOutput } from '../start/print-compiler-output'; +import { createCompiler } from './create-compiler'; +import { closeCompiler, registerShutdown } from './graceful-shutdown'; + export async function runClientDevServer(configuration: Configuration | Configuration[]) { - const clientCompiler = rspack(configuration); + const clientCompiler = createCompiler(configuration); + let clientDevServer: RspackDevServer | undefined; + const stop = async () => { + let stopError: Error | undefined; + + try { + await clientDevServer?.stop(); + } catch (error) { + stopError = error instanceof Error ? error : new Error(String(error)); + } + await closeCompiler(clientCompiler); + if (stopError) throw stopError; + }; + const dispose = registerShutdown(stop); clientCompiler.hooks.invalid.tap('client', () => console.log('Compiling client...')); clientCompiler.hooks.done.tap('client', (stats) => @@ -26,16 +42,25 @@ export async function runClientDevServer(configuration: Configuration | Configur if (!port) { // We have not found a port. - return; + throw new Error('No free port for the client dev server'); } // dev-server слушает порт из конфига как есть, поэтому подменяем его найденным свободным - const clientDevServer = new RspackDevServer({ ...devServerConfig, port }, clientCompiler); + clientDevServer = new RspackDevServer( + { ...devServerConfig, port, setupExitSignals: false }, + clientCompiler, + ); await clientDevServer.start(); console.log(`Client dev server running at http://${HOST}:${port}...`); } catch (err) { + dispose(); + try { + await stop(); + } catch (closeError) { + console.error('Client cleanup failed:', closeError); + } if (types.isNativeError(err)) { console.log(err.message); } diff --git a/packages/arui-scripts/src/commands/util/run-compilers.ts b/packages/arui-scripts/src/commands/util/run-compilers.ts index a791c1eb..35bee6d1 100644 --- a/packages/arui-scripts/src/commands/util/run-compilers.ts +++ b/packages/arui-scripts/src/commands/util/run-compilers.ts @@ -4,42 +4,69 @@ import fs from 'fs-extra'; import { configs } from '../../configs/app-configs'; -export function runCompilers(pathToCompilers: Array) { - if (fs.pathExistsSync(configs.serverOutputPath)) { - fs.removeSync(configs.serverOutputPath); - } - - const compilers = pathToCompilers.map((pathToCompiler) => { - const compiler = spawn( - 'node', - Array.isArray(pathToCompiler) ? pathToCompiler : [pathToCompiler], - { - stdio: 'inherit', - cwd: configs.cwd, - }, - ); +import { SHUTDOWN_TIMEOUT } from './graceful-shutdown'; - compiler.on('error', onProcessError); - compiler.on('close', onProcessClose); +export function runCompilers(pathToCompilers: Array) { + if (fs.pathExistsSync(configs.serverOutputPath)) fs.removeSync(configs.serverOutputPath); + const compilers = pathToCompilers.map((file) => + spawn('node', Array.isArray(file) ? file : [file], { stdio: 'inherit', cwd: configs.cwd }), + ); + const active = new Set(compilers); + let stopping = false; + let exitCode = 0; + let timer: NodeJS.Timeout | undefined; + const dispose = () => { + process.removeListener('SIGINT', onSignal); + process.removeListener('SIGTERM', onSignal); + if (timer) clearTimeout(timer); + }; + const finish = () => { + dispose(); + if (stopping) process.exit(exitCode); + }; + const force = () => { + active.forEach((compiler) => compiler.kill('SIGKILL')); + exitCode = exitCode || 1; + finish(); + }; + const stop = (code: number) => { + if (stopping) return; + stopping = true; + exitCode = code; + if (!active.size) { + finish(); - return compiler; - }); + return; + } + // Children close native compilers and flush caches before they exit. + timer = setTimeout(() => { + console.error('Compiler processes did not stop within 10 seconds.'); + force(); + }, SHUTDOWN_TIMEOUT); + active.forEach((compiler) => compiler.kill('SIGTERM')); + }; - function stopCompilers(exitCode: number) { - compilers.forEach((compiler) => compiler.kill()); - process.exit(exitCode); + function onSignal() { + if (stopping) force(); + else stop(0); } - // в error приходит объект ошибки, а не код для выхода - function onProcessError(error: Error) { - console.error(error.message); - stopCompilers(1); - } + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + compilers.forEach((compiler) => { + compiler.on('error', (error: Error) => { + console.error(error.message); + stop(1); + }); + compiler.on('close', (code: number | null) => { + active.delete(compiler); + if (!stopping && code !== 0) { + stop(code ?? 1); - function onProcessClose(code: number | null) { - if (code !== 0) { - // code === null означает, что процесс убит сигналом - stopCompilers(code ?? 1); - } - } + return; + } + if (stopping && code && !exitCode) exitCode = code; + if (!active.size) finish(); + }); + }); } diff --git a/packages/arui-scripts/src/commands/util/run-server-watch-compiler.ts b/packages/arui-scripts/src/commands/util/run-server-watch-compiler.ts index 32eef7d6..781f18ae 100644 --- a/packages/arui-scripts/src/commands/util/run-server-watch-compiler.ts +++ b/packages/arui-scripts/src/commands/util/run-server-watch-compiler.ts @@ -1,11 +1,16 @@ -import { type Configuration, rspack } from '@rspack/core'; +import { type Configuration } from '@rspack/core'; import { configs } from '../../configs/app-configs'; import { createWatchIgnoreRegex } from '../../configs/util/create-watch-ignore-regex'; import { printCompilerOutput } from '../start/print-compiler-output'; +import { createCompiler } from './create-compiler'; +import { closeCompiler, registerShutdown } from './graceful-shutdown'; + export function runServerWatchCompiler(config: Configuration) { - const serverCompiler = rspack(config); + const serverCompiler = createCompiler(config); + + registerShutdown(() => closeCompiler(serverCompiler)); serverCompiler.hooks.compile.tap('server', () => console.log('Compiling server...')); serverCompiler.hooks.invalid.tap('server', () => console.log('Compiling server...')); diff --git a/packages/arui-scripts/src/configs/app-configs/calculate-dependent-config.ts b/packages/arui-scripts/src/configs/app-configs/calculate-dependent-config.ts index 5b96dd3e..b2f51922 100644 --- a/packages/arui-scripts/src/configs/app-configs/calculate-dependent-config.ts +++ b/packages/arui-scripts/src/configs/app-configs/calculate-dependent-config.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; +import { cacheDirectory } from '../cache/settings'; import { getPolyfills } from '../util/get-polyfills'; import { resolveNodeModuleRelativeTo } from '../util/resolve'; @@ -69,7 +70,11 @@ export function calculateDependentContext(config: AppConfigs, context: AppContex serverOutputPath: path.resolve(context.cwd, config.buildPath), clientOutputPath: path.resolve(context.cwd, config.buildPath, config.assetsPath), statsOutputPath: path.resolve(context.cwd, config.buildPath, config.statsOutputFilename), - watchIgnorePath: ['node_modules', config.buildPath], + watchIgnorePath: [ + 'node_modules', + config.buildPath, + ...(config.persistentCache ? [cacheDirectory({ ...config, ...context })] : []), + ], babelRuntimeVersion, compressionPredefinedDictionaryPath: singleFilesDictionaries, compressionPreviousVersionPath: previousVersionPath, diff --git a/packages/arui-scripts/src/configs/app-configs/get-defaults.ts b/packages/arui-scripts/src/configs/app-configs/get-defaults.ts index 49083dc6..2751d022 100644 --- a/packages/arui-scripts/src/configs/app-configs/get-defaults.ts +++ b/packages/arui-scripts/src/configs/app-configs/get-defaults.ts @@ -61,6 +61,7 @@ export function getDefaultAppConfig(): AppConfigs { keepPropTypes: false, codeLoader: 'swc', experimentalReactCompiler: 'disabled', + persistentCache: false, installServerSourceMaps: false, disableDevRspackTypecheck: true, jestCodeTransformer: 'swc', diff --git a/packages/arui-scripts/src/configs/app-configs/types.ts b/packages/arui-scripts/src/configs/app-configs/types.ts index d90613ea..237932dc 100644 --- a/packages/arui-scripts/src/configs/app-configs/types.ts +++ b/packages/arui-scripts/src/configs/app-configs/types.ts @@ -61,6 +61,7 @@ export type AppConfigs = { /** Значение 'tsc' deprecated; поддерживается только с TypeScript <7. Используйте 'swc' или 'babel'. */ codeLoader: 'babel' | 'tsc' | 'swc'; experimentalReactCompiler: 'disabled' | ReactCompilerOptions; + persistentCache: boolean | PersistentCacheSettings; installServerSourceMaps: boolean; disableDevRspackTypecheck: boolean; /** Значение 'tsc' deprecated; поддерживается только с TypeScript <7. Используйте 'swc' или 'babel'. */ @@ -116,6 +117,17 @@ export type AppConfigs = { nodeExternals?: Omit; }; +export type PersistentCacheSettings = { + compression?: boolean; + modes?: Array<'dev' | 'production'>; + directory?: string; + version?: string; + buildDependencies?: string[]; + env?: string[]; + portable?: boolean; + readonly?: boolean; +}; + export type ModuleConfigBase = { cssPrefix?: false | string; useSeparateBuild?: boolean; diff --git a/packages/arui-scripts/src/configs/app-configs/validate-config.ts b/packages/arui-scripts/src/configs/app-configs/validate-config.ts index 17ff2dff..71dcb66d 100644 --- a/packages/arui-scripts/src/configs/app-configs/validate-config.ts +++ b/packages/arui-scripts/src/configs/app-configs/validate-config.ts @@ -1,9 +1,13 @@ +import { cacheDirectory, cacheSettings } from '../cache/settings'; import { getProjectTypeScript } from '../util/typescript'; import { type AppContextWithConfigs } from './types'; import { validateBuildSizeBudgets } from './validate-build-size-budgets'; export function validateConfig(config: AppContextWithConfigs) { + if (cacheSettings(config)) { + cacheDirectory(config); + } validateBuildSizeBudgets(config.buildSizeBudgets); const tscSettings = (['codeLoader', 'jestCodeTransformer'] as const).filter( diff --git a/packages/arui-scripts/src/configs/cache/__tests__/compression.tests.ts b/packages/arui-scripts/src/configs/cache/__tests__/compression.tests.ts new file mode 100644 index 00000000..934d64f0 --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/__tests__/compression.tests.ts @@ -0,0 +1,135 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { CompressionCache } from '../compression'; + +let directory: string; +const input = Buffer.from('input bytes'); +const compressed = Buffer.from('compressed bytes'); +const entries = () => + fs + .readdirSync(directory, { recursive: true, encoding: 'utf8' }) + .filter((name) => name.endsWith('.bin')) + .map((name) => path.join(directory, name)); + +beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'arui-compression-cache-')); +}); +afterEach(() => fs.rmSync(directory, { recursive: true, force: true })); + +it('reuses bytes in a fresh instance without running the compressor', async () => { + const compress = jest.fn(async () => compressed); + const cold = new CompressionCache({ directory, version: 'v1' }); + + expect(await cold.get(input, 'gzip:level9', compress)).toEqual(compressed); + const warm = new CompressionCache({ directory, version: 'v1' }); + + expect(await warm.get(input, 'gzip:level9', compress)).toEqual(compressed); + expect(compress).toHaveBeenCalledTimes(1); + expect(warm.stats).toEqual({ hits: 1, misses: 0, writes: 0 }); +}); + +it('invalidates changed input, algorithm/options, dictionary and implementation version', async () => { + const compress = jest.fn(async () => compressed); + const cache = new CompressionCache({ directory, version: 'v1' }); + + await cache.get(input, 'gzip:level9', compress); + await cache.get(Buffer.from('changed input'), 'gzip:level9', compress); + await cache.get(input, 'gzip:level1', compress); + await cache.get(input, 'brotli:quality11', compress); + await cache.get(input, 'dcb:dictionary-a', compress); + await cache.get(input, 'dcb:dictionary-b', compress); + await new CompressionCache({ directory, version: 'v2' }).get(input, 'gzip:level9', compress); + expect(compress).toHaveBeenCalledTimes(7); +}); + +it('coalesces concurrent requests for identical input', async () => { + const cache = new CompressionCache({ directory, version: 'v1' }); + const compress = jest.fn(async () => compressed); + + await Promise.all(Array.from({ length: 10 }, () => cache.get(input, 'gzip', compress))); + expect(compress).toHaveBeenCalledTimes(1); + expect(entries()).toHaveLength(1); +}); + +it('detects corruption and regenerates the record', async () => { + const compress = jest.fn(async () => compressed); + const warn = jest.fn(); + + await new CompressionCache({ directory, version: 'v1' }).get(input, 'gzip', compress); + const [file] = entries(); + const record = fs.readFileSync(file); + + record[record.length - 1] = (record[record.length - 1] + 1) % 256; + fs.writeFileSync(file, record); + expect( + await new CompressionCache({ directory, version: 'v1', warn }).get(input, 'gzip', compress), + ).toEqual(compressed); + expect(compress).toHaveBeenCalledTimes(2); + expect(warn).toHaveBeenCalledTimes(1); + await new CompressionCache({ directory, version: 'v1' }).get(input, 'gzip', compress); + expect(compress).toHaveBeenCalledTimes(2); +}); + +it('refuses a valid record stored under another input key', async () => { + const compress = jest.fn(async () => compressed); + const cache = new CompressionCache({ directory, version: 'v1' }); + + await cache.get(input, 'gzip', compress); + const [first] = entries(); + + await cache.get(input, 'brotli', compress); + const second = entries().find((file) => file !== first)!; + + fs.copyFileSync(first, second); + await new CompressionCache({ directory, version: 'v1' }).get(input, 'brotli', compress); + expect(compress).toHaveBeenCalledTimes(3); +}); + +it('reads existing bytes and computes misses without writing in readonly mode', async () => { + const compress = jest.fn(async () => compressed); + + await new CompressionCache({ directory, version: 'v1' }).get(input, 'gzip', compress); + const readonly = new CompressionCache({ directory, version: 'v1', readonly: true }); + + await readonly.get(input, 'gzip', compress); + await readonly.get(input, 'brotli', compress); + expect(readonly.stats).toEqual({ hits: 1, misses: 1, writes: 0 }); + expect(entries()).toHaveLength(1); +}); + +it('continues compressing when cache storage is unavailable', async () => { + const file = path.join(directory, 'not-a-directory'); + + fs.writeFileSync(file, 'keep'); + const warn = jest.fn(); + const cache = new CompressionCache({ directory: file, version: 'v1', warn }); + + expect(await cache.get(input, 'gzip', async () => compressed)).toEqual(compressed); + expect(fs.readFileSync(file, 'utf8')).toBe('keep'); + expect(warn).toHaveBeenCalledTimes(1); +}); + +it('does not cache compressor errors and retries the next request', async () => { + const cache = new CompressionCache({ directory, version: 'v1' }); + const compress = jest + .fn() + .mockRejectedValueOnce(new Error('compression failed')) + .mockResolvedValue(compressed); + + await expect(cache.get(input, 'gzip', compress)).rejects.toThrow('compression failed'); + expect(await cache.get(input, 'gzip', compress)).toEqual(compressed); + expect(compress).toHaveBeenCalledTimes(2); +}); + +it('removes obsolete entries after seven days', async () => { + const compress = jest.fn(async () => compressed); + + await new CompressionCache({ directory, version: 'v1' }).get(input, 'gzip', compress); + const old = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + + entries().forEach((file) => fs.utimesSync(file, old, old)); + await new CompressionCache({ directory, version: 'v1' }).get(input, 'gzip', compress); + expect(compress).toHaveBeenCalledTimes(2); +}); diff --git a/packages/arui-scripts/src/configs/cache/__tests__/identity.tests.ts b/packages/arui-scripts/src/configs/cache/__tests__/identity.tests.ts new file mode 100644 index 00000000..6ad61fed --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/__tests__/identity.tests.ts @@ -0,0 +1,135 @@ +import { type Configuration } from '@rspack/core'; + +import { configs } from '../../app-configs'; +import { finalizePersistentCache, getRspackCache, stableSerialize } from '../../persistent-cache'; + +jest.mock('../../app-configs', () => ({ + appConfigs: { persistentCache: true }, + configs: { + cwd: '/app', + name: 'test', + buildPath: '.build', + appNodeModules: '/app/node_modules', + appPackage: {}, + persistentCache: true, + }, +})); +jest.mock('../dependencies', () => ({ + cacheDependencies: () => [], + linkedDependencyPaths: () => [], +})); + +const configuration = (entry: string, name?: string): Configuration => ({ + name, + entry, + cache: getRspackCache('prod', 'client'), + output: { path: '/app/.build' }, +}); + +beforeEach(() => { + configs.cwd = '/app'; + configs.appNodeModules = '/app/node_modules'; + configs.persistentCache = true; +}); + +it('preserves the existing memory and disabled defaults', () => { + configs.persistentCache = false; + expect(getRspackCache('dev', 'client')).toBe(true); + expect(getRspackCache('prod', 'client')).toBe(false); +}); + +it('enables only the requested modes', () => { + configs.persistentCache = { modes: ['production'] }; + expect(getRspackCache('dev', 'client')).toBe(true); + expect(getRspackCache('prod', 'client')).toEqual({ type: 'persistent' }); + configs.persistentCache = { modes: ['dev'] }; + expect(getRspackCache('prod', 'server')).toBe(false); +}); + +it('isolates unnamed outputs and colliding sanitized names', () => { + const result = finalizePersistentCache([ + configuration('./a.ts'), + configuration('./b.ts'), + configuration('./a.ts', 'a/b'), + configuration('./a.ts', 'a_b'), + ]); + const caches = result.map((item) => item.cache); + + expect( + new Set( + caches.map( + (cache) => typeof cache === 'object' && cache.type === 'persistent' && cache.name, + ), + ).size, + ).toBe(4); +}); + +it('keeps identity when a multiconfig is reordered', () => { + const a = finalizePersistentCache([configuration('./a.ts'), configuration('./b.ts')]); + const b = finalizePersistentCache([configuration('./b.ts'), configuration('./a.ts')]); + + expect(a[0].cache).toEqual(b[1].cache); + expect(a[1].cache).toEqual(b[0].cache); +}); + +it('isolates processing profiles with the same name and entry', () => { + const a = configuration('./a.ts'); + const b = configuration('./a.ts'); + + a.devtool = 'source-map'; + b.devtool = false; + finalizePersistentCache([a, b]); + expect(a.cache).not.toEqual(b.cache); +}); + +it('preserves a low-level cache override', () => { + const config = configuration('./a.ts'); + + config.cache = false; + finalizePersistentCache(config); + expect(config.cache).toBe(false); +}); + +it('falls back for an opaque setting instead of sharing an unknown key', () => { + const warning = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const config = configuration('./a.ts'); + + config.resolve = { alias: { opaque: new Map() } } as unknown as Configuration['resolve']; + finalizePersistentCache(config); + expect(config.cache).toBe(false); + expect(warning).toHaveBeenCalledWith(expect.stringContaining('opaque')); + warning.mockRestore(); +}); + +it('portable names stay stable but native reuse is guarded by checkout', () => { + configs.persistentCache = { portable: true }; + const a = finalizePersistentCache(configuration('/app/src/a.ts')); + + configs.cwd = '/other'; + configs.appNodeModules = '/other/node_modules'; + const b = configuration('/other/src/a.ts'); + + b.output = { path: '/other/.build' }; + finalizePersistentCache(b); + expect(typeof a.cache === 'object' && a.cache.type === 'persistent' && a.cache.name).toEqual( + typeof b.cache === 'object' && b.cache.type === 'persistent' && b.cache.name, + ); + expect( + typeof a.cache === 'object' && a.cache.type === 'persistent' && a.cache.version, + ).not.toEqual(typeof b.cache === 'object' && b.cache.type === 'persistent' && b.cache.version); +}); + +it('stable serialization rejects cycles and bound functions', () => { + const value: { self?: unknown } = {}; + + value.self = value; + expect(() => stableSerialize(value)).toThrow('circular'); + expect(() => stableSerialize(() => 1)).not.toThrow(); + expect(() => stableSerialize(Math.max)).toThrow('native'); +}); + +it('keeps plain plugin tuples and their options in the fingerprint', () => { + expect(stableSerialize({ plugins: [['plugin', { flag: true }]] })).not.toEqual( + stableSerialize({ plugins: [['plugin', { flag: false }]] }), + ); +}); diff --git a/packages/arui-scripts/src/configs/cache/__tests__/storage.tests.ts b/packages/arui-scripts/src/configs/cache/__tests__/storage.tests.ts new file mode 100644 index 00000000..208f6083 --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/__tests__/storage.tests.ts @@ -0,0 +1,100 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { type AppContextWithConfigs } from '../../app-configs/types'; +import { cacheDirectory, cacheSettings } from '../settings'; +import { acquireCache, clearCache, inspectCache } from '../storage'; + +let root: string; +const namespace = { + name: 'client-main', + mode: 'production', + target: 'client', + fingerprint: 'abc', + dependencies: 5, +}; +const config = (settings: unknown = true) => + ({ + cwd: root, + buildPath: '.build', + appNodeModules: path.join(root, 'node_modules'), + persistentCache: settings, + } as AppContextWithConfigs); + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'arui-cache-storage-')); +}); +afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); +}); + +it('holds ownership until release and clear refuses active writers', () => { + const directory = cacheDirectory(config()); + const release = acquireCache(directory, namespace); + + expect(inspectCache(directory).namespaces).toEqual([namespace]); + expect(() => clearCache(directory)).toThrow('active writer'); + expect(() => acquireCache(directory, namespace)).toThrow('active writer'); + release(); + clearCache(directory); + expect(inspectCache(directory).namespaces).toEqual([]); + expect(fs.existsSync(path.join(directory, '.arui-cache.json'))).toBe(true); +}); + +it('clear accepts missing caches but refuses unowned directories', () => { + expect(() => clearCache(path.join(root, 'missing'))).not.toThrow(); + expect(() => clearCache(root)).toThrow(); + fs.writeFileSync(path.join(root, 'keep.txt'), 'keep'); + expect(() => acquireCache(root, namespace)).toThrow('ownership marker'); + expect(fs.readFileSync(path.join(root, 'keep.txt'), 'utf8')).toBe('keep'); +}); + +it('clear preserves unrelated files even inside an owned root', () => { + const directory = cacheDirectory(config()); + + acquireCache(directory, namespace)(); + fs.writeFileSync(path.join(directory, 'keep.txt'), 'keep'); + fs.mkdirSync(path.join(directory, 'other-cache')); + fs.writeFileSync(path.join(directory, 'other-cache/keep'), 'keep'); + clearCache(directory); + expect(fs.readFileSync(path.join(directory, 'keep.txt'), 'utf8')).toBe('keep'); + expect(fs.readFileSync(path.join(directory, 'other-cache/keep'), 'utf8')).toBe('keep'); + expect(fs.existsSync(path.join(directory, namespace.name))).toBe(false); +}); + +it('clear never follows a symlink out of an owned cache', () => { + const directory = cacheDirectory(config()); + + acquireCache(directory, namespace)(); + fs.mkdirSync(path.join(root, 'outside')); + fs.writeFileSync(path.join(root, 'outside/keep'), 'keep'); + fs.symlinkSync(path.join(root, 'outside'), path.join(directory, 'escape')); + expect(() => clearCache(directory)).toThrow('symlink'); + expect(fs.readFileSync(path.join(root, 'outside/keep'), 'utf8')).toBe('keep'); +}); + +it.each(['.', '..', '/', '.build', '.build/cache', 'node_modules'])( + 'rejects unsafe cache directory %s', + (directory) => { + expect(() => cacheDirectory(config({ directory }))).toThrow('unsafe'); + }, +); + +it('rejects a cache path that follows a symlink', () => { + fs.mkdirSync(path.join(root, 'outside')); + fs.symlinkSync(path.join(root, 'outside'), path.join(root, 'cache-link')); + expect(() => cacheDirectory(config({ directory: 'cache-link/rspack' }))).toThrow('symlink'); +}); + +it.each([ + { modes: [] }, + { modes: ['prod'] }, + { env: [2] }, + { readonly: 'yes' }, + { compression: 'yes' }, + { typo: true }, + { buildDependencies: ['missing'] }, +])('rejects invalid settings %j', (settings) => { + expect(() => cacheSettings(config(settings))).toThrow('persistentCache'); +}); diff --git a/packages/arui-scripts/src/configs/cache/compression.ts b/packages/arui-scripts/src/configs/cache/compression.ts new file mode 100644 index 00000000..b02d94c3 --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/compression.ts @@ -0,0 +1,138 @@ +import crypto from 'crypto'; +import fs from 'fs/promises'; +import path from 'path'; + +const MAGIC = Buffer.from('ARUICMP1'); +const HEADER_SIZE = MAGIC.length + 64; +const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; + +function digest(value: Buffer | string) { + return crypto.createHash('sha256').update(value).digest(); +} + +type Options = { + directory: string; + version: string; + readonly?: boolean; + warn?: (error: unknown) => void; +}; + +export class CompressionCache { + private pending = new Map>(); + + private cleanup?: Promise; + + private warned = false; + + stats = { hits: 0, misses: 0, writes: 0 }; + + constructor(private options: Options) {} + + resetStats() { + this.stats = { hits: 0, misses: 0, writes: 0 }; + } + + get(input: Buffer, identity: string, compress: () => Promise): Promise { + const key = digest( + JSON.stringify([this.options.version, identity, digest(input).toString('hex')]), + ).toString('hex'); + const existing = this.pending.get(key); + + if (existing) return existing; + + const result = this.load(key, compress).finally(() => this.pending.delete(key)); + + this.pending.set(key, result); + + return result; + } + + private warn(error: unknown) { + if (!this.warned) { + this.warned = true; + this.options.warn?.(error); + } + } + + private async load(key: string, compress: () => Promise): Promise { + if (!this.options.readonly) { + this.cleanup ??= this.prune().catch((error) => this.warn(error)); + await this.cleanup; + } + const filename = path.join(this.options.directory, key.slice(0, 2), `${key}.bin`); + + try { + const record = await fs.readFile(filename); + const output = record.subarray(HEADER_SIZE); + + if ( + record.length < HEADER_SIZE || + !record.subarray(0, MAGIC.length).equals(MAGIC) || + record.subarray(MAGIC.length, MAGIC.length + 32).toString('hex') !== key || + !record.subarray(MAGIC.length + 32, HEADER_SIZE).equals(digest(output)) + ) + throw new Error(`Invalid compressed cache entry: ${filename}`); + + this.stats.hits += 1; + + return output; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') this.warn(error); + } + + this.stats.misses += 1; + const output = await compress(); + + if (!this.options.readonly) { + const temporary = `${filename}.${crypto.randomBytes(12).toString('hex')}.tmp`; + + try { + await fs.mkdir(path.dirname(filename), { recursive: true }); + await fs.writeFile( + temporary, + Buffer.concat([MAGIC, Buffer.from(key, 'hex'), digest(output), output]), + { flag: 'wx' }, + ); + await fs.rename(temporary, filename); + this.stats.writes += 1; + } catch (error) { + this.warn(error); + } finally { + await fs.rm(temporary, { force: true }).catch(() => {}); + } + } + + return output; + } + + private async prune(): Promise { + const directories = await fs + .readdir(this.options.directory, { withFileTypes: true }) + .catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return []; + throw error; + }); + const cutoff = Date.now() - MAX_AGE_MS; + + await Promise.all( + directories + .filter((entry) => entry.isDirectory() && /^[a-f0-9]{2}$/.test(entry.name)) + .map(async (directory) => { + const location = path.join(this.options.directory, directory.name); + const entries = await fs.readdir(location, { withFileTypes: true }); + + await Promise.all( + entries + .filter( + (entry) => entry.isFile() && /^[a-f0-9]{64}\.bin$/.test(entry.name), + ) + .map(async (entry) => { + const file = path.join(location, entry.name); + + if ((await fs.stat(file)).mtimeMs < cutoff) await fs.unlink(file); + }), + ); + }), + ); + } +} diff --git a/packages/arui-scripts/src/configs/cache/dependencies.ts b/packages/arui-scripts/src/configs/cache/dependencies.ts new file mode 100644 index 00000000..ceed6cf6 --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/dependencies.ts @@ -0,0 +1,106 @@ +import fs from 'fs'; +import path from 'path'; + +import ts from 'typescript'; + +import { getConfigFilePath } from '../app-configs/read-config-file'; +import { type AppContextWithConfigs } from '../app-configs/types'; +import { getPresetsConfigPath } from '../app-configs/update-with-presets'; + +import { cacheSettings } from './settings'; + +/** Collect configuration inputs, including JSONC and package-based TypeScript extends. */ +export function cacheDependencies(config: AppContextWithConfigs): string[] { + const files = new Set(); + const add = (file: string) => { + if (fs.existsSync(file)) files.add(file); + }; + + for ( + let directory = config.cwd; + directory; + directory = directory === path.dirname(directory) ? '' : path.dirname(directory) + ) { + for (const name of [ + 'package.json', + 'yarn.lock', + 'package-lock.json', + 'pnpm-lock.yaml', + 'bun.lock', + 'bun.lockb', + '.browserslistrc', + 'browserslist', + ]) + add(path.join(directory, name)); + } + const roots = [ + getConfigFilePath(config.cwd), + config.presets ? getPresetsConfigPath(config, config.cwd) : undefined, + ...config.overridesPath, + ]; + const visited = new Set(); + const visitModule = (filename: string) => { + if (visited.has(filename)) return; + visited.add(filename); + add(filename); + require.cache[filename]?.children.forEach((child) => visitModule(child.filename)); + }; + + roots.forEach((filename) => { + if (filename) visitModule(filename); + }); + if (config.tsconfig) { + ts.getParsedCommandLineOfConfigFile( + config.tsconfig, + {}, + { + ...ts.sys, + readDirectory: () => [], + readFile: (file) => { + add(file); + + return ts.sys.readFile(file); + }, + onUnRecoverableConfigFileDiagnostic: () => {}, + }, + ); + } + if (process.env.BROWSERSLIST_CONFIG) + add(path.resolve(config.cwd, process.env.BROWSERSLIST_CONFIG)); + if (config.componentsTheme) { + const theme = path.resolve(config.cwd, config.componentsTheme); + + if (fs.existsSync(theme)) add(theme); + else add(require.resolve(config.componentsTheme, { paths: [config.cwd] })); + } + config.dictionaryCompression.dictionaryPath.forEach((file) => + add(path.resolve(config.cwd, file)), + ); + cacheSettings(config)?.buildDependencies?.forEach((file) => + add(path.resolve(config.cwd, file)), + ); + // Also invalidate local changes to the installed arui code, including linked development builds. + add(path.resolve(__dirname, '../../../package.json')); + add(path.resolve(__dirname, '../..')); + + return [...files].sort(); +} + +export function linkedDependencyPaths(config: AppContextWithConfigs): string[] { + const paths = new Set(); + const names = Object.keys({ + ...config.appPackage.dependencies, + ...config.appPackage.devDependencies, + }); + + names.forEach((name) => { + const candidate = path.join(config.appNodeModules, name); + + if (fs.existsSync(candidate) && fs.lstatSync(candidate).isSymbolicLink()) { + paths.add(candidate); + paths.add(fs.realpathSync(candidate)); + } + }); + + return [...paths]; +} diff --git a/packages/arui-scripts/src/configs/cache/settings.ts b/packages/arui-scripts/src/configs/cache/settings.ts new file mode 100644 index 00000000..0481067d --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/settings.ts @@ -0,0 +1,115 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { type AppContextWithConfigs, type PersistentCacheSettings } from '../app-configs/types'; + +export const DEFAULT_CACHE_DIRECTORY = '.cache/arui-scripts/rspack'; + +export function containsPath(parent: string, child: string): boolean { + const relative = path.relative(parent, child); + + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +export function cacheSettings(config: AppContextWithConfigs): PersistentCacheSettings | null { + const setting = config.persistentCache; + + if (setting === false || setting === undefined) return null; + if (setting === true) return {}; + if (!setting || typeof setting !== 'object' || Array.isArray(setting)) { + throw new Error('persistentCache: expected false, true or a settings object'); + } + const allowed = [ + 'modes', + 'directory', + 'version', + 'buildDependencies', + 'env', + 'portable', + 'readonly', + 'compression', + ]; + + Object.keys(setting).forEach((key) => { + if (!allowed.includes(key)) throw new Error(`persistentCache: unknown setting "${key}"`); + }); + if ( + setting.modes !== undefined && + (!Array.isArray(setting.modes) || + !setting.modes.length || + setting.modes.some((mode) => !['dev', 'production'].includes(mode))) + ) { + throw new Error('persistentCache.modes: expected a non-empty array of dev / production'); + } + (['directory', 'version'] as const).forEach((key) => { + if ( + setting[key] !== undefined && + (typeof setting[key] !== 'string' || (key === 'directory' && !setting[key])) + ) + throw new Error(`persistentCache.${key}: expected a string`); + }); + (['portable', 'readonly', 'compression'] as const).forEach((key) => { + if (setting[key] !== undefined && typeof setting[key] !== 'boolean') + throw new Error(`persistentCache.${key}: expected a boolean`); + }); + (['buildDependencies', 'env'] as const).forEach((key) => { + if ( + setting[key] !== undefined && + (!Array.isArray(setting[key]) || + setting[key]?.some((value) => typeof value !== 'string' || !value)) + ) + throw new Error(`persistentCache.${key}: expected an array of strings`); + }); + setting.buildDependencies?.forEach((dependency) => { + if (!fs.existsSync(path.resolve(config.cwd, dependency))) + throw new Error(`persistentCache.buildDependencies: missing path "${dependency}"`); + }); + + return setting; +} + +export function cacheDirectory(config: AppContextWithConfigs): string { + const directory = path.resolve( + config.cwd, + cacheSettings(config)?.directory || DEFAULT_CACHE_DIRECTORY, + ); + const build = path.resolve(config.cwd, config.buildPath); + const forbidden = [ + path.parse(directory).root, + os.homedir(), + config.cwd, + config.appNodeModules, + path.dirname(config.cwd), + ]; + + if ( + forbidden.includes(directory) || + containsPath(directory, config.cwd) || + containsPath(directory, build) || + containsPath(build, directory) || + containsPath(directory, config.appNodeModules) + ) { + throw new Error( + `persistentCache.directory: unsafe directory "${directory}"; use a dedicated cache directory outside buildPath`, + ); + } + + // Refuse symlinks in the path, including missing descendants of an existing symlink. + let current = directory; + + while (current !== path.dirname(current)) { + if ( + fs.existsSync(current) && + fs.lstatSync(current).isSymbolicLink() && + !containsPath(current, config.cwd) + ) + throw new Error(`persistentCache.directory: symlink is not allowed: ${current}`); + current = path.dirname(current); + } + + return directory; +} diff --git a/packages/arui-scripts/src/configs/cache/storage.ts b/packages/arui-scripts/src/configs/cache/storage.ts new file mode 100644 index 00000000..08ea0a99 --- /dev/null +++ b/packages/arui-scripts/src/configs/cache/storage.ts @@ -0,0 +1,173 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +const MARKER = '.arui-cache.json'; +const GUARD = '.operation.lock'; +const LEASE = '.writer.json'; + +function readJson(file: string) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function alive(pid: number): boolean { + if (!Number.isSafeInteger(pid) || pid <= 0) return true; + try { + process.kill(pid, 0); + + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } +} + +function assertTree(directory: string): void { + if (fs.lstatSync(directory).isSymbolicLink()) + throw new Error(`Refusing cache operation: symlink ${directory}`); + fs.readdirSync(directory, { withFileTypes: true }).forEach((entry) => { + const file = path.join(directory, entry.name); + + if (entry.isSymbolicLink()) throw new Error(`Refusing cache operation: symlink ${file}`); + if (entry.isDirectory()) assertTree(file); + }); +} + +function assertOwned(directory: string): void { + const marker = readJson(path.join(directory, MARKER)); + + if (marker.tool !== 'arui-scripts/rspack' || marker.schema !== 1) + throw new Error(`Not an arui-scripts Rspack cache: ${directory}`); +} + +function ownDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true }); + const marker = path.join(directory, MARKER); + + if (!fs.existsSync(marker)) { + if (fs.readdirSync(directory).length) + throw new Error( + `Cache directory is not empty and has no ownership marker: ${directory}`, + ); + try { + fs.writeFileSync(marker, JSON.stringify({ schema: 1, tool: 'arui-scripts/rspack' }), { + flag: 'wx', + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + } + assertOwned(directory); +} + +function withGuard(directory: string, operation: () => T): T { + const file = path.join(directory, GUARD); + + if (fs.existsSync(file) && !alive(readJson(file).pid)) fs.unlinkSync(file); + fs.writeFileSync(file, JSON.stringify({ pid: process.pid }), { flag: 'wx' }); + try { + return operation(); + } finally { + fs.unlinkSync(file); + } +} + +export type CacheNamespace = { + name: string; + mode: string; + target: string; + fingerprint: string; + dependencies: number; + custom?: boolean; + checkout?: string; +}; + +/** The lease lasts until native compiler.close has finished flushing, not just the shutdown hook. */ +export function acquireCache(directory: string, info: CacheNamespace): () => void { + ownDirectory(directory); + + return withGuard(directory, () => { + const location = path.join(directory, info.name); + + if (fs.existsSync(location)) assertTree(location); + fs.mkdirSync(location, { recursive: true }); + const file = path.join(location, LEASE); + + if (fs.existsSync(file)) { + if (alive(readJson(file).pid)) + throw new Error( + `Cache "${info.name}" has an active writer; stop it before reusing this namespace`, + ); + fs.unlinkSync(file); + } + const token = crypto.randomBytes(16).toString('hex'); + + fs.writeFileSync(file, JSON.stringify({ pid: process.pid, token }), { flag: 'wx' }); + try { + fs.writeFileSync(path.join(location, '.namespace.json'), JSON.stringify(info)); + } catch (error) { + fs.unlinkSync(file); + throw error; + } + + return () => { + if (fs.existsSync(file) && readJson(file).token === token) fs.unlinkSync(file); + }; + }); +} + +export function inspectCache(directory: string): { + exists: boolean; + bytes: number; + namespaces: CacheNamespace[]; +} { + if (!fs.existsSync(directory)) return { exists: false, bytes: 0, namespaces: [] }; + assertOwned(directory); + assertTree(directory); + let bytes = 0; + const namespaces: CacheNamespace[] = []; + const visit = (current: string) => { + fs.readdirSync(current, { withFileTypes: true }).forEach((entry) => { + const file = path.join(current, entry.name); + + if (entry.isDirectory()) visit(file); + else { + bytes += fs.statSync(file).size; + if (entry.name === '.namespace.json') namespaces.push(readJson(file)); + } + }); + }; + + visit(directory); + + return { exists: true, bytes, namespaces }; +} + +export function clearCache(directory: string): void { + if (!fs.existsSync(directory)) return; + assertOwned(directory); + assertTree(directory); + withGuard(directory, () => { + const entries = fs.readdirSync(directory).filter((name) => { + const namespace = path.join(directory, name); + const metadata = path.join(namespace, '.namespace.json'); + + if (!fs.statSync(namespace).isDirectory() || !fs.existsSync(metadata)) return false; + if (readJson(metadata).name !== name) + throw new Error(`Invalid cache namespace marker: ${namespace}`); + + return true; + }); + + entries.forEach((name) => { + const lease = path.join(directory, name, LEASE); + + if (fs.existsSync(lease) && alive(readJson(lease).pid)) + throw new Error( + 'Cache has an active writer. Stop build/dev before running cache:clear.', + ); + }); + entries.forEach((name) => + fs.rmSync(path.join(directory, name), { recursive: true, force: true }), + ); + }); +} diff --git a/packages/arui-scripts/src/configs/persistent-cache.ts b/packages/arui-scripts/src/configs/persistent-cache.ts new file mode 100644 index 00000000..204ab4df --- /dev/null +++ b/packages/arui-scripts/src/configs/persistent-cache.ts @@ -0,0 +1,317 @@ +/* eslint-disable no-param-reassign -- Final configuration and cache fallback are deliberately applied in place. */ +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +import { type CacheOptions, type Configuration } from '@rspack/core'; + +import { CompressionCachePlugin } from '../plugins/compression-cache'; + +import { cacheDependencies, linkedDependencyPaths } from './cache/dependencies'; +import { cacheDirectory, cacheSettings } from './cache/settings'; +import { acquireCache, type CacheNamespace } from './cache/storage'; +import { appConfigs, configs } from './app-configs'; + +const requested = new WeakMap(); +const managed = new WeakMap(); +const failures = new WeakMap(); + +/** Explicit unsupported values cause a fallback; distinct settings never share an "unknown" key. */ +export function stableSerialize( + value: unknown, + normalize: (text: string) => string = (text) => text, +): string { + const active = new Set(); + const visit = (item: unknown): unknown => { + if (typeof item === 'string') return normalize(item); + if (item === undefined) return { $undefined: true }; + if (typeof item === 'function') { + const source = Function.prototype.toString.call(item); + + if (source.includes('[native code]')) + throw new Error('a bound/native function cannot be fingerprinted'); + + return { $function: normalize(source) }; + } + if (item === null || typeof item === 'boolean' || typeof item === 'number') return item; + if (item instanceof RegExp) return { $regexp: item.source, flags: item.flags }; + if (typeof item !== 'object') + throw new Error(`unsupported fingerprint value: ${typeof item}`); + if (active.has(item)) throw new Error('a circular setting cannot be fingerprinted'); + if ( + !Array.isArray(item) && + Object.getPrototypeOf(item) !== Object.prototype && + Object.getPrototypeOf(item) !== null + ) + throw new Error('an opaque setting cannot be fingerprinted'); + active.add(item); + const result = Array.isArray(item) + ? item.map(visit) + : Object.fromEntries( + Object.keys(item) + .sort() + .map((key) => [ + key, + ['plugins', 'minimizer'].includes(key) && + Array.isArray((item as Record)[key]) + ? (item as Record)[key] + .filter(Boolean) + .map((plugin) => { + if ( + typeof plugin === 'object' && + plugin !== null && + !Array.isArray(plugin) && + Object.getPrototypeOf(plugin) !== Object.prototype && + Object.getPrototypeOf(plugin) !== null + ) + return { $pluginType: plugin.constructor.name }; + + return visit(plugin); + }) + : visit((item as Record)[key]), + ]), + ); + + active.delete(item); + + return result; + }; + + return JSON.stringify(visit(value)); +} + +function hash(value: string): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +export function getRspackCache(mode: 'dev' | 'prod', target: string): CacheOptions { + const settings = cacheSettings(configs); + const cacheMode = mode === 'dev' ? 'dev' : 'production'; + + if (!settings || !(settings.modes || ['dev', 'production']).includes(cacheMode)) + return mode === 'dev'; + const option: CacheOptions = { type: 'persistent' }; + + requested.set(option, { mode: cacheMode, target }); + + return option; +} + +/** Called after overrides so raw cache overrides retain precedence and final outputs determine identity. */ +export function finalizePersistentCache( + configuration: T, +): T { + const configurations = Array.isArray(configuration) ? configuration : [configuration]; + + configurations.forEach((config) => { + const request = + config.cache && typeof config.cache === 'object' + ? requested.get(config.cache) + : undefined; + + if (!request) return; + const fallback = request.mode === 'dev'; + + try { + const settings = cacheSettings(configs) || {}; + const directory = cacheDirectory(configs); + const normalize = settings.portable + ? (text: string) => + text + .split(configs.cwd) + .join('') + .split(path.resolve(__dirname, '../..')) + .join('') + : (text: string) => text; + const identity = stableSerialize( + { + project: configs.name, + target: request.target, + name: config.name, + mode: request.mode, + entry: config.entry, + output: config.output, + targetPlatform: config.target, + profile: { + module: config.module, + resolve: config.resolve, + optimization: config.optimization, + devtool: config.devtool, + externals: config.externals, + }, + }, + normalize, + ); + const name = `${`${request.target}-${config.name || 'main'}-${request.mode}` + .replace(/[^a-zA-Z0-9_-]/g, '_') + .slice(0, 80)}-${hash(identity).slice(0, 20)}`; + const dependencies = cacheDependencies(configs); + const { plugins, cache, devServer, ...semanticConfig } = config; + const { persistentCache, ...semanticSettings } = appConfigs; + const env = Object.fromEntries( + [ + ...new Set([ + 'NODE_ENV', + 'BABEL_ENV', + 'BROWSERSLIST', + 'BROWSERSLIST_ENV', + 'BROWSERSLIST_CONFIG', + 'USE_ISTANBUL', + ...(settings.env || []), + ]), + ] + .sort() + .map((key) => [key, process.env[key]]), + ); + // Plugin objects are deliberately not serialized. Their definitions/imports are build dependencies; + // authors must declare dynamic env/files used by custom plugins with env/buildDependencies/version. + const version = hash( + stableSerialize( + { + schema: 1, + toolchain: { + node: process.versions.node, + platform: process.platform, + arch: process.arch, + }, + settings: semanticSettings, + config: semanticConfig, + pluginTypes: plugins + ?.filter(Boolean) + .map((plugin: unknown) => + typeof plugin === 'function' + ? plugin.name + : (plugin as object)?.constructor.name, + ), + env, + dependencies: dependencies.map(normalize), + portable: !!settings.portable, + userVersion: settings.version, + }, + normalize, + ), + ); + const location = path.join(directory, name); + // Rspack 2.2.5 restores old readable module paths/source maps after relocation. + // Keep the semantic fingerprint portable, but partition native reuse by checkout + // until upstream relocation passes byte-for-byte artifact/source-map parity. + const checkout = hash(config.context || configs.cwd); + const backendVersion = settings.portable ? `${version}-checkout-${checkout}` : version; + const metadata = path.join(location, '.namespace.json'); + + if (settings.portable && fs.existsSync(metadata)) { + const previous = JSON.parse(fs.readFileSync(metadata, 'utf8')); + + if (previous.checkout && previous.checkout !== checkout) + console.warn( + '[arui-scripts cache] Checkout path changed. Rebuilding to avoid stale paths/source maps in Rspack 2.2.5; subsequent builds in this checkout can reuse the cache.', + ); + } + + config.cache = { + type: 'persistent', + name, + version: backendVersion, + buildDependencies: dependencies, + portable: settings.portable || false, + readonly: settings.readonly || false, + snapshot: { unmanagedPaths: linkedDependencyPaths(configs) }, + storage: { type: 'filesystem', location }, + }; + if (settings.compression !== false) { + config.plugins = [ + new CompressionCachePlugin({ + location, + readonly: settings.readonly || false, + version: settings.version, + }), + ...(config.plugins || []), + ]; + } + managed.set(config, { + directory, + info: { + name, + mode: request.mode, + target: request.target, + fingerprint: version, + dependencies: dependencies.length, + checkout, + }, + }); + } catch (error) { + failures.set(config, 'configuration-unavailable'); + console.warn( + `[arui-scripts cache] Persistent cache disabled for ${request.target}: ${ + (error as Error).message + }`, + ); + config.cache = fallback; + } + }); + + return configuration; +} + +export function acquireCompilerCaches( + configuration: Configuration | Configuration[], +): Array<() => void> { + const releases: Array<() => void> = []; + + (Array.isArray(configuration) ? configuration : [configuration]).forEach((config) => { + const cache = managed.get(config); + + if ( + !cache || + typeof config.cache !== 'object' || + config.cache.type !== 'persistent' || + config.cache.readonly + ) + return; + try { + releases.push(acquireCache(cache.directory, cache.info)); + } catch (error) { + failures.set(config, 'storage-unavailable'); + console.warn( + `[arui-scripts cache] ${cache.info.name}: ${ + (error as Error).message + }. Continuing without persistent cache.`, + ); + config.cache = cache.info.mode === 'dev'; + } + }); + + return releases; +} + +export function describeCompilerCaches(configuration: Configuration | Configuration[]) { + return (Array.isArray(configuration) ? configuration : [configuration]).map((config) => { + const cache = managed.get(config); + const mode = config.mode === 'development' ? 'dev' : 'production'; + const settings = cacheSettings(configs); + const expected = settings?.modes || ['dev', 'production']; + + let policy = 'legacy'; + + if (cache) policy = 'arui'; + else if (settings && expected.includes(mode)) policy = 'custom'; + + let reason = failures.get(config); + + if (!reason && !settings) reason = 'disabled-by-setting'; + else if (!reason && !expected.includes(mode)) reason = 'mode-excluded'; + else if (!reason && !cache) reason = 'cache-override'; + else if (!reason && cache && typeof config.cache !== 'object') + reason = 'cache-overridden-after-finalize'; + + return { + name: config.name || 'main', + mode, + policy, + reason, + target: cache?.info.target || config.target, + namespace: cache?.info.name, + fingerprint: cache?.info.fingerprint, + }; + }); +} diff --git a/packages/arui-scripts/src/configs/rspack.client.dev.ts b/packages/arui-scripts/src/configs/rspack.client.dev.ts index 3fa3235d..0a05f921 100644 --- a/packages/arui-scripts/src/configs/rspack.client.dev.ts +++ b/packages/arui-scripts/src/configs/rspack.client.dev.ts @@ -1,14 +1,17 @@ import { applyOverrides } from './util/apply-overrides'; import { findLoader } from './util/find-loader'; import { createFindPluginFunction } from './util/find-plugin'; +import { finalizePersistentCache } from './persistent-cache'; import { createClientRspackConfig, createSingleClientRspackConfig } from './rspack.client'; -export const config = applyOverrides( - ['rspack', 'rspackClient', 'rspackDev', 'rspackClientDev'], - createClientRspackConfig('dev'), - { - createSingleClientRspackConfig: createSingleClientRspackConfig.bind(null, 'dev'), - findLoader, - findPlugin: createFindPluginFunction<'client'>(), - }, +export const config = finalizePersistentCache( + applyOverrides( + ['rspack', 'rspackClient', 'rspackDev', 'rspackClientDev'], + createClientRspackConfig('dev'), + { + createSingleClientRspackConfig: createSingleClientRspackConfig.bind(null, 'dev'), + findLoader, + findPlugin: createFindPluginFunction<'client'>(), + }, + ), ); diff --git a/packages/arui-scripts/src/configs/rspack.client.prod.ts b/packages/arui-scripts/src/configs/rspack.client.prod.ts index 3953127a..35fe685d 100644 --- a/packages/arui-scripts/src/configs/rspack.client.prod.ts +++ b/packages/arui-scripts/src/configs/rspack.client.prod.ts @@ -1,14 +1,17 @@ import { applyOverrides } from './util/apply-overrides'; import { findLoader } from './util/find-loader'; import { createFindPluginFunction } from './util/find-plugin'; +import { finalizePersistentCache } from './persistent-cache'; import { createClientRspackConfig, createSingleClientRspackConfig } from './rspack.client'; -export const rspackClientConfig = applyOverrides( - ['rspack', 'rspackClient', 'rspackProd', 'rspackClientProd'], - createClientRspackConfig('prod'), - { - createSingleClientRspackConfig: createSingleClientRspackConfig.bind(null, 'prod'), - findLoader, - findPlugin: createFindPluginFunction<'client'>(), - }, +export const rspackClientConfig = finalizePersistentCache( + applyOverrides( + ['rspack', 'rspackClient', 'rspackProd', 'rspackClientProd'], + createClientRspackConfig('prod'), + { + createSingleClientRspackConfig: createSingleClientRspackConfig.bind(null, 'prod'), + findLoader, + findPlugin: createFindPluginFunction<'client'>(), + }, + ), ); diff --git a/packages/arui-scripts/src/configs/rspack.client.ts b/packages/arui-scripts/src/configs/rspack.client.ts index 794ead84..1d2dda10 100644 --- a/packages/arui-scripts/src/configs/rspack.client.ts +++ b/packages/arui-scripts/src/configs/rspack.client.ts @@ -32,6 +32,7 @@ import { checkNodeVersion } from './util/check-node-version'; import { compressionPluginsForDictionaries } from './util/compression-plugins-for-dictionaries'; import { createWatchIgnoreRegex } from './util/create-watch-ignore-regex'; import { type Entry, getEntry } from './util/get-entry'; +import { withPersistentCompression } from './util/persistent-compression-plugin'; import { configs } from './app-configs'; import { babelClientConfig as babelConf } from './babel-client'; import { babelDependencies } from './babel-dependencies'; @@ -41,6 +42,7 @@ import { patchMainRspackConfigForModules, patchRspackConfigForCompat, } from './modules'; +import { getRspackCache } from './persistent-cache'; import { postcssConfig as postcssConf } from './postcss'; import { processAssetsPluginOutput } from './process-assets-plugin-output'; import { swcClientConfig } from './swc'; @@ -196,7 +198,7 @@ export const createSingleClientRspackConfig = ( tsConfig: configs.tsconfig ? { configFile: configs.tsconfig } : undefined, }, resolveLoader: {}, - cache: mode === 'dev', + cache: getRspackCache(mode, 'client'), module: { rules: [ { @@ -367,27 +369,31 @@ export const createSingleClientRspackConfig = ( // production plugins: mode === 'prod' && new RspackManifestPlugin({}), mode === 'prod' && - new CompressionPlugin({ - filename: '[file].gz', - algorithm: 'gzip', - test: /\.js$|\.css$|\.png$|\.svg$/, - threshold: 10240, - minRatio: 0.8, - }), + withPersistentCompression( + new CompressionPlugin({ + filename: '[file].gz', + algorithm: 'gzip', + test: /\.js$|\.css$|\.png$|\.svg$/, + threshold: 10240, + minRatio: 0.8, + }), + ), mode === 'prod' && checkNodeVersion(10) && - new CompressionPlugin({ - filename: '[file].br', - algorithm: 'brotliCompress', - test: /\.(js|css|html|svg|dict)$/, - compressionOptions: { - params: { - [zlib.constants.BROTLI_PARAM_QUALITY]: 11, + withPersistentCompression( + new CompressionPlugin({ + filename: '[file].br', + algorithm: 'brotliCompress', + test: /\.(js|css|html|svg|dict)$/, + compressionOptions: { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: 11, + }, }, - }, - threshold: 10240, - minRatio: 0.8, - }), + threshold: 10240, + minRatio: 0.8, + }), + ), // Ignore prop-types packages in production mode // This should works fine, since proptypes usage should be eliminated in production mode mode === 'prod' && diff --git a/packages/arui-scripts/src/configs/rspack.server.dev.ts b/packages/arui-scripts/src/configs/rspack.server.dev.ts index 4212b541..9dcb82c2 100644 --- a/packages/arui-scripts/src/configs/rspack.server.dev.ts +++ b/packages/arui-scripts/src/configs/rspack.server.dev.ts @@ -1,10 +1,13 @@ import { applyOverrides } from './util/apply-overrides'; import { findLoader } from './util/find-loader'; import { createFindPluginFunction } from './util/find-plugin'; +import { finalizePersistentCache } from './persistent-cache'; import { createServerConfig } from './rspack.server'; -export const rspackServerConfig = applyOverrides( - ['rspack', 'rspackServer', 'rspackDev', 'rspackServerDev'], - createServerConfig('dev'), - { findLoader, findPlugin: createFindPluginFunction<'server'>() }, +export const rspackServerConfig = finalizePersistentCache( + applyOverrides( + ['rspack', 'rspackServer', 'rspackDev', 'rspackServerDev'], + createServerConfig('dev'), + { findLoader, findPlugin: createFindPluginFunction<'server'>() }, + ), ); diff --git a/packages/arui-scripts/src/configs/rspack.server.prod.ts b/packages/arui-scripts/src/configs/rspack.server.prod.ts index 90e100d6..b03422ed 100644 --- a/packages/arui-scripts/src/configs/rspack.server.prod.ts +++ b/packages/arui-scripts/src/configs/rspack.server.prod.ts @@ -1,10 +1,13 @@ import { applyOverrides } from './util/apply-overrides'; import { findLoader } from './util/find-loader'; import { createFindPluginFunction } from './util/find-plugin'; +import { finalizePersistentCache } from './persistent-cache'; import { createServerConfig } from './rspack.server'; -export const rspackServerConfig = applyOverrides( - ['rspack', 'rspackServer', 'rspackProd', 'rspackServerProd'], - createServerConfig('prod'), - { findLoader, findPlugin: createFindPluginFunction<'server'>() }, +export const rspackServerConfig = finalizePersistentCache( + applyOverrides( + ['rspack', 'rspackServer', 'rspackProd', 'rspackServerProd'], + createServerConfig('prod'), + { findLoader, findPlugin: createFindPluginFunction<'server'>() }, + ), ); diff --git a/packages/arui-scripts/src/configs/rspack.server.ts b/packages/arui-scripts/src/configs/rspack.server.ts index da5b92d5..068eb990 100644 --- a/packages/arui-scripts/src/configs/rspack.server.ts +++ b/packages/arui-scripts/src/configs/rspack.server.ts @@ -21,6 +21,7 @@ import { getEntry } from './util/get-entry'; import { configs } from './app-configs'; import { babelDependencies } from './babel-dependencies'; import { config as babelConf } from './babel-server'; +import { getRspackCache } from './persistent-cache'; import { serverPostcssConfig as postcssConf } from './postcss'; import { serverExternalsExemptions } from './server-externals-exemptions'; import { swcServerConfig } from './swc'; @@ -69,7 +70,7 @@ export const createServerConfig = (mode: 'dev' | 'prod'): Configuration => ({ devtoolModuleFilenameTemplate: (info) => path.relative(configs.appSrc, info.absoluteResourcePath).replace(/\\/g, '/'), }, - cache: mode === 'dev', + cache: getRspackCache(mode, 'server'), externalsPresets: { node: true }, externals: [ nodeExternals({ diff --git a/packages/arui-scripts/src/configs/util/__tests__/dcb-cache.tests.ts b/packages/arui-scripts/src/configs/util/__tests__/dcb-cache.tests.ts new file mode 100644 index 00000000..afcee47d --- /dev/null +++ b/packages/arui-scripts/src/configs/util/__tests__/dcb-cache.tests.ts @@ -0,0 +1,96 @@ +import { type Compilation, type Compiler, ModuleFilenameHelpers, sources } from '@rspack/core'; + +import { CustomCompressionPlugin } from '../dcb-compression-plugin'; + +function harness() { + const entries = new Map(); + const compiler = { webpack: { sources, ModuleFilenameHelpers } } as unknown as Compiler; + + return async (plugins: CustomCompressionPlugin[]) => { + const source = new sources.RawSource('x'.repeat(1000)); + const asset = { source, info: {} }; + const emitted: Record = {}; + const compilation = { + getCache: () => ({ + getLazyHashedEtag: () => 'unchanged-input', + getItemCache: (key: string) => ({ + getPromise: async () => entries.get(key), + storePromise: async (value: Buffer) => { + entries.set(key, value); + }, + }), + }), + getAsset: () => asset, + getPath: (value: string) => value, + updateAsset: (_name: string, _source: unknown, info: object) => { + Object.assign(asset.info, info); + }, + emitAsset: (name: string, output: sources.RawSource) => { + emitted[name] = output.buffer(); + }, + errors: [], + } as unknown as Compilation; + + // Plugins share one compilation and run sequentially, as in Rspack. + for (const plugin of plugins) { + // eslint-disable-next-line no-await-in-loop + await plugin.compress(compiler, compilation, { 'main.js': source }); + } + + return emitted; + }; +} + +it('emits identical output on cache hits without recompressing', async () => { + const compile = harness(); + const algorithm = jest.fn(async () => Buffer.from('compressed')); + const plugin = new CustomCompressionPlugin({ + test: /\.js$/, + filename: ({ filename }) => `${filename}.dcb`, + prepare: () => ({ key: 'dictionary-a', compress: algorithm }), + threshold: 10, + minRatio: 0.8, + }); + const cold = await compile([plugin]); + const warm = await compile([plugin]); + + expect(cold['main.js.dcb']).toEqual(Buffer.from('compressed')); + expect(warm).toEqual(cold); + expect(algorithm).toHaveBeenCalledTimes(1); +}); + +it('invalidates the cache when dictionary contents change', async () => { + const compile = harness(); + let dictionary = 'first'; + const algorithm = jest.fn(async () => Buffer.from(dictionary)); + const plugin = new CustomCompressionPlugin({ + test: /\.js$/, + filename: ({ filename }) => `${filename}.dcb`, + prepare: () => ({ key: dictionary, compress: algorithm }), + threshold: 10, + minRatio: 0.8, + }); + + await compile([plugin]); + dictionary = 'second'; + expect((await compile([plugin]))['main.js.dcb']).toEqual(Buffer.from('second')); + expect(algorithm).toHaveBeenCalledTimes(2); +}); + +it('emits independent outputs for algorithms closing over different dictionaries', async () => { + const compile = harness(); + const plugins = ['a', 'b'].map( + (dictionary) => + new CustomCompressionPlugin({ + test: /\.js$/, + filename: ({ filename }) => `${filename}.${dictionary}.dcb`, + prepare: () => ({ key: dictionary, compress: async () => Buffer.from(dictionary) }), + threshold: 10, + minRatio: 0.8, + }), + ); + const cold = await compile(plugins); + + expect(cold).toEqual({ 'main.js.a.dcb': Buffer.from('a'), 'main.js.b.dcb': Buffer.from('b') }); + expect(await compile(plugins)).toEqual(cold); +}); diff --git a/packages/arui-scripts/src/configs/util/__tests__/load-config-file.tests.ts b/packages/arui-scripts/src/configs/util/__tests__/load-config-file.tests.ts index 96635237..e38be741 100644 --- a/packages/arui-scripts/src/configs/util/__tests__/load-config-file.tests.ts +++ b/packages/arui-scripts/src/configs/util/__tests__/load-config-file.tests.ts @@ -76,6 +76,7 @@ describe.each(['6.0.2', '7.0.2'])('загрузка конфигурации в 'util/resolve.ts', 'util/get-polyfills.ts', 'util/apply-overrides.ts', + 'cache/settings.ts', ]; // Compile the scripts themselves as in the package build. Project TS files are diff --git a/packages/arui-scripts/src/configs/util/compression-plugins-for-dictionaries.ts b/packages/arui-scripts/src/configs/util/compression-plugins-for-dictionaries.ts index 0aa35d45..74d9009d 100644 --- a/packages/arui-scripts/src/configs/util/compression-plugins-for-dictionaries.ts +++ b/packages/arui-scripts/src/configs/util/compression-plugins-for-dictionaries.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; import fs from 'fs'; import path from 'path'; @@ -6,15 +7,29 @@ import { configs } from '../app-configs'; import { compressWithDcb } from './compress-with-dcb'; import { CustomCompressionPlugin } from './dcb-compression-plugin'; +// eslint-disable-next-line @typescript-eslint/no-var-requires, global-require +const dictionaryCompressorVersion: string = require('brotli-dict/package.json').version; + +function prepareDictionary(dictionaryContent: Buffer) { + return { + key: JSON.stringify([ + dictionaryCompressorVersion, + compressWithDcb.toString(), + crypto.createHash('sha256').update(dictionaryContent).digest('hex'), + ]), + compress: (input: Buffer) => compressWithDcb(input, dictionaryContent), + }; +} + export function compressionPluginsForDictionaries() { const predefinedPlugins = configs.compressionPredefinedDictionaryPath.map((dictionaryPath) => { const dictionaryName = path.parse(dictionaryPath).name; - const dictionaryContent = fs.readFileSync(dictionaryPath, null); return new CustomCompressionPlugin({ test: /\.js$|\.css$/, filename: ({ filename }) => `${filename}.${dictionaryName}.dcb`, - algorithm: (input) => compressWithDcb(input, dictionaryContent), + fileDependencies: [dictionaryPath], + prepare: () => prepareDictionary(fs.readFileSync(dictionaryPath)), threshold: 10240, minRatio: 0.8, }); @@ -47,24 +62,22 @@ export function compressionPluginsForDictionaries() { return `${parsedFilename.filename}.${matchedDictionary.hash}.dcb`; }, - algorithm: async (input, { filename }) => { - const parsedFilename = parseFilename(filename); - const matchedDictionary = - parsedDictionaries[`${parsedFilename.stableName}.${parsedFilename.ext}`]; + fileDependencies: dictionaries.map((file) => path.join(dictionaryPath, file)), + prepare: (filename) => { + const parsed = parseFilename(filename); + const matched = parsedDictionaries[`${parsed.stableName}.${parsed.ext}`]; - if (!matchedDictionary) { - return input; - } + if (!matched) return { key: 'missing', compress: async (input) => input }; try { const dictionaryContent = fs.readFileSync( - path.join(dictionaryPath, matchedDictionary.filename), + path.join(dictionaryPath, matched.filename), null, ); - return await compressWithDcb(input, dictionaryContent); + return prepareDictionary(dictionaryContent); } catch (e) { - return input; + return { key: 'missing', compress: async (input) => input }; } }, }); diff --git a/packages/arui-scripts/src/configs/util/dcb-compression-plugin.ts b/packages/arui-scripts/src/configs/util/dcb-compression-plugin.ts index f02a05c5..3c5ea291 100644 --- a/packages/arui-scripts/src/configs/util/dcb-compression-plugin.ts +++ b/packages/arui-scripts/src/configs/util/dcb-compression-plugin.ts @@ -10,11 +10,15 @@ import { import { type Rules } from 'compression-webpack-plugin'; import serialize from 'serialize-javascript'; +import { getCompressionCache } from '../../plugins/compression-cache'; + type DcbCompressionOptions = { test?: Rules; include?: Rules; exclude?: Rules; - algorithm: (input: Buffer, options: { filename: string }) => Promise; + /** Capture the dictionary bytes once so the key and compressor use the same input. */ + prepare: (filename: string) => { key: string; compress: (input: Buffer) => Promise }; + fileDependencies?: string[]; filename: (pathdata: PathData) => string; threshold: number; minRatio: number; @@ -33,146 +37,79 @@ Based on compression-webpack-plugin, https://github.com/webpack-contrib/compress export class CustomCompressionPlugin { constructor(protected options: DcbCompressionOptions) {} - runCompressionAlgorithm(input: Buffer, filename: string) { - return this.options.algorithm(input, { filename }); - } - async compress(compiler: Compiler, compilation: Compilation, assets: Assets) { - const cache = compilation.getCache('BrCompressionWebpackPlugin'); - - const assetsForCompression = ( - await Promise.all( - Object.keys(assets).map(async (name) => { - const asset = compilation.getAsset(name); - - if (!asset || asset.info.compressed) { - return false; - } - - if ( - !compiler.webpack.ModuleFilenameHelpers.matchObject.bind( - undefined, - this.options, - )(name) - ) { - return false; - } - - const relatedName = `compression-function-${crypto - .createHash('md5') - .update(serialize(this.options.filename)) - .digest('hex')}`; - - if ( - asset.info.related && - (asset.info.related as Record)[relatedName] - ) { - return false; - } - - const cacheItem = cache.getItemCache( - serialize({ - name, - algorithm: this.options.algorithm, - }), - cache.getLazyHashedEtag(asset.source as HashableObject), - ); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const output = ((await cacheItem.getPromise()) || {}) as any; - - let buffer; - - // No need original buffer for cached files - if (!output.source) { - if (typeof asset.source.buffer === 'function') { - buffer = asset.source.buffer(); - } - // Compatibility with webpack plugins which don't use `webpack-sources` - // See https://github.com/webpack-contrib/compression-webpack-plugin/issues/236 - else { - buffer = asset.source.source(); - - if (!Buffer.isBuffer(buffer)) { - buffer = Buffer.from(buffer); - } - } - - if (buffer.length < this.options.threshold) { - return false; - } - } - - return { - name, - source: asset.source, - info: asset.info, - buffer, - output, - cacheItem, - relatedName, - }; - }), - ) - ).filter(Boolean); - + const cache = compilation.getCache('arui-scripts/dcb-v2'); const { RawSource } = compiler.webpack.sources; - const scheduledTasks = []; - for (const asset of assetsForCompression) { - scheduledTasks.push( - (async () => { - if (!asset || !asset.buffer) { - return; - } - const { name, source, buffer, output, cacheItem, relatedName } = asset; - - if (!output.source) { - if (!output.compressed) { - try { - output.compressed = await this.runCompressionAlgorithm( - buffer, - name, - ); - } catch (error) { - compilation.errors.push(error as RspackError); - - return; - } - } + await Promise.all( + Object.keys(assets).map(async (name) => { + const asset = compilation.getAsset(name); + + if ( + !asset || + asset.info.compressed || + !compiler.webpack.ModuleFilenameHelpers.matchObject(this.options, name) + ) { + return; + } + const newFilename = compilation.getPath(this.options.filename({ filename: name })); + + if (!newFilename) return; + // Different dictionaries must not suppress one another's output. + const relatedName = `dcb-${crypto + .createHash('sha256') + .update(newFilename) + .digest('hex')}`; + + if (Object.prototype.hasOwnProperty.call(asset.info.related || {}, relatedName)) + return; + + const source = asset.source.buffer(); + + if (source.length < this.options.threshold) return; + const task = this.options.prepare(name); + const identity = serialize({ + algorithm: 'dcb', + dictionary: task.key, + implementation: task.compress, + }); + const cacheItem = cache.getItemCache( + serialize({ name, identity }), + cache.getLazyHashedEtag(asset.source as HashableObject), + ); + let compressed: Buffer | undefined = await cacheItem.getPromise(); + + if (!compressed) { + try { + const disk = getCompressionCache(compiler); + + compressed = disk + ? await disk.get(source, identity, () => task.compress(source)) + : await task.compress(source); + } catch (error) { + compilation.errors.push(error as RspackError); - if (output.compressed.length / buffer.length > this.options.minRatio) { - await cacheItem.storePromise({ compressed: output.compressed }); - - return; - } - - output.source = new RawSource(output.compressed); - - await cacheItem.storePromise(output); + return; } - - const newFilename = compilation.getPath( - this.options.filename({ filename: name }), - ); - - const newInfo = { compressed: true }; - - compilation.updateAsset(name, source, { - related: { [relatedName]: newFilename }, - }); - - compilation.emitAsset(newFilename, output.source, newInfo); - })(), - ); - } - - await Promise.all(scheduledTasks); + // Store only bytes: restore a fresh Source on every compilation. + await cacheItem.storePromise(compressed); + } + if (compressed.length / source.length > this.options.minRatio) return; + compilation.updateAsset(name, asset.source, { + related: { ...asset.info.related, [relatedName]: newFilename }, + }); + compilation.emitAsset(newFilename, new RawSource(compressed), { compressed: true }); + }), + ); } apply(compiler: Compiler) { const pluginName = this.constructor.name; compiler.hooks.thisCompilation.tap(pluginName, (compilation) => { + this.options.fileDependencies?.forEach((file) => + compilation.fileDependencies.add(file), + ); compilation.hooks.processAssets.tapPromise( { name: pluginName, diff --git a/packages/arui-scripts/src/configs/util/persistent-compression-plugin.ts b/packages/arui-scripts/src/configs/util/persistent-compression-plugin.ts new file mode 100644 index 00000000..995a422c --- /dev/null +++ b/packages/arui-scripts/src/configs/util/persistent-compression-plugin.ts @@ -0,0 +1,50 @@ +import { brotliCompress, gzip } from 'zlib'; + +import { type Compiler } from '@rspack/core'; +import type CompressionPlugin from 'compression-webpack-plugin'; +import serialize from 'serialize-javascript'; + +import { getCompressionCache } from '../../plugins/compression-cache'; + +type CompressionRuntime = { + algorithm: unknown; + options: { compressionOptions: unknown }; + runCompressionAlgorithm: (input: Buffer) => Promise; +}; + +/** Preserve the plugin instance/name/options consumed by existing overrides. */ +export function withPersistentCompression(plugin: CompressionPlugin): CompressionPlugin { + const { apply } = plugin; + + // compression-webpack-plugin 9 keeps the resolved function separate from public options. + // Give each compiler a receiver so sharing one plugin instance cannot mix cache ownership. + // eslint-disable-next-line no-param-reassign + plugin.apply = (compiler) => { + const instance = Object.create(plugin) as CompressionRuntime; + const run = instance.runCompressionAlgorithm.bind(instance); + + instance.runCompressionAlgorithm = (input) => { + const cache = getCompressionCache(compiler as unknown as Compiler); + let algorithm: string | undefined; + + if (instance.algorithm === gzip) algorithm = 'gzip'; + else if (instance.algorithm === brotliCompress) algorithm = 'brotliCompress'; + + // Custom callback algorithms may capture state not represented by their source. + if (!cache || !algorithm) return run(input); + + let identity: string; + + try { + identity = serialize({ algorithm, options: instance.options.compressionOptions }); + } catch { + return run(input); + } + + return cache.get(input, identity, () => run(input)); + }; + apply.call(instance, compiler); + }; + + return plugin; +} diff --git a/packages/arui-scripts/src/index.ts b/packages/arui-scripts/src/index.ts index 0b02fece..1f3fd465 100644 --- a/packages/arui-scripts/src/index.ts +++ b/packages/arui-scripts/src/index.ts @@ -1,6 +1,11 @@ export type { OverrideFile } from './configs/util/apply-overrides'; -export type { AppConfigs, CompatModuleConfig, PackageSettings } from './configs/app-configs/types'; +export type { + AppConfigs, + CompatModuleConfig, + PackageSettings, + PersistentCacheSettings, +} from './configs/app-configs/types'; /** * Утилиты сборки docker-образа, привязанные к глобальному конфигу arui-scripts. diff --git a/packages/arui-scripts/src/plugins/compression-cache.ts b/packages/arui-scripts/src/plugins/compression-cache.ts new file mode 100644 index 00000000..b3dc437a --- /dev/null +++ b/packages/arui-scripts/src/plugins/compression-cache.ts @@ -0,0 +1,55 @@ +import path from 'path'; + +import { type Compiler } from '@rspack/core'; + +import { CompressionCache } from '../configs/cache/compression'; + +const caches = new WeakMap(); +// eslint-disable-next-line @typescript-eslint/no-var-requires, global-require +const { version } = require('../../package.json') as { version: string }; + +export function getCompressionCache(compiler: Compiler) { + return caches.get(compiler); +} + +export class CompressionCachePlugin { + constructor(private options: { location: string; readonly: boolean; version?: string }) {} + + apply(compiler: Compiler) { + if ( + typeof compiler.options.cache !== 'object' || + compiler.options.cache.type !== 'persistent' + ) + return; + + const logger = compiler.getInfrastructureLogger('arui-scripts.compressionCache'); + const cache = new CompressionCache({ + directory: path.join(this.options.location, 'compression-v1'), + readonly: this.options.readonly, + version: JSON.stringify([ + version, + process.versions, + process.platform, + process.arch, + this.options.version, + ]), + warn: (error) => + logger.warn('Compression cache unavailable; recomputing compressed bytes.', error), + }); + + caches.set(compiler, cache); + compiler.hooks.thisCompilation.tap('CompressionCachePlugin', (compilation) => { + cache.resetStats(); + compilation.hooks.processAssets.tap( + { + name: 'CompressionCachePlugin', + stage: compiler.webpack.Compilation.PROCESS_ASSETS_STAGE_REPORT, + }, + () => + compilation + .getLogger('arui-scripts.compressionCache') + .info(`Compression cache: ${JSON.stringify(cache.stats)}`), + ); + }); + } +} diff --git a/packages/arui-scripts/src/plugins/reload-server-plugin.ts b/packages/arui-scripts/src/plugins/reload-server-plugin.ts index 0f45d36c..1953ec3a 100644 --- a/packages/arui-scripts/src/plugins/reload-server-plugin.ts +++ b/packages/arui-scripts/src/plugins/reload-server-plugin.ts @@ -30,6 +30,25 @@ export class ReloadServerPlugin { } apply(compiler: Compiler) { + compiler.hooks.shutdown.tapAsync('ReloadServerPlugin', (callback) => { + const workers = this.workers.filter((worker) => !worker.isDead()); + + if (!workers.length) { + callback(); + + return; + } + let remaining = workers.length; + + workers.forEach((worker) => { + worker.once('exit', () => { + remaining -= 1; + if (!remaining) callback(); + }); + worker.kill('SIGTERM'); + }); + }); + compiler.hooks.afterEmit.tapAsync('ReloadServerPlugin', (compilation, callback) => { this.done = callback; this.workers.forEach((worker) => { diff --git a/packages/arui-scripts/test/persistent-cache-lifecycle.cjs b/packages/arui-scripts/test/persistent-cache-lifecycle.cjs new file mode 100644 index 00000000..aa36f879 --- /dev/null +++ b/packages/arui-scripts/test/persistent-cache-lifecycle.cjs @@ -0,0 +1,124 @@ +/* Run after building arui-scripts. Exercises the real parent/client/watch shutdown path. */ +/* eslint-disable @typescript-eslint/no-var-requires -- Standalone CommonJS integration fixture. */ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); + +const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'arui-cache-lifecycle-')); +const cli = path.resolve(__dirname, '../build/bin/index.js'); + +fs.mkdirSync(path.join(cwd, 'src/server'), { recursive: true }); +fs.writeFileSync( + path.join(cwd, 'package.json'), + JSON.stringify({ + name: 'cache-lifecycle-fixture', + version: '1.0.0', + aruiScripts: { persistentCache: true, clientServerPort: 0 }, + }), +); +fs.writeFileSync(path.join(cwd, 'src/index.js'), 'console.log("client lifecycle");'); +fs.writeFileSync( + path.join(cwd, 'src/server/index.js'), + 'console.log("server lifecycle"); setInterval(() => {}, 1000);', +); +fs.writeFileSync( + path.join(cwd, 'arui-scripts.overrides.js'), + "const fs = require('fs'); module.exports = { devServer(config) { return {...config, host:'127.0.0.1'}; }, rspack(config) { for (const c of Array.isArray(config) ? config : [config]) { c.plugins.push({ apply(compiler) { compiler.hooks.done.tap('LifecycleFixture', stats => fs.writeFileSync(c.target === 'node' ? 'server-stats.json' : 'client-stats.json', JSON.stringify(stats.toJson({all:false, logging:'verbose', loggingDebug:/rspack\\.persistentCache/})))); }}); } return config; }};", +); + +async function startAndStop(label) { + fs.rmSync(path.join(cwd, 'client-stats.json'), { force: true }); + fs.rmSync(path.join(cwd, 'server-stats.json'), { force: true }); + const child = spawn(process.execPath, [cli, 'start'], { + cwd, + env: { + ...process.env, + CI: 'false', + WATCHPACK_POLLING: '100', + ARUI_SCRIPTS_CACHE_DEBUG: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const completed = new Promise((resolve, reject) => { + child.on('error', reject); + child.on('close', (code, signal) => resolve({ code, signal })); + }); + const deadline = Date.now() + 30000; + + try { + while ( + !fs.existsSync(path.join(cwd, 'client-stats.json')) || + !fs.existsSync(path.join(cwd, 'server-stats.json')) + ) { + if (child.exitCode !== null || Date.now() > deadline) + throw new Error(`${label}: startup failed\n${output.slice(-5000)}`); + // eslint-disable-next-line no-await-in-loop -- Poll this process before sending its stop signal. + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + } + const timer = setTimeout(() => child.kill('SIGKILL'), 12000); + + child.kill('SIGTERM'); + const result = await completed; + + clearTimeout(timer); + assert.equal( + result.code, + 0, + `${label}: shutdown must wait for flush and exit successfully\n${output.slice(-5000)}`, + ); + const files = fs.readdirSync(path.join(cwd, '.cache/arui-scripts/rspack'), { + recursive: true, + }); + + assert.ok(files.some((file) => file.endsWith('.pack'))); + assert.ok( + !files.some((file) => file.endsWith('.writer.json')), + 'leases must be released after native close', + ); + const diagnostics = output + .split('\n') + .filter((line) => line.startsWith('[arui-scripts cache] {')) + .map((line) => JSON.parse(line.slice('[arui-scripts cache] '.length))); + + assert.equal( + diagnostics.filter((event) => event.event === 'close' && event.result === 'completed') + .length, + 2, + ); + if (label === 'warm') + assert.ok( + diagnostics.some( + (event) => event.event === 'compilation' && event.native?.snapshot === 'hit', + ), + 'warm dev must recover from disk', + ); + console.log( + `PASS: ${label} real dev startup, SIGTERM, server worker shutdown and cache flush`, + ); + } finally { + if (child.exitCode === null) child.kill('SIGKILL'); + } +} +(async () => { + try { + await startAndStop('cold'); + await startAndStop('warm'); + } finally { + fs.rmSync(cwd, { recursive: true, force: true }); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/arui-scripts/test/persistent-cache.cjs b/packages/arui-scripts/test/persistent-cache.cjs new file mode 100644 index 00000000..6e66c9b6 --- /dev/null +++ b/packages/arui-scripts/test/persistent-cache.cjs @@ -0,0 +1,528 @@ +/* eslint-disable @typescript-eslint/no-var-requires -- Run directly as CommonJS in Node. */ +/* Integration test: run after building arui-scripts. Every build uses a fresh process. */ +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +let cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'arui-persistent-cache-')); +const cli = path.resolve(__dirname, '../build/bin/index.js'); +const write = (name, content) => fs.writeFileSync(path.join(cwd, name), content); +const json = (name, content) => write(name, JSON.stringify(content)); +function linkTypeScript() { + fs.mkdirSync(path.join(cwd, 'node_modules'), { recursive: true }); + fs.symlinkSync( + path.dirname(require.resolve('typescript/package.json')), + path.join(cwd, 'node_modules/typescript'), + 'dir', + ); +} + +const payload = Array.from({ length: 1000 }, (_, i) => + crypto.createHash('sha256').update(String(i)).digest('hex'), +).join('-'); + +const source = (marker) => + `import "./fixture.css"; declare const AUDIT_MARKER: string; console.log(AUDIT_MARKER, ${JSON.stringify( + marker, + )}, ${JSON.stringify(payload)}); export {};`; + +function artifacts(directory = path.join(cwd, '.build'), prefix = '') { + return Object.fromEntries( + fs + .readdirSync(directory) + .sort() + .flatMap((name) => { + const file = path.join(directory, name); + const key = `${prefix}${name}`; + + if (fs.statSync(file).isDirectory()) + return Object.entries(artifacts(file, `${key}/`)); + if (!/\.(js|css|gz|br|dcb|json|map)$/.test(name) || name === 'env-config.json') + return []; + + return [ + [key, crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')], + ]; + }), + ); +} + +function build(label, env = {}) { + const started = Date.now(); + const result = spawnSync(process.execPath, [cli, 'build'], { + cwd, + env: { ...process.env, CI: 'false', ARUI_SCRIPTS_CACHE_DEBUG: '1', ...env }, + encoding: 'utf8', + timeout: 60000, + }); + + assert.equal( + result.status, + 0, + `${label}: ${result.error || ''}\n${result.stdout.slice(0, 6000)}\n${result.stderr.slice( + 0, + 3000, + )}`, + ); + const prefix = '[arui-scripts cache] '; + const diagnostics = []; + + for (const line of `${result.stdout}\n${result.stderr}`.split('\n')) { + if (line.startsWith(`${prefix}{`)) diagnostics.push(JSON.parse(line.slice(prefix.length))); + else if (line.includes(prefix)) console.log(line); + } + const stats = JSON.parse(fs.readFileSync(path.join(cwd, 'client-stats.json'), 'utf8')); + + console.log(`${label}: ${Date.now() - started} ms`); + + return { stats, output: artifacts(), diagnostics }; +} + +function logs(stats) { + return (stats.logging?.['rspack.persistentCache']?.entries || []) + .map((entry) => entry.message) + .join('\n'); +} + +function compressionStats(stats) { + const entries = stats.logging?.['arui-scripts.compressionCache']?.entries || []; + const message = entries.find((entry) => + entry.message.startsWith('Compression cache: '), + )?.message; + + return message ? JSON.parse(message.slice('Compression cache: '.length)) : undefined; +} + +try { + linkTypeScript(); + fs.mkdirSync(path.join(cwd, 'src/server'), { recursive: true }); + json('package.json', { + name: 'cache-fixture', + version: '1.0.0', + aruiScripts: { + persistentCache: { env: ['AUDIT_ENV'] }, + dictionaryCompression: { dictionaryPath: ['./dictionary.txt'] }, + }, + }); + json('tsconfig.base.json', { + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'bundler', + types: [], + skipLibCheck: true, + }, + }); + json('tsconfig.json', { extends: './tsconfig.base.json', include: ['src'] }); + json('build-values.json', { marker: 'CONFIG_A' }); + write('yarn.lock', '# fixture lock v1\n'); + write('.browserslistrc', 'Chrome 100\n'); + write('dictionary.txt', payload.slice(0, 20000)); + write('src/css.d.ts', 'declare module "*.css"; declare function require(name: string): any;'); + write('src/fixture.css', 'body { color: red; padding: 10px; }'); + // eslint-disable-next-line no-template-curly-in-string -- The runtime placeholder must stay literal. + write('env-config.json', '{"runtime":"${CACHE_RUNTIME}"}'); + write('src/index.ts', source('SOURCE_A')); + write('src/server/index.ts', 'console.log("server-cache"); export {};'); + write( + 'arui-scripts.overrides.js', + ` + const fs = require('fs'); + const values = require('./build-values.json'); + module.exports = { rspack(config) { + for (const c of Array.isArray(config) ? config : [config]) { + if (process.env.COMPRESSION_LEVEL) { + const gzip = c.plugins.find(plugin => plugin?.constructor.name === 'CompressionPlugin' && plugin.options.algorithm === 'gzip'); + if (gzip) gzip.options.compressionOptions.level = Number(process.env.COMPRESSION_LEVEL); + } + c.plugins.push({ apply(compiler) { + new compiler.webpack.DefinePlugin({ AUDIT_MARKER: JSON.stringify(values.marker + (process.env.AUDIT_ENV || '')) }).apply(compiler); + if (c.target !== 'node') compiler.hooks.thisCompilation.tap('FreshCompressionInput', compilation => { + compilation.hooks.processAssets.tap({name: 'FreshCompressionInput', stage: 2990}, () => { + compilation.emitAsset('compression-fixture.js', new compiler.webpack.sources.RawSource(${JSON.stringify( + JSON.stringify(payload), + )})); + }); + }); + compiler.hooks.done.tap('CacheTest', stats => fs.writeFileSync( + c.target === 'node' ? 'server-stats.json' : 'client-stats.json', + JSON.stringify(stats.toJson({ all: false, modules: true, cachedModules: true, logging: 'verbose', loggingDebug: /rspack\\.persistentCache|arui-scripts\\.compressionCache/ })) + )); + }}); + } + return config; + }}; + `, + ); + const cold = build('cold'); + const warm = build('warm'); + + assert.ok( + cold.diagnostics.some( + (event) => event.event === 'compilation' && event.native.snapshot === 'cold', + ), + ); + for (const result of [cold, warm]) { + assert.equal( + result.diagnostics.filter( + (event) => event.event === 'configured' && event.cacheType === 'persistent', + ).length, + 2, + ); + assert.equal( + result.diagnostics.filter( + (event) => event.event === 'close' && event.result === 'completed', + ).length, + 2, + ); + } + assert.equal( + warm.diagnostics.filter( + (event) => event.event === 'compilation' && event.native.snapshot === 'hit', + ).length, + 2, + ); + assert.ok( + warm.diagnostics.some( + (event) => event.compression?.hits >= 3 && event.compression.misses === 0, + ), + ); + const quiet = build('warm diagnostics disabled', { ARUI_SCRIPTS_CACHE_DEBUG: '0' }); + + assert.equal(quiet.diagnostics.length, 0); + assert.match(logs(quiet.stats), /snapshot restored with no changed dependencies/); + assert.deepEqual(quiet.output, warm.output); + + assert.match(logs(warm.stats), /make persistent cache recovery succeeded/); + assert.ok(compressionStats(cold.stats).misses >= 3, 'cold build runs all three compressors'); + assert.ok( + compressionStats(warm.stats).hits >= 3, + 'gzip, Brotli and DCB bytes survive process restart', + ); + assert.equal( + compressionStats(warm.stats).misses, + 0, + 'warm build does not recompress unchanged bytes', + ); + const nativeOnly = build('native cache without compression cache', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: { compression: false } }), + }); + + assert.equal(compressionStats(nativeOnly.stats), undefined); + assert.deepEqual(nativeOnly.output, warm.output); + const serverStats = JSON.parse(fs.readFileSync(path.join(cwd, 'server-stats.json'), 'utf8')); + + assert.match(logs(serverStats), /make persistent cache recovery succeeded/); + const changedLevel = build('gzip level change', { COMPRESSION_LEVEL: '1' }); + + assert.ok(compressionStats(changedLevel.stats).misses > 0); + assert.ok(compressionStats(changedLevel.stats).hits >= 2, 'Brotli and DCB remain reusable'); + assert.notDeepEqual(changedLevel.output, warm.output); + const changedLevelWarm = build('gzip level change warm', { COMPRESSION_LEVEL: '1' }); + + assert.equal(compressionStats(changedLevelWarm.stats).misses, 0); + assert.deepEqual(changedLevelWarm.output, changedLevel.output); + assert.deepEqual( + build('gzip level change uncached', { + COMPRESSION_LEVEL: '1', + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: false }), + }).output, + changedLevel.output, + ); + assert.deepEqual( + warm.output, + cold.output, + 'all emitted artifacts, including DCB, survive cache recovery', + ); + assert.ok( + Object.keys(warm.output).some((file) => file.endsWith('.dcb')), + 'DCB fixture must actually emit a file', + ); + write('src/index.ts', source('SOURCE_B')); + const edited = build('source edit'); + + assert.notDeepEqual(edited.output, warm.output); + const uncached = build('source edit, cache disabled', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: false }), + }); + + assert.ok( + uncached.diagnostics.some( + (event) => event.event === 'configured' && event.reason === 'disabled-by-setting', + ), + ); + assert.ok( + uncached.diagnostics + .filter((event) => event.event === 'compilation') + .every((event) => event.native.snapshot === 'disabled'), + ); + + assert.deepEqual(edited.output, uncached.output); + + write('dictionary.txt', payload.slice(10000, 35000)); + const dictionary = build('dictionary edit'); + + assert.ok( + dictionary.diagnostics.some( + (event) => event.event === 'compilation' && event.native.snapshot === 'invalidated', + ), + ); + + assert.ok(compressionStats(dictionary.stats).misses > 0, 'new dictionary recompresses DCB'); + assert.ok( + compressionStats(dictionary.stats).hits >= 2, + 'unchanged gzip/Brotli bytes are reused', + ); + assert.notDeepEqual(dictionary.output, edited.output); + assert.deepEqual(build('dictionary warm').output, dictionary.output); + + json('build-values.json', { marker: 'CONFIG_B' }); + const configEdit = build('transitive override input'); + + assert.match(logs(configEdit.stats), /invalid|changed/i); + assert.notDeepEqual(configEdit.output, dictionary.output); + json('tsconfig.base.json', { + compilerOptions: { + target: 'ES2021', + module: 'ESNext', + moduleResolution: 'bundler', + types: [], + skipLibCheck: true, + }, + }); + assert.match(logs(build('extended tsconfig').stats), /invalid|changed/i); + write('yarn.lock', '# fixture lock v2\n'); + assert.match(logs(build('lockfile edit').stats), /invalid|changed/i); + write('.browserslistrc', 'Chrome 110\n'); + assert.match(logs(build('browserslist edit').stats), /invalid|changed/i); + assert.match( + logs(build('environment edit', { BROWSERSLIST: 'Chrome 120' }).stats), + /invalid|changed|version/i, + ); + + const declaredEnv = build('declared compile env', { AUDIT_ENV: 'ENV_B' }); + + assert.notDeepEqual(declaredEnv.output, configEdit.output); + build('compile env reset'); + const readonly = build('readonly existing', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: { readonly: true } }), + }); + + assert.match(logs(readonly.stats), /make persistent cache recovery succeeded/); + assert.equal(compressionStats(readonly.stats).writes, 0); + const readonlyMissing = build('readonly missing', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ + persistentCache: { readonly: true, directory: '.cache/readonly-missing' }, + }), + }); + + assert.deepEqual(readonlyMissing.output, readonly.output); + assert.equal(compressionStats(readonlyMissing.stats).writes, 0); + assert.ok( + !fs.existsSync(path.join(cwd, '.cache/readonly-missing')) || + !fs + .readdirSync(path.join(cwd, '.cache/readonly-missing'), { recursive: true }) + .some((name) => name.includes('compression-v1')), + 'readonly misses must not create compression records', + ); + + fs.mkdirSync(path.join(cwd, 'node_modules'), { recursive: true }); + fs.mkdirSync(path.join(cwd, 'linked-package'), { recursive: true }); + json('linked-package/package.json', { + name: 'cache-linked', + version: '1.0.0', + main: 'index.js', + }); + write('linked-package/index.js', 'module.exports = "LINK_A";'); + fs.symlinkSync(path.join(cwd, 'linked-package'), path.join(cwd, 'node_modules/cache-linked')); + const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'))); + + pkg.dependencies = { 'cache-linked': 'file:./linked-package' }; + json('package.json', pkg); + write('src/index.ts', `${source('SOURCE_LINK')}console.log(require("cache-linked"));`); + const linked = build('linked package cold'); + + assert.deepEqual(build('linked package warm').output, linked.output); + write('linked-package/index.js', 'module.exports = "LINK_B";'); + assert.notDeepEqual( + build('linked package source edit without version bump').output, + linked.output, + ); + + write('src/index.ts', `${source('SOURCE_TYPE_ERROR')}const bad: string = 123;`); + const failed = spawnSync(process.execPath, [cli, 'build'], { + cwd, + env: { ...process.env, CI: 'false' }, + encoding: 'utf8', + timeout: 60000, + }); + + assert.notEqual(failed.status, 0, 'type errors must still fail after cache restore'); + assert.match(failed.stdout + failed.stderr, /TS2322|string/); + write('src/index.ts', source('SOURCE_RECOVERED')); + build('after type error'); + + const originalCwd = cwd; + const portableEnv = { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: { portable: true } }), + }; + + build('portable cold', portableEnv); + const portableWarm = build('portable warm', portableEnv); + + assert.match(logs(portableWarm.stats), /make persistent cache recovery succeeded/); + const relocated = fs.mkdtempSync(path.join(os.tmpdir(), 'arui-cache-relocated-')); + + fs.cpSync(cwd, relocated, { + recursive: true, + filter: (file) => !file.includes('/node_modules') && !file.includes('/linked-package'), + }); + cwd = relocated; + try { + linkTypeScript(); + const moved = build('portable new checkout path', portableEnv); + + // Rspack 2.2.5 relocation guard: first build in a new checkout is cold and correct. + assert.match(logs(moved.stats), /invalid|version|changed/i); + const movedWarm = build('portable relocated warm', portableEnv); + + assert.match(logs(movedWarm.stats), /make persistent cache recovery succeeded/); + const movedFresh = build('portable relocated uncached parity', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: false }), + }); + + assert.deepEqual(moved.output, movedFresh.output); + } finally { + cwd = originalCwd; + fs.rmSync(relocated, { recursive: true, force: true }); + } + + const cacheRoot = path.join(cwd, '.cache/arui-scripts/rspack'); + const cacheFiles = fs + .readdirSync(cacheRoot, { recursive: true }) + .filter((file) => file.endsWith('.pack')); + + assert.ok(cacheFiles.length > 0); + cacheFiles.forEach((file) => fs.writeFileSync(path.join(cacheRoot, file), 'CORRUPT')); + const recovered = build('corrupt cache recovery'); + + assert.deepEqual( + recovered.output, + build('corrupt recovery uncached', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ persistentCache: false }), + }).output, + ); + + fs.mkdirSync(path.join(cwd, '.cache/unwritable'), { recursive: true }); + fs.chmodSync(path.join(cwd, '.cache/unwritable'), 0o555); + try { + const fallback = build('unwritable cache fallback', { + ARUI_SCRIPTS_CONFIG: JSON.stringify({ + persistentCache: { directory: '.cache/unwritable' }, + }), + }); + + assert.deepEqual(fallback.output, recovered.output); + assert.ok( + fallback.diagnostics.some( + (event) => + event.event === 'configured' && + event.reason === 'storage-unavailable' && + event.cacheType === 'disabled', + ), + ); + } finally { + fs.chmodSync(path.join(cwd, '.cache/unwritable'), 0o755); + } + + const overridesPath = path.join(cwd, 'arui-scripts.overrides.js'); + const overrideContents = fs.readFileSync(overridesPath, 'utf8'); + + fs.appendFileSync(overridesPath, '\nconsole.log("override diagnostic");\n'); + const info = spawnSync(process.execPath, [cli, 'cache:info', '--json'], { + cwd, + encoding: 'utf8', + }); + + assert.equal(info.status, 0, info.stderr); + const report = JSON.parse(info.stdout); + + assert.ok(report.bytes > 0 && report.namespaces.length >= 2); + assert.doesNotMatch(info.stdout, /ENV_B|runtime-other/); + assert.doesNotMatch(info.stdout, /override diagnostic/); + assert.match(info.stderr, /override diagnostic/); + fs.writeFileSync(overridesPath, overrideContents); + const clear = spawnSync(process.execPath, [cli, 'cache:clear'], { cwd, encoding: 'utf8' }); + + assert.equal(clear.status, 0, clear.stderr); + assert.ok(fs.existsSync(path.join(cwd, '.build')), 'clear must preserve build output'); + assert.ok( + !fs.readdirSync(cacheRoot, { recursive: true }).some((name) => name.endsWith('.bin')), + 'clear removes compressed bytes with native cache', + ); + const devProgram = ` + const { createCompiler } = require(${JSON.stringify( + path.resolve(__dirname, '../build/commands/util/create-compiler.js'), + )}); + const { config } = require(${JSON.stringify( + path.resolve(__dirname, '../build/configs/rspack.client.dev.js'), + )}); + const compiler = createCompiler(config); + compiler.run((error, stats) => { const failed = stats?.hasErrors(); compiler.close(closeError => { + if (error || closeError || failed) { + console.error(error || closeError || 'Compilation failed'); + process.exitCode = 1; + } + }); }); + `; + + for (const label of ['dev cold', 'dev warm']) { + fs.rmSync(path.join(cwd, '.build'), { recursive: true, force: true }); + const result = spawnSync(process.execPath, ['-e', devProgram], { + cwd, + env: { ...process.env, NODE_ENV: 'development' }, + encoding: 'utf8', + timeout: 60000, + }); + + assert.equal(result.status, 0, result.stderr); + if (label === 'dev warm') { + const stats = JSON.parse(fs.readFileSync(path.join(cwd, 'client-stats.json'), 'utf8')); + + assert.match(logs(stats), /make persistent cache recovery succeeded/); + } + console.log(`PASS: ${label}`); + } + for (const [label, value] of [ + ['runtime dev cold', 'runtime-new'], + ['runtime dev warm', 'runtime-other'], + ]) { + const result = spawnSync(process.execPath, ['-e', devProgram], { + cwd, + encoding: 'utf8', + timeout: 60000, + env: { + ...process.env, + NODE_ENV: 'development', + CACHE_RUNTIME: value, + ARUI_SCRIPTS_CONFIG: JSON.stringify({ clientOnly: true }), + }, + }); + + assert.equal(result.status, 0, result.stderr); + assert.equal( + JSON.parse(fs.readFileSync(path.join(cwd, '.build/env-config.json'))).runtime, + value, + ); + console.log(`PASS: ${label}`); + } + console.log( + 'PASS: disk reuse, client/server isolation, source/config/dictionary invalidation and artifact parity', + ); +} finally { + if (process.env.ARUI_CACHE_KEEP_FIXTURE) console.log(`Fixture: ${cwd}`); + else fs.rmSync(cwd, { recursive: true, force: true }); +} diff --git a/packages/arui-scripts/tsconfig.eslint.json b/packages/arui-scripts/tsconfig.eslint.json index 8f000311..79e8194a 100644 --- a/packages/arui-scripts/tsconfig.eslint.json +++ b/packages/arui-scripts/tsconfig.eslint.json @@ -1,5 +1,5 @@ { "extends": "./tsconfig.json", // для корректной работы @typescript-eslint/parser - "include": ["src", "./*.js", "./*.ts", ".eslintrc.js"] + "include": ["src", "test", "./*.js", "./*.ts", ".eslintrc.js"] } diff --git a/packages/example-modules/.prettierignore b/packages/example-modules/.prettierignore index 3b67c90d..cd957637 100644 --- a/packages/example-modules/.prettierignore +++ b/packages/example-modules/.prettierignore @@ -1,2 +1,3 @@ .turbo +.cache/arui-scripts/ .build diff --git a/packages/example/.prettierignore b/packages/example/.prettierignore index 3b67c90d..cd957637 100644 --- a/packages/example/.prettierignore +++ b/packages/example/.prettierignore @@ -1,2 +1,3 @@ .turbo +.cache/arui-scripts/ .build