diff --git a/.github/workflows/check-skip.yaml b/.github/workflows/check-skip.yaml new file mode 100644 index 0000000..6a15c5b --- /dev/null +++ b/.github/workflows/check-skip.yaml @@ -0,0 +1,40 @@ +name: Check for a skip tag + +# Called by the test workflows, to skip their tier if the last commit of a PR says to: +# [skip--tests], [no--tests], [skip-] or [no-], in any case, or +# [skip-tests] or [no-tests] for every tier. Pushes to main always run everything, so +# what's merged still gets tested. +on: + workflow_call: + inputs: + tier: + description: The test tier to check for, like "unit" + required: true + type: string + outputs: + skip: + description: '"true" if this tier should be skipped' + value: ${{ jobs.check.outputs.skip }} + +jobs: + check: + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + skip: ${{ steps.check.outputs.skip }} + steps: + # A pull_request event doesn't include the commit message, so look it up + - id: check + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + TIER: ${{ inputs.tier }} + run: | + message=$(gh api "repos/$REPOSITORY/commits/$SHA" --jq .commit.message) + if grep -qiE "\[(skip|no)-($TIER(-tests)?|tests)\]" <<< "$message"; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "Skipping the $TIER tests, as the commit message asks" + fi diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 0c4d70e..862a497 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,13 +1,28 @@ -name: CI +name: E2E tests on: push: branches: [ main ] + # Any base branch, so stacked PRs get CI too pull_request: - branches: [ main ] + +# A new push to a PR cancels that PR's runs still going for older commits. Pushes to +# main are never cancelled, so every merged commit gets a full run. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + check-skip: + uses: ./.github/workflows/check-skip.yaml + permissions: + contents: read + with: + tier: e2e + e2e: + needs: check-skip + if: needs.check-skip.outputs.skip != 'true' strategy: matrix: os: [ ubuntu-latest, windows-latest, macos-latest ] @@ -35,5 +50,7 @@ jobs: - name: Install dependencies run: uv sync - - name: Run tests with pytest - run: uv run pytest + # Coverage is skipped: the tests run the mkcd2app executable in a subprocess, + # which coverage doesn't measure + - name: Run e2e tests + run: uv run pytest -m e2e --no-cov diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 0000000..a3159e7 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,50 @@ +name: Integration tests + +on: + push: + branches: [ main ] + # Any base branch, so stacked PRs get CI too + pull_request: + +# A new push to a PR cancels that PR's runs still going for older commits. Pushes to +# main are never cancelled, so every merged commit gets a full run. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check-skip: + uses: ./.github/workflows/check-skip.yaml + permissions: + contents: read + with: + tier: integration + + integration: + needs: check-skip + if: needs.check-skip.outputs.skip != 'true' + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + version: "latest" + + - name: Set up Python + run: uv python install 3.14 + + - name: Install dependencies + run: uv sync + + # Offline and fast; npm and mkc are faked, so no Node.js or toolchain install + - name: Run integration tests with coverage + run: uv run pytest -m integration -n 4 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 7d70166..df7faa2 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,8 +3,14 @@ name: CI on: push: branches: [ main ] + # Any base branch, so stacked PRs get CI too pull_request: - branches: [ main ] + +# A new push to a PR cancels that PR's runs still going for older commits. Pushes to +# main are never cancelled, so every merged commit gets a full run. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: lint: diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml new file mode 100644 index 0000000..5d618fb --- /dev/null +++ b/.github/workflows/unit.yaml @@ -0,0 +1,50 @@ +name: Unit tests + +on: + push: + branches: [ main ] + # Any base branch, so stacked PRs get CI too + pull_request: + +# A new push to a PR cancels that PR's runs still going for older commits. Pushes to +# main are never cancelled, so every merged commit gets a full run. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check-skip: + uses: ./.github/workflows/check-skip.yaml + permissions: + contents: read + with: + tier: unit + + unit: + needs: check-skip + if: needs.check-skip.outputs.skip != 'true' + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + version: "latest" + + - name: Set up Python + run: uv python install 3.14 + + - name: Install dependencies + run: uv sync + + # Offline and fast; no Node.js or toolchain install needed + - name: Run unit tests with coverage + run: uv run pytest -m unit diff --git a/.gitignore b/.gitignore index dbe7f58..c253ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ __pycache__/ examples/* !examples/*.yaml !examples/*.png +tests/fixtures/build/ .idea +.coverage +.coverage.* diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..0cb85b5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,125 @@ +# AGENTS.md + +This file provides guidance to AI coding agents when working with code in this repository. + +Rewrite status, branch/PR stack, roadmap, and known bugs live in the project context +doc, [docs/PROJECT_CONTEXT.md](docs/PROJECT_CONTEXT.md). Read it before starting work. +Update that doc (and its "verified as of" line) rather than duplicating its content in +this file. + +## Commands + +Python ≥3.14 and Node.js (LTS) are required. Everything runs through uv. + +```bash +uv sync # install deps + the mkcd2app entry point +uv run ruff check . # lint +uv run ruff format --check . # format check (drop --check to apply) +uv run mypy . # strict type check +uv run pytest # all tests, with a coverage report +uv run pytest -m unit # only the fast offline tests (tests/unit) +uv run pytest -m integration -n 4 # only the offline scheduler tests (tests/integration) +uv run pytest -m e2e --no-cov # only the e2e tests (tests/e2e); these can't use -n +uv run pytest --no-cov # skip coverage (e.g. when using a debugger) +uv run pytest tests/e2e/test_statics.py::test_empty_game_build_to_statics_static +``` + +CI runs exactly these commands: `lint.yaml` the lint/format/mypy checks, and +`unit.yaml` (`-m unit`), `integration.yaml` (`-m integration -n 4`) and `e2e.yaml` +(`-m e2e --no-cov`) the tests, independently, on ubuntu/windows/macos. Only integration +uses xdist: the unit tests are no faster with it, and more than 4 workers didn't help +integration. `-n` isn't in `addopts`, since `--pdb` and `-s` don't work under xdist. +Every test must +be under `tests/unit`, `tests/integration` or `tests/e2e`, or collection fails, since a +test anywhere else would run in no workflow. Test file names must be unique across those +folders, since they have no `__init__.py`. Warnings are errors in pytest. +On a PR, `[skip--tests]`, `[no--tests]`, `[skip-]` or `[no-]` in +the last commit's message skips that tier (`unit`, `integration` or `e2e`; see +`check-skip.yaml`), `[skip-tests]` or `[no-tests]` skips all three, and GitHub's own +`[skip ci]` skips every workflow. Pushes to `main` ignore all but `[skip ci]`. +pre-commit runs ruff check, ruff format --check, and mypy on staged Python files. + +Running the tool end to end (installs must happen first, in this order): + +```bash +uv run mkcd2app toolchain install +uv run mkcd2app target install 4.0.14 +uv run mkcd2app --debug build "examples/Racers.yaml" +``` + +Notes on the tests: +- `tests/conftest.py` marks every test with its tier by its directory, and holds the + shared fixtures, such as `mkcd2app_home` (points `MKCD2APP_HOME` at a temp dir). Keep + it the only `conftest.py`: mypy fails on two modules both named `conftest`. +- Shared helpers (`make_config_yaml`, `FakeRunCmd`, `capture_logger`, ...) are in + `tests/helpers.py`, imported as `from helpers import ...`. + +Notes on the unit tests: +- Call a redun task's plain function with `.func(...)`, and monkeypatch `run_cmd` in + the module under test, so no `npm`/`mkc` runs. +- Our loggers set `propagate = False`, so `caplog` needs `capture_logger` to see them. + +Notes on the integration tests: +- They run the real `main()` and redun scheduler in this process, with `run_cmd` faked, + so no Node.js is needed. Use the `project` fixture (a game with a config and an + installed target) or `cli` (runs any command), which return what ran as a `Ran`. +- `FakeToolchain` does to the disk what each `mkc` and `npm` command would, and fails the + test on any other command, so a new command needs a new case there. +- `TaskSpy` records the tasks that really ran, since a cache hit doesn't call a task's + function. What each change should rerun is the `CHANGES` table in `test_rebuilds.py`. + +Notes on the e2e tests: +- Run pytest from the repo root: tests use repo-relative paths, and shell out to the + installed `mkcd2app` executable rather than importing it. +- They fail under xdist: each worker would run the session fixture, all installing into + the same dir at once, and repeat the module fixtures' builds. So deselect them to use + `-n` across tiers: `uv run pytest -m "not e2e" -n 4`. +- Builds go in temp folders, each with a copy of the game and config next to each other, + since a config's relative paths are relative to it. A full build takes about a minute, + so share one (module fixtures) where a test only needs to read the output. +- `--run-network` also runs the tests marked `network`, which need a public share link. + CI doesn't pass it. +- `tests/fixtures/block_network.cjs` is a Node preload that fails any network connection + and logs it, for testing offline builds, as Node ignores `HTTP_PROXY` by default. + +## Architecture + +Every unit of work is a redun `@task(namespace="mkcd2app")`. `main.py` parses the CLI +and hands a task expression to `run_redun_task()` with the SQLite DB for that command. +redun builds the DAG from how task results feed into other tasks, so data dependencies +between tasks must go through task arguments/returns, not side channels. + +Build pipeline (`build_project/__init__.py`), each step a task: +`fetch_code` → `build_binary_js` (runs `mkc` in its own copy of the source) + +`copy_support_files` (simulator HTML from the target dir, icon) + +`copy_website_template` → `fill_website_template`, plus `install_website_deps` +(`npm ci` from only the template's `package.json` and lockfile) → one `build_website*` +task per entry in `outputs`. A new output type means a new `case` arm in that `match` +plus a field on `BuildProjectResult`; until then `electron` and `tauri` raise +`NotImplementedError`. + +Relative paths in a config are relative to the config file. `main.py` makes them +absolute with `resolve_config_paths` before any task runs. + +Conventions that matter for redun hashing: +- `build_project` takes the whole config as JSON (`model_dump_json`) and passes each + task only what it uses, as strings (models dumped to JSON). Every argument is part of + a task's cache key, so an unused one causes needless reruns. Never pass pydantic + models as task arguments. +- Each stage writes to a fresh directory (named via `_website_dir` in `website.py`) and + returns a `ContentDir`/`ContentFile`; never modify a path another task returned. +- `mkcd2app/__init__.py` imports `_redun_compat` first, which monkey-patches redun and + asserts the pinned version. Changing the redun version means re-checking those patches. + +Paths under the user data/state dirs (toolchain, `js_tools/node_modules/.bin` for `mkc`, +npm cache, per-version target dirs, redun DBs) all come from `utils/paths.py`; use those +helpers instead of building paths by hand. Bundled templates and fixtures under +`resources/` are reached through `utils/resources.py`. + +## Claude Code specific instructions + +When you create a GitHub issue, also add the `found by claude` label (it already exists). + +## ChatGPT Codex specific instructions + +When you create a GitHub issue, also add the `found by chatgpt` label (it already exists). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2c5817d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,11 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Instructions for any AI agent go in `AGENTS.md`, and project status in +`docs/PROJECT_CONTEXT.md`, both imported below. Only add instructions specific to Claude +Code here. + +@AGENTS.md + +@docs/PROJECT_CONTEXT.md diff --git a/docs/PROJECT_CONTEXT.md b/docs/PROJECT_CONTEXT.md new file mode 100644 index 0000000..5e86419 --- /dev/null +++ b/docs/PROJECT_CONTEXT.md @@ -0,0 +1,230 @@ +# MakeCode-Arcade-to-App Rewrite — Project Context + +_Verified as of 2026-11-07, against `toolchain` @ `3bf8a76`. Keep this line to +the latest check, and note any test tier that wasn't run or any OS it wasn't +run on._ + +Commands, test conventions and the build pipeline are in `AGENTS.md`. This doc +holds what the code can't tell you: status, decisions, plans and open problems. +Fixed bugs don't belong here; their reasoning goes in code comments and +regression tests. + +## Goal + +Rewrite `UnsignedArduino/MakeCode-Arcade-to-App` (converts MakeCode Arcade +games into a static website, Electron app, or Tauri app). The original was +basically a bash script zwritten in Python. The rewrite turns it into an +incremental build system that runs in CI and copes with churn in the JS +ecosystem. + +## Branches / PRs + +``` +main (legacy) ← PR #6 `dag` (draft) ← PR #7 `toolchain` (draft, base = dag) +``` + +- The work is on `toolchain`. +- `main` has 2 legacy-only fixes (`d17b052`, `d31e2a5`) to files the rewrite + deleted. Merging gives 2 modify/delete conflicts; resolve them by keeping the + deletions. +- The legacy Electron/Tauri code to use as reference is on **`main`** + (`src/convert/website_to_{electron,tauri}/`, + `src/templates/{electron,tauri}_files/`). The `electron-output`/ + `tauri-output` branches are merged and out of date. The legacy code can't be + ported as is; it has to be reimplemented on the new pipeline. + +## Status + +- **Done:** `static` and `static-singlefile` outputs; all code sources + (`share_link`, + `git`, `github`, `path`, and `file` for a PNG, UF2 or ZIP); the + toolchain/target split. +- **Not implemented:** Electron and Tauri. Their `case` arm in `build_project` + raises + `NotImplementedError` before anything runs. +- **Stubs:** `toolchain status` and `target list`. + +## Layout + +In `src/mkcd2app/`: + +| Path | Purpose | +|-------------------------|----------------------------------------------------------| +| `main.py`, `cli.py` | argparse CLI → redun tasks | +| `_redun_compat.py` | redun monkey-patches | +| `config/`, `models/` | YAML loader; pydantic config and metadata models | +| `build_project/` | `build_project` task, website tasks, `inputs/` (sources) | +| `toolchain/`, `target/` | install/uninstall tasks | +| `utils/` | `run_cmd`, paths, `run_redun_task`, Dulwich `git`, etc. | +| `resources/` | Vite template, `mkc` js_tools, empty project | + +Tests are in `tests/{unit,integration,e2e}`, and an example config is in +`examples/`. + +## Commands + +Run the installs first, in this order: + +```bash +uv sync +uv run mkcd2app toolchain install +uv run mkcd2app target install 4.0.14 +uv run mkcd2app --debug build "examples/Racers.yaml" # add --full-rebuild to start over +uv run mkcd2app target uninstall 4.0.14 +uv run mkcd2app toolchain uninstall +``` + +Checks: + +```bash +uv run ruff check . +uv run ruff format --check . +uv run mypy . +uv run pytest -m unit +uv run pytest -m integration +uv run pytest -m e2e --no-cov +``` + +## Decisions + +- **Caching:** redun's cache is on, and each command has its own SQLite DB + (toolchain, target, or `build_dir/.redun-cache.db`). What each kind of edit + should rerun is the + `CHANGES` table in `tests/integration/test_rebuilds.py`. +- **Out-of-band inputs aren't tracked:** redun can't see changes to a `path` or + `file` + code source, a `git`/`github` branch checkout (or omitted `ref`), an icon + path/URL, or the installed simulator HTML. Editing a path-sourced game + returns the old cached build until `--full-rebuild`. We decided to rely on + that flag for all sources rather than track each one. +- **Website builds are offline:** `npm ci --cache --offline`. + Everything that needs the network is vendored at install time **except + MakeCode extensions**, which are fetched per game. The npm cache holds + platform-specific binaries, so it only works on the OS/arch that ran + `toolchain install`. +- **Uninstalls are never cached:** they return `None`, so a cached result + couldn't notice a reinstall. Install tasks return `ContentDir`/`ContentFile`, + which redun re-validates, so they stay cached. +- **State dirs** are namespaced by mkcd2app version (static `0.1.0`), so a + version bump means re-running both installs. `MKCD2APP_HOME` moves them, but + not `mkc`'s own compiler/extension caches. +- **`github` source** is an older alias of `git`, kept for existing configs. + +## External programs + +`run_cmd` only ever runs `npm` and `mkc`. + +- **Node.js ≥ 22.12 + npm, on PATH, always required.** The floor comes from + `@rolldown/plugin-babel`; vite 8 / rolldown accept `^20.19 || >=22.12`, and + `makecode` + only needs ≥14. `mkc` is a Node script, so `node` must be on PATH too. + Nothing declares or checks this yet. Distro Node packages (e.g. Ubuntu apt) + are too old. +- **Not needed:** git (Dulwich handles clones; an `ssh://` URL needs an `ssh` + binary), system Python (uv downloads 3.14), a C compiler (all Python deps + have wheels except + `redun`, a pure-Python sdist), npm native build tools (rolldown/lightningcss + ship prebuilt binaries). + +## Roadmap + +1. Electron, then Tauri (reference: `main`). +2. Save-state for Electron/Tauri (investigate `pxtsim` postMessage). +3. Toolchain polish: `status`/`list`, `build` pre-flight, extension cache, + version-pinning UX, and cutting external dependencies so that "only uv + installed" works for static and Electron outputs: + - **Node: managed by default, system as an opt-in.** `toolchain install` + downloads the official archive + (`nodejs.org/dist/v/node-v--.{zip|tar.xz}`), verifies + it against `SHASUMS256.txt`, and extracts it to `/node//` + (`.zip` on Windows with binaries at the root; `.tar.xz` on macOS/Linux + with binaries in `bin/`, keeping symlinks and exec bits; musl needs the + unofficial builds). It's a redun task keyed on the version and removed by + `toolchain uninstall`. Nothing is added to the user's PATH. + - Fallback idea: depend on the `nodejs-wheel` PyPI package (official + binaries as wheels) instead of writing download code, at the cost of + not choosing the Node version. + - **Opting into a global Node/Rust:** a per-tool source of + `managed | system | `, set with + `toolchain install --node=… --rust=…` (persisted in the toolchain state + so + `build` uses the same choice), overridable by `MKCD2APP_NODE` / + `MKCD2APP_RUST` env vars (useful in CI, e.g. after `actions/setup-node`). + `system` resolves from the user's PATH; `` points at a specific + install dir. In both, mkcd2app checks the version (Node ≥ 22.12; Rust per + Tauri's MSRV) and fails with a clear message instead of downloading. + `toolchain status` reports which source and version are active. + - **`run_cmd` gets an `env=`**, built by one helper that prepends the + selected Node/cargo bin dirs to the child's `PATH` (npm and the `mkc` + shim run `node` from PATH, so `which_path` alone isn't enough) and + redirects caches into mkcd2app dirs: + `npm_config_cache`, and later `ELECTRON_CACHE` / `electron_config_cache` + and, for a managed Rust, `RUSTUP_HOME` / `CARGO_HOME`. + - **Rust: system by default, managed as an opt-in.** Tauri also needs OS + packages that need an admin install (MSVC Build Tools + WebView2 on + Windows, Xcode CLT on macOS, + `webkit2gtk-4.1`/`libssl-dev` etc. on Linux), so a private rustup alone + doesn't give + "only uv". `--rust=managed` bootstraps rustup into mkcd2app's own + `RUSTUP_HOME`/`CARGO_HOME`. Pre-flight checks list the missing OS + packages with the install command for that OS. Tauri only builds for the + host OS. + - **Electron:** `toolchain install` pre-fetches the Electron (and + electron-builder helper) binaries into a redirected cache so the offline + `npm ci` still works. Cross-building, e.g. Windows targets from + macOS/Linux, needs Wine; out of scope. + - **Private git repos:** need credentials handled by us (no system + credential helper). A token embedded in the URL would work but gets + logged and written to the clone's + `.git/config`. pygit2 (libgit2 bundled in its wheels) is the compiled + alternative. Until then, a ZIP through the `file` source is the way to + build a private repo. + - **Distribution:** publish to PyPI so `uvx mkcd2app` / + `uv tool install mkcd2app` is the install story. A PyInstaller build (one + per OS/arch; `--collect-data mkcd2app`; check that redun works when + frozen; unsigned builds hit Gatekeeper/SmartScreen) is optional and + doesn't remove the Tauri OS prerequisites. + - **Declare the Node floor:** add `"engines": {"node": ">=22.12"}` to the + template's + `package.json` and check it in pre-flight. +4. Later: code signing, auto-update, README (currently a stub; should cover + `--full-rebuild`), schema docs. + +## Known issues + +- **Little pre-flight checking in `build`:** `build_project` raises if the + target isn't installed, naming the install command, but a missing toolchain + only fails with + "Executable 'mkc' not found". +- **`mkc` checks for updates once a day:** it tries `arcade.makecode.com` if + its last check (in `~/.pxt/mkc-cache`, which isn't redirected) is over 24 + hours old. Offline, that fails, and `mkc` logs it and builds from its cache, + so builds still work. +- **The `static` output needs a web server:** its `index.html` loads module + scripts, which browsers don't normally allow from `file://` (not tested + here). `static-singlefile` is the one meant to be opened directly. +- **Files inside `build_dir` aren't all protected:** a `path` source + overlapping + `build_dir` is a config error, but a `file` source or `path` icon inside it + isn't checked, so a build can still delete it. A `path` source next to + `build_dir` still copies its own `built/`, `node_modules/` and + `pxt_modules/`. +- **`path_friendly_name`** still accepts npm's blocked names, `node_modules` + and + `favicon.ico`. +- **`run_cmd` limits:** a failed command's output is logged about 80 lines + above the end of stderr, followed by redun's status table and tracebacks, and + the final `BuildError` line doesn't point back to it. Output also isn't + streamed, so a long `npm ci` shows no progress, and there's no timeout. +- **Test gaps:** + - Nothing checks that a built site really runs. A Playwright smoke test (no + page errors, simulator iframe present) was left out, as it adds a browser + install to CI. + - Integration tests fake `run_cmd`, so its logging is only covered by + `tests/unit/test_run.py`. + - The symlink integration test skips on Windows, so only CI runs it. + - The `share_link` e2e test needs `--run-network` (about 7 minutes), so CI + skips it. + - `Racers-master.zip` and the falling-duck exports aren't in e2e: Racers + pulls GitHub extensions, and the duck was saved from 4.1.27 (it builds on + 4.0.14 with the version-mismatch warning). diff --git a/examples/Racers offline.yaml b/examples/Racers offline.yaml new file mode 100644 index 0000000..1defa59 --- /dev/null +++ b/examples/Racers offline.yaml @@ -0,0 +1,60 @@ +# Example configuration file for MakeCode Arcade to App to compile to static +# HTML, CSS, and JS files + +# Config version (single number, incremented if breaking change to schema is +# made) +version: 1 + +project: + name: Racers + path_friendly_name: racers + description: "Enjoy the high-speed thrills of car racing in MakeCode Arcade! + For the MakeCode Arcade Mini Game Jam #3." + author: Cyrus Yiu + version: 1.3.2 + # This is what the window title will be + # This will also be used for the executable file name for Electron outputs + # You can use {NAME} or {VERSION} or {AUTHOR} to substitute the correct + # values + title: "{NAME} v{VERSION}" + +inputs: + # Can be share_link, git, path, or file (a PNG or UF2 downloaded from the + # editor, or a ZIP of the project such as GitHub's "Download ZIP"). github is + # an older alias for git. + code: + type: path + value: "E:/Racers" + + assets: + # Icon is optional but highly recommended, it will be used as the favicon + # and app icon + icon: + type: path + value: "E:/Racers/Racers icon.png" + +# Path where the build process will take place +# Relative paths in this file are relative to this file's directory +build_dir: "./Racers-offline-build" + +# The MakeCode Arcade version to target +# Must be explicit 3 number sem ver, do not add a "v" at the beginning +# To find it, go to https://arcade.makecode.com/ and click on the settings +# cogwheel button, click About..., and see "arcade version: x.y.z" +target: 2.0.48 + +# You can have multiple outputs +# Currently available options are static, static-singlefile, electron, and +# tauri +outputs: + - type: static + - type: static-singlefile +# - type: electron +# window: +# width: 640 +# height: 480 +# - type: tauri +# identifier: com.unsignedarduino.racers +# window: +# width: 640 +# height: 480 diff --git a/examples/Racers.yaml b/examples/Racers.yaml index 841ee2d..138b214 100644 --- a/examples/Racers.yaml +++ b/examples/Racers.yaml @@ -19,20 +19,28 @@ project: title: "{NAME} v{VERSION}" inputs: - # Can be share_link, github, or path + # Can be share_link, git, path, or file (a PNG or UF2 downloaded from the + # editor, or a ZIP of the project such as GitHub's "Download ZIP"). github is + # an older alias for git. code: type: share_link value: https://arcade.makecode.com/84426-33815-03715-00484 # code: -# type: github -# value: https://github.com/UnsignedArduino/Racers -# checkout: master +# type: git +# url: https://github.com/UnsignedArduino/Racers +# # Branch, tag, or commit SHA. Optional, defaults to the default branch +# ref: master # code: # type: path # value: "E:/Racers" +# code: +# type: file +# # A PNG or UF2 downloaded from the editor, or a ZIP of the repo +# value: "E:/Downloads/Racers-master.zip" + assets: # Icon is optional but highly recommended, it will be used as the favicon # and app icon @@ -43,10 +51,18 @@ inputs: # icon: # type: path -# value: "./examples/Racers icon.png" +# value: "./Racers icon.png" # Path where the build process will take place -build_dir: "./examples/Racers-build" +# Relative paths in this file are relative to this file's directory +build_dir: "./Racers-build" + +# The MakeCode Arcade version to target +# Must be explicit 3 number sem ver, do not add a "v" at the beginning +# To find it, go to https://arcade.makecode.com/ and click on the settings +# cogwheel button, click About..., and see "arcade version: x.y.z" +# As of the time of writing this, the latest version is 4.0.14 +target: 4.0.14 # You can have multiple outputs # Currently available options are static, static-singlefile, electron, and diff --git a/pyproject.toml b/pyproject.toml index eea8454..f42bae0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ dependencies = [ "requests", "beautifulsoup4", "pillow", + "platformdirs>=4.11.0", + "dulwich>=1.2.20", ] [project.scripts] @@ -29,7 +31,9 @@ dev = [ "mypy>=2.3.0", "pre-commit>=4.6.1", "pytest>=9.1.1", + "pytest-cov>=7.1.0", "pytest-timeout>=2.4.0", + "pytest-xdist>=3.8.0", "ruff>=0.16.0", "types-pyyaml>=6.0.12.20260724", ] @@ -40,3 +44,19 @@ target-version = "py314" [tool.mypy] python_version = "3.14" strict = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +# Coverage is on by default; pass --no-cov to turn it off (e.g. when debugging) +addopts = ["--strict-markers", "--cov", "--cov-report=term-missing"] +filterwarnings = ["error"] +# Applied by directory in tests/conftest.py, so tests don't need to mark themselves +markers = [ + "unit: fast, offline tests under tests/unit", + "integration: fast, offline tests under tests/integration that run the real redun scheduler with fake commands", + "e2e: slow tests under tests/e2e that install the toolchain and build real games", + "network: also needs a public game or service; skipped unless --run-network is passed", +] + +[tool.coverage.run] +source = ["mkcd2app"] diff --git a/src/mkcd2app/_redun_compat.py b/src/mkcd2app/_redun_compat.py index f12a059..274c8b0 100644 --- a/src/mkcd2app/_redun_compat.py +++ b/src/mkcd2app/_redun_compat.py @@ -1,5 +1,6 @@ import redun.file -from redun.file import ContentDir, Dir, File +from redun.file import ContentDir, ContentFile, Dir, File, RedunFileNotFoundError +from redun.hashing import hash_struct # Monkey-patch redun.file.get_proto so Windows absolute paths like "E:\..." # aren't parsed as having scheme "e" by urllib.parse.urlparse. @@ -29,7 +30,6 @@ def _patched_get_proto(url: str | None = None) -> str: def _content_dir_calc_hash(self: Dir, files: list[File] | None = None) -> str: if files is None: files = list(self) - from redun.hashing import hash_struct return hash_struct([self.type_basename, self.path] + sorted(f.hash for f in files)) @@ -49,6 +49,22 @@ def _dir_calc_hash(self: Dir, files: list[File] | None = None) -> str: redun.file.Dir._calc_hash = _dir_calc_hash # type: ignore[method-assign] +# Monkey patch ContentFile so a deleted file counts as changed. redun raises from its +# cache check instead, so deleting a task's output would crash the next run rather +# than rerun the task. (File and ContentDir already handle this.) +_orig_content_file_calc_hash = redun.file.ContentFile._calc_hash + + +def _content_file_calc_hash(self: ContentFile) -> str: + try: + return _orig_content_file_calc_hash(self) + except RedunFileNotFoundError: + return hash_struct([self.type_basename, self.path, "missing"]) + + +redun.file.ContentFile._calc_hash = _content_file_calc_hash # type: ignore[method-assign] + + def assert_pinned_redun_version() -> None: assert redun.__version__ == "0.44.1", ( "compat patches assume this exact redun version; verify before bumping" diff --git a/src/mkcd2app/build_project/__init__.py b/src/mkcd2app/build_project/__init__.py index 84c0c52..b92d974 100644 --- a/src/mkcd2app/build_project/__init__.py +++ b/src/mkcd2app/build_project/__init__.py @@ -1,60 +1,39 @@ import logging -import shutil from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path +from typing import assert_never from redun import task from redun.file import ContentDir, ContentFile from mkcd2app.build_project.inputs.code import ( build_binary_js, - download_and_mod_supporting_files, + copy_support_files, fetch_code, ) from mkcd2app.build_project.website import ( + _website_dir, + build_website, + build_website_singlefile, copy_website_template, fill_website_template, - install_deps_and_build_website, - install_deps_and_build_website_singlefile, + install_website_deps, +) +from mkcd2app.models.config import ( + BuildConfig, + ElectronOutput, + StaticOutput, + StaticSinglefileOutput, + TauriOutput, ) -from mkcd2app.config import load_config_from_yaml -from mkcd2app.config.model import StaticOutput, StaticSinglefileOutput from mkcd2app.utils.logger import create_logger -from mkcd2app.utils.resources import get_js_tools_path, get_template_path -from mkcd2app.utils.run import run_cmd +from mkcd2app.utils.paths import get_sim_html_path +from mkcd2app.utils.resources import get_resource_template_path logger = create_logger(name=__name__, level=logging.INFO) -@task(namespace="mkcd2app") -def install_mkcd_build_tools(config_yaml: str, js_tools_src: ContentDir) -> ContentDir: - """ - Installs the MakeCode Arcade build tools. - - :param config_yaml: The raw YAML text of the config file. - :param js_tools_src: A redun.ContentDir pointing to the js_tools directory, - so that redun tracks changes to package.json etc. - :return: A redun.ContentDir that points to node_modules, this is only used so that - redun will see that some tasks depend on `mkc` being installed. - """ - logger.info("Installing MakeCode Arcade build tools") - - config = load_config_from_yaml(config_yaml) - build_path = Path(config.build_dir) - logger.debug(f"Tools will be installed in {build_path}") - - js_tools_path = Path(js_tools_src.path) - shutil.copy(js_tools_path / "package.json", build_path / "package.json") - shutil.copy(js_tools_path / "package-lock.json", build_path / "package-lock.json") - - run_cmd(["npm", "ci"], cwd=build_path) - - logger.debug("All MakeCode Arcade build tools installed") - - return ContentDir(str(build_path / "node_modules")) - - @dataclass class BuildProjectResult: static: ContentDir | None = None @@ -64,58 +43,92 @@ class BuildProjectResult: @task(namespace="mkcd2app") -def build_project(config_yaml: str) -> BuildProjectResult: +def build_project(config_json: str) -> BuildProjectResult: """ Build the project. - :param config_yaml: The raw YAML text of the config file. + Each subtask gets only the parts of the config it uses, as strings, so editing + one part of the config only reruns the tasks that depend on it. + + :param config_json: The BuildConfig, with absolute paths (see + `resolve_config_paths`), dumped to a JSON string. :return BuildProjectResult: A BuildProjectResult object containing paths. """ logger.info("Building project") - config = load_config_from_yaml(config_yaml) + config = BuildConfig.model_validate_json(config_json) build_dir = Path(config.build_dir) + name = config.project.path_friendly_name logger.debug(f"Project build directory is at {build_dir}") build_dir.mkdir(parents=True, exist_ok=True) - with ExitStack() as stack: - js_tools_path = stack.enter_context(get_js_tools_path()) - js_tools_content = ContentDir(str(js_tools_path)) + if not get_sim_html_path(config.target).is_file(): + raise FileNotFoundError( + f"MakeCode Arcade target {config.target} is not installed, install it " + f"with `mkcd2app target install {config.target}`" + ) - template_path = stack.enter_context(get_template_path("vite-project")) - template_content = ContentDir(str(template_path)) + icon = config.inputs.assets.icon - # Install `mkc` with `npm ci` in build dir - node_modules_for_mkc = install_mkcd_build_tools(config_yaml, js_tools_content) - # Fetch game source code with `mkc`, `git`, or copy from disk - code_path = fetch_code(config_yaml, node_modules_for_mkc) - # Build binary.js with `mkc` - bin_js_path = build_binary_js(config_yaml, code_path) - # Download supporting files to run binary.js, including ---simulator.html and all - # it's references, and get favicon.ico if present - support_path = download_and_mod_supporting_files(config_yaml) + with ExitStack() as stack: + template_path = stack.enter_context(get_resource_template_path("vite-project")) + template_content = ContentDir(str(template_path)) + # Fetch game source code with `mkc`, dulwich (git), or copy from disk + code_path = fetch_code( + config.inputs.code.model_dump_json(), + config.target, + str(build_dir / f"{name}-source"), + ) + # Copy the source code + build binary.js with `mkc` + bin_js_path = build_binary_js( + code_path, str(build_dir / f"{name}-binary-js-build") + ) + # Copy ---simulator.html from target dir and get favicon.ico if present + support_path = copy_support_files( + config.target, + icon.model_dump_json() if icon else None, + str(build_dir / f"{name}-binary-js-support"), + ) # Copy website template (clean copy with template files only) - website_path = copy_website_template(config_yaml, template_content) + website_path = copy_website_template( + str(_website_dir(build_dir, name, "")), template_content + ) # Copy + fill (separate dir so stages don't mutate each other's # ContentDir/ContentPath website_filled_path = fill_website_template( - config_yaml, website_path, bin_js_path, support_path + config.project.model_dump_json(), website_path, bin_js_path, support_path + ) + # `npm ci` from only package.json + lockfile, so game/asset changes don't + # bust it; shared by every website output + node_modules = install_website_deps( + str(_website_dir(build_dir, name, "-deps")), + ContentFile(str(template_path / "package.json")), + ContentFile(str(template_path / "package-lock.json")), ) results = BuildProjectResult() - + # Build the website outputs + # Electron and Tauri outputs depend on static_singlefile so redun figures it out logger.debug(f"{config.outputs=}") for output in config.outputs: match output.root: case StaticOutput(): - # Copy + `npm ci` + `npm run build` (separate dir for same reason) - results.static = install_deps_and_build_website(website_filled_path) + # Copy + node_modules + `npm run build` (separate dir for same + # reason) + results.static = build_website(website_filled_path, node_modules) logger.debug("Will build to static website") case StaticSinglefileOutput(): - results.static_singlefile = ( - install_deps_and_build_website_singlefile(website_filled_path) + results.static_singlefile = build_website_singlefile( + website_filled_path, node_modules ) logger.debug("Will build to static single-file website") + case ElectronOutput() | TauriOutput(): + # Tasks are lazy, so nothing has run yet + raise NotImplementedError( + f"The {output.root.type} output is not implemented yet" + ) + case _: + assert_never(output.root) return results diff --git a/src/mkcd2app/build_project/inputs/code.py b/src/mkcd2app/build_project/inputs/code.py index 364cc82..e393f94 100644 --- a/src/mkcd2app/build_project/inputs/code.py +++ b/src/mkcd2app/build_project/inputs/code.py @@ -1,44 +1,63 @@ +import json import logging import shutil from io import BytesIO from pathlib import Path import requests -from bs4 import BeautifulSoup from PIL import Image from redun import task from redun.file import ContentDir, ContentFile -from mkcd2app.config import load_config_from_yaml -from mkcd2app.config.model import ( +from mkcd2app.build_project.inputs.embedded_source import extract_project +from mkcd2app.models.config import ( + AssetSource, + CodeSource, + FileCodeSource, + GitCodeSource, GitHubCodeSource, PathAssetSource, PathCodeSource, ShareLinkCodeSource, UrlAssetSource, ) +from mkcd2app.utils.filesystem import rmtree_robust +from mkcd2app.utils.git import clone_and_checkout from mkcd2app.utils.logger import create_logger -from mkcd2app.utils.paths import rmtree_robust +from mkcd2app.utils.paths import get_js_tools_bin_dir, get_sim_html_path from mkcd2app.utils.run import run_cmd logger = create_logger(name=__name__, level=logging.INFO) +def _warn_on_target_mismatch(pxt_json_path: Path, target: str) -> None: + """Warn if the project was saved with a different Arcade version than target.""" + try: + pxt_json = json.loads(pxt_json_path.read_text(encoding="utf-8")) + saved = pxt_json["targetVersions"]["target"] + except OSError, ValueError, KeyError, TypeError: + logger.debug(f"Could not read targetVersions.target from {pxt_json_path}") + return + if saved != target: + logger.warning( + f"Project was saved with Arcade {saved}, building against {target}" + ) + + @task(namespace="mkcd2app") -def fetch_code(config_yaml: str, node_modules_for_mkc: ContentDir) -> ContentDir: +def fetch_code(code_source_json: str, target: str, code_dir: str) -> ContentDir: """ Download/clone/copy the source code to the build directory. - Example input: config_yaml, ./node_modules + Example input: code_source_json, 4.0.14, ./racers-source Example output: ./racers-source - :param config_yaml: The raw YAML text of the config file. - :param node_modules_for_mkc: redun.ContentDir that points to the node_modules - directory, this ensures that this task depends on `mkc` being installed. + :param code_source_json: The config's CodeSource dumped to a JSON string. + :param target: The MakeCode Arcade version to target. E.g., "4.0.14". + :param code_dir: The directory to put the source code in. :return: A redun.ContentDir that points to the source code. """ - config = load_config_from_yaml(config_yaml) - code_path = Path(config.build_dir) / f"{config.project.path_friendly_name}-source" + code_path = Path(code_dir) logger.info(f"MakeCode Arcade source will be downloaded/copied to {code_path}") # Clean previous output, not wasteful because redun already checks our inputs @@ -46,142 +65,119 @@ def fetch_code(config_yaml: str, node_modules_for_mkc: ContentDir) -> ContentDir if code_path.exists(): rmtree_robust(code_path) - match config.inputs.code.root: + match CodeSource.model_validate_json(code_source_json).root: case ShareLinkCodeSource(value=url): - logger.debug(f"Downloading source code from {url}") + logger.debug(f"Downloading source code from {url} with `mkc` CLI") code_path.mkdir(parents=True) - logger.debug(f"Using `mkc` from {node_modules_for_mkc}") - run_cmd(["npx", "mkc", "download", str(url)], cwd=code_path) - case GitHubCodeSource(value=url, checkout=checkout_target): - logger.debug(f"Cloning source code from {url}@{checkout_target}") - abs_code_path = code_path.resolve() - abs_code_path.parent.mkdir(parents=True, exist_ok=True) run_cmd( - [ - "git", - "clone", - "--filter=blob:none", - "--no-checkout", - str(url), - str(abs_code_path), - ], - cwd=abs_code_path.parent, + ["mkc", "download", str(url)], + cwd=code_path, + which_path=get_js_tools_bin_dir(), ) - run_cmd(["git", "checkout", checkout_target], cwd=abs_code_path) + case ( + GitCodeSource(url=url, ref=ref) | GitHubCodeSource(value=url, checkout=ref) + ): + logger.debug(f"Cloning source code from {url}@{ref or 'default branch'}") + code_path.parent.mkdir(parents=True, exist_ok=True) + clone_and_checkout(str(url), code_path, ref) case PathCodeSource(value=path): logger.debug(f"Copying source code from {path}") shutil.copytree(path, code_path) + case FileCodeSource(value=path): + logger.debug(f"Extracting project from {path}") + src_file = Path(path) + if not src_file.is_file(): + raise FileNotFoundError(f"Code source file not found: {src_file}") + # Raises EmbeddedSourceError (with the file path) if there's no project + files = extract_project(src_file, code_path) + logger.debug(f"Extracted {len(files)} files: {', '.join(files)}") + + _warn_on_target_mismatch(code_path / "pxt.json", target) + + mkc_json_path = code_path / "mkc.json" + logger.debug("Writing mkc.json") + mkc_json = {"targetWebsite": f"https://arcade.makecode.com/v{target}"} + mkc_json_path.write_text(json.dumps(mkc_json), encoding="utf-8") logger.debug("Source code downloaded") return ContentDir(str(code_path)) @task(namespace="mkcd2app") -def build_binary_js(config_yaml: str, code_path: ContentDir) -> ContentFile: +def build_binary_js(code_path: ContentDir, build_dir: str) -> ContentFile: """ - Build the MakeCode Arcade binary.js file + Make a copy of the source code, then build the MakeCode Arcade binary.js file in it. + + We copy so that we never mutate the ContentDir that fetch_code owns. - Example input: config_yaml, ./racers-source - Example output: ./racers-binary-js/binary.js + Example input: ./racers-source, ./racers-binary-js-build + Example output: ./racers-binary-js-build/built/binary.js - :param config_yaml: The raw YAML text of the config file. :param code_path: A redun.ContentDir where the code is stored. + :param build_dir: The directory to copy the source code to and build in. :return: A redun.ContentFile that points to binary.js """ - config = load_config_from_yaml(config_yaml) - logger.info("Building MakeCode Arcade binary.js") - cwd = Path(code_path.path) + cwd = Path(build_dir) + # Robust because a git code source's copy has a .git directory + if cwd.exists(): + rmtree_robust(cwd) + shutil.copytree(code_path.path, cwd) + logger.debug(f"Building in cwd {cwd}") - run_cmd(["npx", "mkc", "build", "-j"], cwd=cwd) + run_cmd( + ["mkc", "build", "-j"], + cwd=cwd, + which_path=get_js_tools_bin_dir(), + ) bin_js_path = cwd / "built" / "binary.js" logger.debug(f"binary.js available at {bin_js_path}") - # Copy binary to a stable location outside code_dir so we don't - # pollute fetch_code's ContentDir hash on subsequent runs. - output_path = ( - cwd.parent / f"{config.project.path_friendly_name}-binary-js" / "binary.js" - ) - output_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(bin_js_path, output_path) - shutil.rmtree(cwd / "built") - - return ContentFile(str(output_path)) + return ContentFile(str(bin_js_path)) @task(namespace="mkcd2app") -def download_and_mod_supporting_files(config_yaml: str) -> ContentDir: +def copy_support_files( + target: str, icon_source_json: str | None, support_dir: str +) -> ContentDir: """ Download and modify all supporting files needed to run binary.js for the website - Example input: config_yaml + Example input: 4.0.14, icon_source_json, ./racers-binary-js-support Example output: ./racers-binary-js-support - :param config_yaml: The raw YAML text of the config file. + :param target: The MakeCode Arcade version to target. E.g., "4.0.14". + :param icon_source_json: The config's icon AssetSource dumped to a JSON string, or + None if there is no icon. + :param support_dir: The directory to put the files in. :return: redun.ContentDir that points to a directory of files that should be copied to the website's public folder as is alongside the binary.js file. """ - config = load_config_from_yaml(config_yaml) - support_path = ( - Path(config.build_dir) - / f"{config.project.path_friendly_name}-binary-js-support" + support_path = Path(support_dir) + logger.info( + f"Copying ---simulator.html for MakeCode Arcade {target} to {support_path}" ) - logger.info(f"Downloading supporting files to {support_path}") # Clean previous output, not wasteful because redun handles caching if support_path.exists(): shutil.rmtree(support_path) support_path.mkdir(parents=True) - - logger.debug("Downloading main simulator file") - res = requests.get("https://trg-arcade.userpxt.io/---simulator") - res.raise_for_status() - sim_html = res.text - - logger.debug( - f"Analyzing sim HTML ({len(sim_html)} chars) for required CSS and JS files" - ) - soup = BeautifulSoup(sim_html, features="html.parser") - css_links = soup.find_all("link", rel="stylesheet") - js_scripts = soup.find_all("script") - logger.debug(f"Found {len(css_links)} CSS links and {len(js_scripts)} JS scripts") - for css in css_links: - url = css.get("href") - if url: - logger.debug(f"Downloading CSS file {url}") - res = requests.get(str(url)) - res.raise_for_status() - style_tag = soup.new_tag("style") - style_tag.string = res.text - css.replace_with(style_tag) - logger.debug(f"Inlined CSS from {url}") - for js in js_scripts: - url = js.get("src") - if url: - logger.debug(f"Downloading JS file {url}") - res = requests.get(str(url)) - res.raise_for_status() - js.string = res.text - del js["src"] - logger.debug(f"Inlined JS from {url}") - new_sim_html = soup.prettify(formatter="html5") - path = support_path / "---simulator.html" - path.write_text(new_sim_html) - logger.debug(f"Wrote modified simulator HTML to {path}") - - if config.inputs.assets.icon: - match config.inputs.assets.icon.root: - case UrlAssetSource(value=url): # type: ignore[misc] - logger.debug(f"Downloading icon from {url}") - res = requests.get(str(url)) + # Only one file to copy + shutil.copy(get_sim_html_path(target), support_path) + + if icon_source_json is not None: + match AssetSource.model_validate_json(icon_source_json).root: + case UrlAssetSource(value=icon_url): + logger.debug(f"Downloading icon from {icon_url}") + res = requests.get(str(icon_url)) res.raise_for_status() buffer = BytesIO(res.content) im = Image.open(buffer) - case PathAssetSource(value=path): # type: ignore[misc] - logger.debug(f"Opening icon from {path}") - im = Image.open(path) + case PathAssetSource(value=icon_path): + logger.debug(f"Opening icon from {icon_path}") + im = Image.open(icon_path) favicon_path = support_path / "favicon.ico" logger.debug(f"Saving favicon to {favicon_path}") im.save(favicon_path) diff --git a/src/mkcd2app/build_project/inputs/embedded_source.py b/src/mkcd2app/build_project/inputs/embedded_source.py new file mode 100644 index 0000000..553615a --- /dev/null +++ b/src/mkcd2app/build_project/inputs/embedded_source.py @@ -0,0 +1,356 @@ +""" +Extract the MakeCode project from a downloaded file: a PNG ("shareable image") or +UF2 from the editor, or a ZIP of the project's repo (GitHub's "Download ZIP"). + +The PNG and UF2 decoders mirror pxt's: + PNG: pxtlib/util.ts decodeBlobAsync -> webapp/src/app.tsx importProjectCoreAsync + UF2: pxtlib/cpp.ts unpackSourceFromHexAsync / extractSourceFromBin +""" + +import json +import lzma +import stat +import struct +import zipfile +from collections.abc import Callable +from pathlib import Path, PurePosixPath + +from PIL import Image, UnidentifiedImageError + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +PNG_MAGIC = 0x59347A7D +PNG_HEADER_SIZE = 36 # 9 x u32 LE: magic, blob_len, added_lines, 6 x reserved + +UF2_MAGIC0 = 0x0A324655 +UF2_MAGIC1 = 0x9E5D5157 +UF2_FLAG_NOT_MAIN_FLASH = 0x00000001 +UF2_BLOCK_SIZE = 512 +UF2_MAX_PAYLOAD = 476 # 512 - 32 byte header - 4 byte end magic +# Blocks further apart than this start a new run instead of being zero-padded +# (pxt's UF2.toBin skips blocks after a gap over 1 MB) +UF2_MAX_GAP = 1024 * 1024 +# The flash images we read are a few MB; bounds the memory a crafted file can ask for +UF2_MAX_IMAGE_SIZE = 64 * 1024 * 1024 +SOURCE_MAGIC = bytes([0x41, 0x14, 0x0E, 0x2F, 0xB8, 0x2F, 0xA2, 0xBB]) + +ZIP_MAGIC = b"PK\x03\x04" +# Limits against zip bombs; a big MakeCode repo is a few MB uncompressed +ZIP_MAX_TOTAL_SIZE = 512 * 1024 * 1024 +ZIP_MAX_ENTRIES = 10_000 + + +class EmbeddedSourceError(ValueError): + """The file has no embedded MakeCode project, or it could not be decoded.""" + + +def _parse_files(text: str | bytes) -> dict[str, str]: + files = json.loads(text) + if not isinstance(files, dict) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in files.items() + ): + raise EmbeddedSourceError("Embedded project is not a {filename: text} map") + return files + + +def _cesu8_to_utf16(buf: bytes) -> bytes: + """ + Decode text written by LZMA-JS's string encoder to UTF-16-LE code units. + + LZMA-JS encodes each UTF-16 code unit on its own (CESU-8): characters outside + the BMP, like emoji, become two 3-byte surrogates that strict UTF-8 rejects, and + U+0000 becomes C0 80. Plain UTF-8 (including 4-byte sequences) decodes the same. + Returning code units keeps JavaScript string offsets (e.g. UF2 ``headerSize``) + valid as ``2 * offset``. + """ + # C0 is never valid UTF-8, so this can't touch anything else + text = buf.replace(b"\xc0\x80", b"\x00").decode("utf-8", "surrogatepass") + return text.encode("utf-16-le", "surrogatepass") + + +def _utf16_to_str(units: bytes) -> str: + """ + Turn UTF-16-LE code units into a str, joining surrogate pairs. + + :raises UnicodeDecodeError: On a lone surrogate, which can't be written as UTF-8. + """ + return units.decode("utf-16-le") + + +def _lzma_decompress(buf: bytes) -> bytes: + """Decompress an LZMA-JS payload to its text, as UTF-16-LE code units.""" + # pxt uses LZMA-JS, which writes the legacy ".lzma" (LZMA_Alone) container. + return _cesu8_to_utf16(lzma.decompress(buf, format=lzma.FORMAT_ALONE)) + + +# ---------------------------------------------------------------- PNG --------- + + +def _png_blob(path: Path) -> bytes: + img = Image.open(path).convert("RGBA") + width, height = img.size + d = img.tobytes() # flat R,G,B,A,R,G,B,A,... like canvas getImageData + + bpp = (d[0] & 1) | ((d[1] & 1) << 1) | ((d[2] & 1) << 2) + if bpp == 0 or bpp > 5: + raise EmbeddedSourceError("PNG does not contain an embedded MakeCode project") + + def decode(ptr: int, bits: int, length: int) -> tuple[bytes, int]: + out = bytearray() + shift = acc = 0 + mask = (1 << bits) - 1 + while len(out) < length: + acc |= (d[ptr] & mask) << shift + ptr += 1 + if ptr & 3 == 3: # skip alpha + ptr += 1 + shift += bits + if shift >= 8: + out.append(acc & 0xFF) + acc >>= 8 + shift -= 8 + return bytes(out), ptr + + header, ptr = decode(4, bpp, PNG_HEADER_SIZE) + magic, blob_len, added_lines = struct.unpack_from("<3I", header) + if magic != PNG_MAGIC: + raise EmbeddedSourceError("Invalid magic in encoded PNG") + + if added_lines == 0: + blob, _ = decode(ptr, bpp, blob_len) + return blob + + # Image was too small: the first chunk lives in the LSBs of the original + # pixels, the rest is stored as full bytes in extra rows appended at the end. + orig_pixels = (height - added_lines) * width + img_cap = ((orig_pixels - 1) * 3 * bpp) >> 3 + first, _ = decode(ptr, bpp, img_cap - PNG_HEADER_SIZE) + rest, _ = decode(orig_pixels * 4, 8, blob_len - len(first)) + return first + rest + + +def files_from_png(path: Path) -> dict[str, str]: + project = json.loads(_utf16_to_str(_lzma_decompress(_png_blob(path)))) + # {"meta": {...name, cloudId, targetVersions, editor}, "source": ""} + if not isinstance(project, dict) or not isinstance(project.get("source"), str): + raise EmbeddedSourceError("Embedded PNG project has no source") + return _parse_files(project["source"]) + + +# ---------------------------------------------------------------- UF2 --------- + + +def _uf2_runs(data: bytes) -> list[tuple[int, bytes]]: + """ + Reassemble a UF2's main-flash blocks into contiguous runs of memory. + + Blocks with small gaps between them are zero-padded into one run, like pxt's + ``UF2.toBin``; a bigger gap starts a new run, so far-apart addresses never turn + into one huge buffer. Everything is validated before anything is allocated. + + :return: (offset from the lowest address, bytes) for each run, in address order. + :raises EmbeddedSourceError: If there are no blocks, a block's payload doesn't fit + in it, or the runs add up to more than ``UF2_MAX_IMAGE_SIZE``. + """ + chunks: dict[int, bytes] = {} + for off in range(0, len(data) - UF2_BLOCK_SIZE + 1, UF2_BLOCK_SIZE): + m0, m1, flags, addr, size = struct.unpack_from("<5I", data, off) + if m0 != UF2_MAGIC0 or m1 != UF2_MAGIC1 or flags & UF2_FLAG_NOT_MAIN_FLASH: + continue + if size > UF2_MAX_PAYLOAD: + raise EmbeddedSourceError( + f"UF2 block at offset {off} declares a {size} byte payload, " + f"over the {UF2_MAX_PAYLOAD} bytes a block can hold" + ) + chunks[addr] = data[off + 32 : off + 32 + size] + if not chunks: + raise EmbeddedSourceError("Not a UF2 file") + + # Lay out the runs as (start, end, [(addr, chunk)]) before allocating + spans: list[tuple[int, int, list[tuple[int, bytes]]]] = [] + for addr in sorted(chunks): + chunk = chunks[addr] + if spans and addr - spans[-1][1] <= UF2_MAX_GAP: + start, end, members = spans[-1] + members.append((addr, chunk)) + spans[-1] = (start, max(end, addr + len(chunk)), members) + else: + spans.append((addr, addr + len(chunk), [(addr, chunk)])) + total = sum(end - start for start, end, _ in spans) + if total > UF2_MAX_IMAGE_SIZE: + raise EmbeddedSourceError( + f"UF2 covers {total} bytes of memory, " + f"over the {UF2_MAX_IMAGE_SIZE // 2**20} MB limit" + ) + + base = spans[0][0] + runs: list[tuple[int, bytes]] = [] + for start, end, members in spans: + buf = bytearray(end - start) + for addr, chunk in members: + buf[addr - start : addr - start + len(chunk)] = chunk + runs.append((start - base, bytes(buf))) + return runs + + +def _find_source(runs: list[tuple[int, bytes]]) -> tuple[bytes, int]: + """ + Find the 16-byte aligned source magic (alignment counted from the lowest + address, like pxt's search over the whole image). + + :return: The run holding it and the magic's position in that run. + """ + for run_offset, run in runs: + p = 0 + while (p := run.find(SOURCE_MAGIC, p)) >= 0: + if (run_offset + p) % 16 == 0: + return run, p + p += 1 + raise EmbeddedSourceError("This UF2 doesn't contain source (binary-only build?)") + + +def files_from_uf2(path: Path) -> dict[str, str]: + bin_, p = _find_source(_uf2_runs(path.read_bytes())) + meta_len, text_len = struct.unpack_from(" PurePosixPath: + """ + Find the folder holding pxt.json: the archive root, or the single top-level + folder that GitHub's "Download ZIP" wraps everything in (-/). + """ + paths = [PurePosixPath(n) for n in names] + if PurePosixPath("pxt.json") in paths: + return PurePosixPath() + tops = {p.parts[0] for p in paths if p.parts} + if len(tops) == 1: + (top,) = tops + if PurePosixPath(top, "pxt.json") in paths: + return PurePosixPath(top) + raise EmbeddedSourceError( + "ZIP has no pxt.json at its root or inside a single top-level folder" + ) + + +def files_from_zip(path: Path) -> dict[str, bytes]: + """Return the project's files as {relative path: bytes}, any wrapper removed.""" + with zipfile.ZipFile(path) as zf: + infos = zf.infolist() + if len(infos) > ZIP_MAX_ENTRIES: + raise EmbeddedSourceError(f"ZIP has more than {ZIP_MAX_ENTRIES} entries") + # Directories and symlinks (stored as small files with a link mode) are + # skipped; a MakeCode project has neither + members = [ + i + for i in infos + if not i.is_dir() and not stat.S_ISLNK(i.external_attr >> 16) + ] + # file_size is enforced while reading, so the header total is a real bound + if sum(i.file_size for i in members) > ZIP_MAX_TOTAL_SIZE: + raise EmbeddedSourceError( + f"ZIP is over {ZIP_MAX_TOTAL_SIZE // 2**20} MB uncompressed" + ) + root = _zip_project_root([i.filename for i in members]) + files: dict[str, bytes] = {} + for info in members: + rel = PurePosixPath(info.filename) + if root.parts: + if rel.parts[: len(root.parts)] != root.parts: + continue + rel = PurePosixPath(*rel.parts[len(root.parts) :]) + files[str(rel)] = zf.read(info) + return files + + +# ------------------------------------------------------------- public -------- + + +def _detect(src: Path) -> str: + """Tell PNG, UF2 and ZIP apart by their magic bytes, not the extension.""" + with src.open("rb") as f: + head = f.read(len(PNG_SIGNATURE)) + if head.startswith(PNG_SIGNATURE): + return "png" + if len(head) >= 4 and struct.unpack_from(" T: + """Run a decoder, turning low-level failures into EmbeddedSourceError.""" + try: + return decode(src) + except EmbeddedSourceError as e: + raise EmbeddedSourceError(f"{src}: {e}") from e + except ( + IndexError, # ran off the end of the pixel data + struct.error, # truncated UF2 source header + lzma.LZMAError, + UnicodeDecodeError, + json.JSONDecodeError, + UnidentifiedImageError, + zipfile.BadZipFile, + NotImplementedError, # ZIP compression method zipfile can't read + RuntimeError, # encrypted ZIP entry + ) as e: + raise EmbeddedSourceError( + f"{src}: embedded MakeCode project is corrupt ({e})" + ) from e + + +def read_embedded_files(src: Path) -> dict[str, str]: + """Return the project in a PNG or UF2 as {filename: text}.""" + match _detect(src): + case "png": + return _wrap_errors(src, files_from_png) + case "uf2": + return _wrap_errors(src, files_from_uf2) + case _: + raise EmbeddedSourceError(f"{src} is not a PNG or UF2") + + +def read_project_files(src: Path) -> dict[str, bytes]: + """Return the project in a PNG, UF2 or ZIP as {relative path: bytes}.""" + if _detect(src) == "zip": + return _wrap_errors(src, files_from_zip) + return {k: v.encode("utf-8") for k, v in read_embedded_files(src).items()} + + +def extract_project(src: Path, dest: Path) -> list[str]: + """Write the project's files (pxt.json, main.ts, ...) from src to dest.""" + files = read_project_files(src) + root = dest.resolve() + targets: dict[Path, bytes] = {} + # Check every name before writing anything, since they come from an untrusted file + for name, content in files.items(): + out = (root / name).resolve() + if root not in out.parents: + raise EmbeddedSourceError(f"Refusing to write outside {dest}: {name}") + targets[out] = content + dest.mkdir(parents=True, exist_ok=True) + for out, content in targets.items(): + out.parent.mkdir(parents=True, exist_ok=True) + # write_bytes so line endings are kept exactly as embedded on every OS + out.write_bytes(content) + return sorted(files) diff --git a/src/mkcd2app/build_project/website.py b/src/mkcd2app/build_project/website.py index 126865c..48e0d45 100644 --- a/src/mkcd2app/build_project/website.py +++ b/src/mkcd2app/build_project/website.py @@ -1,39 +1,113 @@ import base64 +import html import json import logging +import re import shutil from pathlib import Path from redun import task from redun.file import ContentDir, ContentFile -from mkcd2app.config import load_config_from_yaml +from mkcd2app.models.config import Project from mkcd2app.utils.logger import create_logger +from mkcd2app.utils.paths import get_templates_npm_cache_dir from mkcd2app.utils.run import run_cmd logger = create_logger(name=__name__, level=logging.INFO) +TITLE_PLACEHOLDER = re.compile(r"\{(NAME|VERSION|AUTHOR)\}") + +INDEX_HTML_TITLE_MARKER = "vite-project" +INDEX_HTML_FAVICON_MARKER = ( + '' +) + + +def render_title(project: Project) -> str: + """ + Substitute ``{NAME}``, ``{VERSION}`` and ``{AUTHOR}`` in ``project.title``. + + Done in a single pass, so substituted values are never expanded again. Any + other braces are kept as is (this is deliberately not ``str.format``, which + crashes on unknown fields and allows format specs and attribute access). + + :param project: The project metadata. + :return: The rendered (unescaped) title. + """ + values = { + "NAME": project.name, + "VERSION": project.version, + "AUTHOR": project.author, + } + return TITLE_PLACEHOLDER.sub(lambda m: values[m.group(1)], project.title) + + +def _replace_marker_once(text: str, marker: str, replacement: str) -> str: + """ + Replace ``marker`` in ``text``, requiring it to occur exactly once. + + :raises RuntimeError: If the marker is missing or occurs more than once, which + means the template has drifted from what the filler expects. + """ + count = text.count(marker) + if count != 1: + raise RuntimeError( + f"Template index.html must contain {marker!r} exactly once, " + f"found {count} occurrence(s)" + ) + return text.replace(marker, replacement) + + +def fill_index_html( + index_html_text: str, title: str, favicon: bytes | None = None +) -> str: + """ + Set the title and (optionally) inline the favicon in the template's index.html. + + :param index_html_text: The template's index.html. + :param title: The rendered, unescaped title. It is HTML-escaped here. + :param favicon: The favicon.ico bytes to inline as a data URI, or None to leave + the favicon link alone. + :return: The filled index.html. + :raises RuntimeError: If a marker the filler relies on is missing. + """ + index_html_text = _replace_marker_once( + index_html_text, + INDEX_HTML_TITLE_MARKER, + f"{html.escape(title, quote=False)}", + ) + if favicon is not None: + favicon_b64 = base64.b64encode(favicon).decode("ascii") + favicon_data_uri = f"data:image/x-icon;base64,{favicon_b64}" + index_html_text = _replace_marker_once( + index_html_text, + INDEX_HTML_FAVICON_MARKER, + f'', + ) + return index_html_text + + def _website_dir(build_dir: Path, name: str, suffix: str) -> Path: """Return a directory path like ``build_dir/{name}-website{suffix}``.""" return build_dir / f"{name}-website{suffix}" @task(namespace="mkcd2app") -def copy_website_template(config_yaml: str, template_src: ContentDir) -> ContentDir: +def copy_website_template(website_dir: str, template_src: ContentDir) -> ContentDir: """ Copy the template to the build directory. - Example input: config_yaml, template_src + Example input: racers-website, template_src Example output: racers-website - :param config_yaml: The raw YAML text of the config file. + :param website_dir: The directory to copy the template to. :param template_src: A redun.ContentDir pointing to the template source, so that redun tracks changes to template files. :return: A redun.ContentDir that points to the Vite website template directory. """ - config = load_config_from_yaml(config_yaml) - dst = _website_dir(Path(config.build_dir), config.project.path_friendly_name, "") + dst = Path(website_dir) logger.info(f"Website will be copied to {dst}") if dst.exists(): shutil.rmtree(dst) @@ -45,7 +119,7 @@ def copy_website_template(config_yaml: str, template_src: ContentDir) -> Content @task(namespace="mkcd2app") def fill_website_template( - config_yaml: str, + project_json: str, website_path: ContentDir, bin_js_path: ContentFile, support_path: ContentDir, @@ -56,15 +130,15 @@ def fill_website_template( Example intput: racers-website Example output: racers-website-filled - :param config_yaml: The raw YAML text of the config file. - :param website_path: redun.ContentDir pointing to the deps-installed website. + :param project_json: The config's Project dumped to a JSON string. + :param website_path: redun.ContentDir pointing to the copied website template. :param bin_js_path: redun.ContentFile pointing to the binary JS file. :param support_path: redun.ContentDir pointing to the support file directory. :return: A redun.ContentDir pointing to the filled website directory. """ logger.info("Filling website template") - config = load_config_from_yaml(config_yaml) + project = Project.model_validate_json(project_json) src = Path(website_path.path) dst = src.parent / f"{src.name}-filled" if dst.exists(): @@ -83,68 +157,101 @@ def fill_website_template( logger.debug(f"Copying ---simulator.html from {simulator_html_path}") shutil.copy(simulator_html_path, asset_path) - title = config.project.title.format( - NAME=config.project.name, - VERSION=config.project.version, - AUTHOR=config.project.author, - ) + title = render_title(project) logger.debug(f'Using title "{title}" in index.html') + favicon_path = Path(support_path.path) / "favicon.ico" + favicon = favicon_path.read_bytes() if favicon_path.exists() else None index_html_path = dst / "index.html" - index_html_text = index_html_path.read_text() - index_html_text = index_html_text.replace( - "vite-project", f"{title}" + index_html_text = fill_index_html( + index_html_path.read_text(encoding="utf-8"), title, favicon ) - - favicon_path = Path(support_path.path) / "favicon.ico" - if favicon_path.exists(): - favicon_b64 = base64.b64encode(favicon_path.read_bytes()).decode("ascii") - favicon_data_uri = f"data:image/x-icon;base64,{favicon_b64}" - index_html_text = index_html_text.replace( - '', - f'', - ) + if favicon is not None: logger.debug("Inlined favicon.ico into index.html") - - index_html_path.write_text(index_html_text) + index_html_path.write_text(index_html_text, encoding="utf-8") logger.debug("Updating package.json") package_json_path = dst / "package.json" - package_json = json.loads(package_json_path.read_text()) - package_json["name"] = config.project.path_friendly_name - package_json["version"] = config.project.version - package_json["description"] = config.project.description - package_json["authors"] = {"name": config.project.author} - package_json_path.write_text(json.dumps(package_json, indent=2)) + package_json = json.loads(package_json_path.read_text(encoding="utf-8")) + package_json["name"] = project.path_friendly_name + package_json["version"] = project.version + package_json["description"] = project.description + package_json["authors"] = {"name": project.author} + package_json_path.write_text(json.dumps(package_json, indent=2), encoding="utf-8") logger.debug("Website template filled") return ContentDir(str(dst)) @task(namespace="mkcd2app") -def install_deps_and_build_website(website_filled_path: ContentDir) -> ContentDir: +def install_website_deps( + deps_dir: str, package_json: ContentFile, package_lock: ContentFile +) -> ContentDir: """ - Make a copy of the template website, then ``npm ci`` and `npm run build` in it. + ``npm ci`` the template's dependencies in their own directory. + + Only package.json and package-lock.json are inputs, so changing the game or its + assets doesn't bust the cached install. The template's unfilled package.json is + used because filling only changes metadata that ``npm ci`` ignores. + + Example input: racers-website-deps, package.json, package-lock.json + Example output: racers-website-deps/node_modules + + :param deps_dir: The directory to install into. + :param package_json: A redun.ContentFile pointing to the template's package.json. + :param package_lock: A redun.ContentFile pointing to the template's lockfile. + :return: A redun.ContentDir pointing to the installed node_modules. + """ + dst = Path(deps_dir) + logger.info("Installing website dependencies") + + if dst.exists(): + shutil.rmtree(dst) + dst.mkdir(parents=True) + shutil.copy(package_json.path, dst / "package.json") + shutil.copy(package_lock.path, dst / "package-lock.json") + + cache_path = get_templates_npm_cache_dir() + logger.debug(f"Using template npm cache at {cache_path} to install") + + run_cmd(["npm", "ci", "--cache", str(cache_path), "--offline"], cwd=dst) + logger.debug("Website dependencies installed") + + return ContentDir(str(dst / "node_modules")) + + +def _copy_site_with_deps(src: Path, dst: Path, node_modules: ContentDir) -> None: + """Copy the filled site to ``dst`` and put a copy of ``node_modules`` in it.""" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(src, dst) + # Keep symlinks as symlinks: on POSIX node_modules/.bin entries are relative + # symlinks, and copying their targets breaks the scripts' relative imports + shutil.copytree(node_modules.path, dst / "node_modules", symlinks=True) + + +@task(namespace="mkcd2app") +def build_website( + website_filled_path: ContentDir, node_modules: ContentDir +) -> ContentDir: + """ + Make a copy of the filled website with node_modules, then `npm run build` in it. We copy so that we never mutate a ContentDir that another task owns, keeping ContentDir hashes stable for caching. - Example input: racers-website-filled - Example output: racers-website-filled-built + Example input: racers-website-filled, racers-website-deps/node_modules + Example output: racers-website-filled-built/dist - :param website_filled_path: A redun.ContentDir pointing to the template website. + :param website_filled_path: A redun.ContentDir pointing to the filled website. + :param node_modules: A redun.ContentDir pointing to the installed dependencies. :return: A redun.ContentDir pointing to the directory of static HTML/CSS/JS files ready to serve. """ src = Path(website_filled_path.path) dst = src.parent / f"{src.name}-built" - logger.info("Installing website dependencies and building to static HTML/CSS/JS") + logger.info("Building website to static HTML/CSS/JS") - if dst.exists(): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - run_cmd(["npm", "ci"], cwd=dst) - logger.debug("Website dependencies installed") + _copy_site_with_deps(src, dst, node_modules) run_cmd(["npm", "run", "build"], cwd=dst) logger.debug("Website built successfully") @@ -157,31 +264,28 @@ def install_deps_and_build_website(website_filled_path: ContentDir) -> ContentDi @task(namespace="mkcd2app") -def install_deps_and_build_website_singlefile( - website_filled_path: ContentDir, +def build_website_singlefile( + website_filled_path: ContentDir, node_modules: ContentDir ) -> ContentFile: """ - Make a copy of the template website, then ``npm ci`` and `npm run build:singlefile` in it. + Make a copy of the filled website with node_modules, then + `npm run build:singlefile` in it. We copy so that we never mutate a ContentDir that another task owns, keeping ContentDir hashes stable for caching. - Example input: racers-website-filled + Example input: racers-website-filled, racers-website-deps/node_modules Example output: index.html - :param website_filled_path: A redun.ContentDir pointing to the template website. + :param website_filled_path: A redun.ContentDir pointing to the filled website. + :param node_modules: A redun.ContentDir pointing to the installed dependencies. :return: A redun.ContentFile pointing to the single-file HTML. """ src = Path(website_filled_path.path) dst = src.parent / f"{src.name}-singlefile-built" - logger.info("Installing website dependencies and building to static HTML file") + logger.info("Building website to static HTML file") - if dst.exists(): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - run_cmd(["npm", "ci"], cwd=dst) - logger.debug("Website dependencies installed") + _copy_site_with_deps(src, dst, node_modules) run_cmd(["npm", "run", "build:singlefile"], cwd=dst) logger.debug("Website built successfully") diff --git a/src/mkcd2app/cli.py b/src/mkcd2app/cli.py index 4ea7402..db47fde 100644 --- a/src/mkcd2app/cli.py +++ b/src/mkcd2app/cli.py @@ -6,33 +6,83 @@ logger = create_logger(name=__name__, level=logging.INFO) -def generate_and_parse_args() -> Namespace: +def generate_and_parse_args(argv: list[str] | None = None) -> Namespace: """ Parse this CLI tool's arguments. + :param argv: The arguments to parse. Defaults to `sys.argv[1:]`. :return: A `Namespace` object with parsed CLI arguments. """ parser = ArgumentParser( + prog="mkcd2app", description="Convert your MakeCode Arcade games into a " - "standalone offline executable!" + "standalone offline executable!", ) parser.add_argument( "--debug", action="store_true", - help="Enable debug logging. This must go first before the sub command.", + help="Enable debug logging. This must go first before any sub commands.", ) subparsers = parser.add_subparsers(required=True, dest="command") - # build subcommand + parser_build = subparsers.add_parser( "build", help="Build your MakeCode Arcade game." ) - parser_build.add_argument("config", type=str, help="Path to the YAML config file.") parser_build.add_argument( - "--clear-cache", + "config", + type=str, + help="Path to the YAML config file. Relative paths in it are relative to the " + "file's directory.", + ) + parser_build.add_argument( + "--full-rebuild", action="store_true", - help="Delete the entire build directory before building.", + help="Delete the build directory and its cache, then rebuild everything. Use " + "this after changing the game's source files, icon, or git branch, since " + "only changes to the config file are detected automatically. Refuses to " + "delete a non-empty directory that has no build cache in it.", + ) + + parser_toolchain = subparsers.add_parser( + "toolchain", help="Manage the MakeCode CLI toolchain." + ) + toolchain_subparsers = parser_toolchain.add_subparsers( + required=True, dest="toolchain_command" + ) + toolchain_subparsers.add_parser( + "install", + help="Install the MakeCode CLI toolchain for this specific mkcd2app version.", + ) + toolchain_subparsers.add_parser( + "status", help="Show the installed MakeCode CLI toolchain." + ) + toolchain_subparsers.add_parser( + "uninstall", + help="Uninstall the MakeCode CLI toolchain for this specific mkcd2app version.", + ) + + parser_target = subparsers.add_parser( + "target", help="Manage MakeCode Arcade target versions." + ) + target_subparsers = parser_target.add_subparsers( + required=True, dest="target_command" + ) + parser_target_install = target_subparsers.add_parser( + "install", help="Install a MakeCode Arcade target version." + ) + parser_target_install.add_argument( + "version", type=str, help="Target version to install." + ) + target_subparsers.add_parser( + "list", help="List installed MakeCode Arcade target versions." + ) + parser_target_uninstall = target_subparsers.add_parser( + "uninstall", help="Uninstall a MakeCode Arcade target version." + ) + parser_target_uninstall.add_argument( + "version", type=str, help="Target version to uninstall." ) - args = parser.parse_args() + args = parser.parse_args(argv) logger.debug(f"Received arguments: {args}") return args diff --git a/src/mkcd2app/config/__init__.py b/src/mkcd2app/config/__init__.py index 8f1d4c4..2654191 100644 --- a/src/mkcd2app/config/__init__.py +++ b/src/mkcd2app/config/__init__.py @@ -1,8 +1,14 @@ import logging +from pathlib import Path import yaml -from mkcd2app.config.model import BuildConfig +from mkcd2app.models.config import ( + BuildConfig, + FileCodeSource, + PathAssetSource, + PathCodeSource, +) from mkcd2app.utils.logger import create_logger logger = create_logger(name=__name__, level=logging.INFO) @@ -18,10 +24,62 @@ def load_config_from_yaml(config_text: str) -> BuildConfig: logger.debug(f"Loading config from {len(config_text)} characters of YAML text") data = yaml.safe_load(config_text) - if data["version"] == 1: # only one config version so far + version = data.get("version") if isinstance(data, dict) else None + if version == 1: # only one config version so far config = BuildConfig(**data) else: - raise NotImplementedError(f"Unknown config version {data.version}") + raise NotImplementedError(f"Unknown config version {version}") logger.debug(f"Loaded config version {config.version}") return config + + +def _raise_for_overlapping_source(source: Path, build_dir: Path) -> None: + """ + Reject a `path` code source that overlaps the build directory. Copying a source + that contains the build directory would copy the build into itself, and a source + inside the build directory could be deleted by a build or a full rebuild. + + :param source: The absolute path of the code source. + :param build_dir: The absolute path of the build directory. + :raises ValueError: If either directory is inside the other, or they're the same. + """ + if build_dir.is_relative_to(source): + raise ValueError( + f"build_dir {build_dir} is inside the path code source {source}, so the " + "build would copy itself. Set build_dir to a folder outside the source." + ) + if source.is_relative_to(build_dir): + raise ValueError( + f"The path code source {source} is inside build_dir {build_dir}, so a " + "build could delete it. Set build_dir to a folder outside the source." + ) + + +def resolve_config_paths(config: BuildConfig, base_dir: Path) -> BuildConfig: + """ + Make the paths in a config absolute, so it means the same thing wherever it's run + from. Done before the config reaches a task, so that the location of a relative + path is part of what redun hashes. + + :param config: The config. It isn't modified. + :param base_dir: The directory relative paths are relative to, which is the config + file's directory. Absolute paths and URLs are left as they are. + :return: A copy of the config with absolute paths. + :raises ValueError: If a `path` code source overlaps the build directory. + """ + + def resolve(path: str) -> str: + return str((base_dir / path).resolve()) + + resolved = config.model_copy(deep=True) + resolved.build_dir = resolve(resolved.build_dir) + code = resolved.inputs.code.root + if isinstance(code, PathCodeSource | FileCodeSource): + code.value = resolve(code.value) + if isinstance(code, PathCodeSource): + _raise_for_overlapping_source(Path(code.value), Path(resolved.build_dir)) + icon = resolved.inputs.assets.icon + if icon is not None and isinstance(icon.root, PathAssetSource): + icon.root.value = resolve(icon.root.value) + return resolved diff --git a/src/mkcd2app/config/model.py b/src/mkcd2app/config/model.py deleted file mode 100644 index 6e13453..0000000 --- a/src/mkcd2app/config/model.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import Annotated, Literal - -from pydantic import BaseModel, Field, HttpUrl, RootModel - -# Source code can be a share link, GitHub repo, or a path on disk - - -class ShareLinkCodeSource(BaseModel): - type: Literal["share_link"] - value: HttpUrl - - -class GitHubCodeSource(BaseModel): - type: Literal["github"] - value: HttpUrl - checkout: str = "master" - - -class PathCodeSource(BaseModel): - type: Literal["path"] - value: str - - -CodeSource = RootModel[ - Annotated[ - ShareLinkCodeSource | GitHubCodeSource | PathCodeSource, - Field(discriminator="type"), - ] -] - - -# Assets can be a URL or path on disk - - -class UrlAssetSource(BaseModel): - type: Literal["url"] - value: HttpUrl - - -class PathAssetSource(BaseModel): - type: Literal["path"] - value: str - - -AssetSource = RootModel[ - Annotated[UrlAssetSource | PathAssetSource, Field(discriminator="type")] -] - - -# Right now we only have one asset so far, just the icon -class Assets(BaseModel): - icon: AssetSource | None = None - - -# To build a project, we need multiple inputs -class Inputs(BaseModel): - code: CodeSource - assets: Assets - - -# Project metadata -class Project(BaseModel): - name: str - path_friendly_name: str - description: str | None = None - author: str - version: str - title: str = "{NAME} v{VERSION}" - - -class WindowConfig(BaseModel): - width: int = 640 - height: int = 480 - - -# Different types of outputs available that we can build - - -class StaticOutput(BaseModel): - type: Literal["static"] - - -class StaticSinglefileOutput(BaseModel): - type: Literal["static-singlefile"] - - -class ElectronOutput(BaseModel): - type: Literal["electron"] - window: WindowConfig | None = None - - -class TauriOutput(BaseModel): - type: Literal["tauri"] - identifier: str - window: WindowConfig | None = None - - -OutputOption = RootModel[ - Annotated[ - StaticOutput | StaticSinglefileOutput | ElectronOutput | TauriOutput, - Field(discriminator="type"), - ] -] - - -# The entire config -class BuildConfig(BaseModel): - version: int # currently only 1 version so far - project: Project - inputs: Inputs - build_dir: str = Field(..., alias="build_dir") - outputs: list[OutputOption] - - class Config: - populate_by_name = True diff --git a/src/mkcd2app/main.py b/src/mkcd2app/main.py index ef7bbd2..a6e8be3 100644 --- a/src/mkcd2app/main.py +++ b/src/mkcd2app/main.py @@ -1,65 +1,132 @@ +import io import logging -import shutil +import sys from pathlib import Path -import redun -import redun.file -from redun import Scheduler - from mkcd2app.build_project import BuildProjectResult, build_project from mkcd2app.cli import generate_and_parse_args -from mkcd2app.config import load_config_from_yaml +from mkcd2app.config import load_config_from_yaml, resolve_config_paths +from mkcd2app.target.install import install_target +from mkcd2app.target.uninstall import uninstall_target +from mkcd2app.toolchain.install import install_toolchain +from mkcd2app.toolchain.uninstall import uninstall_toolchain +from mkcd2app.utils.filesystem import rmtree_robust from mkcd2app.utils.logger import create_logger, set_all_stdout_logger_levels +from mkcd2app.utils.paths import ( + get_redun_db_for_build_path, + get_redun_db_for_target_path, + get_redun_db_for_toolchain_path, +) +from mkcd2app.utils.run_redun_task import run_redun_task +from mkcd2app.utils.text import raise_for_invalid_strict_semver logger = create_logger(name=__name__, level=logging.INFO) +def _clear_build_dir(build_dir: Path) -> None: + """ + Delete the build directory for a full rebuild. + + Only a directory with our cache DB is deleted, since `build_dir` comes from the + config and a mistake there could otherwise delete an unrelated directory. + + :param build_dir: The build directory. + :raises RuntimeError: If the directory has other files but no cache DB. + """ + if not build_dir.exists() or not any(build_dir.iterdir()): + logger.debug("Build directory is missing or empty; nothing to clear") + return + redun_db_path = get_redun_db_for_build_path(build_dir) + if not redun_db_path.exists(): + raise RuntimeError( + f"Refusing to clear {build_dir}: it isn't empty and has no " + f"{redun_db_path.name}, so it may not be a build directory" + ) + logger.warning(f"Clearing build directory {build_dir}") + rmtree_robust(build_dir) + + def main() -> None: + # Logged command output can have characters that a redirected stdout's encoding + # can't represent (like cp1252 on Windows), which would lose the whole log record. + # This is what Python already does for stderr. + if isinstance(sys.stdout, io.TextIOWrapper): + sys.stdout.reconfigure(errors="backslashreplace") + args = generate_and_parse_args() debug = bool(args.debug) if debug: set_all_stdout_logger_levels(logging.DEBUG) logger.debug(f"Received arguments: {args}") - if args.command == "build": - logger.debug("Building project") + if args.command == "toolchain": + if args.toolchain_command == "install": + logger.debug("Installing MakeCode CLI toolchain") + run_redun_task( + expr=install_toolchain(), + redun_db_path=get_redun_db_for_toolchain_path(), + ) + logger.debug("Toolchain installed") + elif args.toolchain_command == "status": + logger.debug("Checking MakeCode CLI toolchain status") + + elif args.toolchain_command == "uninstall": + logger.debug("Uninstalling MakeCode CLI toolchain") + # noinspection none-function-assignment + run_redun_task( + expr=uninstall_toolchain(), + redun_db_path=get_redun_db_for_toolchain_path(), + ) + logger.debug("Toolchain uninstalled") + elif args.command == "target": + if args.target_command == "install": + install_version: str = args.version + raise_for_invalid_strict_semver(install_version) + logger.debug(f"Installing MakeCode CLI target version {install_version}") + run_redun_task( + expr=install_target(install_version), + redun_db_path=get_redun_db_for_target_path(), + ) + logger.debug(f"Target {install_version} installed") + elif args.target_command == "list": + logger.debug("Listing MakeCode CLI target") + + elif args.target_command == "uninstall": + uninstall_version: str = args.version + raise_for_invalid_strict_semver(uninstall_version) + logger.debug( + f"Uninstalling MakeCode CLI target version {uninstall_version}" + ) + # noinspection none-function-assignment + run_redun_task( + expr=uninstall_target(uninstall_version), + redun_db_path=get_redun_db_for_target_path(), + ) + logger.debug(f"Target {uninstall_version} uninstalled") + elif args.command == "build": config_path = Path(args.config) - logger.debug(f"Loading config from {config_path}") - config_text = config_path.read_text() - - # Parse once only to extract build_dir for the redun DB path. - # The raw YAML text is passed to redun tasks so that argument - # hashing is deterministic (string) rather than pickle-based - # (which is non-deterministic due to pydantic's set fields). - config = load_config_from_yaml(config_text) - build_dir = Path(config.build_dir) + logger.debug(f"Building project with config {config_path}") - if args.clear_cache: - if build_dir.exists(): - logger.warning(f"Clearing build directory {build_dir}") - shutil.rmtree(build_dir) - else: - logger.debug("Build directory does not exist; nothing to clear") - - build_dir.mkdir(parents=True, exist_ok=True) - db_uri = f"sqlite:///{build_dir.resolve() / '.redun-cache.db'}" - logger.debug(f"redun cache DB: {db_uri}") - # noinspection PyUnresolvedReferences - redun_config = redun.config.Config( - { - "scheduler": {"log_level": "DEBUG"}, - "backend": {"db_uri": db_uri}, - } + config_text = config_path.read_text(encoding="utf-8") + + config = resolve_config_paths( + load_config_from_yaml(config_text), config_path.resolve().parent ) - scheduler = Scheduler(config=redun_config) - # Load/migrate the backend so the persistent DB is properly set up. - # Without this, providing a custom db_uri skips the automatic - # engine creation and migration that the in-memory default does. - scheduler.load() - results: BuildProjectResult = scheduler.run( - build_project(config_text), + build_dir = Path(config.build_dir) + + if args.full_rebuild: + _clear_build_dir(build_dir) + + # The config is passed to redun tasks as JSON so that argument hashing is + # deterministic (string) rather than pickle-based (which is non-deterministic + # due to pydantic's set fields), and so that YAML comments and formatting + # aren't part of the hash. + results: BuildProjectResult = run_redun_task( + build_project(config.model_dump_json()), + get_redun_db_for_build_path(build_dir), ) + if results.static: logger.info(f"Static website directory is at {results.static.path}") if results.static_singlefile: diff --git a/tests/fixtures/empty_game/assets.json b/src/mkcd2app/models/__init__.py similarity index 100% rename from tests/fixtures/empty_game/assets.json rename to src/mkcd2app/models/__init__.py diff --git a/src/mkcd2app/models/config.py b/src/mkcd2app/models/config.py new file mode 100644 index 0000000..bd0ad2c --- /dev/null +++ b/src/mkcd2app/models/config.py @@ -0,0 +1,193 @@ +import ntpath +from typing import Annotated, Literal + +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + HttpUrl, + RootModel, + field_validator, +) + +from mkcd2app.utils.text import raise_for_invalid_strict_semver + + +class ConfigModel(BaseModel): + """ + Base for the config's models. Unknown keys are errors, so that a typo like + ``titel:`` fails instead of silently using the default. + """ + + model_config = ConfigDict(extra="forbid") + + +# Source code can be a share link, git repo, a path on disk, or a file: a PNG/UF2 +# downloaded from the editor with the project embedded, or a ZIP of the project + + +class ShareLinkCodeSource(ConfigModel): + type: Literal["share_link"] + value: HttpUrl + + +class GitCodeSource(ConfigModel): + type: Literal["git"] + # Any URL dulwich can clone: https://, http://, git://, file://, or ssh:// (ssh:// + # needs an `ssh` executable on PATH) + url: AnyUrl + # Branch, tag, or (possibly abbreviated) commit SHA. None means the remote's + # default branch. + ref: str | None = None + + +class GitHubCodeSource(ConfigModel): + """ + Alias for GitCodeSource, kept so older configs keep working: `value` is the + URL and `checkout` is the ref. + """ + + type: Literal["github"] + value: HttpUrl + checkout: str = "master" + + +class PathCodeSource(ConfigModel): + type: Literal["path"] + value: str + + +class FileCodeSource(ConfigModel): + """ + A PNG or UF2 downloaded from the MakeCode editor with the project embedded, or a + ZIP of the project (e.g. GitHub's "Download ZIP"). Detected by content. + """ + + type: Literal["file"] + value: str + + +CodeSource = RootModel[ + Annotated[ + ShareLinkCodeSource + | GitCodeSource + | GitHubCodeSource + | PathCodeSource + | FileCodeSource, + Field(discriminator="type"), + ] +] + + +# Assets can be a URL or path on disk + + +class UrlAssetSource(ConfigModel): + type: Literal["url"] + value: HttpUrl + + +class PathAssetSource(ConfigModel): + type: Literal["path"] + value: str + + +AssetSource = RootModel[ + Annotated[UrlAssetSource | PathAssetSource, Field(discriminator="type")] +] + + +# Right now we only have one asset so far, just the icon +class Assets(ConfigModel): + icon: AssetSource | None = None + + +# To build a project, we need multiple inputs +class Inputs(ConfigModel): + code: CodeSource + assets: Assets + + +# Project metadata +class Project(ConfigModel): + """ + ``title`` is the window/page title. The placeholders ``{NAME}``, ``{VERSION}`` + and ``{AUTHOR}`` are replaced with the matching fields, in a single pass (so a + substituted value is never expanded again). Any other text, including other + braces such as ``{Game}`` or ``{}``, is kept literally; there is no brace + escaping, so ``{{NAME}}`` becomes ``{}``. + + ``path_friendly_name`` names the stage directories inside ``build_dir`` and is the + website's ``package.json`` name, so it must be a single path component that's also + lowercase and URL-safe. A ``..`` or path separator would let a build delete and + write outside ``build_dir``. + """ + + name: str + path_friendly_name: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]*$", max_length=64) + description: str | None = None + author: str + version: str + title: str = "{NAME} v{VERSION}" + + @field_validator("path_friendly_name") + @classmethod + def _check_path_friendly_name_is_not_reserved(cls, v: str) -> str: + # ntpath rather than os.path, so configs are rejected the same way on every OS + if ntpath.isreserved(v): + raise ValueError(f"{v!r} is a reserved file name on Windows") + return v + + +class WindowConfig(ConfigModel): + width: int = 640 + height: int = 480 + + +# Different types of outputs available that we can build + + +class StaticOutput(ConfigModel): + type: Literal["static"] + + +class StaticSinglefileOutput(ConfigModel): + type: Literal["static-singlefile"] + + +class ElectronOutput(ConfigModel): + type: Literal["electron"] + window: WindowConfig | None = None + + +class TauriOutput(ConfigModel): + type: Literal["tauri"] + identifier: str + window: WindowConfig | None = None + + +OutputOption = RootModel[ + Annotated[ + StaticOutput | StaticSinglefileOutput | ElectronOutput | TauriOutput, + Field(discriminator="type"), + ] +] + + +# The entire config +class BuildConfig(ConfigModel): + model_config = ConfigDict(populate_by_name=True) + + version: int # currently only 1 version so far + project: Project + inputs: Inputs + build_dir: str = Field(..., alias="build_dir") + target: str = Field(..., alias="target") + outputs: list[OutputOption] = Field(..., min_length=1) + + @field_validator("target") + @classmethod + def _check_target_is_strict_semver(cls, v: str) -> str: + raise_for_invalid_strict_semver(v) + return v diff --git a/src/mkcd2app/models/metadata.py b/src/mkcd2app/models/metadata.py new file mode 100644 index 0000000..d7c929e --- /dev/null +++ b/src/mkcd2app/models/metadata.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, ConfigDict, Field, HttpUrl + + +# { +# "simUrl":"https://trg-arcade.userpxt.io/---simulator", +# "cdnUrl":"https://cdn.makecode.com", +# "version":"v0.0.0", +# "target":"arcade", +# "targetVersion":"4.0.14" +# } +class BinaryJSMetadata(BaseModel): + model_config = ConfigDict(extra="forbid") + + simUrl: HttpUrl = Field(..., description="Simulator URL") + cdnUrl: HttpUrl = Field(..., description="CDN base URL") + version: str = Field(..., description="Package version") + target: str = Field(..., description="Build target name") + targetVersion: str = Field(..., description="Target version string") diff --git a/tests/fixtures/empty_game/README.md b/src/mkcd2app/resources/empty_project/README.md similarity index 100% rename from tests/fixtures/empty_game/README.md rename to src/mkcd2app/resources/empty_project/README.md diff --git a/src/mkcd2app/resources/empty_project/assets.json b/src/mkcd2app/resources/empty_project/assets.json new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/empty_game/main.ts b/src/mkcd2app/resources/empty_project/main.ts similarity index 100% rename from tests/fixtures/empty_game/main.ts rename to src/mkcd2app/resources/empty_project/main.ts diff --git a/src/mkcd2app/resources/empty_project/pxt.json b/src/mkcd2app/resources/empty_project/pxt.json new file mode 100644 index 0000000..782e85f --- /dev/null +++ b/src/mkcd2app/resources/empty_project/pxt.json @@ -0,0 +1,12 @@ +{ + "name": "empty_project", + "description": "", + "dependencies": { + "device": "*" + }, + "files": [ + "main.ts", + "README.md", + "assets.json" + ] +} diff --git a/src/mkcd2app/js_tools/package-lock.json b/src/mkcd2app/resources/js_tools/package-lock.json similarity index 100% rename from src/mkcd2app/js_tools/package-lock.json rename to src/mkcd2app/resources/js_tools/package-lock.json diff --git a/src/mkcd2app/js_tools/package.json b/src/mkcd2app/resources/js_tools/package.json similarity index 100% rename from src/mkcd2app/js_tools/package.json rename to src/mkcd2app/resources/js_tools/package.json diff --git a/src/mkcd2app/templates/vite-project/.gitignore b/src/mkcd2app/resources/templates/vite-project/.gitignore similarity index 100% rename from src/mkcd2app/templates/vite-project/.gitignore rename to src/mkcd2app/resources/templates/vite-project/.gitignore diff --git a/src/mkcd2app/templates/vite-project/.prettierignore b/src/mkcd2app/resources/templates/vite-project/.prettierignore similarity index 100% rename from src/mkcd2app/templates/vite-project/.prettierignore rename to src/mkcd2app/resources/templates/vite-project/.prettierignore diff --git a/src/mkcd2app/templates/vite-project/README.md b/src/mkcd2app/resources/templates/vite-project/README.md similarity index 100% rename from src/mkcd2app/templates/vite-project/README.md rename to src/mkcd2app/resources/templates/vite-project/README.md diff --git a/src/mkcd2app/templates/vite-project/eslint.config.js b/src/mkcd2app/resources/templates/vite-project/eslint.config.js similarity index 100% rename from src/mkcd2app/templates/vite-project/eslint.config.js rename to src/mkcd2app/resources/templates/vite-project/eslint.config.js diff --git a/src/mkcd2app/templates/vite-project/index.html b/src/mkcd2app/resources/templates/vite-project/index.html similarity index 100% rename from src/mkcd2app/templates/vite-project/index.html rename to src/mkcd2app/resources/templates/vite-project/index.html diff --git a/src/mkcd2app/templates/vite-project/package-lock.json b/src/mkcd2app/resources/templates/vite-project/package-lock.json similarity index 100% rename from src/mkcd2app/templates/vite-project/package-lock.json rename to src/mkcd2app/resources/templates/vite-project/package-lock.json diff --git a/src/mkcd2app/templates/vite-project/package.json b/src/mkcd2app/resources/templates/vite-project/package.json similarity index 93% rename from src/mkcd2app/templates/vite-project/package.json rename to src/mkcd2app/resources/templates/vite-project/package.json index 63636b0..47898f3 100644 --- a/src/mkcd2app/templates/vite-project/package.json +++ b/src/mkcd2app/resources/templates/vite-project/package.json @@ -7,8 +7,8 @@ "dev": "vite", "lint": "eslint ", "writeLint": "eslint --fix ", - "format": "prettier --check .", - "writeFormat": "prettier --write .", + "format": "prettier --check ", + "writeFormat": "prettier --write ", "preview": "vite preview", "build": "tsc -b && vite build", "build:singlefile": "tsc -b && vite build --mode singlefile" diff --git a/src/mkcd2app/templates/vite-project/src/App.css b/src/mkcd2app/resources/templates/vite-project/src/App.css similarity index 100% rename from src/mkcd2app/templates/vite-project/src/App.css rename to src/mkcd2app/resources/templates/vite-project/src/App.css diff --git a/src/mkcd2app/templates/vite-project/src/App.tsx b/src/mkcd2app/resources/templates/vite-project/src/App.tsx similarity index 100% rename from src/mkcd2app/templates/vite-project/src/App.tsx rename to src/mkcd2app/resources/templates/vite-project/src/App.tsx diff --git a/src/mkcd2app/templates/vite-project/src/assets/---simulator.html b/src/mkcd2app/resources/templates/vite-project/src/assets/---simulator.html similarity index 100% rename from src/mkcd2app/templates/vite-project/src/assets/---simulator.html rename to src/mkcd2app/resources/templates/vite-project/src/assets/---simulator.html diff --git a/src/mkcd2app/templates/vite-project/src/assets/binary.js b/src/mkcd2app/resources/templates/vite-project/src/assets/binary.js similarity index 100% rename from src/mkcd2app/templates/vite-project/src/assets/binary.js rename to src/mkcd2app/resources/templates/vite-project/src/assets/binary.js diff --git a/src/mkcd2app/templates/vite-project/src/gameConfiguration.ts b/src/mkcd2app/resources/templates/vite-project/src/gameConfiguration.ts similarity index 100% rename from src/mkcd2app/templates/vite-project/src/gameConfiguration.ts rename to src/mkcd2app/resources/templates/vite-project/src/gameConfiguration.ts diff --git a/src/mkcd2app/templates/vite-project/src/global.d.ts b/src/mkcd2app/resources/templates/vite-project/src/global.d.ts similarity index 100% rename from src/mkcd2app/templates/vite-project/src/global.d.ts rename to src/mkcd2app/resources/templates/vite-project/src/global.d.ts diff --git a/src/mkcd2app/templates/vite-project/src/main.tsx b/src/mkcd2app/resources/templates/vite-project/src/main.tsx similarity index 100% rename from src/mkcd2app/templates/vite-project/src/main.tsx rename to src/mkcd2app/resources/templates/vite-project/src/main.tsx diff --git a/src/mkcd2app/templates/vite-project/src/utils/position.ts b/src/mkcd2app/resources/templates/vite-project/src/utils/position.ts similarity index 100% rename from src/mkcd2app/templates/vite-project/src/utils/position.ts rename to src/mkcd2app/resources/templates/vite-project/src/utils/position.ts diff --git a/src/mkcd2app/templates/vite-project/src/utils/toasts.ts b/src/mkcd2app/resources/templates/vite-project/src/utils/toasts.ts similarity index 100% rename from src/mkcd2app/templates/vite-project/src/utils/toasts.ts rename to src/mkcd2app/resources/templates/vite-project/src/utils/toasts.ts diff --git a/src/mkcd2app/templates/vite-project/tsconfig.app.json b/src/mkcd2app/resources/templates/vite-project/tsconfig.app.json similarity index 85% rename from src/mkcd2app/templates/vite-project/tsconfig.app.json rename to src/mkcd2app/resources/templates/vite-project/tsconfig.app.json index e30ef7d..3abe7e5 100644 --- a/src/mkcd2app/templates/vite-project/tsconfig.app.json +++ b/src/mkcd2app/resources/templates/vite-project/tsconfig.app.json @@ -2,9 +2,14 @@ "compilerOptions": { "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", "target": "es2023", - "lib": ["ES2023", "DOM"], + "lib": [ + "ES2023", + "DOM" + ], "module": "esnext", - "types": ["vite/client"], + "types": [ + "vite/client" + ], "allowArbitraryExtensions": true, "skipLibCheck": true, /* Bundler mode */ @@ -20,5 +25,7 @@ "erasableSyntaxOnly": false, "noFallthroughCasesInSwitch": true }, - "include": ["src"] + "include": [ + "src" + ] } diff --git a/src/mkcd2app/templates/vite-project/tsconfig.json b/src/mkcd2app/resources/templates/vite-project/tsconfig.json similarity index 100% rename from src/mkcd2app/templates/vite-project/tsconfig.json rename to src/mkcd2app/resources/templates/vite-project/tsconfig.json diff --git a/src/mkcd2app/templates/vite-project/tsconfig.node.json b/src/mkcd2app/resources/templates/vite-project/tsconfig.node.json similarity index 81% rename from src/mkcd2app/templates/vite-project/tsconfig.node.json rename to src/mkcd2app/resources/templates/vite-project/tsconfig.node.json index 1f8a29b..3bf20da 100644 --- a/src/mkcd2app/templates/vite-project/tsconfig.node.json +++ b/src/mkcd2app/resources/templates/vite-project/tsconfig.node.json @@ -2,8 +2,12 @@ "compilerOptions": { "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", "target": "es2023", - "lib": ["ES2023"], - "types": ["node"], + "lib": [ + "ES2023" + ], + "types": [ + "node" + ], "skipLibCheck": true, /* Bundler mode */ "module": "nodenext", @@ -17,5 +21,7 @@ "erasableSyntaxOnly": true, "noFallthroughCasesInSwitch": true }, - "include": ["vite.config.ts"] + "include": [ + "vite.config.ts" + ] } diff --git a/src/mkcd2app/templates/vite-project/vite.config.ts b/src/mkcd2app/resources/templates/vite-project/vite.config.ts similarity index 92% rename from src/mkcd2app/templates/vite-project/vite.config.ts rename to src/mkcd2app/resources/templates/vite-project/vite.config.ts index 6be7485..6680d8e 100644 --- a/src/mkcd2app/templates/vite-project/vite.config.ts +++ b/src/mkcd2app/resources/templates/vite-project/vite.config.ts @@ -5,6 +5,7 @@ import {viteSingleFile} from "vite-plugin-singlefile"; // https://vite.dev/config/ export default defineConfig(({mode}) => ({ + base: "./", // Relative asset paths, so the site also works from a subfolder plugins: [ react(), babel({presets: [reactCompilerPreset()]}), diff --git a/src/mkcd2app/target/__init__.py b/src/mkcd2app/target/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mkcd2app/target/install.py b/src/mkcd2app/target/install.py new file mode 100644 index 0000000..214a504 --- /dev/null +++ b/src/mkcd2app/target/install.py @@ -0,0 +1,159 @@ +import json +import logging +import shutil +from contextlib import ExitStack +from pathlib import Path +from tempfile import TemporaryDirectory +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup +from redun import task +from redun.file import ContentDir, ContentFile + +from mkcd2app.models.metadata import BinaryJSMetadata +from mkcd2app.utils.logger import create_logger +from mkcd2app.utils.paths import ( + get_js_tools_bin_dir, + get_sim_html_path, +) +from mkcd2app.utils.resources import get_resource_empty_project_path +from mkcd2app.utils.run import run_cmd +from mkcd2app.utils.text import extract_meta_comment + +logger = create_logger(name=__name__, level=logging.INFO) + + +@task(namespace="mkcd2app") +def warm_mkc_cache_for_version_and_get_binary_js_metadata( + empty_prj: ContentDir, version: str +) -> str: + """ + Warm the mkc compiler cache by building an empty project for the target MakeCode + Arcade version in a temporary directory. + + :param empty_prj: The ContentDir pointing to the directory containing the empty + MakeCode Arcade project. + :param version: The MakeCode Arcade version to target. E.g., "4.0.14". Must be + explicit 3 num sem ver, not just like "4.0" or "4", and do not include a "v". + :return: A BinaryJSMetadata model dumped to a JSON string. + """ + logger.info(f"Warming mkc compiler cache for MakeCode Arcade version {version}") + + empty_prj_path = Path(empty_prj.path) + + with TemporaryDirectory() as tmp_dir: + tmp_dir_path = Path(tmp_dir) + + logger.debug(f"Copying {empty_prj_path} to {tmp_dir_path}") + shutil.copytree(empty_prj_path, tmp_dir_path, dirs_exist_ok=True) + + mkc_json_path = tmp_dir_path / "mkc.json" + logger.debug("Writing mkc.json") + mkc_json = {"targetWebsite": f"https://arcade.makecode.com/v{version}"} + mkc_json_path.write_text(json.dumps(mkc_json), encoding="utf-8") + + logger.debug("Running build to warm cache") + run_cmd( + ["mkc", "build", "-j"], cwd=tmp_dir_path, which_path=get_js_tools_bin_dir() + ) + + binary_js_path = tmp_dir_path / "built" / "binary.js" + logger.debug(f"binary.js available at {binary_js_path}") + + logger.debug("Finished mkc cache warming, extracting binary.js metadata") + metadata = extract_meta_comment(binary_js_path) + logger.debug(f"{metadata=}") + return metadata.model_dump_json() + + +def _download_text(url: str) -> str: + """ + Download a text file and decode it as UTF-8. + + ``res.text`` falls back to ISO-8859-1 when the server sends no charset, which + mangles UTF-8 content, so decode the raw bytes ourselves. + + :param url: The URL to download. + :return: The decoded text. + """ + res = requests.get(url) + res.raise_for_status() + return res.content.decode("utf-8") + + +@task(namespace="mkcd2app") +def download_sim(bin_js_metadata: str, version: str) -> ContentFile: + """ + Download the simulator HTML and the supporting files needed to run binary.js for the + website. + + :param bin_js_metadata: The binary JS metadata JSON as a string. + :param version: The MakeCode Arcade version to target. E.g., "4.0.14". Must be + explicit 3 num sem ver, not just like "4.0" or "4", and do not include a "v". + :return: A ContentFile that points to the ---simulator.html. + """ + metadata = BinaryJSMetadata.model_validate_json(bin_js_metadata) + logger.info( + f"Downloading simulator for MakeCode Arcade version {metadata.targetVersion}" + ) + + sim_url = str(metadata.simUrl) + logger.debug(f"Downloading main simulator file from {sim_url}") + sim_html = _download_text(sim_url) + + logger.debug( + f"Analyzing sim HTML ({len(sim_html)} chars) for required CSS and JS files" + ) + soup = BeautifulSoup(sim_html, features="html.parser") + css_links = soup.find_all("link", rel="stylesheet") + js_scripts = soup.find_all("script") + logger.debug(f"Found {len(css_links)} CSS links and {len(js_scripts)} JS scripts") + for css in css_links: + url = css.get("href") + if url: + logger.debug(f"Downloading CSS file {url}") + style_tag = soup.new_tag("style") + style_tag.string = _download_text(urljoin(sim_url, str(url))) + css.replace_with(style_tag) + logger.debug(f"Inlined CSS from {url}") + for js in js_scripts: + url = js.get("src") + if url: + logger.debug(f"Downloading JS file {url}") + js.string = _download_text(urljoin(sim_url, str(url))) + del js["src"] + logger.debug(f"Inlined JS from {url}") + new_sim_html = soup.prettify(formatter="html5") + + path = get_sim_html_path(version) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(new_sim_html, encoding="utf-8") + + logger.debug(f"Wrote simulator HTML to {path}") + + return ContentFile(str(path)) + + +@task(namespace="mkcd2app") +def install_target(version: str) -> ContentFile: + """ + Install a MakeCode Arcade target version for this mkcd2app version. + + :param version: The MakeCode Arcade version to target. E.g., "4.0.14". Must be + explicit 3 num sem ver, not just like "4.0" or "4", and do not include a "v". + :return: A ContentFile that points to the ---simulator.html for the installed version. + """ + logger.info(f"Installing MakeCode Arcade target version {version}") + + with ExitStack() as stack: + empty_prj_path = stack.enter_context(get_resource_empty_project_path()) + + # `mkc build -j` an empty project with the correct version + metadata = warm_mkc_cache_for_version_and_get_binary_js_metadata( + ContentDir(str(empty_prj_path)), version + ) + # Download ---simulator.html and supporting files, mod into single HTML file + sim_html = download_sim(metadata, version) + + return sim_html diff --git a/src/mkcd2app/target/uninstall.py b/src/mkcd2app/target/uninstall.py new file mode 100644 index 0000000..deccde4 --- /dev/null +++ b/src/mkcd2app/target/uninstall.py @@ -0,0 +1,30 @@ +import logging + +from redun import task +from redun.task import CacheScope + +from mkcd2app.utils.filesystem import rmtree_robust +from mkcd2app.utils.logger import create_logger +from mkcd2app.utils.paths import get_target_version_dir + +logger = create_logger(name=__name__, level=logging.INFO) + + +# Never cached: it returns None, so a cached result can't tell that the directory +# was reinstalled since, and a second uninstall would silently do nothing +@task(namespace="mkcd2app", cache_scope=CacheScope.NONE) +def uninstall_target(version: str) -> None: + """ + Uninstall a MakeCode Arcade target version for this mkcd2app version. + + :param version: The MakeCode Arcade version to target. E.g., "4.0.14". Must be + explicit 3 num sem ver, not just like "4.0" or "4", and do not include a "v". + """ + logger.info(f"Uninstall MakeCode Arcade target version {version}") + + target_path = get_target_version_dir(version) + if not target_path.exists(): + logger.warning(f"Target {version} is not installed, nothing to uninstall") + return + logger.debug(f"Removing target directory {target_path}") + rmtree_robust(target_path) diff --git a/src/mkcd2app/toolchain/__init__.py b/src/mkcd2app/toolchain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mkcd2app/toolchain/install.py b/src/mkcd2app/toolchain/install.py new file mode 100644 index 0000000..55a7673 --- /dev/null +++ b/src/mkcd2app/toolchain/install.py @@ -0,0 +1,111 @@ +import logging +import shutil +from contextlib import ExitStack +from pathlib import Path +from tempfile import TemporaryDirectory + +from redun import task +from redun.file import ContentDir + +from mkcd2app.utils.logger import create_logger +from mkcd2app.utils.paths import ( + get_js_tools_dir, + get_templates_npm_cache_dir, +) +from mkcd2app.utils.resources import ( + get_resource_js_tools_path, + get_resource_template_path, +) +from mkcd2app.utils.run import run_cmd + +logger = create_logger(name=__name__, level=logging.INFO) + + +@task(namespace="mkcd2app") +def install_js_tools(js_tools: ContentDir) -> ContentDir: + """ + Installs necessary JS tools for mkcd2app. + + :param js_tools: The ContentDir pointing to the directory containing the + package.json and package-lock.json, which include the tools to install. + :return: A ContentDir pointing to node_modules, to ensure redun sees that this task + has something that depends on its results. + """ + logger.info("Installing JS tools") + + source_path = Path(js_tools.path) + dest_path = get_js_tools_dir() + dest_path.mkdir(parents=True, exist_ok=True) + + logger.debug(f"Copying package files from {source_path} to {dest_path}") + shutil.copy(source_path / "package.json", dest_path / "package.json") + shutil.copy(source_path / "package-lock.json", dest_path / "package-lock.json") + + logger.debug("`npm ci` to download") + run_cmd(["npm", "ci"], cwd=dest_path) + + logger.debug("All JS tools installed") + return ContentDir(str(dest_path / "node_modules")) + + +@task(namespace="mkcd2app") +def warm_npm_cache_for_templates(templates: ContentDir) -> ContentDir: + """ + For every template in the templates directory, copy it to a temporary directory and + run `npm ci --cache CACHE_DIR --prefer-online`, where CACHE_DIR is the templates npm + cache. + + :param templates: The ContentDir pointing to the directory containing the templates. + :return: A ContentDir pointing to the npm cache, to ensure redun sees that this task + has something that depends on its results. + """ + logger.info("Warming npm cache for templates") + + source_path = Path(templates.path) + cache_path = get_templates_npm_cache_dir() + cache_path.mkdir(parents=True, exist_ok=True) + + logger.debug(f"Looking for templates in {source_path}") + + all_templates = list(source_path.iterdir()) + logger.debug(f"Found {len(all_templates)} templates") + + for template in all_templates: + logger.debug(f"Caching packages for template {template.name}") + with TemporaryDirectory() as tmp_dir: + tmp_dir_path = Path(tmp_dir) + logger.debug(f"Copying {template} to {tmp_dir_path}") + shutil.copytree(template, tmp_dir_path, dirs_exist_ok=True) + + logger.debug("Caching npm packages") + run_cmd( + ["npm", "ci", "--cache", str(cache_path), "--prefer-online"], + cwd=tmp_dir_path, + ) + + logger.debug(f"npm cache filled at {cache_path}") + return ContentDir(str(cache_path)) + + +@task(namespace="mkcd2app") +def install_toolchain() -> tuple[ContentDir, ContentDir]: + """ + Install the toolchain for this mkcd2app version. + + :return: A tuple of two ContentDirs, where the first one is the node_modules folder + for the JS tools, and the other one is the cache directory for the templates. + """ + logger.info("Installing toolchain") + + with ExitStack() as stack: + js_tools_path = stack.enter_context(get_resource_js_tools_path()) + templates_path = stack.enter_context(get_resource_template_path()) + + # `npm ci` the necessary tools (`mkc` CLI itself) + js_tools_node_modules = install_js_tools(ContentDir(str(js_tools_path))) + # `npm ci --cache CACHE_DIR --prefer-online` for all templates + templates_npm_cache = warm_npm_cache_for_templates( + ContentDir(str(templates_path)) + ) + + return js_tools_node_modules, templates_npm_cache diff --git a/src/mkcd2app/toolchain/uninstall.py b/src/mkcd2app/toolchain/uninstall.py new file mode 100644 index 0000000..d27a1a7 --- /dev/null +++ b/src/mkcd2app/toolchain/uninstall.py @@ -0,0 +1,28 @@ +import logging + +from redun import task +from redun.task import CacheScope + +from mkcd2app.utils.filesystem import rmtree_robust +from mkcd2app.utils.logger import create_logger +from mkcd2app.utils.paths import get_toolchain_dir + +logger = create_logger(name=__name__, level=logging.INFO) + + +# Never cached: it returns None, so a cached result can't tell that the directory +# was reinstalled since, and a second uninstall would silently do nothing +@task(namespace="mkcd2app", cache_scope=CacheScope.NONE) +def uninstall_toolchain() -> None: + """ + Uninstall the toolchain for this mkcd2app version by removing the toolchain + directory. The toolchain redun DB lives in the state dir and is left in place. + """ + logger.info("Uninstalling toolchain") + + toolchain_path = get_toolchain_dir() + if not toolchain_path.exists(): + logger.warning("Toolchain is not installed, nothing to uninstall") + return + logger.debug(f"Removing toolchain directory {toolchain_path}") + rmtree_robust(toolchain_path) diff --git a/src/mkcd2app/utils/filesystem.py b/src/mkcd2app/utils/filesystem.py new file mode 100644 index 0000000..5a9d0b8 --- /dev/null +++ b/src/mkcd2app/utils/filesystem.py @@ -0,0 +1,25 @@ +import os +import shutil +import stat +from pathlib import Path +from types import TracebackType +from typing import Any + + +def _remove_readonly( + func: Any, + path: str, + exc_info: tuple[type[BaseException], BaseException, TracebackType], +) -> None: + """shutil.rmtree error handler: clear read-only bit and retry. + + Needed on Windows because git marks files under .git/objects (and + sometimes .git itself) read-only, which makes os.unlink/os.rmdir + raise PermissionError (WinError 5) even though we own the files. + """ + os.chmod(path, stat.S_IWRITE) + func(path) + + +def rmtree_robust(path: Path) -> None: + shutil.rmtree(path, onerror=_remove_readonly) diff --git a/src/mkcd2app/utils/git.py b/src/mkcd2app/utils/git.py new file mode 100644 index 0000000..ef92d76 --- /dev/null +++ b/src/mkcd2app/utils/git.py @@ -0,0 +1,63 @@ +import logging +import os +import sys +from pathlib import Path +from typing import BinaryIO + +from dulwich import porcelain + +from mkcd2app.utils.logger import create_logger + +logger = create_logger(name=__name__, level=logging.INFO) + + +def clone_and_checkout(url: str, path: Path, ref: str | None = None) -> None: + """ + Clone a git repository and check out a branch, tag, or commit, using dulwich + (pure Python) so no `git` executable is needed. + + The default branch, branches, and tags get a shallow (depth 1) clone. Anything + else is treated as a (possibly abbreviated) commit SHA, which needs a full clone + because a shallow clone can only fetch refs the server advertises. + + :param url: The URL (or local path) of the repository to clone. + :param path: The directory to clone into. Must not exist or be empty. + :param ref: A branch name, tag name, or (possibly abbreviated) commit SHA. None + checks out the remote's default branch. + """ + # dulwich writes \r-separated progress to errstream, which floods non-TTY logs + # (like CI), so only show it with --debug + show_progress = logger.isEnabledFor(logging.DEBUG) + with open(os.devnull, "wb") as devnull: + errstream: BinaryIO = sys.stderr.buffer if show_progress else devnull + try: + logger.debug(f"Shallow cloning {url}@{ref or 'default branch'} to {path}") + with porcelain.clone( + url, + str(path), + depth=1, + branch=ref, + errstream=errstream, + ): + return + except ValueError: + # dulwich raises ValueError (and removes the directory) when ref isn't + # a branch or tag on the remote. That can't happen with ref=None. + if ref is None: + raise + logger.debug(f"{ref!r} is not a branch or tag, assuming a commit SHA") + + logger.debug(f"Cloning {url} to {path}") + # Check out the default branch first: dulwich's checkout only writes files + # that differ from the current HEAD tree, so after a no-checkout clone + # (empty worktree), a target with the same tree as HEAD would write nothing. + with porcelain.clone(url, str(path), errstream=errstream): + pass + + logger.debug(f"Checking out {ref}") + try: + porcelain.checkout(str(path), ref) + except KeyError: + raise ValueError( + f"Could not find branch, tag, or commit {ref!r} in {url}" + ) from None diff --git a/src/mkcd2app/utils/logger.py b/src/mkcd2app/utils/logger.py index 2e6351d..f29d3ef 100644 --- a/src/mkcd2app/utils/logger.py +++ b/src/mkcd2app/utils/logger.py @@ -16,6 +16,8 @@ def create_logger(name: str, level: int = logging.DEBUG) -> logging.Logger: logger = logging.getLogger(name=name) logger.setLevel(level=level) logger.propagate = False + # So creating a logger twice doesn't log every message twice + logger.handlers.clear() console_formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s" @@ -26,14 +28,12 @@ def create_logger(name: str, level: int = logging.DEBUG) -> logging.Logger: stdout_handler.setLevel(level=level) stdout_handler.addFilter(lambda record: record.levelno < logging.WARNING) stdout_handler.setFormatter(fmt=console_formatter) - if stdout_handler not in logger.handlers: - logger.addHandler(hdlr=stdout_handler) + logger.addHandler(hdlr=stdout_handler) stderr_handler = logging.StreamHandler(stream=sys.stderr) stderr_handler.setLevel(level=logging.WARNING) stderr_handler.setFormatter(fmt=console_formatter) - if stderr_handler not in logger.handlers: - logger.addHandler(hdlr=stderr_handler) + logger.addHandler(hdlr=stderr_handler) logger.debug(f"Created logger named {name!r} with level {level!r}") logger.debug(f"Handlers for {name!r}: {logger.handlers!r}") diff --git a/src/mkcd2app/utils/paths.py b/src/mkcd2app/utils/paths.py index f0d6ab8..f560c0d 100644 --- a/src/mkcd2app/utils/paths.py +++ b/src/mkcd2app/utils/paths.py @@ -1,27 +1,88 @@ import os -import shutil -import stat +from importlib.metadata import version from pathlib import Path -from types import TracebackType -from typing import Any +from platformdirs import PlatformDirs -def _remove_readonly( - func: Any, - path: str, - exc_info: tuple[type[BaseException], BaseException, TracebackType], -) -> None: - """shutil.rmtree error handler: clear read-only bit and retry. - Needed on Windows because git marks files under .git/objects (and - sometimes .git itself) read-only, which makes os.unlink/os.rmdir - raise PermissionError (WinError 5) even though we own the files. +def get_mkcd2app_version() -> str: + return version("mkcd2app") + + +DIRS = PlatformDirs("mkcd2app", appauthor=False) + +# If set, data and state go under this directory instead of the platformdirs user +# dirs (e.g. so tests don't touch the real install) +HOME_ENV_VAR = "MKCD2APP_HOME" + + +def get_home_override_dir() -> Path | None: """ - os.chmod(path, stat.S_IWRITE) - func(path) + The ``MKCD2APP_HOME`` override, made absolute (with ``~`` expanded). + + Relative paths would break subprocesses: they run in other directories, so a + relative executable or ``--cache`` path would point somewhere else. + """ + home = os.environ.get(HOME_ENV_VAR) + return Path(home).expanduser().resolve() if home else None + + +def get_user_data_dir() -> Path: + home = get_home_override_dir() + base = home / "data" if home else Path(DIRS.user_data_dir) + return base / get_mkcd2app_version() + + +def get_user_state_dir() -> Path: + home = get_home_override_dir() + base = home / "state" if home else Path(DIRS.user_state_dir) + return base / get_mkcd2app_version() + + +# Toolchain stuff + + +def get_toolchain_dir() -> Path: + return get_user_data_dir() / "toolchain" + + +def get_redun_db_for_toolchain_path() -> Path: + return get_user_state_dir() / "redun_db_for_toolchain.sqlite3" + + +def get_js_tools_dir() -> Path: + return get_toolchain_dir() / "js_tools" + + +def get_js_tools_bin_dir() -> Path: + return get_js_tools_dir() / "node_modules" / ".bin" + + +def get_templates_npm_cache_dir() -> Path: + return get_toolchain_dir() / "templates_npm_cache" + + +# Target stuff + + +def get_target_dir() -> Path: + return get_user_data_dir() / "target" + + +def get_redun_db_for_target_path() -> Path: + return get_user_state_dir() / "redun_db_for_target.sqlite3" + + +def get_target_version_dir(v: str) -> Path: + return get_target_dir() / v + + +def get_sim_html_path(v: str) -> Path: + return get_target_version_dir(v) / "---simulator.html" + + +# Build stuff -def rmtree_robust(path: Path) -> None: - if not path.exists(): - return - shutil.rmtree(path, onerror=_remove_readonly) +def get_redun_db_for_build_path(build_dir: Path) -> Path: + return build_dir / ".redun-cache.db" diff --git a/src/mkcd2app/utils/resources.py b/src/mkcd2app/utils/resources.py index 60efddb..8640518 100644 --- a/src/mkcd2app/utils/resources.py +++ b/src/mkcd2app/utils/resources.py @@ -5,14 +5,23 @@ @contextmanager -def get_template_path(name: str) -> Iterator[Path]: - ref = files("mkcd2app").joinpath("templates", name) +def get_resource_template_path(name: str | None = None) -> Iterator[Path]: + ref = files("mkcd2app").joinpath("resources", "templates") + if name: + ref = ref.joinpath(name) with as_file(ref) as path: yield path @contextmanager -def get_js_tools_path() -> Iterator[Path]: - ref = files("mkcd2app").joinpath("js_tools") +def get_resource_js_tools_path() -> Iterator[Path]: + ref = files("mkcd2app").joinpath("resources", "js_tools") + with as_file(ref) as path: + yield path + + +@contextmanager +def get_resource_empty_project_path() -> Iterator[Path]: + ref = files("mkcd2app").joinpath("resources", "empty_project") with as_file(ref) as path: yield path diff --git a/src/mkcd2app/utils/run.py b/src/mkcd2app/utils/run.py index 08721cb..8f0c7b2 100644 --- a/src/mkcd2app/utils/run.py +++ b/src/mkcd2app/utils/run.py @@ -22,7 +22,11 @@ def __init__( super().__init__(f"{command} failed ({return_code}) in {cwd}") -def run_cmd(command: list[str], cwd: Path | str) -> str: +def run_cmd( + command: list[str], + cwd: Path | str, + which_path: str | os.PathLike[str] | None = None, +) -> str: """ Runs a command (as a list, no shell) and captures its output. @@ -32,11 +36,12 @@ def run_cmd(command: list[str], cwd: Path | str) -> str: :param command: The command as a list of arguments, e.g. ``["npm", "ci"]``. :param cwd: The working directory to execute the command in. + :param which_path: The path to pass to shutil.which, to search for the binary. :return: The stdout. :raises BuildError: If the command fails. :raises FileNotFoundError: If the executable cannot be found on PATH. """ - resolved = shutil.which(command[0]) + resolved = shutil.which(command[0], path=which_path) if resolved is None: raise FileNotFoundError( f"Executable '{command[0]}' not found on PATH. " @@ -45,8 +50,20 @@ def run_cmd(command: list[str], cwd: Path | str) -> str: cmd_list = [resolved, *command[1:]] logger.debug(f"Running {command} in {cwd}") proc = subprocess.run( - cmd_list, shell=False, cwd=cwd, capture_output=True, text=True, check=False + cmd_list, + shell=False, + cwd=cwd, + capture_output=True, + check=False, + encoding="utf-8", + errors="replace", ) + output = f"{proc.stdout}\n{proc.stderr}" if proc.returncode != 0: - raise BuildError(command, cwd, proc.returncode, proc.stdout + proc.stderr) - return proc.stdout + logger.error( + f"{command} in {cwd} failed ({proc.returncode}), output is below:\n{output}" + ) + raise BuildError(command, cwd, proc.returncode, output) + else: + logger.debug(f"{command} in {cwd} output:\n{output}") + return proc.stdout diff --git a/src/mkcd2app/utils/run_redun_task.py b/src/mkcd2app/utils/run_redun_task.py new file mode 100644 index 0000000..1572619 --- /dev/null +++ b/src/mkcd2app/utils/run_redun_task.py @@ -0,0 +1,37 @@ +import logging +from pathlib import Path + +import redun +from redun import Scheduler +from redun.backends.db import RedunBackendDb +from redun.expression import Expression, Result + +from mkcd2app.utils.logger import create_logger + +logger = create_logger(name=__name__, level=logging.INFO) + + +def run_redun_task(expr: Expression[Result] | Result, redun_db_path: Path) -> Result: + redun_db_path.parent.mkdir(parents=True, exist_ok=True) + db_uri = f"sqlite:///{redun_db_path.resolve()}" + logger.debug(f"redun cache DB: {db_uri}") + # noinspection PyUnresolvedReferences + redun_config = redun.config.Config( + { + "scheduler": {"log_level": "DEBUG"}, + "backend": {"db_uri": db_uri}, + } + ) + scheduler = Scheduler(config=redun_config) + scheduler.load() + try: + return scheduler.run(expr) + finally: + # Close the SQLite connections, so a process that runs several commands + # (like the tests) doesn't leak one per run + backend = scheduler.backend + if isinstance(backend, RedunBackendDb): + if backend.session is not None: + backend.session.close() + if backend.engine is not None: + backend.engine.dispose() diff --git a/src/mkcd2app/utils/text.py b/src/mkcd2app/utils/text.py new file mode 100644 index 0000000..d113473 --- /dev/null +++ b/src/mkcd2app/utils/text.py @@ -0,0 +1,43 @@ +import re +from pathlib import Path + +from mkcd2app.models.metadata import BinaryJSMetadata + + +def extract_meta_comment(file_path: Path) -> BinaryJSMetadata: + """ + Extracts the // meta={} data from the binary.js file. + + :param file_path: Path to the binary.js + :return: The metadata (a BinaryJSMetadata) + :raises ValueError: If the meta comment is not found in file. + """ + with file_path.open("rt", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line.startswith("// meta="): + # Extract everything after '// meta=' + json_str = line.split("// meta=", 1)[1] + return BinaryJSMetadata.model_validate_json(json_str) + + raise ValueError("Meta comment not found in file.") + + +# Thanks Gemini +STRICT_NUMERIC_SEMVER = re.compile(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)") + + +def is_strict_semver(version: str) -> bool: + """ + Whether the string is exactly MAJOR.MINOR.PATCH (no prefix, pre-release, build + metadata, or surrounding whitespace). + """ + # fullmatch, not match with ^...$: $ also matches before a trailing newline + return STRICT_NUMERIC_SEMVER.fullmatch(version) is not None + + +def raise_for_invalid_strict_semver(version: str) -> None: + if not is_strict_semver(version): + raise ValueError( + f"Version '{version}' is not a valid strict numeric semver (e.g., 1.2.3)." + ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3408cc0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,134 @@ +import logging +import shutil +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest +from helpers import Cli, FakeToolchain, GameProject, TaskSpy, install_fake_target + +import mkcd2app.build_project as build_project_module +from mkcd2app.build_project import website +from mkcd2app.build_project.inputs import code +from mkcd2app.target import install as target_install +from mkcd2app.toolchain import install as toolchain_install +from mkcd2app.utils.resources import get_resource_template_path + +TESTS_DIR = Path(__file__).parent +TIERS = ("unit", "integration", "e2e") + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-network", + action="store_true", + help="Also run the tests marked network, which need more than the toolchain " + "install does, like a public MakeCode share link.", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """ + Mark each test with its tier by its directory, so `-m unit` selects them. + + CI runs each tier as a separate workflow, so a test outside these directories would + run in none of them; fail collection instead of skipping it quietly. + """ + run_network = config.getoption("--run-network") + for item in items: + if not run_network and item.get_closest_marker("network"): + item.add_marker(pytest.mark.skip(reason="needs --run-network")) + parts = item.path.relative_to(TESTS_DIR).parts + if len(parts) < 2 or parts[0] not in TIERS: + raise pytest.UsageError( + f"{item.nodeid}: tests must live under " + f"{', '.join(f'tests/{tier}' for tier in TIERS)}" + ) + item.add_marker(parts[0]) + + +@pytest.fixture +def mkcd2app_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point MKCD2APP_HOME at a temp dir, so the test can't touch the real user dirs.""" + home = tmp_path / "mkcd2app-home" + monkeypatch.setenv("MKCD2APP_HOME", str(home)) + return home + + +@pytest.fixture +def restore_log_levels() -> Iterator[None]: + """Put back every logger's level, since set_all_stdout_logger_levels changes all.""" + all_loggers = logging.Logger.manager.loggerDict + before = { + name: (lgr.level, [(h, h.level) for h in lgr.handlers]) + for name, lgr in all_loggers.items() + if isinstance(lgr, logging.Logger) + } + yield + for name, lgr in all_loggers.items(): + if isinstance(lgr, logging.Logger): + level, handlers = before.get(name, (logging.NOTSET, [])) + lgr.setLevel(level) + for handler, handler_level in handlers: + handler.setLevel(handler_level) + + +# ---------------------------------------------------- integration test harness --- + + +@pytest.fixture +def fake_toolchain(monkeypatch: pytest.MonkeyPatch) -> FakeToolchain: + """Replace `run_cmd` everywhere it's used, so no `npm` or `mkc` really runs.""" + fake = FakeToolchain() + for module in (code, website, toolchain_install, target_install): + monkeypatch.setattr(module, "run_cmd", fake) + return fake + + +@pytest.fixture +def task_spy(monkeypatch: pytest.MonkeyPatch) -> TaskSpy: + return TaskSpy(monkeypatch) + + +@pytest.fixture +def cli( + mkcd2app_home: Path, + monkeypatch: pytest.MonkeyPatch, + fake_toolchain: FakeToolchain, + task_spy: TaskSpy, +) -> Cli: + return Cli(monkeypatch, fake_toolchain, task_spy) + + +@pytest.fixture +def template_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """ + A copy of the real bundled template that builds use, so a test can change it. The + tests that change nothing about it still get drift in the real one caught. + """ + dst = tmp_path / "vite-project" + with get_resource_template_path("vite-project") as src: + shutil.copytree(src, dst, ignore=shutil.ignore_patterns("node_modules", "dist")) + + @contextmanager + def fake_get_resource_template_path(name: str | None = None) -> Iterator[Path]: + assert name == "vite-project" + yield dst + + monkeypatch.setattr( + build_project_module, + "get_resource_template_path", + fake_get_resource_template_path, + ) + return dst + + +@pytest.fixture +def project(tmp_path: Path, cli: Cli, template_dir: Path) -> GameProject: + """A game ready to build: its config is written and its target is installed.""" + install_fake_target() + root = tmp_path / "project" + root.mkdir() + return GameProject(root, cli) diff --git a/tests/e2e/test_statics.py b/tests/e2e/test_statics.py index 7033c07..d60b6b4 100644 --- a/tests/e2e/test_statics.py +++ b/tests/e2e/test_statics.py @@ -1,40 +1,163 @@ +import os +import re import shutil import subprocess -import time +import zipfile from collections.abc import Generator +from dataclasses import dataclass from pathlib import Path +from typing import Any import pytest +import yaml +from bs4 import BeautifulSoup +from dulwich import porcelain +from mkcd2app.build_project.inputs.embedded_source import read_embedded_files +from mkcd2app.utils.paths import get_sim_html_path -def run_build(config_path: Path) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["mkcd2app", "build", str(config_path)], +TARGET_VERSION = "4.0.14" +FIXTURES = Path("tests/fixtures") +MKCD_PROJECTS = "mkcd_projects" # Next to the configs, which use relative paths +EMPTY_GAME_DIR = FIXTURES / MKCD_PROJECTS / "empty_game" +STATICS_CONFIG = "empty_game_path_build_statics.yaml" +FILE_SOURCE_CONFIG = "empty_game_file_source_build_statics.yaml" +RACERS_SHARE_LINK = "https://arcade.makecode.com/84426-33815-03715-00484" +EMPTY_GAME_TITLE = "Empty Game v0.0.1" +FAVICON = 'href="data:image/x-icon;base64,' +# From binary.js and the simulator, as a build can only have these if it has those +BINARY_JS_MARKER = f'"targetVersion":"{TARGET_VERSION}"' +SIMULATOR_MARKER = "initCurrentRuntime" + + +def run_mkcd2app( + *args: str, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + ["mkcd2app", *args], capture_output=True, text=True, - timeout=300, - check=True, + encoding="utf-8", + errors="replace", + timeout=600, + env=env, + check=False, # pytest.fail below, as CalledProcessError hides the output ) + if result.returncode != 0: + pytest.fail( + f"`mkcd2app {' '.join(args)}` failed with exit code {result.returncode}\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) + return result -def run_empty_game_build() -> subprocess.CompletedProcess[str]: - yaml_path = Path.cwd() / "tests" / "fixtures" / "empty_game_build_to_statics.yaml" - return run_build(yaml_path) +def parse_job_status(redun_output: str) -> dict[str, dict[str, int]]: + """ + Parse the last job status table redun logs after a run, as {task or "ALL": {column: + count}}. + This relies on redun's log format, which is stable because redun is pinned. + Example:: -@pytest.fixture(scope="module") -def empty_game_build_to_statics() -> Generator[None]: - run_empty_game_build() + [redun] | TASK PENDING RUNNING FAILED CACHED DONE TOTAL + [redun] | + [redun] | ALL 0 0 0 7 0 7 + [redun] | mkcd2app.fetch_code 0 0 0 1 0 1 + """ + lines = redun_output.splitlines() + headers = [i for i, line in enumerate(lines) if "| TASK " in line] + assert headers, f"no redun job status table in output:\n{redun_output}" + columns = lines[headers[-1]].split("| TASK", 1)[1].split() + status: dict[str, dict[str, int]] = {} + for line in lines[headers[-1] + 1 :]: + row = re.search(r"\| (\S+)((?:\s+\d+)+)\s*$", line) + if row: + counts = map(int, row[2].split()) + status[row[1]] = dict(zip(columns, counts, strict=True)) + assert "ALL" in status, f"no ALL row in job status table:\n{redun_output}" + return status + + +def tasks_that_ran(redun_output: str) -> set[str]: + return { + task.removeprefix("mkcd2app.") + for task, counts in parse_job_status(redun_output).items() + if task != "ALL" and counts["DONE"] + } + + +@pytest.fixture(scope="session", autouse=True) +def installed_toolchain_and_target(pytestconfig: pytest.Config) -> Generator[None]: + # Each xdist worker would run this fixture, all installing into the same dir below + if "PYTEST_XDIST_WORKER" in os.environ: + pytest.fail("The e2e tests can't run under xdist; drop -n or deselect them") + # Install into a dir under .pytest_cache instead of the real user dirs; it + # persists between sessions so reruns hit the install's redun cache + home = pytestconfig.cache.mkdir("mkcd2app-home") + with pytest.MonkeyPatch.context() as mp: + mp.setenv("MKCD2APP_HOME", str(home)) + run_mkcd2app("toolchain", "install") + run_mkcd2app("target", "install", TARGET_VERSION) + yield + + +@dataclass +class Build: + config: Path + + @property + def dir(self) -> Path: + return self.config.parent / "build" + + @property + def source(self) -> Path: + return self.dir / "empty-game-source" + + @property + def static(self) -> Path: + return self.dir / "empty-game-website-filled-built" / "dist" + + @property + def singlefile(self) -> Path: + return self.dir / "empty-game-website-filled-singlefile-built" / "dist" + + +def make_project(root: Path) -> Build: + """Copy the empty game and its config to a folder, as the config's paths are relative.""" + shutil.copytree(EMPTY_GAME_DIR, root / MKCD_PROJECTS / "empty_game") + return Build(Path(shutil.copy(FIXTURES / STATICS_CONFIG, root))) + + +def read_config(build: Build) -> dict[str, Any]: + config: dict[str, Any] = yaml.safe_load(build.config.read_text(encoding="utf-8")) + return config + + +def write_config(build: Build, config: dict[str, Any]) -> None: + build.config.write_text(yaml.safe_dump(config), encoding="utf-8") + + +def assert_is_a_static_website(dist: Path) -> None: + assert (dist / "index.html").is_file(), "expected index.html present" + assert list((dist / "assets").glob("*.js")), "expected js files" - yield - dir_path = Path.cwd() / "examples" / "testing" / "empty_game_build" - shutil.rmtree(dir_path) +# ---------------------------------------------------------- the shared build ----- -def test_empty_game_build_to_statics_static(empty_game_build_to_statics: None) -> None: - build_dir = Path("examples/testing/empty_game_build") - static_dist = build_dir / "empty-game-website-filled-built" / "dist" +@pytest.fixture(scope="module") +def empty_game_build_to_statics(tmp_path_factory: pytest.TempPathFactory) -> Build: + build = make_project(tmp_path_factory.mktemp("statics")) + shutil.copy(Path("examples") / "Racers icon.png", build.config.parent / "icon.png") + config = read_config(build) + config["inputs"]["assets"] = {"icon": {"type": "path", "value": "icon.png"}} + write_config(build, config) + run_mkcd2app("build", str(build.config)) + return build + + +def test_empty_game_build_to_statics_static(empty_game_build_to_statics: Build) -> None: + static_dist = empty_game_build_to_statics.static # check for index.html and assets dir assert (static_dist / "index.html").is_file(), "expected index.html file present" @@ -44,15 +167,31 @@ def test_empty_game_build_to_statics_static(empty_game_build_to_statics: None) - js_files = list((static_dist / "assets").glob("*.js")) assert css_files, "expected css files present" assert js_files, "expected js files present" + # Relative asset paths, so the site also works from a subfolder like // + index = (static_dist / "index.html").read_text(encoding="utf-8") + soup = BeautifulSoup(index, features="html.parser") + tags = soup.select("script[src], link[rel=stylesheet], link[rel=modulepreload]") + urls = [str(tag.get("src") or tag.get("href")) for tag in tags] + assert urls and all( + url.startswith("./") and (static_dist / url).is_file() for url in urls + ), f"expected relative asset paths to files in the output, got {urls}" + + +def test_static_output_has_the_game_and_simulator( + empty_game_build_to_statics: Build, +) -> None: + assets = empty_game_build_to_statics.static / "assets" + # vite.config.ts splits these into their own chunks + [binary_js] = assets.glob("binary-*.js") + [simulator] = assets.glob("simulator-*.js") + assert BINARY_JS_MARKER in binary_js.read_text(encoding="utf-8") + assert SIMULATOR_MARKER in simulator.read_text(encoding="utf-8") def test_empty_game_build_to_statics_static_singlefile( - empty_game_build_to_statics: None, + empty_game_build_to_statics: Build, ) -> None: - build_dir = Path("examples/testing/empty_game_build") - static_singlefile_dist = ( - build_dir / "empty-game-website-filled-singlefile-built" / "dist" - ) + static_singlefile_dist = empty_game_build_to_statics.singlefile # ensure no other files or directories are present in the dist items = list(static_singlefile_dist.iterdir()) @@ -64,14 +203,242 @@ def test_empty_game_build_to_statics_static_singlefile( assert only.name == "index.html", "expected only index.html present" +def test_singlefile_output_has_the_game_and_simulator_inside_it( + empty_game_build_to_statics: Build, +) -> None: + text = (empty_game_build_to_statics.singlefile / "index.html").read_text("utf-8") + assert BINARY_JS_MARKER in text + assert SIMULATOR_MARKER in text + soup = BeautifulSoup(text, features="html.parser") + assert soup.find_all("script", src=True) == [] + assert soup.find_all("link", rel="stylesheet") == [] + + +@pytest.mark.parametrize( + "get_dist", + [lambda build: build.static, lambda build: build.singlefile], + ids=["static", "singlefile"], +) +def test_title_and_icon_reach_the_built_page( + empty_game_build_to_statics: Build, get_dist: Any +) -> None: + text = (get_dist(empty_game_build_to_statics) / "index.html").read_text("utf-8") + assert EMPTY_GAME_TITLE in text + assert FAVICON in text + + +@pytest.mark.parametrize( + "get_page", + [ + lambda build: build.static / "index.html", + lambda build: build.singlefile / "index.html", + lambda build: get_sim_html_path(TARGET_VERSION), + ], + ids=["static", "singlefile", "installed-simulator"], +) +def test_nothing_loads_scripts_or_styles_from_the_internet( + empty_game_build_to_statics: Build, get_page: Any +) -> None: + # The offline promise + text = get_page(empty_game_build_to_statics).read_text(encoding="utf-8") + soup = BeautifulSoup(text, features="html.parser") + urls = [str(tag["src"]) for tag in soup.find_all("script", src=True)] + urls += [str(tag.get("href")) for tag in soup.find_all("link", rel="stylesheet")] + external = [url for url in urls if re.match(r"(https?:)?//", url)] + assert external == [] + + def test_empty_game_build_to_statics_second_build_caches( - empty_game_build_to_statics: None, + empty_game_build_to_statics: Build, ) -> None: - start_time = time.monotonic() - run_empty_game_build() # this should do nothing file wise, all hit caches - end_time = time.monotonic() - elapsed_time = end_time - start_time + # Nothing changed, so every job should be served from redun's cache + result = run_mkcd2app("build", str(empty_game_build_to_statics.config)) + totals = parse_job_status(result.stderr)["ALL"] - assert elapsed_time < 5, ( - f"expected second build to all hit cache and take less than 5s, took {elapsed_time}s" + assert totals["TOTAL"] > 0, f"expected redun to schedule jobs, got {totals}" + assert totals["DONE"] == 0 and totals["CACHED"] == totals["TOTAL"], ( + f"expected every job on the second build to hit the cache, got {totals}" ) + + +# ------------------------------------------------------------- code sources ----- + + +def test_empty_game_file_source_build_to_static(tmp_path: Path) -> None: + # Same empty game, but exported as a PNG from the 4.0.14 editor (`file` source) + (tmp_path / MKCD_PROJECTS).mkdir() + png = FIXTURES / MKCD_PROJECTS / "empty-game.png" + shutil.copy(png, tmp_path / MKCD_PROJECTS) + build = Build(Path(shutil.copy(FIXTURES / FILE_SOURCE_CONFIG, tmp_path))) + + result = run_mkcd2app("build", str(build.config)) + + # Extraction keeps the embedded bytes exactly, so compare against the PNG's + # own contents; the checked-in fixture may have CRLF from git on Windows + embedded = read_embedded_files(png) + assert (build.source / "main.ts").read_bytes() == embedded["main.ts"].encode() + expected = EMPTY_GAME_DIR / "main.ts" + assert (build.source / "main.ts").read_text(encoding="utf-8") == expected.read_text( + encoding="utf-8" + ) + assert (build.source / "mkc.json").is_file(), "expected mkc.json written" + # The PNG was saved with the same Arcade version as `target` + output = result.stdout + result.stderr + assert "Project was saved with Arcade" not in output + + assert_is_a_static_website(build.static) + + +def test_empty_game_zip_file_source_build_to_static(tmp_path: Path) -> None: + # Zip the checked-in empty game the way GitHub's "Download ZIP" lays it out (one + # -/ wrapper folder) and build it as a `file` source + game_dir = EMPTY_GAME_DIR + zip_path = tmp_path / "empty_game-master.zip" + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.mkdir("empty_game-master") + for f in sorted(game_dir.iterdir()): + zf.write(f, f"empty_game-master/{f.name}") + + template = (FIXTURES / FILE_SOURCE_CONFIG).read_text(encoding="utf-8") + yaml_text = template.replace(f"{MKCD_PROJECTS}/empty-game.png", zip_path.name) + assert zip_path.name in yaml_text + build = Build(tmp_path / "empty_game_zip_source.yaml") + build.config.write_text(yaml_text, encoding="utf-8") + + result = run_mkcd2app("build", str(build.config)) + + # The wrapper is stripped and every file comes through byte for byte (the ZIP + # was made from the checkout, so this holds with CRLF on Windows too) + extracted = {p.name for p in build.source.iterdir()} + assert extracted == {p.name for p in game_dir.iterdir()} | {"mkc.json"} + for f in game_dir.iterdir(): + assert (build.source / f.name).read_bytes() == f.read_bytes(), f.name + output = result.stdout + result.stderr + assert "Project was saved with Arcade" not in output + + assert_is_a_static_website(build.static) + assert [p.name for p in build.singlefile.iterdir()] == ["index.html"] + + +def test_git_source_build_to_static(tmp_path: Path) -> None: + # A local repo, so this runs real dulwich without needing the network + game_dir = EMPTY_GAME_DIR + repo = (tmp_path / "repo").resolve() + shutil.copytree(game_dir, repo) + porcelain.init(str(repo)) + porcelain.add(str(repo), [str(path) for path in repo.iterdir() if path.is_file()]) + # sign=False so a commit.gpgsign in the user's git config doesn't apply + identity = b"Test " + porcelain.commit( + str(repo), + message=b"Add game", + author=identity, + committer=identity, + sign=False, + ) + + (tmp_path / "project").mkdir() + build = Build(Path(shutil.copy(FIXTURES / STATICS_CONFIG, tmp_path / "project"))) + config = read_config(build) + config["inputs"]["code"] = {"type": "git", "url": repo.as_uri()} + write_config(build, config) + + run_mkcd2app("build", str(build.config)) + + for f in game_dir.iterdir(): + assert (build.source / f.name).read_bytes() == f.read_bytes(), f.name + assert (build.source / ".git").is_dir() + assert_is_a_static_website(build.static) + + +@pytest.mark.network +def test_share_link_source_build_to_static(tmp_path: Path) -> None: + build = make_project(tmp_path) + config = read_config(build) + config["inputs"]["code"] = {"type": "share_link", "value": RACERS_SHARE_LINK} + write_config(build, config) + + run_mkcd2app("build", str(build.config)) + + assert (build.source / "pxt.json").is_file() + assert_is_a_static_website(build.static) + + +# ------------------------------------------------------------ without a network ----- + + +def blocked_env(log: Path) -> dict[str, str]: + """The environment for a subprocess that can't reach the network.""" + preload = (FIXTURES / "block_network.cjs").resolve().as_posix() + dead_proxy = "http://127.0.0.1:9" # Python's requests honours these + return { + **os.environ, + "NODE_OPTIONS": f'--require "{preload}"', + "MKCD2APP_TEST_BLOCKED_LOG": str(log), + "HTTP_PROXY": dead_proxy, + "HTTPS_PROXY": dead_proxy, + "npm_config_offline": "true", + } + + +def read_blocked(log: Path) -> list[str]: + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + +def test_network_blocker_really_blocks(tmp_path: Path) -> None: + # Guards the fixture: the tests below would pass by accident if this did nothing + log = tmp_path / "blocked.log" + script = "fetch('https://example.com').then(() => process.exit(0), () => process.exit(3))" + result = subprocess.run( + ["node", "-e", script], env=blocked_env(log), timeout=60, check=False + ) + assert result.returncode == 3 + assert read_blocked(log) == ["example.com:443"] + + +@dataclass +class OfflineBuilds: + build: Build + blocked: list[str] + title_edit_output: str + + +@pytest.fixture(scope="module") +def offline_builds(tmp_path_factory: pytest.TempPathFactory) -> OfflineBuilds: + """A full build, then a rebuild after editing the title, with no network.""" + root = tmp_path_factory.mktemp("offline") + log = root / "blocked.log" + env = blocked_env(log) + build = make_project(root) + run_mkcd2app("build", str(build.config), env=env) + + config = read_config(build) + config["project"]["title"] = "Edited Title" + write_config(build, config) + result = run_mkcd2app("build", str(build.config), env=env) + + return OfflineBuilds(build, read_blocked(log), result.stderr) + + +def test_build_needs_no_network(offline_builds: OfflineBuilds) -> None: + # The empty game has no extensions, so everything should be vendored. The one + # exception is `mkc`'s update check, once a day: when it fails, `mkc` logs that and + # carries on, so whether it's tried here depends on the age of `mkc`'s cache. + assert set(offline_builds.blocked) <= {"arcade.makecode.com:443"} + assert_is_a_static_website(offline_builds.build.static) + assert [p.name for p in offline_builds.build.singlefile.iterdir()] == ["index.html"] + + +def test_title_edit_reaches_the_output(offline_builds: OfflineBuilds) -> None: + for dist in (offline_builds.build.static, offline_builds.build.singlefile): + text = (dist / "index.html").read_text(encoding="utf-8") + assert "Edited Title" in text + + +def test_title_edit_only_rebuilds_the_website(offline_builds: OfflineBuilds) -> None: + assert tasks_that_ran(offline_builds.title_edit_output) == { + "build_project", + "fill_website_template", + "build_website", + "build_website_singlefile", + } diff --git a/tests/fixtures/block_network.cjs b/tests/fixtures/block_network.cjs new file mode 100644 index 0000000..8d388f9 --- /dev/null +++ b/tests/fixtures/block_network.cjs @@ -0,0 +1,29 @@ +// Preload with `node --require` (NODE_OPTIONS) to make any network connection fail, as +// it would offline, and append "host:port" to the file in MKCD2APP_TEST_BLOCKED_LOG. Node +// ignores HTTP_PROXY by default, so a dead proxy alone wouldn't block anything. Local +// connections and pipes are allowed, since build tools use them to talk to themselves. +const fs = require("node:fs"); +const net = require("node:net"); + +const LOOPBACK = new Set(["localhost", "127.0.0.1", "::1"]); +const connect = net.Socket.prototype.connect; + +net.Socket.prototype.connect = function (...args) { + // net.connect() passes [options, callback] as one array, internally + const first = Array.isArray(args[0]) ? args[0][0] : args[0]; + const options = + typeof first === "object" ? first : { port: first, host: args[1] }; + const host = options.host ?? "localhost"; + if (options.path || LOOPBACK.has(host)) { + return connect.apply(this, args); + } + + fs.appendFileSync( + process.env.MKCD2APP_TEST_BLOCKED_LOG, + `${host}:${options.port}\n`, + ); + const error = new Error(`Network blocked by test: ${host}:${options.port}`); + error.code = "ENETUNREACH"; + process.nextTick(() => this.destroy(error)); + return this; +}; diff --git a/tests/fixtures/empty_game/pxt.json b/tests/fixtures/empty_game/pxt.json deleted file mode 100644 index 0015c50..0000000 --- a/tests/fixtures/empty_game/pxt.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "test", - "description": "", - "dependencies": { - "device": "*" - }, - "files": [ - "main.blocks", - "main.ts", - "README.md", - "assets.json" - ], - "targetVersions": { - "branch": "stable4.0", - "tag": "v4.0.14", - "commits": "https://github.com/microsoft/pxt-arcade/commits/a33ec9537bc6c5fada2a3b02aac57a4bc9a10803", - "target": "4.0.14", - "pxt": "12.2.34" - }, - "preferredEditor": "blocksprj" -} diff --git a/tests/fixtures/empty_game_file_source_build_statics.yaml b/tests/fixtures/empty_game_file_source_build_statics.yaml new file mode 100644 index 0000000..0994bf8 --- /dev/null +++ b/tests/fixtures/empty_game_file_source_build_statics.yaml @@ -0,0 +1,61 @@ +# E2E config that builds the empty game from a PNG downloaded from the editor +# (a `file` code source) + +# Config version (single number, incremented if breaking change to schema is +# made) +version: 1 + +project: + name: Empty Game + path_friendly_name: empty-game + description: "Empty game, exported as a PNG from the 4.0.14 editor, used for E2E testing" + author: Cyrus Yiu + version: 0.0.1 + # This is what the window title will be + # This will also be used for the executable file name for Electron outputs + # You can use {NAME} or {VERSION} or {AUTHOR} to substitute the correct + # values + title: "{NAME} v{VERSION}" + +inputs: + # Can be share_link, git, path, or file (a PNG or UF2 downloaded from the + # editor, or a ZIP of the project such as GitHub's "Download ZIP"). github is + # an older alias for git. + code: + type: file + value: "mkcd_projects/empty-game.png" + + assets: {} + # Icon is optional but highly recommended, it will be used as the favicon + # and app icon +# icon: +# Can be url or path +# type: url +# value: https://raw.githubusercontent.com/UnsignedArduino/MakeCode-Arcade-to-App/refs/heads/main/examples/Racers%20icon.png + +# icon: +# type: path +# value: "../../examples/Racers icon.png" + +# Path where the build process will take place +# Relative paths in this file are relative to this file's directory +build_dir: "./build" + +# MakeCode Arcade target version (must be installed with `mkcd2app target install`) +target: 4.0.14 + +# You can have multiple outputs +# Currently available options are static, static-singlefile, electron, and +# tauri +outputs: + - type: static + - type: static-singlefile +# - type: electron +# window: +# width: 640 +# height: 480 +# - type: tauri +# identifier: com.unsignedarduino.racers +# window: +# width: 640 +# height: 480 diff --git a/tests/fixtures/empty_game_build_to_statics.yaml b/tests/fixtures/empty_game_path_build_statics.yaml similarity index 74% rename from tests/fixtures/empty_game_build_to_statics.yaml rename to tests/fixtures/empty_game_path_build_statics.yaml index 3650c37..44ef400 100644 --- a/tests/fixtures/empty_game_build_to_statics.yaml +++ b/tests/fixtures/empty_game_path_build_statics.yaml @@ -18,10 +18,12 @@ project: title: "{NAME} v{VERSION}" inputs: - # Can be share_link, github, or path + # Can be share_link, git, path, or file (a PNG or UF2 downloaded from the + # editor, or a ZIP of the project such as GitHub's "Download ZIP"). github is + # an older alias for git. code: type: path - value: "tests/fixtures/empty_game" + value: "mkcd_projects/empty_game" assets: {} # Icon is optional but highly recommended, it will be used as the favicon @@ -33,10 +35,14 @@ inputs: # icon: # type: path -# value: "./examples/Racers icon.png" +# value: "../../examples/Racers icon.png" # Path where the build process will take place -build_dir: "./examples/testing/empty_game_build" +# Relative paths in this file are relative to this file's directory +build_dir: "./build" + +# MakeCode Arcade target version (must be installed with `mkcd2app target install`) +target: 4.0.14 # You can have multiple outputs # Currently available options are static, static-singlefile, electron, and diff --git a/tests/fixtures/lzma-js/files.json b/tests/fixtures/lzma-js/files.json new file mode 100644 index 0000000..bc53886 --- /dev/null +++ b/tests/fixtures/lzma-js/files.json @@ -0,0 +1 @@ +{"pxt.json":"{\"name\":\"emoji 😀 game\",\"dependencies\":{\"device\":\"*\"}}","main.ts":"game.splash(\"hello 😀\")\r\nlet s = \"é ✓ 𝄞\"\n","README.md":"# 🦆 Duck\n"} \ No newline at end of file diff --git a/tests/fixtures/lzma-js/generate.cjs b/tests/fixtures/lzma-js/generate.cjs new file mode 100644 index 0000000..5422359 --- /dev/null +++ b/tests/fixtures/lzma-js/generate.cjs @@ -0,0 +1,38 @@ +// Regenerates the LZMA-JS payloads used by tests/unit/test_embedded_source.py. +// +// MakeCode compresses projects with LZMA-JS, which encodes strings as CESU-8 (each +// UTF-16 code unit on its own, so emoji become two 3-byte surrogates). Python's +// lzma can't produce that, so these payloads come from the real library: +// +// npm install --no-save lzma@2.3.2 +// node generate.cjs +// +// Output: png-project.lzma (the PNG blob), uf2-source.lzma (the UF2 LZMA text), +// uf2-header-size.txt (headerSize, in UTF-16 code units), and files.json (the +// expected {filename: text} map, as normal UTF-8). +const fs = require("fs"); +const path = require("path"); +const LZMA = require("lzma"); + +const files = { + "pxt.json": JSON.stringify({ name: "emoji 😀 game", dependencies: { device: "*" } }), + "main.ts": 'game.splash("hello 😀")\r\nlet s = "é ✓ 𝄞"\n', + "README.md": "# 🦆 Duck\n", +}; +const meta = { name: "emoji 😀 game", editor: "tsprj" }; + +function compress(text) { + return Buffer.from(Uint8Array.from(LZMA.compress(text, 9), (b) => b & 0xff)); +} + +const out = __dirname; +// PNG: pxt's compressProjectAsync -> {meta, source: JSON of files} +fs.writeFileSync( + path.join(out, "png-project.lzma"), + compress(JSON.stringify({ meta, source: JSON.stringify(files) })), +); +// UF2: header JSON then files JSON, headerSize = header's JS string length +const header = JSON.stringify(meta); +fs.writeFileSync(path.join(out, "uf2-source.lzma"), compress(header + JSON.stringify(files))); +fs.writeFileSync(path.join(out, "uf2-header-size.txt"), String(header.length)); +fs.writeFileSync(path.join(out, "files.json"), JSON.stringify(files)); diff --git a/tests/fixtures/lzma-js/png-project.lzma b/tests/fixtures/lzma-js/png-project.lzma new file mode 100644 index 0000000..5e0a151 Binary files /dev/null and b/tests/fixtures/lzma-js/png-project.lzma differ diff --git a/tests/fixtures/lzma-js/uf2-header-size.txt b/tests/fixtures/lzma-js/uf2-header-size.txt new file mode 100644 index 0000000..aaa6442 --- /dev/null +++ b/tests/fixtures/lzma-js/uf2-header-size.txt @@ -0,0 +1 @@ +41 \ No newline at end of file diff --git a/tests/fixtures/lzma-js/uf2-source.lzma b/tests/fixtures/lzma-js/uf2-source.lzma new file mode 100644 index 0000000..e63b09c Binary files /dev/null and b/tests/fixtures/lzma-js/uf2-source.lzma differ diff --git a/tests/fixtures/mkcd_projects/Racers-master.zip b/tests/fixtures/mkcd_projects/Racers-master.zip new file mode 100644 index 0000000..ff3872e Binary files /dev/null and b/tests/fixtures/mkcd_projects/Racers-master.zip differ diff --git a/tests/fixtures/mkcd_projects/empty-game.png b/tests/fixtures/mkcd_projects/empty-game.png new file mode 100644 index 0000000..362a568 Binary files /dev/null and b/tests/fixtures/mkcd_projects/empty-game.png differ diff --git a/tests/fixtures/mkcd_projects/empty_game/README.md b/tests/fixtures/mkcd_projects/empty_game/README.md new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/tests/fixtures/mkcd_projects/empty_game/README.md @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fixtures/mkcd_projects/empty_game/assets.json b/tests/fixtures/mkcd_projects/empty_game/assets.json new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/empty_game/main.blocks b/tests/fixtures/mkcd_projects/empty_game/main.blocks similarity index 100% rename from tests/fixtures/empty_game/main.blocks rename to tests/fixtures/mkcd_projects/empty_game/main.blocks diff --git a/tests/fixtures/mkcd_projects/empty_game/main.ts b/tests/fixtures/mkcd_projects/empty_game/main.ts new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/fixtures/mkcd_projects/empty_game/main.ts @@ -0,0 +1 @@ + diff --git a/tests/fixtures/mkcd_projects/empty_game/pxt.json b/tests/fixtures/mkcd_projects/empty_game/pxt.json new file mode 100644 index 0000000..70a52b0 --- /dev/null +++ b/tests/fixtures/mkcd_projects/empty_game/pxt.json @@ -0,0 +1,21 @@ +{ + "name": "empty_game", + "description": "", + "dependencies": { + "device": "*" + }, + "files": [ + "main.blocks", + "main.ts", + "README.md", + "assets.json" + ], + "targetVersions": { + "branch": "stable4.0", + "tag": "v4.0.14", + "commits": "https://github.com/microsoft/pxt-arcade/commits/a33ec9537bc6c5fada2a3b02aac57a4bc9a10803", + "target": "4.0.14", + "pxt": "12.2.34" + }, + "preferredEditor": "blocksprj" +} diff --git a/tests/fixtures/mkcd_projects/falling-duck.png b/tests/fixtures/mkcd_projects/falling-duck.png new file mode 100644 index 0000000..6aaa2a4 Binary files /dev/null and b/tests/fixtures/mkcd_projects/falling-duck.png differ diff --git a/tests/fixtures/mkcd_projects/falling-duck.uf2 b/tests/fixtures/mkcd_projects/falling-duck.uf2 new file mode 100644 index 0000000..0d95fc2 Binary files /dev/null and b/tests/fixtures/mkcd_projects/falling-duck.uf2 differ diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..562c469 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,366 @@ +"""Shared helpers for the tests (a plain module, since pytest won't collect it).""" + +import hashlib +import json +import logging +import os +import shutil +import sys +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from functools import wraps +from pathlib import Path +from typing import Any + +import pytest +import requests +import yaml +from redun.file import ContentDir, ContentFile +from redun.task import Task, get_task_registry + +from mkcd2app.main import main +from mkcd2app.utils.paths import get_sim_html_path +from mkcd2app.utils.run import BuildError + +TARGET_VERSION = "4.0.14" +FIXTURES = Path(__file__).parent / "fixtures" +FAKE_SIM_URL = "https://sim.example.com/---simulator" + + +def make_config_yaml( + build_dir: Path, + code: dict[str, Any] | None = None, + icon: dict[str, Any] | None = None, + project: dict[str, Any] | None = None, + target: str = TARGET_VERSION, + outputs: list[dict[str, Any]] | None = None, +) -> str: + """ + Build a version 1 config as YAML text, like the contents of a config file. + + :param build_dir: The config's build_dir. + :param code: The inputs.code source. Defaults to a path source that doesn't exist, + for tests that never read it. + :param icon: The inputs.assets.icon source, or None for no icon. + :param project: Keys to override in the project section. + :param target: The MakeCode Arcade target version. + :param outputs: The outputs. Defaults to only a static website. + """ + assets: dict[str, Any] = {"icon": icon} if icon else {} + return yaml.safe_dump( + { + "version": 1, + "project": { + "name": "Test Game", + "path_friendly_name": "test", + "description": "A test game", + "author": "Tester", + "version": "1.2.3", + **(project or {}), + }, + "inputs": { + "code": code or {"type": "path", "value": str(build_dir / "nowhere")}, + "assets": assets, + }, + "build_dir": str(build_dir), + "target": target, + "outputs": outputs or [{"type": "static"}], + } + ) + + +@dataclass +class CmdCall: + command: list[str] + cwd: Path + which_path: str | os.PathLike[str] | None + + +class FakeRunCmd: + """ + Stands in for ``run_cmd``: records each call, then runs ``on_run`` (if given) so + the test can create the files the real command would have. + """ + + def __init__(self, on_run: Callable[[CmdCall], None] | None = None) -> None: + self.calls: list[CmdCall] = [] + self.on_run = on_run + + def __call__( + self, + command: list[str], + cwd: Path | str, + which_path: str | os.PathLike[str] | None = None, + ) -> str: + call = CmdCall(command, Path(cwd), which_path) + self.calls.append(call) + if self.on_run: + self.on_run(call) + return "" + + +class FakeResponse: + """Stands in for the ``requests.Response`` that ``requests.get`` returns.""" + + def __init__(self, content: bytes, status_code: int = 200) -> None: + self.content = content + self.status_code = status_code + + @property + def text(self) -> str: + # What requests falls back to when the server sends no charset + return self.content.decode("iso-8859-1") + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise requests.HTTPError(f"{self.status_code} error") + + +def tree_digest(root: Path) -> dict[str, str]: + """Map each file under ``root`` (relative POSIX path) to its SHA-256.""" + return { + p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +@contextmanager +def capture_logger( + caplog: pytest.LogCaptureFixture, logger: logging.Logger +) -> Iterator[pytest.LogCaptureFixture]: + """ + Capture records from one of our loggers. ``create_logger`` sets + ``propagate = False``, so caplog's root handler never sees them otherwise. + """ + logger.addHandler(caplog.handler) + try: + yield caplog + finally: + logger.removeHandler(caplog.handler) + + +# ---------------------------------------------------- integration test harness --- + + +def _digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def command_name(command: list[str]) -> str: + """A command without its arguments, like ``mkc build`` or ``npm run build``.""" + return " ".join(command[:3] if command[:2] == ["npm", "run"] else command[:2]) + + +class FakeToolchain: + """ + Stands in for ``run_cmd`` everywhere: does to the disk what each ``mkc`` and ``npm`` + command would, and fails the test on any other command. Like the real commands, + these need their inputs to exist, and their output depends on those inputs, so a + changed game really does produce a different website. + """ + + def __init__(self) -> None: + self.commands: list[str] = [] + # Names (see command_name) of the commands that should fail + self.failing: set[str] = set() + + def __call__( + self, + command: list[str], + cwd: Path | str, + which_path: str | os.PathLike[str] | None = None, + ) -> str: + name = command_name(command) + self.commands.append(name) + if name in self.failing: + raise BuildError(command, cwd, 1, f"fake {name} output") + path = Path(cwd) + match command: + case ["mkc", "build", "-j"]: + self._mkc_build(path) + case ["mkc", "download", url]: + self._mkc_download(path, url) + case ["npm", "ci", *args]: + self._npm_ci(path, args) + case ["npm", "run", "build"]: + self._npm_run_build(path) + case ["npm", "run", "build:singlefile"]: + self._npm_run_build_singlefile(path) + case _: + raise AssertionError(f"Unexpected command {command} in {cwd}") + return "" + + @staticmethod + def _mkc_build(cwd: Path) -> None: + mkc_json = json.loads((cwd / "mkc.json").read_text(encoding="utf-8")) + meta = { + "simUrl": FAKE_SIM_URL, + "cdnUrl": "https://cdn.example.com/", + "version": "v0.0.0", + "target": "arcade", + "targetVersion": mkc_json["targetWebsite"].rsplit("/v", 1)[1], + } + source = _digest(json.dumps(tree_digest(cwd), sort_keys=True).encode()) + (cwd / "built").mkdir() + (cwd / "built" / "binary.js").write_text( + f"// meta={json.dumps(meta)}\n// built from {source}\n", encoding="utf-8" + ) + + @staticmethod + def _mkc_download(cwd: Path, url: str) -> None: + (cwd / "pxt.json").write_text('{"name": "downloaded"}', encoding="utf-8") + (cwd / "main.ts").write_text(f"// downloaded from {url}\n", encoding="utf-8") + + @staticmethod + def _npm_ci(cwd: Path, args: list[str]) -> None: + lockfile = _digest((cwd / "package-lock.json").read_bytes()) + package = cwd / "node_modules" / "package" + package.mkdir(parents=True) + (package / "index.js").write_text(f"// {lockfile}\n", encoding="utf-8") + (cwd / "node_modules" / ".bin").mkdir() + if sys.platform != "win32": # npm makes .cmd shims on Windows instead + (cwd / "node_modules" / ".bin" / "tool").symlink_to( + Path("..") / "package" / "index.js" + ) + if "--prefer-online" in args: + cache = Path(args[args.index("--cache") + 1]) + cache.mkdir(parents=True, exist_ok=True) + (cache / lockfile).write_text("cached", encoding="utf-8") + + @staticmethod + def _built_index_html(cwd: Path) -> str: + assert (cwd / "node_modules" / "package" / "index.js").is_file() + index_html = (cwd / "index.html").read_text(encoding="utf-8") + assets = cwd / "src" / "assets" + binary = (assets / "binary.js").read_text(encoding="utf-8") + sim = (assets / "---simulator.html").read_text(encoding="utf-8") + return f"{index_html}\n\n\n" + + def _npm_run_build(self, cwd: Path) -> None: + index_html = self._built_index_html(cwd) + assets = cwd / "dist" / "assets" + assets.mkdir(parents=True) + (cwd / "dist" / "index.html").write_text(index_html, encoding="utf-8") + (assets / "index.js").write_text("run()\n", encoding="utf-8") + + def _npm_run_build_singlefile(self, cwd: Path) -> None: + index_html = self._built_index_html(cwd) + (cwd / "dist").mkdir() + (cwd / "dist" / "index.html").write_text(index_html, encoding="utf-8") + + +class TaskSpy: + """ + Wraps every mkcd2app task to record the ones that really run (a cache hit doesn't + call the function), and what each ContentDir/ContentFile held when it was returned. + """ + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.ran: list[str] = [] + self.returned: dict[Path, dict[str, str]] = {} + for task in get_task_registry(): # type: ignore[no-untyped-call] + if task.namespace == "mkcd2app": + monkeypatch.setattr(task, "func", self._wrap(task)) + + def _wrap(self, task: Task[Any, Any]) -> Callable[..., Any]: + func = task.func + + @wraps(func) + def spy(*args: Any, **kwargs: Any) -> Any: + result = func(*args, **kwargs) + self.ran.append(task.name) + if isinstance(result, ContentDir | ContentFile): + path = Path(result.path) + self.returned[path] = current_contents(path) + return result + + return spy + + +def current_contents(path: Path) -> dict[str, str]: + """Map each file under ``path``, or ``path`` itself if a file, to its SHA-256.""" + return {"": _digest(path.read_bytes())} if path.is_file() else tree_digest(path) + + +@dataclass +class Ran: + """What one CLI run did: the tasks that really ran, and the commands they ran.""" + + tasks: set[str] + commands: list[str] + + +class Cli: + """Runs the real ``main()`` in this process, with the toolchain faked.""" + + def __init__( + self, monkeypatch: pytest.MonkeyPatch, toolchain: FakeToolchain, spy: TaskSpy + ) -> None: + self.monkeypatch = monkeypatch + self.toolchain = toolchain + self.spy = spy + + def run(self, *argv: str) -> Ran: + """ + :param argv: The CLI arguments. + :return: What this run did. If it raises instead, ``toolchain.commands`` and + ``spy.ran`` still say what it did. + """ + self.toolchain.commands.clear() + self.spy.ran.clear() + self.monkeypatch.setattr(sys, "argv", ["mkcd2app", *argv]) + main() + return Ran(set(self.spy.ran), sorted(self.toolchain.commands)) + + +def install_fake_target(version: str = TARGET_VERSION) -> None: + """Put a simulator where ``mkcd2app target install`` would.""" + path = get_sim_html_path(version) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"simulator {version}", encoding="utf-8") + + +class GameProject: + """A game in a folder of its own, with a config that uses paths relative to it.""" + + def __init__(self, root: Path, cli: Cli) -> None: + self.root = root + self.cli = cli + self.game = root / "game" + self.build_dir = root / "build" + self.config_path = root / "config.yaml" + shutil.copytree(FIXTURES / "mkcd_projects" / "empty_game", self.game) + self.config: dict[str, Any] = yaml.safe_load( + make_config_yaml( + Path("build"), + code={"type": "path", "value": "./game"}, + outputs=[{"type": "static"}, {"type": "static-singlefile"}], + ) + ) + self.write_config() + + def write_config(self, prefix: str = "") -> None: + """Write ``self.config`` to the config file, after the text in ``prefix``.""" + self.config_path.write_text( + prefix + yaml.safe_dump(self.config), encoding="utf-8" + ) + + def build(self, *args: str) -> Ran: + return self.cli.run("build", str(self.config_path), *args) + + @property + def website(self) -> Path: + """The static website's index.html.""" + return self.build_dir / "test-website-filled-built" / "dist" / "index.html" + + @property + def singlefile(self) -> Path: + """The single-file website.""" + return ( + self.build_dir + / "test-website-filled-singlefile-built" + / "dist" + / "index.html" + ) diff --git a/tests/integration/test_build_wiring.py b/tests/integration/test_build_wiring.py new file mode 100644 index 0000000..0acb772 --- /dev/null +++ b/tests/integration/test_build_wiring.py @@ -0,0 +1,220 @@ +"""The real task graph through the real scheduler, with only `mkc` and `npm` faked.""" + +import sys +from pathlib import Path + +import pytest +import requests +from helpers import ( + FakeResponse, + FakeToolchain, + GameProject, + TaskSpy, + current_contents, +) +from PIL import Image + +from mkcd2app.build_project import BuildProjectResult, build_project +from mkcd2app.config import load_config_from_yaml, resolve_config_paths +from mkcd2app.utils.paths import get_redun_db_for_build_path +from mkcd2app.utils.run import BuildError +from mkcd2app.utils.run_redun_task import run_redun_task + +ALL_TASKS = { + "build_project", + "fetch_code", + "build_binary_js", + "copy_support_files", + "copy_website_template", + "fill_website_template", + "install_website_deps", + "build_website", + "build_website_singlefile", +} +ALL_COMMANDS = ["mkc build", "npm ci", "npm run build", "npm run build:singlefile"] + + +def build_for_result(project: GameProject) -> BuildProjectResult: + """Build like main() does, but keep the result, which main() only logs.""" + config = resolve_config_paths( + load_config_from_yaml(project.config_path.read_text("utf-8")), project.root + ) + result: BuildProjectResult = run_redun_task( + build_project(config.model_dump_json()), + get_redun_db_for_build_path(project.build_dir), + ) + return result + + +# ----------------------------------------------------------------- wiring ----- + + +def test_full_build_runs_every_task_once( + project: GameProject, task_spy: TaskSpy +) -> None: + ran = project.build() + assert sorted(task_spy.ran) == sorted(ALL_TASKS) + assert ran.commands == ALL_COMMANDS # includes one `npm ci`, shared by both + assert project.website.is_file() + assert project.singlefile.is_file() + + +def test_result_fields_point_at_the_outputs(project: GameProject) -> None: + result = build_for_result(project) + assert result.static is not None and result.static_singlefile is not None + assert Path(result.static.path) == project.website.parent + assert Path(result.static_singlefile.path) == project.singlefile + assert result.electron is None and result.tauri is None + + +def test_only_static_requested( + project: GameProject, fake_toolchain: FakeToolchain +) -> None: + project.config["outputs"] = [{"type": "static"}] + project.write_config() + result = build_for_result(project) + assert result.static is not None and result.static_singlefile is None + assert sorted(fake_toolchain.commands) == ["mkc build", "npm ci", "npm run build"] + assert not project.singlefile.exists() + + +def test_only_singlefile_requested( + project: GameProject, fake_toolchain: FakeToolchain +) -> None: + project.config["outputs"] = [{"type": "static-singlefile"}] + project.write_config() + result = build_for_result(project) + assert result.static is None and result.static_singlefile is not None + assert sorted(fake_toolchain.commands) == [ + "mkc build", + "npm ci", + "npm run build:singlefile", + ] + assert not project.website.exists() + + +def test_game_project_details_reach_the_website(project: GameProject) -> None: + project.config["project"]["title"] = "My Game" + project.write_config() + project.build() + for output in (project.website, project.singlefile): + text = output.read_text("utf-8") + assert "My Game" in text + assert "// built from" in text # binary.js + assert "simulator 4.0.14" in text + + +def test_path_icon_relative_to_the_config_reaches_the_website( + project: GameProject, +) -> None: + Image.new("RGBA", (16, 16), (255, 0, 0, 255)).save(project.root / "icon.png") + project.config["inputs"]["assets"]["icon"] = {"type": "path", "value": "icon.png"} + project.write_config() + project.build() + assert "data:image/x-icon;base64," in project.website.read_text("utf-8") + + +def test_url_icon_reaches_the_website( + project: GameProject, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + png = tmp_path / "served.png" + Image.new("RGBA", (16, 16), (0, 255, 0, 255)).save(png) + monkeypatch.setattr(requests, "get", lambda url: FakeResponse(png.read_bytes())) + project.config["inputs"]["assets"]["icon"] = { + "type": "url", + "value": "https://example.com/icon.png", + } + project.write_config() + project.build() + assert "data:image/x-icon;base64," in project.website.read_text("utf-8") + + +def test_share_link_source(project: GameProject) -> None: + url = "https://arcade.makecode.com/S12345-67890-12345-67890" + project.config["inputs"]["code"] = {"type": "share_link", "value": url} + project.write_config() + ran = project.build() + assert ran.commands == sorted([*ALL_COMMANDS, "mkc download"]) + source = project.build_dir / "test-source" / "main.ts" + assert url in source.read_text("utf-8") + + +def test_outputs_built_from_a_file_source(project: GameProject) -> None: + fixtures = Path(__file__).parent.parent / "fixtures" / "mkcd_projects" + project.config["inputs"]["code"] = { + "type": "file", + "value": str(fixtures / "empty-game.png"), + } + project.write_config() + project.build() + assert (project.build_dir / "test-source" / "pxt.json").is_file() + assert project.website.is_file() + + +@pytest.mark.skipif(sys.platform == "win32", reason="the fake npm makes no symlinks") +def test_node_modules_symlinks_stay_symlinks(project: GameProject) -> None: + # Regression: copying their targets breaks the scripts' relative imports + project.build() + tool = project.website.parent.parent / "node_modules" / ".bin" / "tool" + assert tool.is_symlink() + assert tool.resolve().is_file() + + +# ----------------------------------------------------------- copy-on-write ----- + + +def test_no_task_changes_what_another_task_returned( + project: GameProject, task_spy: TaskSpy +) -> None: + project.build() + assert len(task_spy.returned) == len(ALL_TASKS) - 1 # all but build_project + for path, contents in task_spy.returned.items(): + assert current_contents(path) == contents, f"{path} changed after it returned" + + +def test_mkc_builds_outside_the_fetched_source(project: GameProject) -> None: + project.build() + assert not (project.build_dir / "test-source" / "built").exists() + assert ( + project.build_dir / "test-binary-js-build" / "built" / "binary.js" + ).is_file() + + +# ------------------------------------------------------- failure and retry ----- + + +@pytest.mark.parametrize( + ("command", "task"), + [ + ("mkc build", "build_binary_js"), + ("npm ci", "install_website_deps"), + ("npm run build", "build_website"), + ("npm run build:singlefile", "build_website_singlefile"), + ], +) +def test_failed_command_raises_and_the_next_build_retries_it( + project: GameProject, fake_toolchain: FakeToolchain, command: str, task: str +) -> None: + fake_toolchain.failing.add(command) + with pytest.raises(BuildError) as exc: + project.build() + assert exc.value.output == f"fake {command} output" + + fake_toolchain.failing.clear() + assert task in project.build().tasks # the failure wasn't cached + assert project.website.is_file() and project.singlefile.is_file() + assert project.build().tasks == set() + + +def test_finished_tasks_stay_cached_after_a_failure( + project: GameProject, fake_toolchain: FakeToolchain +) -> None: + # fetch_code has to finish before `mkc build` can run + fake_toolchain.failing.add("mkc build") + with pytest.raises(BuildError): + project.build() + + fake_toolchain.failing.clear() + ran = project.build() + assert "fetch_code" not in ran.tasks + assert "build_binary_js" in ran.tasks diff --git a/tests/integration/test_cli_commands.py b/tests/integration/test_cli_commands.py new file mode 100644 index 0000000..5f57807 --- /dev/null +++ b/tests/integration/test_cli_commands.py @@ -0,0 +1,173 @@ +"""The real ``main()``, from arguments to tasks, with only `mkc` and `npm` faked.""" + +import logging +from pathlib import Path + +import pytest +from helpers import Cli, FakeToolchain, GameProject, TaskSpy +from pydantic import ValidationError + +from mkcd2app.utils.paths import get_sim_html_path + + +def assert_nothing_ran(fake_toolchain: FakeToolchain, task_spy: TaskSpy) -> None: + assert fake_toolchain.commands == [] + assert task_spy.ran == [] + + +# --------------------------------------------------------------- arguments ----- + + +@pytest.mark.parametrize("command", ["install", "uninstall"]) +@pytest.mark.parametrize("version", ["4.0", "v4.0.14", "4.0.14-beta", "latest"]) +def test_target_version_must_be_strict_semver( + cli: Cli, + fake_toolchain: FakeToolchain, + task_spy: TaskSpy, + command: str, + version: str, +) -> None: + with pytest.raises(ValueError, match="strict numeric semver"): + cli.run("target", command, version) + assert_nothing_ran(fake_toolchain, task_spy) + + +@pytest.mark.parametrize("argv", [("toolchain", "status"), ("target", "list")]) +def test_stub_commands_run_nothing(cli: Cli, argv: tuple[str, ...]) -> None: + ran = cli.run(*argv) + assert ran.tasks == set() + assert ran.commands == [] + + +def test_debug_turns_on_debug_logging( + project: GameProject, restore_log_levels: None +) -> None: + logger = logging.getLogger("mkcd2app.main") + assert logger.level == logging.INFO + project.cli.run("--debug", "build", str(project.config_path)) + assert logger.level == logging.DEBUG + + +# ------------------------------------------------------------------ configs ----- + + +def test_missing_config_file( + cli: Cli, tmp_path: Path, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + with pytest.raises(FileNotFoundError): + cli.run("build", str(tmp_path / "missing.yaml")) + assert_nothing_ran(fake_toolchain, task_spy) + + +def test_unknown_config_version( + project: GameProject, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + project.config["version"] = 2 + project.write_config() + with pytest.raises(NotImplementedError, match="Unknown config version 2"): + project.build() + assert_nothing_ran(fake_toolchain, task_spy) + assert not project.build_dir.exists() + + +def test_unknown_config_key( + project: GameProject, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + project.config["project"]["titel"] = "Typo" + project.write_config() + with pytest.raises(ValidationError, match="titel"): + project.build() + assert_nothing_ran(fake_toolchain, task_spy) + + +@pytest.mark.parametrize( + "output", + [{"type": "electron"}, {"type": "tauri", "identifier": "com.example.game"}], + ids=["electron", "tauri"], +) +def test_unimplemented_output_builds_nothing( + project: GameProject, + fake_toolchain: FakeToolchain, + task_spy: TaskSpy, + output: dict[str, str], +) -> None: + project.config["outputs"].append(output) + project.write_config() + with pytest.raises(NotImplementedError, match=output["type"]): + project.build() + assert fake_toolchain.commands == [] + assert not project.website.exists() + + +def test_target_not_installed_builds_nothing( + project: GameProject, fake_toolchain: FakeToolchain +) -> None: + get_sim_html_path("4.0.14").unlink() + with pytest.raises(FileNotFoundError, match="mkcd2app target install 4.0.14"): + project.build() + assert fake_toolchain.commands == [] + + +# ---------------------------------------------------------- --full-rebuild ----- + + +def test_full_rebuild_clears_the_build_dir_and_reruns_everything( + project: GameProject, +) -> None: + project.build() + stale = project.build_dir / "stale.txt" + stale.write_text("left over", encoding="utf-8") + + ran = project.build("--full-rebuild") + + assert not stale.exists() + assert (project.build_dir / ".redun-cache.db").is_file() + assert "fetch_code" in ran.tasks + assert "mkc build" in ran.commands and "npm ci" in ran.commands + assert project.website.is_file() + + +def test_full_rebuild_with_no_previous_build(project: GameProject) -> None: + assert not project.build_dir.exists() + assert project.build("--full-rebuild").commands + assert project.website.is_file() + + +def test_full_rebuild_refuses_a_folder_that_is_not_a_build_dir( + project: GameProject, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + # A mistake in `build_dir` that points at something else the user owns + documents = project.root / "documents" + documents.mkdir() + (documents / "notes.txt").write_text("keep me", encoding="utf-8") + project.config["build_dir"] = "documents" + project.write_config() + with pytest.raises(RuntimeError, match="Refusing to clear"): + project.build("--full-rebuild") + assert_nothing_ran(fake_toolchain, task_spy) + assert (documents / "notes.txt").is_file() + + +def test_full_rebuild_rejects_a_build_dir_containing_the_source( + project: GameProject, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + # `build_dir: "."`, say: this would delete the game and the config + project.config["build_dir"] = "." + project.write_config() + with pytest.raises(ValueError, match="is inside build_dir"): + project.build("--full-rebuild") + assert_nothing_ran(fake_toolchain, task_spy) + assert project.config_path.is_file() + assert (project.game / "main.ts").is_file() + + +def test_build_rejects_a_build_dir_inside_the_source( + project: GameProject, fake_toolchain: FakeToolchain, task_spy: TaskSpy +) -> None: + # The config's own folder is the game, as it would be for a local game + project.config["inputs"]["code"]["value"] = "." + project.write_config() + with pytest.raises(ValueError, match="is inside the path code source"): + project.build() + assert_nothing_ran(fake_toolchain, task_spy) + assert not project.build_dir.exists() diff --git a/tests/integration/test_installs.py b/tests/integration/test_installs.py new file mode 100644 index 0000000..82f3bbd --- /dev/null +++ b/tests/integration/test_installs.py @@ -0,0 +1,178 @@ +"""Toolchain and target installs through the real scheduler and ``main()``.""" + +from html import unescape + +import pytest +import requests +from helpers import FAKE_SIM_URL, Cli, FakeResponse, GameProject, install_fake_target + +from mkcd2app.utils.paths import ( + get_js_tools_dir, + get_sim_html_path, + get_target_version_dir, + get_templates_npm_cache_dir, + get_toolchain_dir, +) +from mkcd2app.utils.resources import get_resource_template_path + +VERSION = "4.0.14" +SIM_PAGES = { + FAKE_SIM_URL: b'' + + "

héllo ✓

" + "".encode(), + "https://sim.example.com/sim.css": "p::after { content: 'é' }".encode(), + "https://cdn.example.com/sim.js": "run('✓', 1 < 2)".encode(), +} + + +@pytest.fixture +def sim_requests(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Serve the simulator from fake URLs; return the list of requested URLs.""" + requested: list[str] = [] + + def fake_get(url: str) -> FakeResponse: + requested.append(url) + return FakeResponse(SIM_PAGES[url]) + + monkeypatch.setattr(requests, "get", fake_get) + return requested + + +# -------------------------------------------------------------- toolchain ----- + + +def install_commands() -> list[str]: + """The commands a toolchain install runs: one `npm ci` per template, and the tools.""" + with get_resource_template_path() as templates: + return ["npm ci"] * (len(list(templates.iterdir())) + 1) + + +def test_toolchain_install(cli: Cli) -> None: + ran = cli.run("toolchain", "install") + assert ran.tasks == { + "install_toolchain", + "install_js_tools", + "warm_npm_cache_for_templates", + } + assert ran.commands == install_commands() + assert (get_js_tools_dir() / "node_modules").is_dir() + assert list(get_templates_npm_cache_dir().iterdir()) + + +def test_second_toolchain_install_is_cached(cli: Cli) -> None: + cli.run("toolchain", "install") + ran = cli.run("toolchain", "install") + assert ran.tasks == set() + assert ran.commands == [] + + +def test_toolchain_uninstall_then_install(cli: Cli) -> None: + cli.run("toolchain", "install") + cli.run("toolchain", "uninstall") + assert not get_toolchain_dir().exists() + + ran = cli.run("toolchain", "install") + + assert ran.commands == install_commands() + assert (get_js_tools_dir() / "node_modules").is_dir() + + +def test_toolchain_uninstall_when_not_installed_is_fine(cli: Cli) -> None: + # Regression: this crashed with a FileNotFoundError + cli.run("toolchain", "uninstall") + assert not get_toolchain_dir().exists() + + +def test_toolchain_uninstall_twice_is_fine(cli: Cli) -> None: + cli.run("toolchain", "install") + cli.run("toolchain", "uninstall") + cli.run("toolchain", "uninstall") + assert not get_toolchain_dir().exists() + + +# ----------------------------------------------------------------- target ----- + + +def test_target_install_writes_a_self_contained_simulator( + cli: Cli, sim_requests: list[str] +) -> None: + ran = cli.run("target", "install", VERSION) + + assert ran.commands == ["mkc build"] + assert ran.tasks == { + "install_target", + "warm_mkc_cache_for_version_and_get_binary_js_metadata", + "download_sim", + } + assert sim_requests == list(SIM_PAGES) + text = get_sim_html_path(VERSION).read_text(encoding="utf-8") + assert "src=" not in text and "" in text and "héllo ✓" in unescape(text) + + +def test_second_target_install_is_cached(cli: Cli, sim_requests: list[str]) -> None: + cli.run("target", "install", VERSION) + sim_requests.clear() + ran = cli.run("target", "install", VERSION) + assert ran.tasks == set() + assert ran.commands == [] + assert sim_requests == [] + + +def test_target_uninstall_then_install(cli: Cli, sim_requests: list[str]) -> None: + # Regression: the simulator is a ContentFile, and a deleted one crashed the next run + cli.run("target", "install", VERSION) + cli.run("target", "uninstall", VERSION) + assert not get_target_version_dir(VERSION).exists() + sim_requests.clear() + + ran = cli.run("target", "install", VERSION) + + assert "download_sim" in ran.tasks + assert sim_requests == list(SIM_PAGES) + assert get_sim_html_path(VERSION).is_file() + + +def test_each_target_version_installs_separately( + cli: Cli, sim_requests: list[str] +) -> None: + cli.run("target", "install", VERSION) + ran = cli.run("target", "install", "4.1.27") + assert "download_sim" in ran.tasks + assert get_sim_html_path(VERSION).is_file() + assert get_sim_html_path("4.1.27").is_file() + + +def test_target_uninstall_when_not_installed_is_fine(cli: Cli) -> None: + # Regression: this crashed with a FileNotFoundError + cli.run("target", "uninstall", VERSION) + assert not get_target_version_dir(VERSION).exists() + + +def test_target_uninstall_only_affects_that_version(cli: Cli) -> None: + install_fake_target(VERSION) + install_fake_target("4.1.27") + cli.run("target", "uninstall", VERSION) + assert not get_sim_html_path(VERSION).exists() + assert get_sim_html_path("4.1.27").exists() + + +# ------------------------------------------------------------ all together ----- + + +def test_install_then_build( + project: GameProject, cli: Cli, sim_requests: list[str] +) -> None: + # The project fixture installs a fake target, so start from nothing + get_sim_html_path(VERSION).unlink() + with pytest.raises(FileNotFoundError, match="target install"): + project.build() + + cli.run("toolchain", "install") + cli.run("target", "install", VERSION) + project.build() + + # The simulator that `target install` made is what ends up in the website + assert "run('✓', 1 < 2)" in project.website.read_text(encoding="utf-8") diff --git a/tests/integration/test_rebuilds.py b/tests/integration/test_rebuilds.py new file mode 100644 index 0000000..ed6972e --- /dev/null +++ b/tests/integration/test_rebuilds.py @@ -0,0 +1,313 @@ +""" +What a change reruns. Each test builds, makes one change, builds again, and checks +exactly which tasks and commands ran the second time. +""" + +import os +import shutil +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest +from helpers import GameProject, install_fake_target +from PIL import Image + +SITE_TASKS = {"fill_website_template", "build_website", "build_website_singlefile"} +SITE_COMMANDS = ["npm run build", "npm run build:singlefile"] +ALL_TASKS = SITE_TASKS | { + "build_project", + "fetch_code", + "build_binary_js", + "copy_support_files", + "copy_website_template", + "install_website_deps", +} + + +@dataclass(frozen=True) +class Change: + id: str + apply: Callable[[GameProject, Path], None] + tasks: set[str] + commands: list[str] + + +def set_project_field(field: str, value: str) -> Callable[[GameProject, Path], None]: + def apply(project: GameProject, template: Path) -> None: + project.config["project"][field] = value + project.write_config() + + return apply + + +def add_comment(project: GameProject, template: Path) -> None: + project.write_config(prefix="# A new comment\n\n") + + +def add_icon(project: GameProject, template: Path) -> None: + Image.new("RGBA", (16, 16), (255, 0, 0, 255)).save(project.root / "icon.png") + project.config["inputs"]["assets"]["icon"] = {"type": "path", "value": "icon.png"} + project.write_config() + + +def change_target(project: GameProject, template: Path) -> None: + install_fake_target("4.1.27") + project.config["target"] = "4.1.27" + project.write_config() + + +def rename_project(project: GameProject, template: Path) -> None: + project.config["project"]["path_friendly_name"] = "renamed" + project.write_config() + + +def change_build_dir(project: GameProject, template: Path) -> None: + project.config["build_dir"] = "other-build" + project.write_config() + + +def remove_singlefile_output(project: GameProject, template: Path) -> None: + project.config["outputs"] = [{"type": "static"}] + project.write_config() + + +def change_template_source(project: GameProject, template: Path) -> None: + with (template / "src" / "App.tsx").open("a", encoding="utf-8") as f: + f.write("// edited\n") + + +def change_template_lockfile(project: GameProject, template: Path) -> None: + with (template / "package-lock.json").open("a", encoding="utf-8") as f: + f.write("\n") + + +def touch_everything(project: GameProject, template: Path) -> None: + for root in (template, project.root): + for path in root.rglob("*"): + if path.is_file() and not path.is_symlink(): + os.utime(path, (1_000_000, 1_000_000)) + + +def edit_path_source(project: GameProject, template: Path) -> None: + with (project.game / "main.ts").open("a", encoding="utf-8") as f: + f.write("// edited\n") + + +CHANGES = [ + Change("comment", add_comment, set(), []), + Change( + "title", + set_project_field("title", "New Title"), + {"build_project"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "author", + set_project_field("author", "Someone Else"), + {"build_project"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "project-version", + set_project_field("version", "9.9.9"), + {"build_project"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "description", + set_project_field("description", "New description"), + {"build_project"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "icon", + add_icon, + {"build_project", "copy_support_files"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "target", + change_target, + { + "build_project", + "fetch_code", + "build_binary_js", + "copy_support_files", + } + | SITE_TASKS, + ["mkc build", *SITE_COMMANDS], + ), + Change( + "path-friendly-name", + rename_project, + ALL_TASKS, + ["mkc build", "npm ci", *SITE_COMMANDS], + ), + Change( + "build-dir", + change_build_dir, + ALL_TASKS, + ["mkc build", "npm ci", *SITE_COMMANDS], + ), + Change("output-removed", remove_singlefile_output, {"build_project"}, []), + # build_project reruns too, as the template in its cached result has changed + Change( + "template-source", + change_template_source, + {"build_project", "copy_website_template"} | SITE_TASKS, + SITE_COMMANDS, + ), + Change( + "template-lockfile", + change_template_lockfile, + {"build_project", "copy_website_template", "install_website_deps"} | SITE_TASKS, + ["npm ci", *SITE_COMMANDS], + ), + Change("mtimes-only", touch_everything, set(), []), + # By design: only the config is tracked, so this needs --full-rebuild + Change("path-source-edit", edit_path_source, set(), []), +] + + +@pytest.mark.parametrize("change", CHANGES, ids=lambda change: change.id) +def test_change_reruns_only_what_depends_on_it( + project: GameProject, template_dir: Path, change: Change +) -> None: + project.build() + change.apply(project, template_dir) + ran = project.build() + assert ran.tasks == change.tasks + assert ran.commands == sorted(change.commands) + + +# -------------------------------------------------------------- no changes ----- + + +def test_unchanged_rebuild_runs_nothing(project: GameProject) -> None: + project.build() + ran = project.build() + assert ran.tasks == set() + assert ran.commands == [] + + +def test_running_from_another_directory_runs_nothing( + project: GameProject, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project.build() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + assert project.build().tasks == set() + assert list(elsewhere.iterdir()) == [] + + +def test_relative_config_argument_runs_nothing( + project: GameProject, monkeypatch: pytest.MonkeyPatch +) -> None: + project.build() + monkeypatch.chdir(project.root) + assert project.cli.run("build", project.config_path.name).tasks == set() + + +# ------------------------------------------------------------- the results ----- + + +def test_adding_an_output_only_builds_that_output(project: GameProject) -> None: + project.config["outputs"] = [{"type": "static"}] + project.write_config() + project.build() + + project.config["outputs"].append({"type": "static-singlefile"}) + project.write_config() + ran = project.build() + + assert ran.tasks == {"build_project", "build_website_singlefile"} + assert ran.commands == ["npm run build:singlefile"] + assert project.singlefile.is_file() + + +def test_new_title_reaches_both_outputs(project: GameProject) -> None: + project.build() + set_project_field("title", "New Title")(project, Path()) + project.build() + for output in (project.website, project.singlefile): + assert "New Title" in output.read_text("utf-8") + + +def test_new_target_reaches_the_game_and_simulator( + project: GameProject, template_dir: Path +) -> None: + project.build() + change_target(project, template_dir) + project.build() + text = project.website.read_text("utf-8") + assert '"targetVersion": "4.1.27"' in text + assert "simulator 4.1.27" in text + + +def test_build_dir_moves_with_the_config(project: GameProject) -> None: + project.build() + change_build_dir(project, Path()) + project.build() + assert (project.root / "other-build" / ".redun-cache.db").is_file() + + +def test_path_source_edit_is_stale_until_full_rebuild(project: GameProject) -> None: + project.build() + before = project.website.read_text("utf-8") + edit_path_source(project, Path()) + + project.build() + assert project.website.read_text("utf-8") == before + + ran = project.build("--full-rebuild") + assert ran.tasks == ALL_TASKS + assert project.website.read_text("utf-8") != before + + +@pytest.mark.parametrize( + ("get_output", "delete_parent", "task", "commands"), + [ + (lambda p: p.website, False, "build_website", ["npm run build"]), + (lambda p: p.website, True, "build_website", ["npm run build"]), + ( + lambda p: p.singlefile, + False, + "build_website_singlefile", + ["npm run build:singlefile"], + ), + ( + lambda p: p.singlefile, + True, + "build_website_singlefile", + ["npm run build:singlefile"], + ), + # Same source, so the same binary.js: nothing after it needs to rerun + ( + lambda p: p.build_dir / "test-binary-js-build" / "built" / "binary.js", + False, + "build_binary_js", + ["mkc build"], + ), + ], + ids=["static", "static-folder", "singlefile", "singlefile-folder", "binary-js"], +) +def test_deleted_output_is_rebuilt( + project: GameProject, + get_output: Callable[[GameProject], Path], + delete_parent: bool, + task: str, + commands: list[str], +) -> None: + # Regression: a deleted ContentFile crashed the next run instead of rerunning + project.build() + output = get_output(project) + if delete_parent: + shutil.rmtree(output.parent) + else: + output.unlink() + ran = project.build() + assert task in ran.tasks + assert ran.commands == commands + assert output.is_file() diff --git a/tests/unit/test_build_project.py b/tests/unit/test_build_project.py new file mode 100644 index 0000000..06525f6 --- /dev/null +++ b/tests/unit/test_build_project.py @@ -0,0 +1,77 @@ +""" +What ``build_project`` checks before anything runs. How its tasks are wired together +needs the redun scheduler, since calling a task here only gives an unevaluated +expression. +""" + +from pathlib import Path +from typing import Any + +import pytest +import yaml +from helpers import TARGET_VERSION, make_config_yaml + +from mkcd2app.build_project import BuildProjectResult, build_project +from mkcd2app.config import load_config_from_yaml +from mkcd2app.utils.paths import get_sim_html_path + + +def config_json(build_dir: Path, *outputs: dict[str, Any]) -> str: + data = yaml.safe_load(make_config_yaml(build_dir)) + data["outputs"] = list(outputs) + return load_config_from_yaml(yaml.safe_dump(data)).model_dump_json() + + +@pytest.fixture +def installed_target(mkcd2app_home: Path) -> None: + path = get_sim_html_path(TARGET_VERSION) + path.parent.mkdir(parents=True) + path.write_text("sim", encoding="utf-8") + + +@pytest.mark.parametrize( + ("output", "name"), + [ + ({"type": "electron"}, "electron"), + ({"type": "tauri", "identifier": "com.example.game"}, "tauri"), + ], + ids=["electron", "tauri"], +) +@pytest.mark.parametrize("with_static", [False, True], ids=["alone", "with-static"]) +def test_unimplemented_output_raises( + tmp_path: Path, + installed_target: None, + output: dict[str, Any], + name: str, + with_static: bool, +) -> None: + outputs = [{"type": "static"}, output] if with_static else [output] + with pytest.raises(NotImplementedError, match=name): + build_project.func(config_json(tmp_path / "build", *outputs)) + + +@pytest.mark.parametrize( + ("output", "field"), + [ + ({"type": "static"}, "static"), + ({"type": "static-singlefile"}, "static_singlefile"), + ], + ids=["static", "static-singlefile"], +) +def test_implemented_outputs_only_fill_their_field( + tmp_path: Path, installed_target: None, output: dict[str, Any], field: str +) -> None: + result: BuildProjectResult = build_project.func( + config_json(tmp_path / "build", output) + ) + filled = {k for k, v in vars(result).items() if v is not None} + assert filled == {field} + + +def test_missing_target_names_the_install_command( + tmp_path: Path, mkcd2app_home: Path +) -> None: + with pytest.raises( + FileNotFoundError, match=f"mkcd2app target install {TARGET_VERSION}" + ): + build_project.func(config_json(tmp_path / "build", {"type": "static"})) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..17a0627 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,72 @@ +import sys + +import pytest + +from mkcd2app.cli import generate_and_parse_args + + +@pytest.mark.parametrize( + "argv", + [ + [], # a subcommand is required + ["nonsense"], + ["build"], # config is required + ["toolchain"], # these need their own subcommand + ["toolchain", "nonsense"], + ["target"], + ["target", "nonsense"], + ["target", "install"], # version is required + ["target", "uninstall"], + ["build", "c.yaml", "--debug"], # --debug must come before the subcommand + ["target", "list", "--debug"], + ["--full-rebuild", "build", "c.yaml"], # --full-rebuild belongs to build + ["target", "install", "4.0.14", "--full-rebuild"], + ], +) +def test_invalid_arguments_exit_2(argv: list[str]) -> None: + with pytest.raises(SystemExit) as exc: + generate_and_parse_args(argv) + assert exc.value.code == 2 + + +def test_build_defaults() -> None: + args = generate_and_parse_args(["build", "c.yaml"]) + assert (args.command, args.config) == ("build", "c.yaml") + assert args.full_rebuild is False + assert args.debug is False + + +def test_build_full_rebuild() -> None: + assert generate_and_parse_args(["build", "c.yaml", "--full-rebuild"]).full_rebuild + + +def test_debug_before_subcommand() -> None: + assert generate_and_parse_args(["--debug", "build", "c.yaml"]).debug is True + + +@pytest.mark.parametrize("command", ["install", "status", "uninstall"]) +def test_toolchain_commands(command: str) -> None: + args = generate_and_parse_args(["toolchain", command]) + assert (args.command, args.toolchain_command) == ("toolchain", command) + + +@pytest.mark.parametrize("command", ["install", "uninstall"]) +def test_target_commands_take_version(command: str) -> None: + args = generate_and_parse_args(["target", command, "4.0.14"]) + assert (args.command, args.target_command) == ("target", command) + assert args.version == "4.0.14" + + +def test_target_list() -> None: + args = generate_and_parse_args(["target", "list"]) + assert (args.command, args.target_command) == ("target", "list") + + +def test_version_is_not_validated_by_the_parser() -> None: + # main.py checks it's strict semver, so that's where the error comes from + assert generate_and_parse_args(["target", "install", "4.0"]).version == "4.0" + + +def test_defaults_to_sys_argv(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["mkcd2app", "build", "from-argv.yaml"]) + assert generate_and_parse_args().config == "from-argv.yaml" diff --git a/tests/unit/test_code.py b/tests/unit/test_code.py new file mode 100644 index 0000000..4fe73da --- /dev/null +++ b/tests/unit/test_code.py @@ -0,0 +1,339 @@ +import json +import logging +from collections.abc import Generator +from pathlib import Path +from typing import Any + +import pytest +import requests +from helpers import ( + TARGET_VERSION, + CmdCall, + FakeResponse, + FakeRunCmd, + capture_logger, + tree_digest, +) +from PIL import Image +from redun.file import ContentDir, ContentFile + +from mkcd2app.build_project.inputs import code +from mkcd2app.build_project.inputs.code import ( + _warn_on_target_mismatch, + build_binary_js, + copy_support_files, + fetch_code, +) +from mkcd2app.utils.paths import get_js_tools_bin_dir + +FIXTURES = Path(__file__).parent.parent / "fixtures" +EMPTY_GAME_DIR = FIXTURES / "mkcd_projects" / "empty_game" +EMPTY_PNG = FIXTURES / "mkcd_projects" / "empty-game.png" # saved with 4.0.14 +DUCK_PNG = FIXTURES / "mkcd_projects" / "falling-duck.png" # saved with 4.1.27 +MKC_JSON = {"targetWebsite": f"https://arcade.makecode.com/v{TARGET_VERSION}"} + + +def read_mkc_json(code_path: Path) -> Any: + return json.loads((code_path / "mkc.json").read_text(encoding="utf-8")) + + +def run_fetch_code( + build_dir: Path, source: dict[str, Any], target: str = TARGET_VERSION +) -> Path: + result: ContentDir = fetch_code.func( + json.dumps(source), target, str(build_dir / "test-source") + ) + return Path(result.path) + + +# ------------------------------------------------------------ fetch_code ------ + + +def test_path_source_copies_project(tmp_path: Path) -> None: + code_path = run_fetch_code( + tmp_path / "build", {"type": "path", "value": str(EMPTY_GAME_DIR)} + ) + assert code_path == tmp_path / "build" / "test-source" + copied = tree_digest(code_path) + assert copied.pop("mkc.json") + assert copied == tree_digest(EMPTY_GAME_DIR) + assert read_mkc_json(code_path) == MKC_JSON + + +def test_fetch_code_replaces_previous_source(tmp_path: Path) -> None: + stale = tmp_path / "build" / "test-source" / "stale.ts" + stale.parent.mkdir(parents=True) + stale.write_text("old", encoding="utf-8") + run_fetch_code(tmp_path / "build", {"type": "path", "value": str(EMPTY_GAME_DIR)}) + assert not stale.exists() + + +def test_mkc_json_uses_target(tmp_path: Path) -> None: + code_path = run_fetch_code( + tmp_path / "build", + {"type": "path", "value": str(EMPTY_GAME_DIR)}, + target="4.1.27", + ) + assert read_mkc_json(code_path) == { + "targetWebsite": "https://arcade.makecode.com/v4.1.27" + } + + +def test_share_link_source_runs_mkc_download( + tmp_path: Path, mkcd2app_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake = FakeRunCmd() + monkeypatch.setattr(code, "run_cmd", fake) + url = "https://arcade.makecode.com/S12345-67890-12345-67890" + + code_path = run_fetch_code(tmp_path / "build", {"type": "share_link", "value": url}) + + assert len(fake.calls) == 1 + call = fake.calls[0] + assert call.command == ["mkc", "download", url] + assert call.cwd == code_path + assert call.which_path == get_js_tools_bin_dir() + assert read_mkc_json(code_path) == MKC_JSON + + +@pytest.mark.parametrize( + ("source", "expected_ref"), + [ + ({"type": "git", "url": "https://example.com/a/b.git", "ref": "v1"}, "v1"), + ({"type": "git", "url": "https://example.com/a/b.git"}, None), + ({"type": "github", "value": "https://github.com/a/b", "checkout": "dev"}, "dev"), + ({"type": "github", "value": "https://github.com/a/b"}, "master"), + ], +) # fmt: skip +def test_git_sources_clone( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + source: dict[str, str], + expected_ref: str | None, +) -> None: + # Cloning itself is covered by test_git.py; this checks fetch_code's dispatch + calls: list[tuple[str, Path, str | None]] = [] + + def fake_clone(url: str, path: Path, ref: str | None = None) -> None: + # The parent must exist, and the clone target must not + assert path.parent.is_dir() + assert not path.exists() + calls.append((url, path, ref)) + path.mkdir() + + monkeypatch.setattr(code, "clone_and_checkout", fake_clone) + code_path = run_fetch_code(tmp_path / "build", source) + + url = source.get("url") or source["value"] + assert calls == [(url, code_path, expected_ref)] + assert read_mkc_json(code_path) == MKC_JSON + + +def test_file_source_extracts_project(tmp_path: Path) -> None: + code_path = run_fetch_code( + tmp_path / "build", {"type": "file", "value": str(EMPTY_PNG)} + ) + assert (code_path / "pxt.json").is_file() + assert (code_path / "main.ts").is_file() + assert read_mkc_json(code_path) == MKC_JSON + + +def test_file_source_missing_file(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="nope.png"): + run_fetch_code( + tmp_path / "build", {"type": "file", "value": str(tmp_path / "nope.png")} + ) + + +@pytest.mark.parametrize(("png", "warns"), [(EMPTY_PNG, False), (DUCK_PNG, True)]) +def test_file_source_warns_on_target_mismatch( + tmp_path: Path, caplog: pytest.LogCaptureFixture, png: Path, warns: bool +) -> None: + with capture_logger(caplog, code.logger): + run_fetch_code(tmp_path / "build", {"type": "file", "value": str(png)}) + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + if warns: + assert [r.getMessage() for r in warnings] == [ + f"Project was saved with Arcade 4.1.27, building against {TARGET_VERSION}" + ] + else: + assert warnings == [] + + +# ---------------------------------------------- _warn_on_target_mismatch ------ + + +@pytest.mark.parametrize( + "pxt_json", + [ + None, # file missing + "not json", + "[]", + "{}", + '{"targetVersions": []}', + '{"targetVersions": {"targetId": "arcade"}}', + f'{{"targetVersions": {{"target": "{TARGET_VERSION}"}}}}', + ], +) +def test_target_mismatch_no_warning( + tmp_path: Path, caplog: pytest.LogCaptureFixture, pxt_json: str | None +) -> None: + path = tmp_path / "pxt.json" + if pxt_json is not None: + path.write_text(pxt_json, encoding="utf-8") + with capture_logger(caplog, code.logger): + _warn_on_target_mismatch(path, TARGET_VERSION) + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + + +def test_target_mismatch_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + path = tmp_path / "pxt.json" + path.write_text(json.dumps({"targetVersions": {"target": "2.0.48"}}), "utf-8") + with capture_logger(caplog, code.logger): + _warn_on_target_mismatch(path, TARGET_VERSION) + assert [(r.levelno, r.getMessage()) for r in caplog.records] == [ + ( + logging.WARNING, + f"Project was saved with Arcade 2.0.48, building against {TARGET_VERSION}", + ) + ] + + +# ------------------------------------------------------- build_binary_js ------ + + +def test_build_binary_js( + tmp_path: Path, mkcd2app_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + code_path = tmp_path / "build" / "test-source" + code_path.mkdir(parents=True) + (code_path / "main.ts").write_text("// game", encoding="utf-8") + before = tree_digest(code_path) + + def fake_mkc_build(call: CmdCall) -> None: + assert (call.cwd / "main.ts").is_file() + (call.cwd / "built").mkdir() + (call.cwd / "built" / "binary.js").write_text("// built", encoding="utf-8") + (call.cwd / "built" / "other.txt").write_text("x", encoding="utf-8") + + fake = FakeRunCmd(fake_mkc_build) + monkeypatch.setattr(code, "run_cmd", fake) + build_dir = tmp_path / "build" / "test-binary-js-build" + + result: ContentFile = build_binary_js.func( + ContentDir(str(code_path)), str(build_dir) + ) + + assert [(c.command, c.cwd, c.which_path) for c in fake.calls] == [ + (["mkc", "build", "-j"], build_dir, get_js_tools_bin_dir()) + ] + output = build_dir / "built" / "binary.js" + assert Path(result.path) == output + assert output.read_text(encoding="utf-8") == "// built" + # Built in a copy, so fetch_code's ContentDir hash doesn't change + assert tree_digest(code_path) == before + + +def test_build_binary_js_replaces_previous_build( + tmp_path: Path, mkcd2app_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + code_path = tmp_path / "build" / "test-source" + code_path.mkdir(parents=True) + build_dir = tmp_path / "build" / "test-binary-js-build" + stale = build_dir / "built" / "binary.js" + stale.parent.mkdir(parents=True) + stale.write_text("old", encoding="utf-8") + + def check_no_stale_build(call: CmdCall) -> None: + assert not stale.exists() + + monkeypatch.setattr(code, "run_cmd", FakeRunCmd(check_no_stale_build)) + build_binary_js.func(ContentDir(str(code_path)), str(build_dir)) + + +# ---------------------------------------------------- copy_support_files ------ + + +@pytest.fixture +def sim_html(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[Path]: + path = tmp_path / "target" / TARGET_VERSION / "---simulator.html" + path.parent.mkdir(parents=True) + path.write_text("sim", encoding="utf-8") + requested: list[str] = [] + + def fake_get_sim_html_path(v: str) -> Path: + requested.append(v) + return path + + monkeypatch.setattr(code, "get_sim_html_path", fake_get_sim_html_path) + yield path + assert requested == [TARGET_VERSION] + + +def make_png(path: Path, size: tuple[int, int] = (32, 32)) -> Path: + Image.new("RGBA", size, (255, 0, 0, 255)).save(path) + return path + + +def run_copy_support_files(build_dir: Path, icon: dict[str, str] | None) -> Path: + result: ContentDir = copy_support_files.func( + TARGET_VERSION, + json.dumps(icon) if icon else None, + str(build_dir / "test-binary-js-support"), + ) + return Path(result.path) + + +def test_support_files_without_icon(tmp_path: Path, sim_html: Path) -> None: + support = run_copy_support_files(tmp_path / "build", None) + assert support == tmp_path / "build" / "test-binary-js-support" + assert sorted(p.name for p in support.iterdir()) == ["---simulator.html"] + assert (support / "---simulator.html").read_bytes() == sim_html.read_bytes() + + +def test_support_files_replaces_previous_output(tmp_path: Path, sim_html: Path) -> None: + stale = tmp_path / "build" / "test-binary-js-support" / "favicon.ico" + stale.parent.mkdir(parents=True) + stale.write_bytes(b"old icon") + support = run_copy_support_files(tmp_path / "build", None) + assert not (support / "favicon.ico").exists() + + +def test_support_files_icon_from_path(tmp_path: Path, sim_html: Path) -> None: + icon = make_png(tmp_path / "icon.png") + support = run_copy_support_files( + tmp_path / "build", {"type": "path", "value": str(icon)} + ) + with Image.open(support / "favicon.ico") as im: + assert im.format == "ICO" + + +def test_support_files_icon_from_url( + tmp_path: Path, sim_html: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + png = make_png(tmp_path / "icon.png").read_bytes() + urls: list[str] = [] + + def fake_get(url: str) -> FakeResponse: + urls.append(url) + return FakeResponse(png) + + monkeypatch.setattr(requests, "get", fake_get) + support = run_copy_support_files( + tmp_path / "build", {"type": "url", "value": "https://example.com/icon.png"} + ) + assert urls == ["https://example.com/icon.png"] + with Image.open(support / "favicon.ico") as im: + assert im.format == "ICO" + + +def test_support_files_icon_url_http_error( + tmp_path: Path, sim_html: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(requests, "get", lambda url: FakeResponse(b"", 404)) + with pytest.raises(requests.HTTPError): + run_copy_support_files( + tmp_path / "build", {"type": "url", "value": "https://example.com/x.png"} + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..b25ae49 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,453 @@ +from pathlib import Path +from typing import Any + +import pytest +import yaml +from helpers import make_config_yaml +from pydantic import ValidationError + +from mkcd2app.config import load_config_from_yaml, resolve_config_paths +from mkcd2app.models.config import ( + AssetSource, + BuildConfig, + CodeSource, + ElectronOutput, + FileCodeSource, + PathAssetSource, + PathCodeSource, + Project, + TauriOutput, + UrlAssetSource, + WindowConfig, +) + + +def base_config() -> dict[str, Any]: + data: dict[str, Any] = yaml.safe_load(make_config_yaml(Path("build"))) + return data + + +def load(data: object) -> BuildConfig: + return load_config_from_yaml(yaml.safe_dump(data)) + + +def load_outputs(*outputs: dict[str, Any]) -> list[Any]: + data = base_config() + data["outputs"] = list(outputs) + return [output.root for output in load(data).outputs] + + +# --------------------------------------------------------- config version ----- + + +@pytest.mark.parametrize("version", [0, 2, "1", None]) +def test_unknown_version(version: object) -> None: + data = base_config() + data["version"] = version + with pytest.raises(NotImplementedError, match="Unknown config version"): + load(data) + + +def test_missing_version() -> None: + data = base_config() + del data["version"] + with pytest.raises(NotImplementedError, match="Unknown config version None"): + load(data) + + +@pytest.mark.parametrize("text", ["", "# only a comment\n", "- a\n- b\n", "just text"]) +def test_empty_or_non_mapping_yaml(text: str) -> None: + with pytest.raises(NotImplementedError, match="Unknown config version None"): + load_config_from_yaml(text) + + +# ------------------------------------------------------------ valid config ----- + + +def test_valid_config() -> None: + config = load(base_config()) + assert config.version == 1 + assert config.project.name == "Test Game" + assert config.build_dir == "build" + assert config.target == "4.0.14" + + +def test_project_title_has_default() -> None: + assert load(base_config()).project.title == "{NAME} v{VERSION}" + + +@pytest.mark.parametrize("key", ["description", "title"]) +def test_optional_project_fields(key: str) -> None: + data = base_config() + data["project"].pop(key, None) + load(data) + + +def test_icon_is_optional() -> None: + assert load(base_config()).inputs.assets.icon is None + + +# ---------------------------------------------------------- required fields ----- + + +@pytest.mark.parametrize("key", ["project", "inputs", "build_dir", "target", "outputs"]) +def test_missing_top_level_field(key: str) -> None: + data = base_config() + del data[key] + with pytest.raises(ValidationError, match=key): + load(data) + + +@pytest.mark.parametrize("key", ["name", "path_friendly_name", "author", "version"]) +def test_missing_project_field(key: str) -> None: + data = base_config() + del data["project"][key] + with pytest.raises(ValidationError, match=key): + load(data) + + +@pytest.mark.parametrize("key", ["code", "assets"]) +def test_missing_inputs_field(key: str) -> None: + data = base_config() + del data["inputs"][key] + with pytest.raises(ValidationError, match=key): + load(data) + + +# ------------------------------------------------------- path_friendly_name ----- + + +@pytest.mark.parametrize( + "name", ["racers", "empty-game", "game_2", "v1.2", "9lives", "a" * 64] +) +def test_valid_path_friendly_name(name: str) -> None: + data = base_config() + data["project"]["path_friendly_name"] = name + assert load(data).project.path_friendly_name == name + + +@pytest.mark.parametrize( + "name", + [ + # Would escape build_dir + "../victim", + "..", + ".", + "a/b", + "a\\b", + "/abs", + "C:evil", + "C:\\evil", + # Not allowed in an npm package name + "", + "Racers", + "my game", + ".hidden", + "_private", + "-flag", + "@scope/pkg", + "racers\n", + "a" * 65, + # Reserved on Windows, whichever OS the config is loaded on + "con", + "nul", + "com1", + "con.txt", + "trailing.", + ], +) +def test_invalid_path_friendly_name(name: str) -> None: + data = base_config() + data["project"]["path_friendly_name"] = name + with pytest.raises(ValidationError, match="path_friendly_name"): + load(data) + + +# -------------------------------------------------------------- unknown keys ----- + + +def add_key(data: dict[str, Any], *path: str | int) -> None: + section: Any = data + for key in path: + section = section[key] + section["typo"] = "oops" + + +@pytest.mark.parametrize( + "path", + [ + (), + ("project",), + ("inputs",), + ("inputs", "code"), + ("inputs", "assets"), + ("outputs", 0), + ], + ids=str, +) +def test_unknown_key_is_rejected(path: tuple[str | int, ...]) -> None: + data = base_config() + add_key(data, *path) + with pytest.raises(ValidationError, match="typo"): + load(data) + + +def test_misspelled_key_is_rejected_instead_of_using_the_default() -> None: + data = base_config() + data["project"]["titel"] = "My Game" + with pytest.raises(ValidationError, match="titel"): + load(data) + + +def test_unknown_key_in_icon_is_rejected() -> None: + data = base_config() + data["inputs"]["assets"]["icon"] = {"type": "path", "value": "i.png", "typo": 1} + with pytest.raises(ValidationError, match="typo"): + load(data) + + +def test_unknown_key_in_window_is_rejected() -> None: + with pytest.raises(ValidationError, match="depth"): + load_outputs({"type": "electron", "window": {"depth": 32}}) + + +# ------------------------------------------------------------------ outputs ----- + + +def test_empty_outputs_are_rejected() -> None: + with pytest.raises(ValidationError, match="outputs"): + load_outputs() + + +def test_unknown_output_type_is_rejected() -> None: + with pytest.raises(ValidationError, match="nonsense"): + load_outputs({"type": "nonsense"}) + + +def test_tauri_requires_identifier() -> None: + with pytest.raises(ValidationError, match="identifier"): + load_outputs({"type": "tauri"}) + + +def test_tauri_with_identifier() -> None: + [output] = load_outputs({"type": "tauri", "identifier": "com.example.game"}) + assert isinstance(output, TauriOutput) + assert output.identifier == "com.example.game" + assert output.window is None + + +def test_electron_window_is_optional() -> None: + [output] = load_outputs({"type": "electron"}) + assert isinstance(output, ElectronOutput) + assert output.window is None + + +def test_window_config_defaults() -> None: + assert WindowConfig() == WindowConfig(width=640, height=480) + + +def test_window_config_fills_in_missing_side() -> None: + [output] = load_outputs({"type": "electron", "window": {"width": 800}}) + assert output.window == WindowConfig(width=800, height=480) + + +# ------------------------------------------------------------- code sources ----- + + +def test_unknown_code_source_type_is_rejected() -> None: + data = base_config() + data["inputs"]["code"] = {"type": "ftp", "value": "ftp://example.com"} + with pytest.raises(ValidationError, match="ftp"): + load(data) + + +def test_unknown_icon_source_type_is_rejected() -> None: + data = base_config() + data["inputs"]["assets"]["icon"] = {"type": "ftp", "value": "ftp://example.com"} + with pytest.raises(ValidationError, match="ftp"): + load(data) + + +# ---------------------------------------------------- resolve_config_paths ----- + + +def load_with_paths( + code: dict[str, Any], icon: dict[str, Any] | None = None, build_dir: str = "build" +) -> BuildConfig: + data = base_config() + data["inputs"]["code"] = code + if icon: + data["inputs"]["assets"]["icon"] = icon + data["build_dir"] = build_dir + return load(data) + + +@pytest.mark.parametrize( + "code_type", + [("path", PathCodeSource), ("file", FileCodeSource)], + ids=lambda t: t[0], +) +def test_resolve_makes_relative_paths_absolute( + tmp_path: Path, code_type: tuple[str, type[PathCodeSource | FileCodeSource]] +) -> None: + name, source_type = code_type + config = load_with_paths( + {"type": name, "value": "./game"}, + {"type": "path", "value": "assets/icon.png"}, + "./out/build", + ) + + resolved = resolve_config_paths(config, tmp_path) + + base = tmp_path.resolve() + assert resolved.build_dir == str(base / "out" / "build") + assert resolved.inputs.code.root == source_type( + type=name, # type: ignore[arg-type] + value=str(base / "game"), + ) + assert resolved.inputs.assets.icon is not None + assert resolved.inputs.assets.icon.root == PathAssetSource( + type="path", value=str(base / "assets" / "icon.png") + ) + + +def test_resolve_follows_parent_directories(tmp_path: Path) -> None: + config = load_with_paths({"type": "path", "value": "../game"}, None, "../../build") + base = tmp_path / "a" / "b" + resolved = resolve_config_paths(config, base) + assert resolved.build_dir == str((tmp_path / "build").resolve()) + assert resolved.inputs.code.root == PathCodeSource( + type="path", value=str((tmp_path / "a" / "game").resolve()) + ) + + +def test_resolve_keeps_absolute_paths(tmp_path: Path) -> None: + game = (tmp_path / "elsewhere" / "game").resolve() + build = (tmp_path / "elsewhere" / "build").resolve() + config = load_with_paths({"type": "path", "value": str(game)}, None, str(build)) + resolved = resolve_config_paths(config, tmp_path / "config-dir") + assert resolved.build_dir == str(build) + assert resolved.inputs.code.root == PathCodeSource(type="path", value=str(game)) + + +@pytest.mark.parametrize( + "code", + [ + {"type": "share_link", "value": "https://arcade.makecode.com/S12345/"}, + {"type": "git", "url": "https://github.com/a/b", "ref": "main"}, + {"type": "git", "url": "file:///a/b"}, + {"type": "github", "value": "https://github.com/a/b"}, + ], + ids=lambda code: code["type"], +) +def test_resolve_leaves_urls_alone(tmp_path: Path, code: dict[str, Any]) -> None: + icon = {"type": "url", "value": "https://example.com/icon.png"} + config = load_with_paths(code, icon) + resolved = resolve_config_paths(config, tmp_path) + assert resolved.inputs.code == config.inputs.code + assert resolved.inputs.assets.icon is not None + assert isinstance(resolved.inputs.assets.icon.root, UrlAssetSource) + assert resolved.inputs.assets.icon == config.inputs.assets.icon + + +@pytest.mark.parametrize( + ("source", "build_dir", "message"), + [ + (".", "build", "is inside the path code source"), + (".", "a/b/build", "is inside the path code source"), + ("../game", "build", "is inside the path code source"), + ("game", "game", "is inside the path code source"), + ("build/game", "build", "is inside build_dir"), + ("build/g-source/game", "build", "is inside build_dir"), + ], +) +def test_resolve_rejects_overlapping_source_and_build_dir( + tmp_path: Path, source: str, build_dir: str, message: str +) -> None: + base = tmp_path / "game" + config = load_with_paths({"type": "path", "value": source}, None, build_dir) + with pytest.raises(ValueError, match=message): + resolve_config_paths(config, base) + + +def test_resolve_rejects_build_dir_inside_a_linked_source(tmp_path: Path) -> None: + game = tmp_path / "game" + game.mkdir() + link = tmp_path / "link" + try: + link.symlink_to(game, target_is_directory=True) + except OSError: + pytest.skip("Creating symlinks isn't allowed here") + config = load_with_paths({"type": "path", "value": str(link)}, None, "build") + with pytest.raises(ValueError, match="is inside the path code source"): + resolve_config_paths(config, game) + + +def test_resolve_allows_names_that_only_share_a_prefix(tmp_path: Path) -> None: + config = load_with_paths({"type": "path", "value": "game"}, None, "game-build") + resolved = resolve_config_paths(config, tmp_path) + assert resolved.build_dir == str((tmp_path / "game-build").resolve()) + + +def test_resolve_does_not_modify_the_config(tmp_path: Path) -> None: + config = load_with_paths( + {"type": "path", "value": "game"}, {"type": "path", "value": "icon.png"} + ) + before = config.model_dump_json() + resolve_config_paths(config, tmp_path) + assert config.model_dump_json() == before + + +def test_resolve_does_not_depend_on_the_current_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config = load_with_paths({"type": "path", "value": "game"}) + results = [] + for cwd in (tmp_path / "one", tmp_path / "two"): + cwd.mkdir() + monkeypatch.chdir(cwd) + results.append(resolve_config_paths(config, tmp_path).model_dump_json()) + assert results[0] == results[1] + + +# ------------------------------------------------- the JSON tasks are given ----- + + +def test_comments_and_formatting_do_not_change_the_json() -> None: + data = base_config() + plain = yaml.safe_dump(data) + reformatted = "# A comment\n\n" + yaml.safe_dump(data, sort_keys=False, indent=4) + assert plain != reformatted + assert ( + load_config_from_yaml(plain).model_dump_json() + == load_config_from_yaml(reformatted).model_dump_json() + ) + + +@pytest.mark.parametrize( + "code", + [ + {"type": "share_link", "value": "https://arcade.makecode.com/S12345/"}, + {"type": "git", "url": "https://github.com/a/b", "ref": "main"}, + {"type": "github", "value": "https://github.com/a/b", "checkout": "dev"}, + {"type": "path", "value": "game"}, + {"type": "file", "value": "game.png"}, + ], + ids=lambda code: code["type"], +) +def test_config_and_its_parts_survive_json(code: dict[str, Any]) -> None: + icon = {"type": "url", "value": "https://example.com/icon.png"} + config = load_with_paths(code, icon) + assert BuildConfig.model_validate_json(config.model_dump_json()) == config + assert ( + CodeSource.model_validate_json(config.inputs.code.model_dump_json()) + == config.inputs.code + ) + assert config.inputs.assets.icon is not None + assert ( + AssetSource.model_validate_json(config.inputs.assets.icon.model_dump_json()) + == config.inputs.assets.icon + ) + assert ( + Project.model_validate_json(config.project.model_dump_json()) == config.project + ) diff --git a/tests/unit/test_config_code_source.py b/tests/unit/test_config_code_source.py new file mode 100644 index 0000000..a144acc --- /dev/null +++ b/tests/unit/test_config_code_source.py @@ -0,0 +1,83 @@ +from typing import Any + +import pytest +import yaml +from pydantic import ValidationError + +from mkcd2app.config import load_config_from_yaml +from mkcd2app.models.config import GitCodeSource, GitHubCodeSource + + +def config_yaml_with_code(code: dict[str, Any]) -> str: + return yaml.safe_dump( + { + "version": 1, + "project": { + "name": "Test", + "path_friendly_name": "test", + "author": "Test", + "version": "1.0.0", + }, + "inputs": {"code": code, "assets": {}}, + "build_dir": "build", + "target": "4.0.14", + "outputs": [{"type": "static"}], + } + ) + + +def test_git_source_with_ref() -> None: + config = load_config_from_yaml( + config_yaml_with_code( + {"type": "git", "url": "https://gitlab.com/a/b.git", "ref": "v1.0.0"} + ) + ) + source = config.inputs.code.root + assert isinstance(source, GitCodeSource) + assert str(source.url) == "https://gitlab.com/a/b.git" + assert source.ref == "v1.0.0" + + +def test_git_source_ref_defaults_to_default_branch() -> None: + config = load_config_from_yaml( + config_yaml_with_code({"type": "git", "url": "https://codeberg.org/a/b"}) + ) + source = config.inputs.code.root + assert isinstance(source, GitCodeSource) + assert source.ref is None + + +@pytest.mark.parametrize( + "url", ["ssh://git@example.com/a/b.git", "git://example.com/a/b", "file:///a/b"] +) +def test_git_source_accepts_non_http_urls(url: str) -> None: + config = load_config_from_yaml(config_yaml_with_code({"type": "git", "url": url})) + assert isinstance(config.inputs.code.root, GitCodeSource) + + +def test_git_source_requires_url() -> None: + with pytest.raises(ValidationError): + load_config_from_yaml( + config_yaml_with_code({"type": "git", "value": "https://github.com/a/b"}) + ) + + +def test_github_alias_still_parses() -> None: + config = load_config_from_yaml( + config_yaml_with_code( + {"type": "github", "value": "https://github.com/a/b", "checkout": "dev"} + ) + ) + source = config.inputs.code.root + assert isinstance(source, GitHubCodeSource) + assert str(source.value) == "https://github.com/a/b" + assert source.checkout == "dev" + + +def test_github_alias_checkout_defaults_to_master() -> None: + config = load_config_from_yaml( + config_yaml_with_code({"type": "github", "value": "https://github.com/a/b"}) + ) + source = config.inputs.code.root + assert isinstance(source, GitHubCodeSource) + assert source.checkout == "master" diff --git a/tests/unit/test_embedded_source.py b/tests/unit/test_embedded_source.py new file mode 100644 index 0000000..890a2bf --- /dev/null +++ b/tests/unit/test_embedded_source.py @@ -0,0 +1,730 @@ +import hashlib +import json +import lzma +import stat +import struct +import zipfile +from collections.abc import Callable +from pathlib import Path + +import pytest +from PIL import Image + +from mkcd2app.build_project.inputs import embedded_source +from mkcd2app.build_project.inputs.embedded_source import ( + PNG_HEADER_SIZE, + PNG_MAGIC, + SOURCE_MAGIC, + UF2_FLAG_NOT_MAIN_FLASH, + UF2_MAGIC0, + UF2_MAGIC1, + UF2_MAX_GAP, + UF2_MAX_IMAGE_SIZE, + UF2_MAX_PAYLOAD, + ZIP_MAX_TOTAL_SIZE, + EmbeddedSourceError, + extract_project, + read_embedded_files, + read_project_files, +) +from mkcd2app.config import load_config_from_yaml +from mkcd2app.models.config import FileCodeSource + +FIXTURES = Path(__file__).parent.parent / "fixtures" +MKCD_PROJECTS = FIXTURES / "mkcd_projects" +DUCK_UF2 = MKCD_PROJECTS / "falling-duck.uf2" +DUCK_PNG = MKCD_PROJECTS / "falling-duck.png" +EMPTY_PNG = MKCD_PROJECTS / "empty-game.png" +# Made by the real LZMA-JS (see generate.cjs there), which writes emoji as CESU-8 +LZMA_JS = FIXTURES / "lzma-js" +RACERS_ZIP = MKCD_PROJECTS / "Racers-master.zip" # GitHub "Download ZIP" of Racers +EMPTY_GAME_DIR = MKCD_PROJECTS / "empty_game" +DUCK_FILES = {"README.md", "assets.json", "main.blocks", "main.ts", "pxt.json"} +DUCK_MAIN_TS_SHA256 = "2100e5d1f310a31abac094e6173c66b7394c18a67c466baa9a655623feded775" + +SAMPLE_FILES = { + "pxt.json": json.dumps({"name": "sample", "dependencies": {"device": "*"}}), + "main.ts": "let x = 1\r\ngame.splash('héllo')\n", + "README.md": " ", +} + + +# ------------------------------------------------------------ helpers --------- + + +def _encode(img: bytearray, ptr: int, bpp: int, data: bytes) -> int: + """Port of pxt's encodeBlobAsync bit packer.""" + shift = dp = 0 + v = data[dp] + dp += 1 + mask = (1 << bpp) - 1 + keep_going = True + while keep_going: + bits = (v >> shift) & mask + left = 8 - shift + if left <= bpp: + if dp >= len(data): + if left == 0: + break + keep_going = False + v = data[dp] if dp < len(data) else 0 + dp += 1 + bits |= (v << left) & mask + shift = bpp - left + else: + shift += bpp + img[ptr] = ((img[ptr] & ~mask) | bits) & 0xFF + ptr += 1 + if ptr & 3 == 3: + img[ptr] = 0xFF + ptr += 1 + return ptr + + +def make_project_png(w: int, h: int, blob: bytes, path: Path) -> int: + """ + Write a PNG the way pxt's encodeBlobAsync does. + + :param blob: lzma.compress(json_text, format=lzma.FORMAT_ALONE). + :return: The number of overflow rows added. + """ + need = PNG_HEADER_SIZE + len(blob) + usable = (w * h - 1) * 3 + bpp = 1 + while bpp < 4 and usable * bpp < need * 8: + bpp += 1 + cap = (usable * bpp) >> 3 + added = max(0, -(-(need - cap) // (w * 3))) + added_offset = w * h * 4 + img = bytearray(Image.new("RGBA", (w, h + added), (120, 30, 200, 255)).tobytes()) + header = struct.pack("<9I", PNG_MAGIC, len(blob), added, *[0] * 6) + _encode(img, 0, 1, bytes([bpp])) + p = _encode(img, 4, bpp, header) + if added == 0: + _encode(img, p, bpp, blob) + else: + first = cap - PNG_HEADER_SIZE + _encode(img, p, bpp, blob[:first]) + _encode(img, added_offset, 8, blob[first:]) + Image.frombytes("RGBA", (w, h + added), bytes(img)).save(path) + return added + + +def _png_blob_for(files: dict[str, str], padding: str = "") -> bytes: + project = { + "meta": {"name": "sample", "editor": "blocksprj", "pad": padding}, + "source": json.dumps(files), + } + return lzma.compress(json.dumps(project).encode(), format=lzma.FORMAT_ALONE) + + +def make_project_uf2( + path: Path, + meta: dict[str, object], + text: bytes, + pad: int = 1000, + meta_bytes: bytes | None = None, +) -> None: + """ + Embed source the way pxt does, then chunk it into 256-byte UF2 blocks. + + :param meta_bytes: The encoded meta JSON, instead of ``json.dumps(meta)``. + """ + meta_bytes = json.dumps(meta).encode() if meta_bytes is None else meta_bytes + pre = b"\xaa" * (pad + (-pad % 16)) # source magic must be 16-byte aligned + binary = ( + pre + + SOURCE_MAGIC + + struct.pack(" None: + chunks = [binary[i : i + 256] for i in range(0, len(binary), 256)] + out = bytearray() + for i, chunk in enumerate(chunks): + block = struct.pack( + "<8I", UF2_MAGIC0, UF2_MAGIC1, 0, base + i * 256, 256, i, len(chunks), 0 + ) + block += chunk.ljust(476, b"\x00") + struct.pack(" None: + assert set(files) == DUCK_FILES + main_ts = files["main.ts"] + assert main_ts.startswith("enum ActionKind {") + assert len(main_ts.splitlines()) == 539 + assert hashlib.sha256(main_ts.encode()).hexdigest() == DUCK_MAIN_TS_SHA256 + pxt = json.loads(files["pxt.json"]) + assert pxt["name"] == "falling duck" + assert set(pxt["dependencies"]) == {"device", "animation"} + assert pxt["targetVersions"]["target"] == "4.1.27" + + +def test_duck_uf2() -> None: + _check_duck(read_embedded_files(DUCK_UF2)) + + +def test_duck_png_matches_uf2() -> None: + # Both were exported from 4.1.27 in the same session, so every file matches + png = read_embedded_files(DUCK_PNG) + _check_duck(png) + assert png == read_embedded_files(DUCK_UF2) + + +def test_empty_game_png_matches_fixture_dir() -> None: + # Exported from the 4.0.14 editor; only the formatting of pxt.json differs from + # the checked-in tests/fixtures/mkcd_projects/empty_game + files = read_embedded_files(EMPTY_PNG) + assert set(files) == {p.name for p in EMPTY_GAME_DIR.iterdir()} + for name, content in files.items(): + expected = (EMPTY_GAME_DIR / name).read_text(encoding="utf-8") + if name == "pxt.json": + assert json.loads(content) == json.loads(expected) + else: + assert content == expected + + +def test_duck_extract(tmp_path: Path) -> None: + names = extract_project(DUCK_UF2, tmp_path / "out") + assert set(names) == DUCK_FILES + data = (tmp_path / "out" / "main.ts").read_bytes() + assert hashlib.sha256(data).hexdigest() == DUCK_MAIN_TS_SHA256 + + +# ---------------------------------------------------------- synthetic PNG ----- + + +def test_png_fits_in_image(tmp_path: Path) -> None: + path = tmp_path / "game.png" + assert make_project_png(160, 120, _png_blob_for(SAMPLE_FILES), path) == 0 + assert read_embedded_files(path) == SAMPLE_FILES + + +def test_png_overflow_rows(tmp_path: Path) -> None: + path = tmp_path / "game.png" + blob = _png_blob_for(SAMPLE_FILES, padding="".join(map(str, range(200)))) + assert len(blob) > 300 + assert make_project_png(6, 6, blob, path) > 0 + assert read_embedded_files(path) == SAMPLE_FILES + + +@pytest.mark.parametrize("size", [(20, 20), (40, 30), (64, 64)]) +def test_png_various_bpp(tmp_path: Path, size: tuple[int, int]) -> None: + path = tmp_path / "game.png" + make_project_png(*size, _png_blob_for(SAMPLE_FILES), path) + assert read_embedded_files(path) == SAMPLE_FILES + + +# ---------------------------------------------------------- synthetic UF2 ----- + + +def test_uf2_lzma(tmp_path: Path) -> None: + header = json.dumps({"name": "sample", "editor": "blocksprj"}) + text = lzma.compress( + (header + json.dumps(SAMPLE_FILES)).encode(), format=lzma.FORMAT_ALONE + ) + path = tmp_path / "game.uf2" + make_project_uf2(path, {"compression": "LZMA", "headerSize": len(header)}, text) + assert read_embedded_files(path) == SAMPLE_FILES + + +def test_uf2_uncompressed(tmp_path: Path) -> None: + path = tmp_path / "game.uf2" + make_project_uf2(path, {"name": "sample"}, json.dumps(SAMPLE_FILES).encode()) + assert read_embedded_files(path) == SAMPLE_FILES + + +def test_extract_writes_bytes_exactly(tmp_path: Path) -> None: + src = tmp_path / "game.uf2" + make_project_uf2(src, {}, json.dumps(SAMPLE_FILES).encode()) + dest = tmp_path / "out" + assert extract_project(src, dest) == sorted(SAMPLE_FILES) + for name, content in SAMPLE_FILES.items(): + assert (dest / name).read_bytes() == content.encode("utf-8") + + +# ------------------------------------------------- LZMA-JS text encoding ------ + + +def _lzma_js_files() -> dict[str, str]: + files: dict[str, str] = json.loads((LZMA_JS / "files.json").read_text("utf-8")) + return files + + +def test_lzma_js_fixture_is_cesu8() -> None: + # Guards the fixture itself: strict UTF-8 must reject it, or it tests nothing + raw = lzma.decompress( + (LZMA_JS / "png-project.lzma").read_bytes(), format=lzma.FORMAT_ALONE + ) + assert "😀".encode("utf-16-be").hex().startswith("d83d") + assert b"\xed\xa0\xbd\xed\xb8\x80" in raw # 😀 as two surrogates + with pytest.raises(UnicodeDecodeError): + raw.decode("utf-8") + + +def test_lzma_js_png_with_emoji(tmp_path: Path) -> None: + src = tmp_path / "game.png" + make_project_png(160, 120, (LZMA_JS / "png-project.lzma").read_bytes(), src) + files = _lzma_js_files() + assert read_embedded_files(src) == files + assert 'game.splash("hello 😀")' in files["main.ts"] + + dest = tmp_path / "out" + extract_project(src, dest) + for name, content in files.items(): + assert (dest / name).read_bytes() == content.encode("utf-8") + + +def test_lzma_js_uf2_with_emoji_in_header(tmp_path: Path) -> None: + header_size = int((LZMA_JS / "uf2-header-size.txt").read_text("utf-8")) + text = (LZMA_JS / "uf2-source.lzma").read_bytes() + src = tmp_path / "game.uf2" + make_project_uf2(src, {"compression": "LZMA", "headerSize": header_size}, text) + files = _lzma_js_files() + + # The header holds an emoji, so its JS length (UTF-16 code units) is longer than + # its Python length: slicing by characters would cut into the files JSON + header = '{"name":"emoji 😀 game","editor":"tsprj"}' + assert header_size == len(header.encode("utf-16-le")) // 2 == len(header) + 1 + assert read_embedded_files(src) == files + + dest = tmp_path / "out" + extract_project(src, dest) + for name, content in files.items(): + assert (dest / name).read_bytes() == content.encode("utf-8") + + +def _cesu8(text: str) -> bytes: + """Encode like LZMA-JS: each UTF-16 code unit on its own, U+0000 as C0 80.""" + units = text.encode("utf-16-le") + split = "".join( + chr(int.from_bytes(units[i : i + 2], "little")) for i in range(0, len(units), 2) + ) + return split.encode("utf-8", "surrogatepass").replace(b"\x00", b"\xc0\x80") + + +@pytest.mark.parametrize("encode", [str.encode, _cesu8], ids=["utf8", "cesu8"]) +def test_uf2_meta_with_emoji(tmp_path: Path, encode: Callable[[str], bytes]) -> None: + meta: dict[str, object] = {"name": "emoji 😀 game"} + src = tmp_path / "game.uf2" + make_project_uf2( + src, + meta, + json.dumps(SAMPLE_FILES).encode(), + meta_bytes=encode(json.dumps(meta, ensure_ascii=False)), + ) + assert read_embedded_files(src) == SAMPLE_FILES + + +@pytest.mark.parametrize("encode", [str.encode, _cesu8], ids=["utf8", "cesu8"]) +@pytest.mark.parametrize("text", ["a😀b", "é ✓ 𝄞", "plain"]) +def test_uncompressed_uf2_text_encodings( + tmp_path: Path, encode: Callable[[str], bytes], text: str +) -> None: + src = tmp_path / "game.uf2" + make_project_uf2(src, {}, encode(json.dumps({"main.ts": text}, ensure_ascii=False))) + assert read_embedded_files(src) == {"main.ts": text} + + +def test_cesu8_nul() -> None: + # JSON.stringify escapes U+0000, so it never reaches a project, but LZMA-JS + # would write it as C0 80 + assert _cesu8("a\x00b") == b"a\xc0\x80b" + units = embedded_source._cesu8_to_utf16(b"a\xc0\x80b") + assert embedded_source._utf16_to_str(units) == "a\x00b" + + +def test_lone_surrogate_is_corrupt(tmp_path: Path) -> None: + # Can't be written as UTF-8, so it must fail cleanly rather than at write time + project = '{"meta": {}, "source": "{\\"main.ts\\": \\"x\ud83dy\\"}"}' + blob = lzma.compress( + project.encode("utf-8", "surrogatepass"), format=lzma.FORMAT_ALONE + ) + src = tmp_path / "game.png" + make_project_png(160, 120, blob, src) + with pytest.raises(EmbeddedSourceError, match="corrupt"): + read_embedded_files(src) + + +@pytest.mark.parametrize("header_size", [-1, 10_000, "5"]) +def test_uf2_invalid_header_size(tmp_path: Path, header_size: object) -> None: + text = lzma.compress(json.dumps(SAMPLE_FILES).encode(), format=lzma.FORMAT_ALONE) + src = tmp_path / "game.uf2" + make_project_uf2(src, {"compression": "LZMA", "headerSize": header_size}, text) + with pytest.raises(EmbeddedSourceError, match="headerSize"): + read_embedded_files(src) + + +# ------------------------------------------------------------ UF2 layout ------ + + +def _uf2_block(addr: int, payload: bytes, size: int | None = None) -> bytes: + header = struct.pack( + "<8I", + UF2_MAGIC0, + UF2_MAGIC1, + 0, + addr, + len(payload) if size is None else size, + 0, + 1, + 0, + ) + return header + payload.ljust(476, b"\x00") + struct.pack(" bytes: + text = json.dumps(SAMPLE_FILES).encode() + meta = b"{}" + return SOURCE_MAGIC + struct.pack(" list[int]: + """Record the size of each bytearray the decoder allocates.""" + sizes: list[int] = [] + + def spy(size: int = 0) -> bytearray: + sizes.append(size) + assert size <= UF2_MAX_IMAGE_SIZE, f"tried to allocate {size} bytes" + return bytearray(size) + + monkeypatch.setattr(embedded_source, "bytearray", spy, raising=False) + return sizes + + +def test_uf2_far_apart_blocks_not_densified( + tmp_path: Path, allocations: list[int] +) -> None: + # Regression: addresses 0 and 0xFFFF0000 asked for a ~4 GiB buffer + payload = _source_payload() + chunks = [payload[i : i + 256] for i in range(0, len(payload), 256)] + blocks = _uf2_block(0, b"\xaa" * 256) + for i, chunk in enumerate(chunks): + blocks += _uf2_block(0xFFFF0000 + i * 256, chunk) + src = tmp_path / "game.uf2" + src.write_bytes(blocks) + + assert read_embedded_files(src) == SAMPLE_FILES + assert max(allocations) <= len(blocks) + + +def test_uf2_small_gap_is_padded(tmp_path: Path) -> None: + # Blocks with a gap (like pxt's toBin pads) still form one image, so the + # source's 16-byte alignment is counted from the lowest address + payload = _source_payload() + chunks = [payload[i : i + 256] for i in range(0, len(payload), 256)] + blocks = _uf2_block(0x1000, b"\xaa" * 256) + for i, chunk in enumerate(chunks): + blocks += _uf2_block(0x4000 + i * 256, chunk) + src = tmp_path / "game.uf2" + src.write_bytes(blocks) + assert read_embedded_files(src) == SAMPLE_FILES + + +def test_uf2_too_much_memory(tmp_path: Path, allocations: list[int]) -> None: + # Each block sits just within the padding gap, so they'd all join one run + count = UF2_MAX_IMAGE_SIZE // UF2_MAX_GAP + 2 + blocks = b"".join( + _uf2_block(i * (UF2_MAX_GAP + 256), b"\xaa" * 256) for i in range(count) + ) + src = tmp_path / "game.uf2" + src.write_bytes(blocks) + with pytest.raises(EmbeddedSourceError, match="MB limit"): + read_embedded_files(src) + assert allocations == [] # rejected before allocating anything + + +@pytest.mark.parametrize("size", [UF2_MAX_PAYLOAD + 1, 0xFFFFFFFF]) +def test_uf2_oversized_payload( + tmp_path: Path, allocations: list[int], size: int +) -> None: + src = tmp_path / "game.uf2" + src.write_bytes(_uf2_block(0x2000, _source_payload()[:256], size=size)) + with pytest.raises(EmbeddedSourceError, match="payload"): + read_embedded_files(src) + assert allocations == [] + + +def test_uf2_max_payload_accepted(tmp_path: Path) -> None: + payload = _source_payload() + assert len(payload) <= 2 * UF2_MAX_PAYLOAD + src = tmp_path / "game.uf2" + src.write_bytes( + _uf2_block(0x2000, payload[:UF2_MAX_PAYLOAD]) + + _uf2_block(0x2000 + UF2_MAX_PAYLOAD, payload[UF2_MAX_PAYLOAD:]) + ) + assert read_embedded_files(src) == SAMPLE_FILES + + +# ----------------------------------------------------------------- errors ----- + + +def test_plain_png_has_no_project(tmp_path: Path) -> None: + path = tmp_path / "plain.png" + Image.new("RGBA", (32, 32), (0, 0, 0, 255)).save(path) + with pytest.raises(EmbeddedSourceError): + read_embedded_files(path) + + +def test_noisy_png_has_no_project(tmp_path: Path) -> None: + path = tmp_path / "noise.png" + Image.new("RGBA", (32, 32), (255, 255, 255, 255)).save(path) # bpp == 7 + with pytest.raises(EmbeddedSourceError): + read_embedded_files(path) + + +def test_png_with_low_bits_but_no_project(tmp_path: Path) -> None: + # The first pixel looks like it says how many bits are used, as in a normal photo + path = tmp_path / "photo.png" + Image.new("RGBA", (32, 32), (1, 0, 0, 255)).save(path) + with pytest.raises(EmbeddedSourceError, match="Invalid magic"): + read_embedded_files(path) + + +@pytest.mark.parametrize( + "project", [[], {"meta": {}}, {"source": {"main.ts": ""}}], ids=str +) +def test_png_project_without_source_string(tmp_path: Path, project: object) -> None: + blob = lzma.compress(json.dumps(project).encode(), format=lzma.FORMAT_ALONE) + path = tmp_path / "game.png" + make_project_png(160, 120, blob, path) + with pytest.raises(EmbeddedSourceError, match="has no source"): + read_embedded_files(path) + + +@pytest.mark.parametrize("files", [["main.ts"], "main.ts", {"main.ts": 1}], ids=str) +def test_files_not_a_map_of_strings(tmp_path: Path, files: object) -> None: + path = tmp_path / "game.uf2" + make_project_uf2(path, {}, json.dumps(files).encode()) + with pytest.raises(EmbeddedSourceError, match="not a {filename: text} map"): + read_embedded_files(path) + + +def test_uf2_magic_without_a_full_block(tmp_path: Path) -> None: + path = tmp_path / "game.uf2" + path.write_bytes(struct.pack("<2I", UF2_MAGIC0, UF2_MAGIC1) + b"\x00" * 100) + with pytest.raises(EmbeddedSourceError, match="Not a UF2 file"): + read_embedded_files(path) + + +def test_uf2_without_main_flash_blocks(tmp_path: Path) -> None: + block = struct.pack( + "<8I", UF2_MAGIC0, UF2_MAGIC1, UF2_FLAG_NOT_MAIN_FLASH, 0, 256, 0, 1, 0 + ).ljust(512, b"\x00") + path = tmp_path / "game.uf2" + path.write_bytes(block) + with pytest.raises(EmbeddedSourceError, match="Not a UF2 file"): + read_embedded_files(path) + + +def test_uf2_skips_unaligned_source_magic(tmp_path: Path) -> None: + # 5 + 8 + 3 bytes puts the real source at offset 16, after a decoy at offset 5 + decoy = b"\xaa" * 5 + SOURCE_MAGIC + b"\xaa" * 3 + path = tmp_path / "game.uf2" + _write_uf2(path, decoy + _source_payload()) + assert read_embedded_files(path) == SAMPLE_FILES + + +@pytest.mark.parametrize("meta_bytes", [b"[]", b'"text"', b"1"]) +def test_uf2_metadata_not_an_object(tmp_path: Path, meta_bytes: bytes) -> None: + path = tmp_path / "game.uf2" + make_project_uf2(path, {}, json.dumps(SAMPLE_FILES).encode(), meta_bytes=meta_bytes) + with pytest.raises(EmbeddedSourceError, match="not a JSON object"): + read_embedded_files(path) + + +def test_uf2_unsupported_compression(tmp_path: Path) -> None: + path = tmp_path / "game.uf2" + make_project_uf2(path, {"compression": "gzip"}, b"") + with pytest.raises(EmbeddedSourceError, match="Unsupported compression gzip"): + read_embedded_files(path) + + +def test_uf2_without_source(tmp_path: Path) -> None: + path = tmp_path / "binary-only.uf2" + _write_uf2(path, b"\x00" * 4096) + with pytest.raises(EmbeddedSourceError, match="doesn't contain source"): + read_embedded_files(path) + + +def test_neither_format(tmp_path: Path) -> None: + path = tmp_path / "game.txt" + path.write_text("hello") + with pytest.raises(EmbeddedSourceError, match="not a PNG, UF2 or ZIP"): + read_embedded_files(path) + + +def test_truncated_png_payload(tmp_path: Path) -> None: + path = tmp_path / "game.png" + blob = _png_blob_for(SAMPLE_FILES) + make_project_png(160, 120, blob[: len(blob) // 2], path) + with pytest.raises(EmbeddedSourceError, match="corrupt"): + read_embedded_files(path) + + +@pytest.mark.parametrize("name", ["../evil.ts", "sub/../../evil.ts", "/abs/evil.ts"]) +def test_path_traversal_rejected(tmp_path: Path, name: str) -> None: + src = tmp_path / "game.uf2" + make_project_uf2(src, {}, json.dumps({"main.ts": "", name: "x"}).encode()) + dest = tmp_path / "out" + with pytest.raises(EmbeddedSourceError, match="Refusing to write outside"): + extract_project(src, dest) + assert not dest.exists() # nothing written before the check fails + assert not (tmp_path / "evil.ts").exists() + + +# -------------------------------------------------------------------- ZIP ----- + + +def make_zip(path: Path, entries: dict[str, bytes]) -> None: + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in entries.items(): + zf.writestr(name, data) + + +SAMPLE_BYTES = {k: v.encode() for k, v in SAMPLE_FILES.items()} + + +def test_racers_github_zip(tmp_path: Path) -> None: + dest = tmp_path / "out" + names = extract_project(RACERS_ZIP, dest) + # The Racers-master/ wrapper is stripped and the whole repo is kept + assert {"pxt.json", "main.ts", "main.blocks", "test.ts"} <= set(names) + assert "assets/js/binary.js" in names + assert ".github/workflows/makecode.yml" in names + assert not any(n.startswith("Racers-master") for n in names) + pxt = json.loads((dest / "pxt.json").read_text(encoding="utf-8")) + assert pxt["name"] == "Racers v1.3.2" + for name in pxt["files"] + pxt["testFiles"]: + assert (dest / name).is_file(), name + # Binary files come through untouched + with zipfile.ZipFile(RACERS_ZIP) as zf: + png = zf.read("Racers-master/.github/makecode/blocks.png") + assert (dest / ".github" / "makecode" / "blocks.png").read_bytes() == png + + +@pytest.mark.parametrize("prefix", ["", "Game-main/"]) +def test_zip_root_or_wrapper(tmp_path: Path, prefix: str) -> None: + src = tmp_path / "game.zip" + entries = {prefix + k: v for k, v in SAMPLE_BYTES.items()} + entries[prefix + "assets/img.bin"] = bytes(range(256)) + make_zip(src, entries) + expected = SAMPLE_BYTES | {"assets/img.bin": bytes(range(256))} + assert read_project_files(src) == expected + + +def test_zip_keeps_line_endings(tmp_path: Path) -> None: + src = tmp_path / "game.zip" + make_zip(src, {"pxt.json": b"{}", "main.ts": b"a\r\nb\n"}) + extract_project(src, tmp_path / "out") + assert (tmp_path / "out" / "main.ts").read_bytes() == b"a\r\nb\n" + + +@pytest.mark.parametrize( + "entries", + [ + {"main.ts": b""}, # no pxt.json at all + {"a/b/pxt.json": b"{}"}, # nested too deep + {"a/pxt.json": b"{}", "b/main.ts": b""}, # two top-level folders + ], +) +def test_zip_without_project_root(tmp_path: Path, entries: dict[str, bytes]) -> None: + src = tmp_path / "game.zip" + make_zip(src, entries) + with pytest.raises(EmbeddedSourceError, match="no pxt.json"): + read_project_files(src) + + +def test_zip_skips_symlinks(tmp_path: Path) -> None: + src = tmp_path / "game.zip" + with zipfile.ZipFile(src, "w") as zf: + zf.writestr("pxt.json", b"{}") + link = zipfile.ZipInfo("evil") + link.external_attr = (stat.S_IFLNK | 0o777) << 16 + zf.writestr(link, "/etc/passwd") + assert read_project_files(src) == {"pxt.json": b"{}"} + + +def test_zip_path_traversal_rejected(tmp_path: Path) -> None: + src = tmp_path / "game.zip" + make_zip(src, {"pxt.json": b"{}", "../evil.ts": b"x"}) + dest = tmp_path / "out" + with pytest.raises(EmbeddedSourceError, match="Refusing to write outside"): + extract_project(src, dest) + assert not dest.exists() + assert not (tmp_path / "evil.ts").exists() + + +def test_zip_bomb_rejected(tmp_path: Path) -> None: + src = tmp_path / "bomb.zip" + with zipfile.ZipFile(src, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("pxt.json", b"{}") + with zf.open("big.bin", "w", force_zip64=True) as f: + chunk = b"\0" * 2**20 + for _ in range(ZIP_MAX_TOTAL_SIZE // len(chunk) + 1): + f.write(chunk) + with pytest.raises(EmbeddedSourceError, match="uncompressed"): + read_project_files(src) + + +def test_zip_too_many_entries_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(embedded_source, "ZIP_MAX_ENTRIES", len(SAMPLE_BYTES) - 1) + src = tmp_path / "game.zip" + make_zip(src, SAMPLE_BYTES) + with pytest.raises(EmbeddedSourceError, match="entries"): + read_project_files(src) + + +def test_zip_entry_outside_wrapper_is_skipped(tmp_path: Path) -> None: + # An entry named "." has no path parts, so it isn't under the wrapper folder + src = tmp_path / "game.zip" + make_zip(src, {"Game-main/pxt.json": b"{}", ".": b"junk"}) + assert read_project_files(src) == {"pxt.json": b"{}"} + + +def test_corrupt_zip(tmp_path: Path) -> None: + src = tmp_path / "game.zip" + make_zip(src, SAMPLE_BYTES) + src.write_bytes(src.read_bytes()[:-30]) # cut off the central directory + with pytest.raises(EmbeddedSourceError, match="corrupt"): + read_project_files(src) + + +def test_read_embedded_files_rejects_zip(tmp_path: Path) -> None: + src = tmp_path / "game.zip" + make_zip(src, SAMPLE_BYTES) + with pytest.raises(EmbeddedSourceError, match="not a PNG or UF2"): + read_embedded_files(src) + + +# ----------------------------------------------------------------- config ----- + + +def test_config_file_source() -> None: + config_path = FIXTURES / "empty_game_path_build_statics.yaml" + text = config_path.read_text(encoding="utf-8") + text = text.replace( + 'type: path\n value: "mkcd_projects/empty_game"', + 'type: file\n value: "mkcd_projects/falling-duck.png"', + ) + config = load_config_from_yaml(text) + assert config.inputs.code.root == FileCodeSource( + type="file", value="mkcd_projects/falling-duck.png" + ) diff --git a/tests/unit/test_filesystem.py b/tests/unit/test_filesystem.py new file mode 100644 index 0000000..21bc110 --- /dev/null +++ b/tests/unit/test_filesystem.py @@ -0,0 +1,28 @@ +import os +import stat +import sys +from pathlib import Path + +from mkcd2app.utils.filesystem import _remove_readonly, rmtree_robust + + +def make_readonly_tree(root: Path) -> Path: + (root / "objects" / "ab").mkdir(parents=True) + readonly = root / "objects" / "ab" / "cdef" + readonly.write_bytes(b"git object") + os.chmod(readonly, stat.S_IREAD) + return readonly + + +def test_removes_tree_with_readonly_files(tmp_path: Path) -> None: + root = tmp_path / ".git" + make_readonly_tree(root) + rmtree_robust(root) + assert not root.exists() + + +def test_remove_readonly_handler_clears_bit_and_retries(tmp_path: Path) -> None: + # Called directly, since only Windows refuses to unlink read-only files + readonly = make_readonly_tree(tmp_path) + _remove_readonly(os.unlink, str(readonly), sys.exc_info()) # type: ignore[arg-type] + assert not readonly.exists() diff --git a/tests/unit/test_git.py b/tests/unit/test_git.py new file mode 100644 index 0000000..fb6e249 --- /dev/null +++ b/tests/unit/test_git.py @@ -0,0 +1,105 @@ +from dataclasses import dataclass +from pathlib import Path + +import pytest +from dulwich import porcelain +from dulwich.repo import Repo + +from mkcd2app.utils.git import clone_and_checkout + +IDENTITY = b"Test " + + +@dataclass +class SourceRepo: + path: Path + first_sha: str # tagged v1, content "1" + second_sha: str # tip of the "feature" branch, content "2" + head_sha: str # tip of the default branch, content "3" + + +def commit_file(repo_path: Path, content: str) -> str: + (repo_path / "file.txt").write_text(content, encoding="utf-8") + porcelain.add(str(repo_path), [str(repo_path / "file.txt")]) + # sign=False so a commit.gpgsign in the user's git config doesn't apply + sha = porcelain.commit( + str(repo_path), + message=f"Write {content}".encode(), + author=IDENTITY, + committer=IDENTITY, + sign=False, + ) + return sha.decode() + + +@pytest.fixture(scope="module") +def source_repo(tmp_path_factory: pytest.TempPathFactory) -> SourceRepo: + path = tmp_path_factory.mktemp("source") + porcelain.init(str(path)) + first = commit_file(path, "1") + porcelain.tag_create(str(path), b"v1", sign=False) + second = commit_file(path, "2") + porcelain.branch_create(str(path), b"feature") + head = commit_file(path, "3") + return SourceRepo(path, first, second, head) + + +def read_clone(path: Path) -> tuple[str, str]: + """Return the checked-out file content and the commit HEAD points at.""" + with Repo(str(path)) as repo: + head = repo.head().decode() + return (path / "file.txt").read_text(encoding="utf-8"), head + + +def test_default_branch(source_repo: SourceRepo, tmp_path: Path) -> None: + clone_and_checkout(str(source_repo.path), tmp_path / "clone") + assert read_clone(tmp_path / "clone") == ("3", source_repo.head_sha) + + +def test_branch(source_repo: SourceRepo, tmp_path: Path) -> None: + clone_and_checkout(str(source_repo.path), tmp_path / "clone", "feature") + assert read_clone(tmp_path / "clone") == ("2", source_repo.second_sha) + + +def test_tag(source_repo: SourceRepo, tmp_path: Path) -> None: + clone_and_checkout(str(source_repo.path), tmp_path / "clone", "v1") + assert read_clone(tmp_path / "clone") == ("1", source_repo.first_sha) + + +@pytest.mark.parametrize("length", [7, 40]) +def test_commit_sha(source_repo: SourceRepo, tmp_path: Path, length: int) -> None: + sha = source_repo.second_sha[:length] + clone_and_checkout(str(source_repo.path), tmp_path / "clone", sha) + assert read_clone(tmp_path / "clone") == ("2", source_repo.second_sha) + + +def test_commit_sha_matching_default_branch( + source_repo: SourceRepo, tmp_path: Path +) -> None: + # Regression: dulwich only writes files that differ from the current HEAD tree, + # so this used to leave an empty worktree after a no-checkout clone + clone_and_checkout( + str(source_repo.path), tmp_path / "clone", source_repo.head_sha[:7] + ) + assert read_clone(tmp_path / "clone") == ("3", source_repo.head_sha) + + +def test_unknown_ref(source_repo: SourceRepo, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Could not find"): + clone_and_checkout(str(source_repo.path), tmp_path / "clone", "nope") + + +def test_value_error_without_ref_is_not_retried_as_a_sha( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Only a ref can be mistaken for a commit SHA, so anything else must surface + calls: list[str] = [] + + def fake_clone(url: str, *args: object, **kwargs: object) -> None: + calls.append(url) + raise ValueError("some other problem") + + monkeypatch.setattr(porcelain, "clone", fake_clone) + with pytest.raises(ValueError, match="some other problem"): + clone_and_checkout("https://example.com/a/b", tmp_path / "clone") + assert calls == ["https://example.com/a/b"] diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py new file mode 100644 index 0000000..42fc8ae --- /dev/null +++ b/tests/unit/test_logger.py @@ -0,0 +1,88 @@ +import logging +import sys +from collections.abc import Iterator + +import pytest + +from mkcd2app.utils.logger import create_logger, set_all_stdout_logger_levels + + +@pytest.fixture +def logger_name(request: pytest.FixtureRequest) -> Iterator[str]: + """A logger name unique to the test, removed afterwards (loggers are global).""" + name = f"test-logger-{request.node.name}" + yield name + logging.Logger.manager.loggerDict.pop(name, None) + + +def test_create_logger_setup(logger_name: str) -> None: + logger = create_logger(logger_name, logging.INFO) + assert logger.name == logger_name + assert logger.level == logging.INFO + assert logger.propagate is False + assert len(logger.handlers) == 2 + + +def test_info_goes_to_stdout_and_warning_to_stderr( + logger_name: str, capsys: pytest.CaptureFixture[str] +) -> None: + logger = create_logger(logger_name, logging.INFO) + logger.info("some info") + logger.warning("some warning") + logger.debug("some debug") + captured = capsys.readouterr() + assert "some info" in captured.out and "some warning" not in captured.out + assert "some warning" in captured.err and "some info" not in captured.err + assert "some debug" not in captured.out + captured.err + + +def test_creating_a_logger_twice_does_not_log_twice( + logger_name: str, capsys: pytest.CaptureFixture[str] +) -> None: + # Regression: the check for an existing handler compared against new objects + create_logger(logger_name, logging.INFO) + logger = create_logger(logger_name, logging.INFO) + assert len(logger.handlers) == 2 + logger.info("some info") + logger.warning("some warning") + captured = capsys.readouterr() + assert captured.out.count("some info") == 1 + assert captured.err.count("some warning") == 1 + + +def test_creating_a_logger_again_uses_the_new_level( + logger_name: str, capsys: pytest.CaptureFixture[str] +) -> None: + create_logger(logger_name, logging.INFO) + logger = create_logger(logger_name, logging.DEBUG) + logger.debug("some debug") + assert "some debug" in capsys.readouterr().out + + +def test_set_all_stdout_logger_levels( + logger_name: str, restore_log_levels: None, capsys: pytest.CaptureFixture[str] +) -> None: + logger = create_logger(logger_name, logging.INFO) + stdout_handler, stderr_handler = logger.handlers + assert isinstance(stdout_handler, logging.StreamHandler) + assert stdout_handler.stream == sys.stdout + + set_all_stdout_logger_levels(logging.DEBUG) + + assert logger.level == logging.DEBUG + assert stdout_handler.level == logging.DEBUG + assert stderr_handler.level == logging.WARNING # only stdout handlers change + logger.debug("some debug") + assert "some debug" in capsys.readouterr().out + + +def test_set_all_stdout_logger_levels_covers_our_module_loggers( + logger_name: str, restore_log_levels: None +) -> None: + other = create_logger(f"{logger_name}-other", logging.INFO) + try: + set_all_stdout_logger_levels(logging.DEBUG) + assert logging.getLogger("mkcd2app.utils.logger").level == logging.DEBUG + assert other.level == logging.DEBUG + finally: + logging.Logger.manager.loggerDict.pop(other.name, None) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py new file mode 100644 index 0000000..2e3bceb --- /dev/null +++ b/tests/unit/test_main.py @@ -0,0 +1,50 @@ +import stat +from pathlib import Path + +import pytest + +from mkcd2app.main import _clear_build_dir +from mkcd2app.utils.paths import get_redun_db_for_build_path + + +def test_clear_build_dir_missing_is_fine(tmp_path: Path) -> None: + _clear_build_dir(tmp_path / "missing") + assert not (tmp_path / "missing").exists() + + +def test_clear_build_dir_empty_is_fine(tmp_path: Path) -> None: + build_dir = tmp_path / "build" + build_dir.mkdir() + _clear_build_dir(build_dir) + assert build_dir.is_dir() + + +def test_clear_build_dir_with_cache_db_deletes_it(tmp_path: Path) -> None: + build_dir = tmp_path / "build" + (build_dir / "game-source").mkdir(parents=True) + (build_dir / "game-source" / "main.ts").write_text("x", encoding="utf-8") + get_redun_db_for_build_path(build_dir).write_text("db", encoding="utf-8") + _clear_build_dir(build_dir) + assert not build_dir.exists() + assert tmp_path.is_dir() + + +def test_clear_build_dir_deletes_read_only_files(tmp_path: Path) -> None: + # A git source's .git has read-only files, which fail a plain rmtree on Windows + build_dir = tmp_path / "build" + read_only = build_dir / "game-source" / ".git" / "objects" / "pack" + read_only.parent.mkdir(parents=True) + read_only.write_text("x", encoding="utf-8") + read_only.chmod(stat.S_IREAD) + get_redun_db_for_build_path(build_dir).write_text("db", encoding="utf-8") + _clear_build_dir(build_dir) + assert not build_dir.exists() + + +def test_clear_build_dir_refuses_without_cache_db(tmp_path: Path) -> None: + # e.g. a typo in build_dir, or `build_dir: "."` in a project folder + important = tmp_path / "important.txt" + important.write_text("keep me", encoding="utf-8") + with pytest.raises(RuntimeError, match="Refusing to clear"): + _clear_build_dir(tmp_path) + assert important.read_text("utf-8") == "keep me" diff --git a/tests/unit/test_meta_comment.py b/tests/unit/test_meta_comment.py new file mode 100644 index 0000000..d53c3b3 --- /dev/null +++ b/tests/unit/test_meta_comment.py @@ -0,0 +1,72 @@ +import json +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from mkcd2app.utils.text import extract_meta_comment + +# Copied from a real `mkc build -j` of the empty game on 4.0.14 +META: dict[str, Any] = { + "simUrl": "https://trg-arcade.userpxt.io/---simulator", + "cdnUrl": "https://cdn.makecode.com", + "version": "v0.0.0", + "target": "arcade", + "targetVersion": "4.0.14", +} + + +def write_binary_js(tmp_path: Path, *lines: str) -> Path: + path = tmp_path / "binary.js" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def test_extracts_metadata(tmp_path: Path) -> None: + path = write_binary_js( + tmp_path, "// start", f"// meta={json.dumps(META)}", "var x = 1;" + ) + metadata = extract_meta_comment(path) + assert str(metadata.simUrl) == META["simUrl"] + assert metadata.target == "arcade" + assert metadata.targetVersion == "4.0.14" + + +def test_indented_comment(tmp_path: Path) -> None: + path = write_binary_js(tmp_path, f" // meta={json.dumps(META)} ") + assert extract_meta_comment(path).targetVersion == "4.0.14" + + +def test_first_meta_comment_wins(tmp_path: Path) -> None: + other = {**META, "targetVersion": "9.9.9"} + path = write_binary_js( + tmp_path, f"// meta={json.dumps(META)}", f"// meta={json.dumps(other)}" + ) + assert extract_meta_comment(path).targetVersion == "4.0.14" + + +def test_missing_comment(tmp_path: Path) -> None: + path = write_binary_js(tmp_path, "// no metadata here", "var meta = {};") + with pytest.raises(ValueError, match="Meta comment not found"): + extract_meta_comment(path) + + +@pytest.mark.parametrize( + "meta", + [ + {**META, "unexpected": 1}, # extra="forbid" + {k: v for k, v in META.items() if k != "simUrl"}, + {**META, "cdnUrl": "not a url"}, + ], +) +def test_invalid_metadata(tmp_path: Path, meta: dict[str, Any]) -> None: + path = write_binary_js(tmp_path, f"// meta={json.dumps(meta)}") + with pytest.raises(ValidationError): + extract_meta_comment(path) + + +def test_invalid_json(tmp_path: Path) -> None: + path = write_binary_js(tmp_path, "// meta={not json") + with pytest.raises(ValidationError): + extract_meta_comment(path) diff --git a/tests/unit/test_paths.py b/tests/unit/test_paths.py new file mode 100644 index 0000000..a04dd2f --- /dev/null +++ b/tests/unit/test_paths.py @@ -0,0 +1,93 @@ +from collections.abc import Callable +from importlib.metadata import version +from pathlib import Path + +import pytest + +from mkcd2app.utils import paths + +VERSION = version("mkcd2app") + +DATA_PATHS: list[tuple[Callable[[], Path], str]] = [ + (paths.get_toolchain_dir, "toolchain"), + (paths.get_js_tools_dir, "toolchain/js_tools"), + (paths.get_js_tools_bin_dir, "toolchain/js_tools/node_modules/.bin"), + (paths.get_templates_npm_cache_dir, "toolchain/templates_npm_cache"), + (paths.get_target_dir, "target"), + (lambda: paths.get_target_version_dir("4.0.14"), "target/4.0.14"), + (lambda: paths.get_sim_html_path("4.0.14"), "target/4.0.14/---simulator.html"), +] +STATE_PATHS: list[tuple[Callable[[], Path], str]] = [ + (paths.get_redun_db_for_toolchain_path, "redun_db_for_toolchain.sqlite3"), + (paths.get_redun_db_for_target_path, "redun_db_for_target.sqlite3"), +] + + +def test_version_matches_package() -> None: + assert paths.get_mkcd2app_version() == VERSION + + +def test_home_override(mkcd2app_home: Path) -> None: + assert paths.get_home_override_dir() == mkcd2app_home + assert paths.get_user_data_dir() == mkcd2app_home / "data" / VERSION + assert paths.get_user_state_dir() == mkcd2app_home / "state" / VERSION + + +@pytest.mark.parametrize(("get_path", "relative"), DATA_PATHS) +def test_data_paths_under_home( + mkcd2app_home: Path, get_path: Callable[[], Path], relative: str +) -> None: + assert get_path() == mkcd2app_home / "data" / VERSION / relative + + +@pytest.mark.parametrize(("get_path", "relative"), STATE_PATHS) +def test_state_paths_under_home( + mkcd2app_home: Path, get_path: Callable[[], Path], relative: str +) -> None: + assert get_path() == mkcd2app_home / "state" / VERSION / relative + + +def test_build_redun_db_is_in_build_dir(tmp_path: Path) -> None: + assert paths.get_redun_db_for_build_path(tmp_path) == tmp_path / ".redun-cache.db" + + +@pytest.mark.parametrize("value", [None, ""]) +def test_no_override_uses_platformdirs( + monkeypatch: pytest.MonkeyPatch, value: str | None +) -> None: + if value is None: + monkeypatch.delenv(paths.HOME_ENV_VAR, raising=False) + else: + monkeypatch.setenv(paths.HOME_ENV_VAR, value) + assert paths.get_home_override_dir() is None + assert paths.get_user_data_dir() == Path(paths.DIRS.user_data_dir) / VERSION + assert paths.get_user_state_dir() == Path(paths.DIRS.user_state_dir) / VERSION + + +# ------------------------------------------------ relative/tilde overrides ---- + + +@pytest.fixture +def relative_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Set MKCD2APP_HOME to a relative path from tmp_path; return where it points.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv(paths.HOME_ENV_VAR, "home") + return tmp_path / "home" + + +def test_relative_override_is_made_absolute(relative_home: Path) -> None: + home = paths.get_home_override_dir() + assert home is not None and home.is_absolute() + assert home == relative_home.resolve() + for get_path, _ in DATA_PATHS + STATE_PATHS: + assert get_path().is_absolute() + assert get_path().is_relative_to(relative_home.resolve()) + + +def test_tilde_override_is_expanded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows + monkeypatch.setenv(paths.HOME_ENV_VAR, str(Path("~") / "mkcd2app-home")) + assert paths.get_home_override_dir() == (tmp_path / "mkcd2app-home").resolve() diff --git a/tests/unit/test_redun_compat.py b/tests/unit/test_redun_compat.py new file mode 100644 index 0000000..a85e476 --- /dev/null +++ b/tests/unit/test_redun_compat.py @@ -0,0 +1,159 @@ +import os +from collections.abc import Callable +from pathlib import Path + +import pytest +import redun.file +from redun.file import ContentDir, ContentFile, Dir, File + +# Importing mkcd2app applies the patches +from mkcd2app._redun_compat import ( + _orig_get_proto, + _patched_get_proto, + assert_pinned_redun_version, +) + + +def test_pinned_redun_version() -> None: + assert_pinned_redun_version() + + +# ------------------------------------------------------------- get_proto ------ + + +def test_get_proto_is_patched() -> None: + assert redun.file.get_proto is _patched_get_proto + + +@pytest.mark.parametrize( + "url", ["C:\\Users\\me\\game", "c:\\x", "E:/build/x", "z:/"] +) # fmt: skip +def test_windows_drive_paths_are_local(url: str) -> None: + assert _orig_get_proto(url) != "local" # the bug the patch works around + assert redun.file.get_proto(url) == "local" + + +@pytest.mark.parametrize( + ("url", "proto"), + [ + ("/tmp/x", "local"), + ("relative/x", "local"), + ("", "local"), + (None, "local"), + ("s3://bucket/key", "s3"), + ("gs://bucket/key", "gs"), + ("http://example.com/x", "http"), + ("C:x", "c"), # drive-relative, not a drive path: left to urlparse + ("1:/x", "local"), # not a letter, so not patched; urlparse has no scheme + ], +) +def test_other_urls_unchanged(url: str | None, proto: str) -> None: + assert redun.file.get_proto(url) == _orig_get_proto(url) == proto + + +def test_file_with_drive_letter_uses_local_filesystem() -> None: + assert File("C:\\Users\\me\\game\\main.ts").filesystem.name == "local" + + +# --------------------------------------------------------- content hashing ---- + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + root = tmp_path / "tree" + (root / "sub").mkdir(parents=True) + (root / "a.txt").write_text("a", encoding="utf-8") + (root / "sub" / "b.txt").write_text("b", encoding="utf-8") + return root + + +def content_hash(path: Path) -> str: + # A new object each time: redun caches the hash on the instance + return ContentDir(str(path)).hash + + +def dir_hash(path: Path) -> str: + return Dir(str(path)).hash + + +def touch_all(root: Path, mtime: float) -> None: + for p in root.rglob("*"): + os.utime(p, (mtime, mtime)) + + +@pytest.mark.parametrize("hash_fn", [content_hash, dir_hash]) +def test_hash_ignores_mtime(tree: Path, hash_fn: Callable[[Path], str]) -> None: + touch_all(tree, 1_000_000_000) + before = hash_fn(tree) + touch_all(tree, 2_000_000_000) + assert hash_fn(tree) == before + + +def test_hash_is_stable(tree: Path) -> None: + assert content_hash(tree) == content_hash(tree) + + +def test_dir_hash_matches_content_dir(tree: Path) -> None: + assert dir_hash(tree) == content_hash(tree) + + +def test_hash_changes_with_content(tree: Path) -> None: + before = content_hash(tree) + (tree / "sub" / "b.txt").write_text("B", encoding="utf-8") + assert content_hash(tree) != before + + +def test_hash_changes_with_new_file(tree: Path) -> None: + before = content_hash(tree) + (tree / "c.txt").write_text("c", encoding="utf-8") + assert content_hash(tree) != before + + +def test_hash_changes_with_removed_file(tree: Path) -> None: + before = content_hash(tree) + (tree / "a.txt").unlink() + assert content_hash(tree) != before + + +def test_hash_changes_with_rename(tree: Path) -> None: + before = content_hash(tree) + (tree / "a.txt").rename(tree / "sub" / "a.txt") + assert content_hash(tree) != before + + +def test_hash_includes_path(tree: Path, tmp_path: Path) -> None: + # Same contents somewhere else is a different output dir, so a different value + copy = tmp_path / "copy" + (copy / "sub").mkdir(parents=True) + (copy / "a.txt").write_text("a", encoding="utf-8") + (copy / "sub" / "b.txt").write_text("b", encoding="utf-8") + assert content_hash(copy) != content_hash(tree) + + +# ------------------------------------------------------ deleted ContentFile ---- + + +def test_deleted_content_file_is_invalid_instead_of_raising(tree: Path) -> None: + # Regression: this raised from redun's cache check, crashing the next run + path = tree / "a.txt" + file = ContentFile(str(path)) + assert file.hash + assert file.is_valid() + path.unlink() + assert not file.is_valid() + + +def test_recreated_content_file_is_valid_again(tree: Path) -> None: + path = tree / "a.txt" + file = ContentFile(str(path)) + assert file.hash + path.unlink() + path.write_text("a", encoding="utf-8") + assert file.is_valid() + + +def test_content_file_hash_still_changes_with_content(tree: Path) -> None: + path = tree / "a.txt" + before = ContentFile(str(path)).hash + path.write_text("A", encoding="utf-8") + assert ContentFile(str(path)).hash != before diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py new file mode 100644 index 0000000..941ff67 --- /dev/null +++ b/tests/unit/test_resources.py @@ -0,0 +1,30 @@ +from mkcd2app.utils.resources import ( + get_resource_empty_project_path, + get_resource_js_tools_path, + get_resource_template_path, +) + + +def test_templates_dir() -> None: + with get_resource_template_path() as path: + assert (path / "vite-project").is_dir() + + +def test_vite_template() -> None: + with get_resource_template_path("vite-project") as path: + for name in ("package.json", "package-lock.json", "index.html"): + assert (path / name).is_file(), name + # fill_website_template copies binary.js and the simulator in here + assert (path / "src" / "assets").is_dir() + + +def test_js_tools() -> None: + with get_resource_js_tools_path() as path: + assert (path / "package.json").is_file() + assert (path / "package-lock.json").is_file() + + +def test_empty_project() -> None: + with get_resource_empty_project_path() as path: + assert (path / "pxt.json").is_file() + assert (path / "main.ts").is_file() diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py new file mode 100644 index 0000000..9349c55 --- /dev/null +++ b/tests/unit/test_run.py @@ -0,0 +1,151 @@ +import logging +import sys +from pathlib import Path + +import pytest +from helpers import capture_logger + +from mkcd2app.utils import paths, run +from mkcd2app.utils.run import BuildError, run_cmd + + +def python(code: str) -> list[str]: + return [sys.executable, "-c", code] + + +def test_returns_stdout(tmp_path: Path) -> None: + assert run_cmd(python("print('hello')"), cwd=tmp_path).strip() == "hello" + + +def test_runs_in_cwd(tmp_path: Path) -> None: + out = run_cmd(python("import os; print(os.getcwd())"), cwd=tmp_path) + assert Path(out.strip()).samefile(tmp_path) + + +def test_accepts_str_cwd(tmp_path: Path) -> None: + assert run_cmd(python("print(1)"), cwd=str(tmp_path)).strip() == "1" + + +def test_failure_raises_build_error(tmp_path: Path) -> None: + command = python( + "import sys; print('to stdout'); print('to stderr', file=sys.stderr); " + "sys.exit(3)" + ) + with pytest.raises(BuildError) as exc_info: + run_cmd(command, cwd=tmp_path) + err = exc_info.value + assert err.returncode == 3 + assert err.command == command + assert err.cwd == tmp_path + assert "to stdout" in err.output + assert "to stderr" in err.output + assert "(3)" in str(err) + + +def test_failure_logs_output_at_error_level( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # The markers are split so that they only match the output, as the logged message + # also has the command in it + command = python( + "import sys; print('to' + '_stdout'); " + "print('to' + '_stderr', file=sys.stderr); sys.exit(1)" + ) + with capture_logger(caplog, run.logger), pytest.raises(BuildError): + run_cmd(command, cwd=tmp_path) + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 + assert "to_stdout" in errors[0] + assert "to_stderr" in errors[0] + + +def test_success_logs_output_only_at_debug_level( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + command = python("print('to' + '_stdout')") + with capture_logger(caplog, run.logger): + run_cmd(command, cwd=tmp_path) + assert "to_stdout" not in caplog.text + with caplog.at_level(logging.DEBUG, logger=run.logger.name): + run_cmd(command, cwd=tmp_path) + assert [r.levelno for r in caplog.records if "to_stdout" in r.getMessage()] == [ + logging.DEBUG + ] + + +def test_output_is_decoded_as_utf8_replacing_invalid_bytes(tmp_path: Path) -> None: + # A checkmark in UTF-8, then a byte that's never valid in UTF-8 + code = r"import sys; sys.stdout.buffer.write(b'\xe2\x9c\x93 \xff')" + assert run_cmd(python(code), cwd=tmp_path) == "✓ �" + + +def test_missing_executable(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="definitely-not-a-real-program"): + run_cmd(["definitely-not-a-real-program"], cwd=tmp_path) + + +def test_which_path_limits_search(tmp_path: Path) -> None: + # The interpreter's own name is on PATH, but not in an empty which_path dir + name = Path(sys.executable).name + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(FileNotFoundError): + run_cmd([name, "-c", "pass"], cwd=tmp_path, which_path=empty) + + +def test_which_path_finds_executable(tmp_path: Path) -> None: + exe = Path(sys.executable) + out = run_cmd( + [exe.name, "-c", "print('found')"], cwd=tmp_path, which_path=exe.parent + ) + assert out.strip() == "found" + + +def _write_fake_mkc(bin_dir: Path) -> None: + """A tiny ``mkc`` that prints ok, runnable through shutil.which on any OS.""" + bin_dir.mkdir(parents=True) + if sys.platform == "win32": + (bin_dir / "mkc.cmd").write_text("@echo ok\r\n", encoding="utf-8") + else: + exe = bin_dir / "mkc" + exe.write_text("#!/bin/sh\necho ok\n", encoding="utf-8") + exe.chmod(0o755) + + +def test_relative_home_executable_runs_from_other_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Regression: a relative MKCD2APP_HOME gave a relative which_path, so the + # resolved executable didn't exist from the subprocess's cwd + monkeypatch.chdir(tmp_path) + monkeypatch.setenv(paths.HOME_ENV_VAR, "home") + _write_fake_mkc(paths.get_js_tools_bin_dir()) + other = tmp_path / "build" + other.mkdir() + + out = run_cmd( + ["mkc", "build", "-j"], cwd=other, which_path=paths.get_js_tools_bin_dir() + ) + + assert out.strip() == "ok" + + +def test_relative_home_cache_dir_is_same_from_other_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Regression: npm got a relative --cache, which it resolved against its own cwd + monkeypatch.chdir(tmp_path) + monkeypatch.setenv(paths.HOME_ENV_VAR, "home") + cache = paths.get_templates_npm_cache_dir() + other = tmp_path / "build" + other.mkdir() + + run_cmd( + python("import pathlib, sys; pathlib.Path(sys.argv[1]).mkdir(parents=True)") + + [str(cache)], + cwd=other, + ) + + assert (tmp_path / "home").resolve() in cache.parents + assert cache.is_dir() + assert not (other / "home").exists() diff --git a/tests/unit/test_semver.py b/tests/unit/test_semver.py new file mode 100644 index 0000000..c4fc539 --- /dev/null +++ b/tests/unit/test_semver.py @@ -0,0 +1,54 @@ +from pathlib import Path + +import pytest +from helpers import make_config_yaml +from pydantic import ValidationError + +from mkcd2app.config import load_config_from_yaml +from mkcd2app.utils.text import is_strict_semver, raise_for_invalid_strict_semver + +VALID = ["4.0.14", "0.0.0", "10.20.30"] +INVALID = [ + "v4.0.14", + "4.0", + "4", + "01.0.0", + "4.00.1", + "4.0.14\n", # $ in Python's re matches before a trailing newline + " 4.0.14", + "4.0.14 ", + "4.0.14-beta", + "4.0.14+build", + "", +] + + +@pytest.mark.parametrize("version", VALID) +def test_is_strict_semver_accepts(version: str) -> None: + assert is_strict_semver(version) + + +@pytest.mark.parametrize("version", INVALID) +def test_is_strict_semver_rejects(version: str) -> None: + assert not is_strict_semver(version) + + +def test_raise_for_invalid_strict_semver_accepts_valid() -> None: + raise_for_invalid_strict_semver("4.0.14") # doesn't raise + + +@pytest.mark.parametrize("version", INVALID) +def test_raise_for_invalid_strict_semver_raises(version: str) -> None: + with pytest.raises(ValueError, match="not a valid strict numeric semver"): + raise_for_invalid_strict_semver(version) + + +def test_config_accepts_strict_semver_target(tmp_path: Path) -> None: + config = load_config_from_yaml(make_config_yaml(tmp_path, target="4.0.14")) + assert config.target == "4.0.14" + + +@pytest.mark.parametrize("target", ["4.0", "v4.0.14", "4.0.14-beta", "4.0.14\n"]) +def test_config_rejects_non_semver_target(tmp_path: Path, target: str) -> None: + with pytest.raises(ValidationError, match="target"): + load_config_from_yaml(make_config_yaml(tmp_path, target=target)) diff --git a/tests/unit/test_target_install.py b/tests/unit/test_target_install.py new file mode 100644 index 0000000..5467d93 --- /dev/null +++ b/tests/unit/test_target_install.py @@ -0,0 +1,270 @@ +import json +from html import unescape +from pathlib import Path +from typing import Any + +import pytest +import requests +from helpers import ( + TARGET_VERSION, + CmdCall, + FakeResponse, + FakeRunCmd, + tree_digest, +) +from redun.file import ContentDir + +from mkcd2app.target import install +from mkcd2app.target.install import ( + _download_text, + download_sim, + warm_mkc_cache_for_version_and_get_binary_js_metadata, +) +from mkcd2app.utils.paths import get_js_tools_bin_dir, get_sim_html_path +from mkcd2app.utils.resources import get_resource_empty_project_path +from mkcd2app.utils.run import BuildError + +SIM_URL = "https://sim.example.com/---simulator" +METADATA = { + "simUrl": SIM_URL, + "cdnUrl": "https://cdn.example.com/", + "version": "v0.0.0", + "target": "arcade", + "targetVersion": TARGET_VERSION, +} +METADATA_JSON = json.dumps(METADATA) + + +def serve(monkeypatch: pytest.MonkeyPatch, pages: dict[str, bytes]) -> list[str]: + """Fake ``requests.get`` with these pages; return the list of requested URLs.""" + requested: list[str] = [] + + def fake_get(url: str) -> FakeResponse: + requested.append(url) + return FakeResponse(pages[url]) + + monkeypatch.setattr(requests, "get", fake_get) + return requested + + +def run_download_sim(monkeypatch: pytest.MonkeyPatch, sim_html: str) -> str: + """Run download_sim on this simulator page, and return the HTML it wrote.""" + serve(monkeypatch, {SIM_URL: sim_html.encode()}) + return Path(download_sim.func(METADATA_JSON, TARGET_VERSION).path).read_text( + encoding="utf-8" + ) + + +# --------------------------------------------------------- _download_text ----- + + +def test_download_text_decodes_utf8_without_charset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Regression: res.text decodes as ISO-8859-1 when the server sends no charset + text = "héllo ✓ 😀" + serve(monkeypatch, {"https://example.com/x": text.encode()}) + assert _download_text("https://example.com/x") == text + + +def test_download_text_raises_on_http_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(requests, "get", lambda url: FakeResponse(b"", 404)) + with pytest.raises(requests.HTTPError): + _download_text("https://example.com/x") + + +# ----------------------------------------------------------- download_sim ----- + + +def test_download_sim_inlines_css_and_js( + mkcd2app_home: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + requested = serve( + monkeypatch, + { + SIM_URL: b'' + b'', + "https://cdn.example.com/a.css": b"body { color: red }", + "https://cdn.example.com/b.js": b"run()", + }, + ) + + result = download_sim.func(METADATA_JSON, TARGET_VERSION) + + assert Path(result.path) == get_sim_html_path(TARGET_VERSION) + assert requested == [ + SIM_URL, + "https://cdn.example.com/a.css", + "https://cdn.example.com/b.js", + ] + html = Path(result.path).read_text(encoding="utf-8") + assert "