diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09837a6..7ec7c4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: branches: [main] + tags: ["v*"] pull_request: permissions: @@ -11,3 +12,51 @@ permissions: jobs: docs: uses: BenchCommons/.github/.github/workflows/docs-ci.yml@main + + tests: + name: Python tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --locked + - name: Tests (never use the network) + run: uv run pytest -q + + install-from-git: + name: Install from a git URL (spec 004, R1.4) + runs-on: ubuntu-24.04 + steps: + - uses: astral-sh/setup-uv@v6 + - name: Install the commit under test into a fresh uv project and import every module + env: + REF: ${{ github.event.pull_request.head.sha || github.sha }} + REPO: ${{ github.server_url }}/${{ github.repository }} + run: | + mkdir "$RUNNER_TEMP/consumer" && cd "$RUNNER_TEMP/consumer" + uv init --bare --python 3.12 + uv add "benchcommons-tools-dataset @ git+$REPO@$REF" + uv run python -c "import datasettool, datasettool.files, datasettool.sources.adapters, datasettool.sources.cache, datasettool.sources.registry, datasettool.sources.spreadsheet; print(datasettool.__version__)" + + version-matches-tag: + name: Package version equals the tag (ADR 0004) + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Compare pyproject version with the tag + env: + TAG: ${{ github.ref_name }} + run: | + python3 - <<'PY' + import os, sys, tomllib + version = tomllib.load(open("pyproject.toml", "rb"))["project"]["version"] + tag = os.environ["TAG"] + if tag != f"v{version}": + print(f"::error::tag {tag} does not match pyproject version {version}") + sys.exit(1) + print(f"tag {tag} matches version {version}") + PY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0b53ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +dist/ +build/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/AGENTS.md b/AGENTS.md index 865acc3..34f0157 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,29 @@ -# AGENTS.md — dataset-tool +# AGENTS.md — tools-dataset Follow the org-wide rules in [BenchCommons/.github AGENTS.md](https://github.com/BenchCommons/.github/blob/main/AGENTS.md) and the decisions in [.product](https://github.com/BenchCommons/.product). -Repo-specific stack and runbooks will be added here with spec 001 (issue #1). +## What this repo is + +The shared tooling of the `dataset-*` repos: the source toolkit (spec 004) and, next, the input contract with its reference builder and validator (spec 001). Specs: [`docs/specs/ROADMAP.md`](docs/specs/ROADMAP.md). Decisions: [`docs/adr/`](docs/adr/). + +## Stack + +Python 3.12, `uv`, `pytest`, built with `hatchling`. Distribution `benchcommons-tools-dataset`, import package `datasettool` ([ADR 0003](docs/adr/0003-python-package-name.md)). Runtime dependencies pinned: `pyyaml==6.0.3`, `xlrd==2.0.2` ([ADR 0005](docs/adr/0005-source-toolkit-runtime-dependencies.md)); a new one needs an ADR. + +## Commands + +```sh +uv sync --locked # install +uv run pytest -q # tests (never use the network; downloads go to a server on 127.0.0.1) +``` + +## Releases + +SemVer tags `vX.Y.Z` on `main`, created by the maintainer after merge, never moved. `version` in `pyproject.toml` equals the tag without `v`. Dataset repos pin a tag through `[tool.uv.sources]` ([ADR 0004](docs/adr/0004-dataset-repos-depend-on-a-git-tag.md)). + +## Conventions + +- Docs, specs, ADRs, code, comments and messages in English. +- Conventional Commits; signed commits; AI co-authorship only as a `Co-Authored-By:` trailer. +- Code ported from another repo names its origin (`ported from BenchCommons/@ `) in the commit message. +- Open items go in issues, never in docs or code. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index f709767..358bc66 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,26 @@ -# dataset-tool +# tools-dataset Builds, validates and exports BenchCommons datasets and delivers pinned inputs to the catalogs and benchmark-tool. -This repo owns the **input contract** for every `dataset-*` repo (see [CONTRACTS.md](https://github.com/BenchCommons/.product/blob/main/CONTRACTS.md)). +This repo owns the **input contract** for every `dataset-*` repo (see [CONTRACTS.md](https://github.com/BenchCommons/.product/blob/main/CONTRACTS.md), spec [001](docs/specs/001-input-contract/spec.md)) and the **source toolkit** they share (spec [004](docs/specs/004-source-toolkit/spec.md)): a download cache, a spreadsheet reader, the adapter base and the source registry. -Status: starting. The first spec is being defined in issue #1. Specs live in `docs/specs/-/`. +## Use from a dataset repo + +```toml +[project] +dependencies = ["benchcommons-tools-dataset==0.1.0"] + +[tool.uv.sources] +benchcommons-tools-dataset = { git = "https://github.com/BenchCommons/tools-dataset", tag = "v0.1.0" } +``` + +```python +from datasettool.sources.cache import fetch_resource +from datasettool.sources.spreadsheet import rows +``` + +See [ADR 0004](docs/adr/0004-dataset-repos-depend-on-a-git-tag.md). Specs live in `docs/specs/-/`. + +## License + +Code: Apache-2.0 ([`LICENSE`](LICENSE)). diff --git a/docs/adr/0003-python-package-name.md b/docs/adr/0003-python-package-name.md new file mode 100644 index 0000000..ef37a75 --- /dev/null +++ b/docs/adr/0003-python-package-name.md @@ -0,0 +1,26 @@ +# 0003. Python package name: `benchcommons-tools-dataset`, imported as `datasettool` + +## Status + +Accepted (2026-10-12). Spec [004](../specs/004-source-toolkit/spec.md). + +## Context + +Spec 004 makes this repo an installable Python package, first for the source toolkit and later for the reference builder and validator of spec 001. Spec 001's plan already places the code in `src/datasettool/`. The package needs a distribution name (what `uv` and `pip` install) and an import name (what code imports). + +This repo was renamed from `dataset-tool` to `tools-dataset` by [.product ADR 0005](https://github.com/BenchCommons/.product/blob/main/docs/adr/0005-tools-prefix.md). A bare `tools-dataset` is generic enough that an unrelated project on PyPI may hold it, and a git dependency still declares a distribution name that must not collide with anything a dataset repo also installs. Import names cannot contain `-`. + +Where the toolkit lives (here rather than in `tools-commons`) is [ADR 0007](0007-source-toolkit-lives-in-tools-dataset.md). + +## Decision + +- Distribution name: **`benchcommons-tools-dataset`**: the org prefix plus the repo name, so a lock file shows where the code comes from. +- Import package: **`datasettool`**, as in spec 001's plan, so the two specs share one package and no import changes with the repo rename. +- Layout: the source toolkit lives in the subpackage `datasettool.sources` (`cache`, `spreadsheet`, `adapters`, `registry`); helpers shared with spec 001 (checksums, row counts) live at the top level (`datasettool.files`). Spec 001 adds its own modules next to them. +- The command name of spec 001's CLI is decided by spec 001. + +## Consequences + +- Dataset repos write `benchcommons-tools-dataset` in their dependencies and `from datasettool.sources import cache` in code. +- The distribution and import names differ, which is common in Python but must be documented in `AGENTS.md` and `README.md`. +- Renaming later breaks every dataset repo's imports and lock file, so the names are fixed from `v0.1.0`. diff --git a/docs/adr/0004-dataset-repos-depend-on-a-git-tag.md b/docs/adr/0004-dataset-repos-depend-on-a-git-tag.md new file mode 100644 index 0000000..97e4560 --- /dev/null +++ b/docs/adr/0004-dataset-repos-depend-on-a-git-tag.md @@ -0,0 +1,41 @@ +# 0004. Dataset repos depend on tools-dataset through a uv git source pinned to a tag + +## Status + +Accepted (2026-10-12). Spec [004](../specs/004-source-toolkit/spec.md). + +## Context + +`dataset-names` and later `dataset-*` repos must import the source toolkit. Their builds have to be reproducible: the same commit of a dataset repo must always run the same toolkit code. The options are: + +| Option | Reproducible | Cost | +|---|---|---| +| Git dependency pinned to a tag, resolved by `uv` and locked to the commit in `uv.lock` | Yes (`uv.lock` records the commit SHA) | None beyond tagging releases | +| Wheel published to PyPI | Yes, by version | A PyPI project, trusted publishing, name ownership, a release job; nothing else in the org needs PyPI today | +| Wheel attached to GitHub Releases, installed by URL | Yes | A build job and a URL per version in every consumer | +| Git submodule or vendored copy | Yes | Copies drift, which is what spec 004 removes | +| Git dependency on a branch | No | A push to the branch changes every consumer | + +## Decision + +- tools-dataset releases are git tags `vX.Y.Z` (SemVer) on `main`, created by the maintainer after the release's pull request merges. Tags are never moved or deleted. +- A dataset repo declares the dependency by version and pins its source to the tag with `uv`: + + ```toml + [project] + dependencies = ["benchcommons-tools-dataset==0.1.0"] + + [tool.uv.sources] + benchcommons-tools-dataset = { git = "https://github.com/BenchCommons/tools-dataset", tag = "v0.1.0" } + ``` + + and commits the resulting `uv.lock`. Upgrading is a change of both lines in one pull request. +- The package version in `pyproject.toml` equals the tag without the `v`; CI fails a tag push where it does not. CI also checks that a fresh project installs the package from this repo's git URL. +- No PyPI publication for now. Publishing a wheel later does not change the import or distribution name (ADR 0003), so consumers would only drop the `[tool.uv.sources]` entry. + +## Consequences + +- Consumers need network access to GitHub when they resolve or sync without a cache, as they already do for PyPI. +- A tag must exist before a consumer can adopt a version; the pull request that changes the package ends with a tagging step for the maintainer. +- `pip` users can install the same ref with `pip install "benchcommons-tools-dataset @ git+https://github.com/BenchCommons/tools-dataset@v0.1.0"`. +- A tag that points at a broken commit cannot be fixed in place; the fix is a new patch tag. diff --git a/docs/adr/0005-source-toolkit-runtime-dependencies.md b/docs/adr/0005-source-toolkit-runtime-dependencies.md new file mode 100644 index 0000000..50bd4a8 --- /dev/null +++ b/docs/adr/0005-source-toolkit-runtime-dependencies.md @@ -0,0 +1,24 @@ +# 0005. Runtime dependencies of the source toolkit: `pyyaml` and `xlrd`, pinned + +## Status + +Accepted (2026-10-12). Spec [004](../specs/004-source-toolkit/spec.md). + +## Context + +The source toolkit is ported from `dataset-names`, which uses `pyyaml` to read `sources.yaml` and `xlrd` 2.0.2 to read XLS. Its [ADR 0005](https://github.com/BenchCommons/dataset-names/blob/master/docs/adr/0005-leitor-de-planilhas.md) compared spreadsheet readers (`openpyxl`, `pandas`, `python-calamine`, `xlrd`, an own XLSX reader) and chose: XLSX through the standard library (`zipfile` and `iterparse`, streamed, with DTD and ZIP-bomb guards), XLS through `xlrd`, the only pure-Python reader of BIFF8 with no further dependencies (BSD licence). Everything else (HTTP, hashing, CSV, XML) uses the standard library. + +`dataset-names` also depends on `pycountry` and `pyarrow`, but only for names-specific code (country codes and Parquet output); they are not needed by the generic toolkit. + +## Decision + +- Runtime dependencies: `pyyaml==6.0.3` (registry files) and `xlrd==2.0.2` (XLS). Both are pinned exactly in `pyproject.toml`, and `uv.lock` is committed. +- The reasoning of dataset-names ADR 0005 is carried over, including its amendment in dataset-names PR #39: XLSX stays on the standard library, without `defusedxml`. expat in Python 3.12 does not fetch external entities and limits expansion, and the reader refuses DTDs in two independent layers: (1) a pre-scan of the first 64 KiB, which must be valid UTF-8 or UTF-16 with an XML declaration naming no other encoding, so no encoding expat accepts can hide a ``: rules V1 to V10; V5 rebuilds from the upstream artifact in the same directory. - `dataset-tool vectors --check`: runs the implementation against `vectors/`. - **Layout:** `src/datasettool/{definition,population,sampling,formats,manifest,licensing,validate,cli}.py`, `tests/`, `schema/`, `vectors/`. -- **Use from dataset repos:** pinned by tag, for example `uvx --from git+https://github.com/BenchCommons/dataset-tool@v0.1.0 dataset-tool build …`. A reusable release workflow comes in spec 002. +- **Use from dataset repos:** pinned by tag, for example `uvx --from git+https://github.com/BenchCommons/tools-dataset@vX.Y.Z dataset-tool build …`, where `vX.Y.Z` is the first tag that ships this CLI (`v0.1.0` is the source toolkit of spec 004 and has no CLI). A reusable release workflow comes in spec 002. - **Tests never use the network**, as in `dataset-names`. ## 10. Decisions diff --git a/docs/specs/001-input-contract/spec.md b/docs/specs/001-input-contract/spec.md index b351026..d1f5d24 100644 --- a/docs/specs/001-input-contract/spec.md +++ b/docs/specs/001-input-contract/spec.md @@ -1,6 +1,6 @@ # 001. The input contract -Issue: [#1](https://github.com/BenchCommons/dataset-tool/issues/1). Org contract index: [.product CONTRACTS.md](https://github.com/BenchCommons/.product/blob/main/CONTRACTS.md), row "Benchmark inputs". Roadmap: [`../ROADMAP.md`](../ROADMAP.md). +Issue: [#1](https://github.com/BenchCommons/tools-dataset/issues/1). Org contract index: [.product CONTRACTS.md](https://github.com/BenchCommons/.product/blob/main/CONTRACTS.md), row "Benchmark inputs". Roadmap: [`../ROADMAP.md`](../ROADMAP.md). ## Why diff --git a/docs/specs/001-input-contract/tasks.md b/docs/specs/001-input-contract/tasks.md index a08d196..673445a 100644 --- a/docs/specs/001-input-contract/tasks.md +++ b/docs/specs/001-input-contract/tasks.md @@ -3,7 +3,7 @@ Spec: [`spec.md`](spec.md). Plan: [`plan.md`](plan.md). Each task ends with tests passing and is one pull request or less. - [ ] **T1. ADRs.** Write `docs/adr/0001-inputs-in-dataset-releases.md` (plan D1) and `docs/adr/0002-hash-based-sampling.md` (plan D2). Done when both are reviewed. -- [ ] **T2. Project skeleton.** `pyproject.toml` (Python 3.12, `uv`, `pytest`, `hatchling`), `src/datasettool/`, `tests/`, CI running `uv run pytest -q`. Update `AGENTS.md` with stack and commands. Test: an empty test suite runs in CI. +- [ ] **T2. Project skeleton.** The package skeleton (`pyproject.toml` with Python 3.12, `uv`, `pytest`, `hatchling`; `src/datasettool/`; `tests/`; CI running `uv run pytest -q`; `AGENTS.md` with stack and commands) is delivered by [spec 004](../004-source-toolkit/tasks.md) T2, as distribution `benchcommons-tools-dataset`. This task adds spec 001's runtime dependencies (`jsonschema`, `license-expression`, by ADR) and its module files. Test: the suite runs in CI. - [ ] **T3. JSON Schemas.** `schema/inputs-definition.schema.json`, `schema/input-manifest.schema.json`, `schema/input-reference.schema.json` (plan section 7). Tests: valid and invalid fixtures for each schema, including bad size names, bad checksums and a `lines` record with two fields. - [ ] **T4. Formats.** `formats.py`: writers and checkers for `lines` and `csv` (spec R4). Tests: BOM, CRLF, missing or double trailing LF, non-NFC, control characters, surrounding whitespace, CSV quoting of commas and quotes, integer and decimal rendering. Each rejection names the rule. - [ ] **T5. Population.** `definition.py` and `population.py`: read `inputs.yaml`, verify the upstream checksum against the dataset manifest, filter, project, check, deduplicate, sort, compute `population.sha256` and `scripts` (plan section 3). Tests: fixture subset of `given_names.csv` with Latin, Cyrillic, Han names and a name with a space; build fails on an invalid value. diff --git a/docs/specs/004-source-toolkit/plan.md b/docs/specs/004-source-toolkit/plan.md new file mode 100644 index 0000000..4854b23 --- /dev/null +++ b/docs/specs/004-source-toolkit/plan.md @@ -0,0 +1,189 @@ +# 004. The source toolkit: plan + +Spec: [`spec.md`](spec.md). Tasks: [`tasks.md`](tasks.md). Origin of the ported code: `BenchCommons/dataset-names@e7e8405` (`master`, 2026-10-12). + +## 1. Layout + +``` +pyproject.toml benchcommons-tools-dataset 0.1.0, hatchling, pins (ADR 0003, 0005) +uv.lock, .python-version (3.12) +LICENSE Apache-2.0, the same text as dataset-names +src/datasettool/__init__.py __version__ from the installed distribution +src/datasettool/files.py sha256_file, count_csv_records, describe_files (spec R6) +src/datasettool/sources/__init__.py +src/datasettool/sources/adapters.py Resource, SourceFiles, errors, checks, parse_count/parse_code (R4) +src/datasettool/sources/cache.py fetch_resource and friends (R2) +src/datasettool/sources/spreadsheet.py sheet_names, rows, cell_count (R3) +src/datasettool/sources/registry.py Field, RegistrySchema, Source, load_registry (R5) +tests/conftest.py LocalServer on 127.0.0.1 (ported from dataset-names tests/conftest.py) +tests/example_adapters/ test-only adapter package (load_adapter tests) +tests/names_registry.py test-only registry schema with dataset-names' fields (acceptance criterion 3) +tests/fixtures/spreadsheets/types.xls synthetic XLS, byte-identical copy of dataset-names tests/fixtures/planilhas/tipos.xls +tests/test_*.py +``` + +Dependency direction inside the package: `cache`, `registry` and `spreadsheet` import `adapters` (for `Resource`, `FormatError`, `is_http_url`, `parse_count`); `adapters` imports nothing from the package. `files` is independent. + +## 2. Module contracts + +Signatures are the public API of `v0.1.0`. Messages are English translations of the dataset-names messages; the tests match on them. + +### 2.1 `datasettool.sources.adapters` + +```python +class AdapterError(Exception) +class FormatError(ValueError) +@dataclass(frozen=True) class Resource: url: str; filename: str +class SourceFiles(dict) # __missing__ -> AdapterError listing available files +def module_name(source_id: str) -> str # "us-ssa-births" -> "us_ssa_births" +def is_http_url(url: object) -> bool # never raises +def is_safe_filename(name: object) -> bool +def check_adapter(mod: object, source_id: str, optional: Iterable[str] = ()) -> None +def load_adapter(source_id: str, package: str, optional: Iterable[str] = ()) -> ModuleType +def parse_count(value: str, missing: Collection[str] = (), field: str = "count") -> int | None +def parse_code(value: str, mapping: Mapping[str, str], field: str) -> str +``` + +Generic records: the module docstring states the adapter protocol (`SOURCE_ID`, `RESOURCES`, `parse(files) -> Iterator[R]`) and that `R` is the dataset's own type (`RawRecord` and `RawLink` in dataset-names). `package` has no default: the toolkit does not know where a dataset keeps its adapters. `optional` lists extra functions a dataset's protocol allows (dataset-names: `("links",)`); each is checked only for callability when present. The absent/broken distinction is kept exactly: `ModuleNotFoundError` whose `name` is the adapter module or one of its parents means absent; any other exception, including a `ModuleNotFoundError` for a different module, means broken. + +### 2.2 `datasettool.sources.cache` + +```python +class FetchError(Exception) +class BlockedError(FetchError) +DEFAULT_USER_AGENT: str # "tools-dataset/ (+https://github.com/BenchCommons/tools-dataset)" +def make_user_agent(name: str, version: str, url: str) -> str # "/ (+)" +def cached_path(cache_dir: Path, source_id: str, filename: str) -> Path +def fetch_resource(cache_dir, source_id, resource, *, refresh=False, offline=False, timeout=60.0, + attempts=3, backoff=None, sleep=time.sleep, user_agent=None) -> Path +``` + +`attempts`, `backoff` and `sleep` are the dataset-names parameters `tentativas`, `espera` and `dormir`; `backoff=None` means 2.0 s. `user_agent=None` means `DEFAULT_USER_AGENT`; dataset-names passes `make_user_agent("namescommons", , "https://github.com/BenchCommons/dataset-names")`, which reproduces its current header byte for byte. The manual-download hints say "rerun with --offline": dataset CLIs keep the `--offline` and `--refresh` flag names. + +### 2.3 `datasettool.sources.spreadsheet` + +```python +def sheet_names(path) -> list[str] +def rows(path, sheet) -> Iterator[tuple[int, list]] +def cell_count(value, missing: Iterable[str] = (), field: str = "count") -> int | None +``` + +A straight port of `namescommons.spreadsheet` at `dataset-names@eaaa4d1` (PR #39, which adds the UTF-8/UTF-16 encoding guard and the expat-level DTD refusal through `xml.parsers.expat` and `ElementTree.TreeBuilder`), limits unchanged (`_MAX_MEMBER` 1 GiB, ratio 100:1 above 1 MiB, prologue 64 KiB, 16,384 columns). Private names are translated; the tests that monkeypatch `_MAX_MEMBER`, `_events` and `_prologue` follow the new names. + +### 2.4 `datasettool.sources.registry` + +```python +ID_PATTERN = re.compile(r"[a-z0-9]+(-[a-z0-9]+)*") +ATTRIBUTION_ONLY_LICENSES: frozenset[str] # ADR 0006 +class RegistryError(ValueError) + +@dataclass(frozen=True) +class Field: + name: str + check: Callable[[object], str | None] | None = None # returns an error text or None + required: bool = True + default: object = None # used when optional and absent + +@dataclass(frozen=True) +class Source: # the default factory's result + id: str; name: str; url: str; license: str; attribution: str; retrieved: date + adapter: bool = True + extra: Mapping[str, object] = {} # dataset fields, read-only + +@dataclass(frozen=True) +class RegistrySchema(Generic[S]): + accepted_licenses: frozenset[str] + license_policy: str # e.g. "ADR 0001", shown in the licence error + fields: tuple[Field, ...] = () + checks: tuple[Callable[[Mapping[str, S]], Iterable[str]], ...] = () + factory: Callable[[dict], S] = Source.from_entry + +def parse_registry(entries: list, schema: RegistrySchema[S]) -> dict[str, S] +def load_registry(path: Path, schema: RegistrySchema[S]) -> dict[str, S] +``` + +Validation order per entry: not a mapping; missing required fields (generic, then dataset fields; stop this entry); unknown fields; empty strings; `id` format; `url`; licence; `retrieved`; `adapter`; then each dataset field's `check` in declaration order. This differs from dataset-names, which checks its own fields (`country`, `count_type`, `primary`, `suppression`) between the licence and `retrieved`, and `uniform_case` before `adapter`: the set of messages is the same, only their order inside a multi-error `RegistryError` changes. The adoption ([#5](https://github.com/BenchCommons/tools-dataset/issues/5)) must not assert on that order. An entry with any error is not built. Duplicate ids are reported. Then each cross-entry check runs over the entries that were built, and every message is collected into one `RegistryError`. A schema whose `accepted_licenses` is empty, or a dataset field that redefines a generic field or is declared twice, raises `ValueError` when the schema is created; a `str` given for `accepted_licenses`, `fields` or `checks` (or for `optional` in `check_adapter`) raises `TypeError`, and the collections are coerced to `frozenset` and tuples. The default `Source` is hashable; `extra` is left out of the hash. + +How dataset-names maps onto it (the test-only `tests/names_registry.py` is exactly this, with a fixed set of country codes instead of `pycountry`): + +| dataset-names field | Toolkit | +|---|---| +| `id`, `name`, `url`, `license`, `attribution`, `retrieved`, `adapter` | generic | +| `country`, `count_type`, `primary`, `suppression` | `Field(..., required=True)` with dataset checks | +| `uniform_case` | `Field("uniform_case", check=boolean, required=False, default=False)` | +| one primary per (country, count type), except `attested` | `checks` | +| `LICENCAS_COMPATIVEIS` | `ATTRIBUTION_ONLY_LICENSES \| {"statcan-open-licence"}`, policy "ADR 0001" | +| `Source` dataclass | `factory` building dataset-names' own `Source` | + +### 2.5 `datasettool.files` + +```python +def sha256_file(path) -> str +def count_csv_records(path) -> int +def describe_files(root, exclude: Collection[str] = ("manifest.json",), + counters: Mapping[str, Callable[[Path], int]] = {".csv": count_csv_records}) -> dict[str, dict] +``` + +`describe_files` is the file loop of dataset-names' `build_manifest`; dataset-names registers `{".parquet": }` itself. The release format, `schema_version` and the `sources` list of the names manifest stay in dataset-names. + +## 3. Port map + +Tests are ported with their inputs and expected values unchanged (spec R7.1). Names in parentheses are the dataset-names tests. + +| dataset-names test file | Ported to | Not ported (stays in dataset-names) | +|---|---|---| +| `tests/test_cache.py` (all 22 tests) | `tests/test_cache.py`; the User-Agent test checks the default and the explicit dataset-names header | none | +| `tests/test_spreadsheet.py` (at `eaaa4d1`, PR #39) | `tests/test_spreadsheet.py`, all tests | `test_r1_3_amostras_do_ine` (INE adapter samples, CC BY 4.0 data of dataset-names) | +| `tests/test_adapters.py` | `tests/test_adapters.py`: module name, valid adapter, the 15 invalid-adapter cases, missing attribute, loading a test adapter (renamed to `example` with English file names), absent adapter | `test_parse_da_fixture_sem_rede` (uses `RawRecord`), `test_pacote_padrao_ainda_nao_tem_adaptadores` (names default package) | +| `tests/test_contrato_adaptador.py` | `parse_count`, `parse_code`, `SourceFiles`, absent adapter path, broken adapter, missing package | R2.1, R2.3, R2.5 and the ingest test (names pipeline) | +| `tests/test_sources.py`, `tests/test_sources_extensoes.py` (all) | `tests/test_registry.py` through `tests/names_registry.py` | none | +| `tests/test_manifest.py` | `tests/test_files.py` (`count_csv_records`) | none | +| `tests/test_robustez.py` | `test_manifesto_conta_registros_csv_e_nao_linhas_fisicas` → `tests/test_files.py` (`describe_files`) | the other tests (aggregation, Parquet, names manifest) | +| `tests/conftest.py` | `LocalServer` and the `server` fixture | names observations fixtures, `servir_fixtures` | + +New tests cover what is new in the toolkit: `optional` callables, the schema rules of R5.4 and R5.5 (no licence list, redefined generic field, custom factory), `describe_files` with an unknown suffix and a custom counter, `DEFAULT_USER_AGENT`, and that `pyproject.toml` pins `xlrd==2.0.2` and `pyyaml==6.0.3`. + +## 4. CI + +`.github/workflows/ci.yml` keeps the `docs` job and adds: + +- `tests`: `actions/checkout@v4`, `astral-sh/setup-uv@v6`, `uv sync --locked`, `uv run pytest -q`. +- `install-from-git`: in an empty directory, `uv init --bare --python 3.12`, `uv add "benchcommons-tools-dataset @ git+https://github.com/@"`, then `uv run python -c "import datasettool.files, datasettool.sources.adapters, …"`. On a pull request the head SHA is `github.event.pull_request.head.sha`; on a push, `github.sha`. + +## 5. Release + +After the pull request merges, the maintainer tags the merge commit on `main`: + +```sh +git tag -s v0.1.0 -m "benchcommons-tools-dataset 0.1.0 (spec 004)" +git push origin v0.1.0 +``` + +and checks acceptance criterion 4 at the tag in an empty directory: + +```sh +uv init --bare --python 3.12 && uv add "benchcommons-tools-dataset @ git+https://github.com/BenchCommons/tools-dataset@v0.1.0" +uv run python -c "import datasettool.sources.cache, datasettool.sources.spreadsheet, datasettool.sources.adapters, datasettool.sources.registry, datasettool.files" +``` + +Issue #5 (dataset-names adoption) starts only after the tag exists. + +## 6. Decisions + +| # | Decision | Alternatives rejected | Reason | +|---|---|---|---| +| D1 | Distribution `benchcommons-tools-dataset`, import `datasettool` (ADR 0003) | `benchcommons-dataset-tool` (the pre-rename repo name), `dataset_tool` | Matches the repo name (.product ADR 0005) and spec 001's `src/datasettool/`; org prefix avoids collisions. | +| D2 | uv git source pinned to a tag (ADR 0004) | PyPI, release wheels, vendoring | Reproducible through `uv.lock` with no publishing infrastructure. | +| D3 | Accepted licences are per dataset, with a shared `ATTRIBUTION_ONLY_LICENSES` starting set (ADR 0006) | One org-wide list | Compatibility depends on each dataset's data licence; .product ADR 0003 is still proposed. | +| D4 | Registry extension by `Field` list, cross-entry checks and a factory | Subclassing a base `Source`; JSON Schema | Keeps dataset-names' typed `Source` and its exact messages; no new dependency. | +| D5 | Messages and identifiers translated to English | Keep Portuguese | This repo's language convention; vectors (inputs and expected values) stay identical. | +| D6 | INE spreadsheet samples stay in dataset-names; the synthetic `tipos.xls` moves (as `types.xls`) | Copy the INE samples | They are adapter fixtures carrying CC BY 4.0 data with attribution; the reader is fully covered by synthetic files. | +| D7 | `describe_files` refuses files without a registered counter | Count unknown files as CSV (dataset-names' fallback) | A generic tool cannot guess; a wrong row count would go unnoticed. | +| D8 | No CLI in this spec | Port `fetch` | The CLIs differ per dataset; spec 001 owns the CLI. | + +## 7. Risks + +- **Translated messages break consumers that match text.** dataset-names tests match Portuguese messages; its adoption (issue #5) updates those assertions or keeps its own wrappers. The adoption spec in dataset-names decides. +- **Spec 001 builds on the same package.** Its task T2 (project skeleton) is delivered here; 001 adds its dependencies through its own ADR when it starts. +- **Overlap with `tools-commons`.** [.product ADR 0005](https://github.com/BenchCommons/.product/blob/main/docs/adr/0005-tools-prefix.md) creates `tools-commons` for code shared by several tools, and its scope issue ([tools-commons#1](https://github.com/BenchCommons/tools-commons/issues/1)) lists retry wrappers for flaky downloads. The toolkit stays here because its consumers are the `dataset-*` repos ([ADR 0007](../../adr/0007-source-toolkit-lives-in-tools-dataset.md)); if `tools-benchmark` needs the download cache, moving it is a `tools-commons` spec, and this package then depends on it. +- **GitHub fetch by SHA in CI.** The install job fetches the PR head commit by SHA from GitHub; if that ever fails, the fallback is a `git+file://` URL to the checkout, which still proves installability from git. diff --git a/docs/specs/004-source-toolkit/spec.md b/docs/specs/004-source-toolkit/spec.md new file mode 100644 index 0000000..5d5c63f --- /dev/null +++ b/docs/specs/004-source-toolkit/spec.md @@ -0,0 +1,106 @@ +# 004. The source toolkit + +Issue: [#4](https://github.com/BenchCommons/tools-dataset/issues/4). Roadmap: [`../ROADMAP.md`](../ROADMAP.md). Adoption by dataset-names: [#5](https://github.com/BenchCommons/tools-dataset/issues/5). + +## Why + +Every `dataset-*` repo does the same three things before it can build anything: it downloads upstream files politely and reproducibly, it reads the formats institutes publish (CSV, XLS, XLSX), and it keeps a registry of its sources with their licences and attributions. `dataset-names` already does all three, with tests hardened over several reviews (its specs 002, 003, 003b and 003e, ADRs 0001 and 0005). `dataset-commons` and later datasets would otherwise copy that code and let the copies drift. + +Owner decision (2026-10-12): the generic parts move to this repo (`tools-dataset`, formerly `dataset-tool`) as an installable Python package, the **source toolkit**; the names-specific parts (record types, output schema, the names manifest, the names licence list) stay in `dataset-names`. This spec defines the toolkit's behaviour and how dataset repos depend on it. `tools-dataset` owns this contract; dataset repos link to it and never redefine it. + +## Terms + +- **Source**: one upstream publisher's dataset, registered once in the dataset repo's registry file (for `dataset-names`, `sources.yaml`). +- **Resource**: one file of a source, given by URL and local file name. +- **Adapter**: the dataset repo's module that lists a source's resources and parses the downloaded files into the dataset's own record type. +- **Registry schema**: the generic registry fields plus the fields, licence list and cross-entry checks a dataset adds. + +## Requirements + +### R1. Package + +- **R1.1** The toolkit is the Python distribution `benchcommons-tools-dataset`, import package `datasettool`, in this repo ([ADR 0003](../../adr/0003-python-package-name.md); placement: [ADR 0007](../../adr/0007-source-toolkit-lives-in-tools-dataset.md)). It needs Python 3.12 or later, is built with `hatchling` and developed with `uv` and `pytest`. +- **R1.2** The toolkit modules are `datasettool.sources.cache` (R2), `datasettool.sources.spreadsheet` (R3), `datasettool.sources.adapters` (R4), `datasettool.sources.registry` (R5) and `datasettool.files` (R6). +- **R1.3** Versions follow SemVer and are released as git tags `vX.Y.Z` of this repo. The first release of this spec is `v0.1.0`. While the major version is 0, a minor bump may break the API; dataset repos pin an exact tag. +- **R1.4** Dataset repos depend on the package through a `uv` git source pinned to a tag ([ADR 0004](../../adr/0004-dataset-repos-depend-on-a-git-tag.md)). The package is not published to PyPI. +- **R1.5** Runtime dependencies are `pyyaml` and `xlrd`, both pinned to an exact version ([ADR 0005](../../adr/0005-source-toolkit-runtime-dependencies.md)). Any other runtime dependency needs a new ADR. +- **R1.6** User-facing messages and identifiers are in English. Each ported behaviour keeps its meaning and its test vectors (R7). + +### R2. Download cache + +Ported from `dataset-names` spec 002 R4, spec 003 R4 (403 hint) and spec 003e R4.1. + +- **R2.1** A resource is cached at `//`, with `.meta.json` next to it holding `url`, `downloaded_at` (ISO 8601 with time zone, seconds), `sha256` and `size`. A source id outside the registry id format (R5.3), a file name that fails the R4.2 rule, or a path that resolves outside the cache directory is a `ValueError` before any file access or request. +- **R2.2** Without refresh, a file present in the cache is returned without any request. With refresh, it is downloaded again. With offline, no request is made and a missing file is a `FetchError` naming the path. Offline and refresh together are a `ValueError`. +- **R2.3** Every request sends a `User-Agent`. The default identifies `tools-dataset`, its version and this repo's URL; a dataset passes its own (name, version and repo URL) so upstream operators know who is downloading. +- **R2.4** A download has a timeout and up to three attempts by default, waiting 2 s, then 4 s (doubling). A 4xx response is final and is not retried, except 408 and 429. The number of attempts, the first wait and the sleep function are parameters; fewer than one attempt is a `ValueError`. +- **R2.5** A partial download never enters the cache: bytes go to `.part` and are renamed only when complete. A `Content-Length` that is malformed or differs from the bytes received is a `FetchError`. The old `.meta.json` is removed before the new file is renamed into place, so a failure in between leaves a file without metadata, never metadata describing another file. A failed refresh keeps the previous version. +- **R2.6** A `Content-Type` of `text/html` for a resource whose file name does not end in `.html` or `.htm` is a `BlockedError` (a subclass of `FetchError`): nothing is cached, there is no retry, and the message names the URL and the exact cache path where to save a file downloaded with a browser, then rerun offline. +- **R2.7** After the last failed attempt, the `FetchError` names the URL, the number of attempts and the last cause. For a 403, the message adds the same manual-download hint as R2.6; other errors do not carry it. + +### R3. Spreadsheet reader + +Ported from `dataset-names` spec 003b R1 and ADR 0005, unchanged in behaviour. + +- **R3.1** `sheet_names(path)` lists sheets in file order. `rows(path, sheet)` yields `(Excel row number, values by column)` for each non-empty row; column A is index 0. A missing sheet is a `FormatError` naming the sheet and listing the sheets. +- **R3.2** The format is detected from the first bytes, not the extension: `PK\x03\x04` is XLSX, the OLE2 signature is XLS. Anything else, including an HTML error page saved under a spreadsheet name, is a `FormatError`. +- **R3.3** Both formats return the same value types: `str`, `int` (integral numbers), `float`, `bool`, `None` for empty cells. Trailing `None` values are cut and empty rows skipped. Formula errors, the XLSX date type `d` and unknown cell types are `FormatError` naming the sheet and cell. +- **R3.4** XLSX is read in a stream with the standard library (`zipfile`, and `xml.parsers.expat` feeding an `ElementTree.TreeBuilder`). Rejected with `FormatError`: a DTD (`.`, where the dataset chooses ``. It exposes `SOURCE_ID` (equal to the registry id), `RESOURCES` (a non-empty list of `Resource(url, filename)`) and `parse(files)`, which reads the downloaded files without network and yields records of the **dataset's own record type**. The toolkit defines no record type. +- **R4.2** `check_adapter(module, source_id, optional=())` rejects with `AdapterError`: a missing `SOURCE_ID`, `RESOURCES` or `parse`; a `SOURCE_ID` different from the registry id; `RESOURCES` that is empty or not made of `Resource`; a URL that is not `http` or `https` with a host, or that contains whitespace or control characters; a file name that is empty, `.` or `..`, starts with a dot, contains `/`, `\` or a non-printable character, or ends in `.meta.json` or `.part`; a repeated file name; a `parse` that is not callable; and any name in `optional` that the module defines but is not callable (for `dataset-names`, `links`). +- **R4.3** `load_adapter(source_id, package, optional=())` imports and checks the adapter. A module that does not exist is reported as **absent**, with the path where it was expected; a module that exists but fails to import, including when it imports something missing, is reported as **broken**, with the exception type and message. A package that does not exist is reported by name. +- **R4.4** `parse_count(value, missing=(), field="count")` accepts a non-negative decimal integer (surrounding whitespace allowed) or a declared marker (returns `None`); anything else is a `FormatError` naming the field, the value and the markers, so a new suppression marker never becomes zero. `parse_code(value, mapping, field)` maps a source code through an explicit mapping; an unknown code is a `FormatError` listing the expected codes. +- **R4.5** `SourceFiles` is the mapping passed to `parse`; asking it for a file outside `RESOURCES` is an `AdapterError` listing the available files. + +### R5. Source registry and licence policy + +Ported from the generic part of `dataset-names` `src/namescommons/sources.py` (its spec 001 R2, spec 002 R2 and R3, spec 003 R2.7). + +- **R5.1** The registry is a YAML file whose root is a mapping with a `sources` list. An unreadable file, invalid YAML, a root that is not a mapping and a `sources` that is not a list are each a `RegistryError` naming the file. An empty or missing list is an empty registry. +- **R5.2** Generic required fields of every entry: `id`, `name`, `url`, `license`, `attribution`, `retrieved`. Generic optional field: `adapter` (boolean, default `true`; `false` registers a source used only by hand). +- **R5.3** Generic rules: `id`, `name`, `url` and `attribution` are non-empty strings; `id` matches `[a-z0-9]+(-[a-z0-9]+)*`; `url` is `http` or `https` with a host (the R4.2 rule); `license` is a string in the dataset's accepted list; `retrieved` is a date without time; ids are unique; fields outside the schema are rejected. +- **R5.4** A dataset extends the schema with its own fields (each required or optional with a default, each with its own check), its accepted licence list, a text naming the policy that decides that list (shown in the licence error, for example "ADR 0001"), cross-entry checks over the parsed sources, and a factory that builds its own source type. `dataset-names` uses this for `country`, `count_type`, `primary`, `suppression`, `uniform_case` and its one-primary rule. +- **R5.5** The accepted licence list belongs to each dataset, because it depends on that dataset's data licence ([ADR 0006](../../adr/0006-accepted-licences-per-dataset.md); org policy in [.product ADR 0003](https://github.com/BenchCommons/.product/blob/main/docs/adr/0003-default-branch-and-licenses.md)). There is no implicit default: a schema without a list is an error. The toolkit exports `ATTRIBUTION_ONLY_LICENSES`, the licences that ask for no more than attribution (`public-domain`, `CC0-1.0`, `CC-BY-3.0`, `CC-BY-4.0`, `ODC-By-1.0`, `OGL-UK-3.0`, `etalab-2.0`), as a starting point a dataset copies or extends by its own ADR. +- **R5.6** All problems of a registry are collected and reported together in one `RegistryError`, each prefixed with the entry id (or `#` when the id is unusable). + +### R6. File helpers + +Ported from the generic part of `dataset-names` `src/namescommons/manifest.py` (its spec 001 R5 and spec 002 R9). + +- **R6.1** `sha256_file(path)` returns the lowercase hex SHA-256 of a file, read in blocks. +- **R6.2** `count_csv_records(path)` counts CSV records with a CSV reader, not physical lines: the header and blank lines are not counted, and a quoted value spanning lines is one record. +- **R6.3** `describe_files(root, exclude=…, counters=…)` maps each file under `root` (POSIX relative path, sorted) to `{sha256, rows}`, counting rows with the counter registered for the file's suffix (`.csv` by default); a file with no counter is a `ValueError` naming it, never a guessed count. +- **R6.4** The dataset's manifest layout, release version format and output schema are not part of the toolkit. + +### R7. Faithful port + +- **R7.1** Each test of `dataset-names` that exercises a ported behaviour is ported with its inputs and expected values unchanged (test vectors), at `dataset-names@e7e8405` (the spreadsheet reader at `eaaa4d1`, its PR #39). Only the language of messages and identifiers changes. Tests that depend on names-specific code (record types, the names pipeline, the INE adapter samples) stay in `dataset-names`. +- **R7.2** Each commit that ports a module names its origin, for example `ported from BenchCommons/dataset-names@e7e8405 src/namescommons/cache.py`. Both repos' code is Apache-2.0, and this repo carries the Apache-2.0 `LICENSE`. +- **R7.3** Tests never use the network: downloads are tested against an HTTP server on `127.0.0.1` started by the tests. + +### R8. CI + +The CI workflow keeps the org docs job and adds a Python job that runs `uv sync --locked` and `uv run pytest -q`, and a job that installs the package into a fresh project from this repo by git URL at the commit under test and imports every module. + +## Acceptance criteria + +1. Every test listed in the plan's port map passes in this repo with the same test vectors as in `dataset-names@e7e8405`. +2. `uv run pytest -q` passes locally and in CI, with no network access other than `127.0.0.1`. +3. A registry schema built only from the public API, with the fields and rules of `dataset-names` (`country`, `count_type`, `primary`, `suppression`, `uniform_case`, one primary per country and count type), passes the ported `dataset-names` registry tests. +4. In a fresh `uv` project, the package installs from a git URL of this repo pinned to a ref, and every module imports. The CI job proves it at the commit under test; after merge, the same command at tag `v0.1.0` proves it for the release. +5. The CI workflow runs the docs job and the Python jobs, and all pass on the pull request of this spec. +6. Issue [#5](https://github.com/BenchCommons/tools-dataset/issues/5) tracks the adoption by `dataset-names`. + +## Out of scope + +- Any change in `dataset-names` (issue #5; its own spec goes in that repo). +- A command-line interface. Dataset repos keep their CLIs and pass the offline and refresh flags through. +- The input contract and its build and validate commands (spec 001), the release workflow (002) and consumer fetch (003). +- Publishing to PyPI, Parquet row counting (a dataset registers its own counter), and the names record types, output schema and manifest. diff --git a/docs/specs/004-source-toolkit/tasks.md b/docs/specs/004-source-toolkit/tasks.md new file mode 100644 index 0000000..35a5a26 --- /dev/null +++ b/docs/specs/004-source-toolkit/tasks.md @@ -0,0 +1,13 @@ +# 004. The source toolkit: tasks + +Spec: [`spec.md`](spec.md). Plan: [`plan.md`](plan.md). Each task is one commit with its tests passing (test first). Port commits name their origin: `ported from BenchCommons/dataset-names@e7e8405 `. + +- [x] **T1. Spec and ADRs.** `spec.md`, ADRs 0003 to 0006, roadmap row. Done when committed. +- [x] **T2. Package skeleton and CI.** `pyproject.toml` (plan section 1), `.python-version`, `uv.lock`, `LICENSE`, `src/datasettool/__init__.py`, `src/datasettool/sources/__init__.py`, `tests/conftest.py` with the local HTTP server, the `tests` and `install-from-git` CI jobs, `AGENTS.md` and `README.md` with stack and commands. Tests: `__version__` equals the distribution version; `pyproject.toml` pins `xlrd==2.0.2` and `pyyaml==6.0.3`. This also delivers spec 001's T2 skeleton. +- [x] **T3. Adapter base.** `datasettool.sources.adapters` (plan 2.1). Tests: the adapter rows of the port map, plus `optional` callables. +- [x] **T4. Download cache.** `datasettool.sources.cache` (plan 2.2). Tests: the 22 ported cache tests, plus `DEFAULT_USER_AGENT` and the dataset-names header through `make_user_agent`. +- [x] **T5. Spreadsheet reader.** `datasettool.sources.spreadsheet` (plan 2.3) and `tests/fixtures/spreadsheets/types.xls`. Tests: the ported spreadsheet tests, including the encoding and expat-level DTD guards of dataset-names PR #39. +- [x] **T6. Source registry.** `datasettool.sources.registry` (plan 2.4) and `tests/names_registry.py`. Tests: every test of dataset-names' `test_sources.py` and `test_sources_extensoes.py` through the names schema, plus schema rules (no licence list, redefined generic field, generic `Source` with `extra`, custom factory). +- [x] **T7. File helpers.** `datasettool.files` (plan 2.5). Tests: the ported row-count vectors, `describe_files` with quoted multi-line CSV, an excluded manifest, an unknown suffix and a custom counter. +- [x] **T8. Acceptance.** Run the suite; install from the pushed branch by git URL in an empty `uv` project and import every module; tick the tasks; open the pull request (closes #4). +- [ ] **T9. Release tag (maintainer, after merge).** Tag `v0.1.0` on the merge commit and run the check of plan section 5. Then issue #5 can start. diff --git a/docs/specs/ROADMAP.md b/docs/specs/ROADMAP.md index 7ff12a1..1fc7689 100644 --- a/docs/specs/ROADMAP.md +++ b/docs/specs/ROADMAP.md @@ -1,11 +1,12 @@ -# Roadmap: dataset-tool +# Roadmap: tools-dataset -Specs of this repo, in order. Each one goes through its own cycle: spec, plan, implementation, review. Tracking: issue [#1](https://github.com/BenchCommons/dataset-tool/issues/1). +Specs of this repo, in order. Each one goes through its own cycle: spec, plan, implementation, review. Tracking: issue [#1](https://github.com/BenchCommons/tools-dataset/issues/1). | Order | Spec | Depends on | Delivers | |---|---|---|---| | 1 | [`001-input-contract`](001-input-contract/spec.md) | none | The input contract, its JSON Schemas and test vectors, and the reference `build` and `validate` commands | | 2 | `002-release-workflow` | 001 | A reusable GitHub workflow that dataset repos call to build, validate and attach inputs to their releases | | 3 | `003-consumer-fetch` | 001 | A `fetch` command (and/or GitHub Action) for catalog-algorithms and benchmark-tool: resolve input references, download, verify, cache, offline mode | +| 4 | [`004-source-toolkit`](004-source-toolkit/spec.md) | none | The shared Python package `benchcommons-tools-dataset` that every `dataset-*` repo uses to download, read and register upstream sources (cache, spreadsheet reader, adapter base, source registry), installed from a git tag ([#4](https://github.com/BenchCommons/tools-dataset/issues/4)) | -Adoption by each dataset is specified in that dataset's repo: [dataset-names#24](https://github.com/BenchCommons/dataset-names/issues/24), [dataset-commons#1](https://github.com/BenchCommons/dataset-commons/issues/1). +Spec 004 does not depend on 001 and can be implemented first; 001 then adds its modules to the same package. Adoption by each dataset is specified in that dataset's repo: [dataset-names#24](https://github.com/BenchCommons/dataset-names/issues/24), [dataset-commons#1](https://github.com/BenchCommons/dataset-commons/issues/1); for the source toolkit, [#5](https://github.com/BenchCommons/tools-dataset/issues/5). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3ed004f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "benchcommons-tools-dataset" +version = "0.1.0" +description = "Shared tooling for BenchCommons dataset repos: source download cache, spreadsheet reader, adapter base and source registry" +readme = "README.md" +requires-python = ">=3.12" +license = "Apache-2.0" +license-files = ["LICENSE"] +dependencies = [ + "pyyaml==6.0.3", + "xlrd==2.0.2", +] + +[project.urls] +Repository = "https://github.com/BenchCommons/tools-dataset" + +[dependency-groups] +dev = ["pytest==9.1.1"] + +[build-system] +requires = ["hatchling==1.32.4"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/datasettool"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] diff --git a/src/datasettool/__init__.py b/src/datasettool/__init__.py new file mode 100644 index 0000000..c1fc3e2 --- /dev/null +++ b/src/datasettool/__init__.py @@ -0,0 +1,7 @@ +"""BenchCommons tools-dataset: shared tooling for dataset repos (spec 004).""" + +from importlib.metadata import version + +DISTRIBUTION = "benchcommons-tools-dataset" +REPOSITORY = "https://github.com/BenchCommons/tools-dataset" +__version__ = version(DISTRIBUTION) diff --git a/src/datasettool/files.py b/src/datasettool/files.py new file mode 100644 index 0000000..43520e2 --- /dev/null +++ b/src/datasettool/files.py @@ -0,0 +1,53 @@ +"""File checksums and row counts for release manifests (spec 004, R6). + +The manifest layout, release version format and output schema belong to each dataset. + +Ported from BenchCommons/dataset-names@e7e8405 src/namescommons/manifest.py (generic part). +""" + +import csv +import hashlib +from collections.abc import Callable, Collection, Mapping +from pathlib import Path + + +def sha256_file(path: str | Path) -> str: + """Lowercase hex SHA-256 of a file, read in 1 MiB blocks.""" + h = hashlib.sha256() + with open(path, "rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def count_csv_records(path: str | Path) -> int: + """CSV records, read with a CSV reader: no header, no blank lines; a quoted multi-line value is one record.""" + with open(path, encoding="utf-8", newline="") as fh: + records = sum(1 for row in csv.reader(fh) if row) + return max(records - 1, 0) + + +DEFAULT_COUNTERS: Mapping[str, Callable[[Path], int]] = {".csv": count_csv_records} + + +def describe_files( + root: str | Path, + exclude: Collection[str] = ("manifest.json",), + counters: Mapping[str, Callable[[Path], int]] = DEFAULT_COUNTERS, +) -> dict[str, dict]: + """{POSIX relative path: {"sha256", "rows"}} of every file under root, sorted. + + Rows are counted by the counter registered for the file's suffix; a file without one is a + ValueError, never a guessed count. Files whose name is in `exclude` are left out. + """ + root = Path(root) + files = {} + for p in sorted(root.rglob("*")): + if not p.is_file() or p.name in exclude: + continue + relative = p.relative_to(root).as_posix() + counter = counters.get(p.suffix) + if counter is None: + raise ValueError(f"{relative}: no row counter for {p.suffix!r}; register one in counters") + files[relative] = {"sha256": sha256_file(p), "rows": counter(p)} + return files diff --git a/src/datasettool/sources/__init__.py b/src/datasettool/sources/__init__.py new file mode 100644 index 0000000..ef07633 --- /dev/null +++ b/src/datasettool/sources/__init__.py @@ -0,0 +1 @@ +"""Source toolkit (spec 004): download, read and register the upstream sources of a dataset.""" diff --git a/src/datasettool/sources/adapters.py b/src/datasettool/sources/adapters.py new file mode 100644 index 0000000..a00a3d9 --- /dev/null +++ b/src/datasettool/sources/adapters.py @@ -0,0 +1,160 @@ +"""Adapter base (spec 004, R4). + +An adapter is the module . of a dataset repo. It exposes +SOURCE_ID (the registry id), RESOURCES (a non-empty list of Resource) and parse(files), which reads +the downloaded files without network and yields records of the dataset's own record type. +The toolkit defines no record type: dataset-names, for example, yields RawRecord and RawLink. + +Ported from BenchCommons/dataset-names@e7e8405 src/namescommons/adapters/__init__.py (generic part). +""" + +import importlib +import importlib.util +import re +from collections.abc import Collection, Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from urllib.parse import urlsplit + + +# Source id format, shared by the registry and the cache. +ID_PATTERN = re.compile(r"[a-z0-9]+(-[a-z0-9]+)*") + + +class AdapterError(Exception): + """Adapter absent, broken or outside the contract of R4.""" + + +class FormatError(ValueError): + """A source file is not in the format the adapter expects.""" + + +@dataclass(frozen=True) +class Resource: + url: str + filename: str + + +def module_name(source_id: str) -> str: + return source_id.replace("-", "_") + + +def is_http_url(url: object) -> bool: + """http or https URL with a host, without whitespace or control characters; never raises.""" + if not isinstance(url, str) or any(c.isspace() or not c.isprintable() for c in url): + return False + try: + parts = urlsplit(url) + return parts.scheme.lower() in ("http", "https") and bool(parts.netloc) + except ValueError: + return False + + +def is_safe_filename(name: object) -> bool: + """A plain file name that cannot escape the cache directory or collide with cache bookkeeping.""" + return ( + isinstance(name, str) and name.isprintable() and name not in ("", ".", "..") and not name.startswith(".") + and "/" not in name and "\\" not in name + and not name.endswith((".meta.json", ".part")) + ) + + +def check_adapter(mod: object, source_id: str, optional: Iterable[str] = ()) -> None: + """Raise AdapterError if the module breaks the contract of R4. + + `optional` names extra functions the dataset's own protocol allows; each is checked only when present. + """ + optional = _names(optional) + for attribute in ("SOURCE_ID", "RESOURCES", "parse"): + if not hasattr(mod, attribute): + raise AdapterError(f"adapter '{source_id}' has no {attribute}") + if mod.SOURCE_ID != source_id: + raise AdapterError(f"SOURCE_ID '{mod.SOURCE_ID}' differs from the id '{source_id}' in the registry") + resources = mod.RESOURCES + if not isinstance(resources, (list, tuple)) or not resources or not all( + isinstance(r, Resource) for r in resources + ): + raise AdapterError(f"'{source_id}': RESOURCES must be a non-empty list of Resource") + seen: set[str] = set() + for r in resources: + if not is_http_url(r.url): + raise AdapterError(f"'{source_id}': url {r.url!r} must use http or https") + if not is_safe_filename(r.filename): + raise AdapterError(f"'{source_id}': filename {r.filename!r} is invalid") + if r.filename in seen: + raise AdapterError(f"'{source_id}': filename {r.filename!r} is repeated") + seen.add(r.filename) + if not callable(mod.parse): + raise AdapterError(f"'{source_id}': parse must be a function") + for name in optional: + if hasattr(mod, name) and not callable(getattr(mod, name)): + raise AdapterError(f"'{source_id}': {name} must be a function") + + +def _names(optional: Iterable[str]) -> tuple[str, ...]: + if isinstance(optional, (str, bytes)): + raise TypeError("optional must be a collection of names, not a string") + return tuple(optional) + + +def _expected_path(package: str, module: str) -> str: + try: + spec = importlib.util.find_spec(package) + except ImportError: + spec = None + if spec is None or not spec.submodule_search_locations: + return f"package {package!r} not found" + return str(Path(list(spec.submodule_search_locations)[0]) / f"{module}.py") + + +def load_adapter(source_id: str, package: str, optional: Iterable[str] = ()) -> ModuleType: + """Import and check the adapter of `source_id` from `package`. + + A module that does not exist is reported as absent, with the expected path; a module that exists + but fails to import (even because it imports something missing) is reported as broken. + """ + optional = _names(optional) + name = f"{package}.{module_name(source_id)}" + try: + mod = importlib.import_module(name) + except ModuleNotFoundError as e: + if e.name is not None and (name == e.name or name.startswith(e.name + ".")): + expected = _expected_path(package, module_name(source_id)) + raise AdapterError(f"adapter {name} absent (expected at {expected})") from e + raise AdapterError(f"adapter {name} broken: {type(e).__name__}: {e}") from e + except Exception as e: + raise AdapterError(f"adapter {name} broken: {type(e).__name__}: {e}") from e + check_adapter(mod, source_id, optional) + return mod + + +class SourceFiles(dict): + """Downloaded files of a source (key = filename); a key outside RESOURCES raises AdapterError.""" + + def __missing__(self, key): + raise AdapterError(f"parse asked for file {key!r}, outside RESOURCES; available: {sorted(self)}") + + +_INTEGER = re.compile(r"[0-9]+") + + +def parse_count(value: str, missing: Collection[str] = (), field: str = "count") -> int | None: + """Published integer (a count or a year); a marker from `missing` (suppressed or absent value) becomes None. + + Any other text raises FormatError, so a new marker of the source never passes silently. + """ + text = value.strip() + if text in missing: + return None + if _INTEGER.fullmatch(text): + return int(text) + raise FormatError(f"{field} {value!r} is neither an integer nor a known marker {sorted(missing)}") + + +def parse_code(value: str, mapping: Mapping[str, str], field: str) -> str: + """Translate a source code (for example sex '1' and '2') through the mapping; an unknown code is FormatError.""" + try: + return mapping[value.strip()] + except KeyError: + raise FormatError(f"{field} {value!r} unknown; expected one of {sorted(mapping)}") from None diff --git a/src/datasettool/sources/cache.py b/src/datasettool/sources/cache.py new file mode 100644 index 0000000..4fb7804 --- /dev/null +++ b/src/datasettool/sources/cache.py @@ -0,0 +1,159 @@ +"""Download cache of source resources (spec 004, R2). + +Each resource is cached at //, with .meta.json next to it. + +Ported from BenchCommons/dataset-names@e7e8405 src/namescommons/cache.py. +""" + +import hashlib +import json +import os +import time +import urllib.error +import urllib.request +from collections.abc import Callable +from datetime import datetime, timezone +from http.client import HTTPException +from pathlib import Path + +from datasettool import REPOSITORY, __version__ +from datasettool.sources.adapters import ID_PATTERN, Resource, is_safe_filename + +BLOCK = 1 << 16 +BACKOFF = 2.0 # seconds before the second attempt; doubles at each new attempt + + +def make_user_agent(name: str, version: str, url: str) -> str: + """User-Agent that tells upstream operators who is downloading: '/ (+)'.""" + return f"{name}/{version} (+{url})" + + +DEFAULT_USER_AGENT = make_user_agent("tools-dataset", __version__, REPOSITORY) + + +class FetchError(Exception): + """Failure to obtain a resource, from the network or from the cache.""" + + +class BlockedError(FetchError): + """The server returned an HTML page instead of the file (R2.6); retrying does not help.""" + + +_PAGES = (".html", ".htm") + + +def cached_path(cache_dir: Path, source_id: str, filename: str) -> Path: + """//; ValueError unless both parts are safe and the path stays in the cache.""" + if not isinstance(source_id, str) or not ID_PATTERN.fullmatch(source_id): + raise ValueError(f"invalid source id {source_id!r} (use [a-z0-9]+(-[a-z0-9]+)*)") + if not is_safe_filename(filename): + raise ValueError(f"invalid file name {filename!r} for the cache") + root = Path(cache_dir) + path = root / source_id / filename + if not path.resolve().is_relative_to(root.resolve()): + raise ValueError(f"{path} resolves outside the cache {root}") + return path + + +def _write_atomic(dest: Path, content: bytes) -> None: + tmp = dest.with_name(dest.name + ".part") + tmp.write_bytes(content) + os.replace(tmp, dest) + + +def _download(url: str, dest: Path, timeout: float, user_agent: str) -> dict: + """Download url to dest through a temporary file; return the metadata.""" + tmp = dest.with_name(dest.name + ".part") + request = urllib.request.Request(url, headers={"User-Agent": user_agent}) + h = hashlib.sha256() + size = 0 + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + content_type = response.headers.get_content_type() # lowercase, without parameters + if content_type == "text/html" and not dest.name.lower().endswith(_PAGES): + raise BlockedError(f"the server returned an HTML page ({content_type}) instead of {dest.name}") + with open(tmp, "wb") as fh: + announced = response.headers.get("Content-Length") + while block := response.read(BLOCK): + fh.write(block) + h.update(block) + size += len(block) + if announced is not None: + try: + expected = int(announced) + except ValueError: + raise FetchError(f"Content-Length {announced!r} is invalid for {url}") from None + if size != expected: + raise FetchError(f"incomplete download of {url}: {size} of {announced} bytes") + meta = { + "url": url, + "downloaded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "sha256": h.hexdigest(), + "size": size, + } + # The old metadata goes before the new file comes in: a failure in between leaves the file + # without metadata (the same state as a file saved by hand), never metadata describing another file. + meta_file = dest.with_name(dest.name + ".meta.json") + meta_file.unlink(missing_ok=True) + os.replace(tmp, dest) + _write_atomic(meta_file, (json.dumps(meta, ensure_ascii=False, indent=2) + "\n").encode("utf-8")) + return meta + finally: + tmp.unlink(missing_ok=True) + + +def _final(e: Exception) -> bool: + """A 4xx (except 408 and 429) is a final answer from the server: trying again changes nothing.""" + return isinstance(e, urllib.error.HTTPError) and 400 <= e.code < 500 and e.code not in (408, 429) + + +def fetch_resource( + cache_dir: Path, + source_id: str, + resource: Resource, + *, + refresh: bool = False, + offline: bool = False, + timeout: float = 60.0, + attempts: int = 3, + backoff: float | None = None, + sleep: Callable[[float], None] = time.sleep, + user_agent: str | None = None, +) -> Path: + """Path of the resource in the cache, downloading it when needed (R2).""" + if attempts < 1: + raise ValueError(f"attempts must be at least 1, got {attempts}") + backoff = BACKOFF if backoff is None else backoff + user_agent = DEFAULT_USER_AGENT if user_agent is None else user_agent + if offline and refresh: + raise ValueError("--offline and --refresh are incompatible") + dest = cached_path(cache_dir, source_id, resource.filename) + if dest.is_file() and not refresh: + return dest + if offline: + raise FetchError(f"{dest} missing from the cache and --offline prevents the download") + dest.parent.mkdir(parents=True, exist_ok=True) + last: Exception | None = None + for i in range(attempts): + try: + _download(resource.url, dest, timeout, user_agent) + return dest + except BlockedError as e: + raise BlockedError( + f"failed to download {resource.url}: {e}, probably an automatic-download block: " + f"download the file with a browser, save it at {dest} and rerun with --offline" + ) from e + except (OSError, HTTPException, FetchError) as e: + last = e + done = i + 1 + if _final(e): + break + if done < attempts: + sleep(backoff * 2**i) + message = f"failed to download {resource.url} after {done} attempt(s): {last}" + if isinstance(last, urllib.error.HTTPError) and last.code == 403: + message += ( + "; the server refused the automatic download: download the file with a browser, " + f"save it at {dest} and rerun with --offline" + ) + raise FetchError(message) diff --git a/src/datasettool/sources/registry.py b/src/datasettool/sources/registry.py new file mode 100644 index 0000000..8f29090 --- /dev/null +++ b/src/datasettool/sources/registry.py @@ -0,0 +1,179 @@ +"""Source registry and licence policy (spec 004, R5; ADR 0006). + +A registry is a YAML file with a `sources` list. Every entry has the generic fields; each dataset +extends the schema with its own fields, its accepted licences, cross-entry checks and a factory +that builds its own source type. + +Ported from BenchCommons/dataset-names@e7e8405 src/namescommons/sources.py (generic part). +""" + +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date, datetime +from pathlib import Path +from types import MappingProxyType +from typing import Any, Generic, TypeVar + +import yaml + +from datasettool.sources.adapters import ID_PATTERN, is_http_url + +# Licences that ask for no more than attribution, compatible with a CC BY 4.0 dataset (ADR 0006). +# A starting set: each dataset declares its own list and extends it by its own ADR. +ATTRIBUTION_ONLY_LICENSES = frozenset({ + "public-domain", "CC0-1.0", "CC-BY-3.0", "CC-BY-4.0", "ODC-By-1.0", "OGL-UK-3.0", "etalab-2.0", +}) + +REQUIRED = ("id", "name", "url", "license", "attribution", "retrieved") +OPTIONAL = {"adapter": True} +GENERIC = REQUIRED + tuple(OPTIONAL) + + +class RegistryError(ValueError): + """Invalid registry. The message lists every problem found, one per line.""" + + +@dataclass(frozen=True) +class Field: + """A dataset field: `check` returns an error text or None; optional fields take `default` when absent.""" + + name: str + check: Callable[[Any], str | None] | None = None + required: bool = True + default: Any = None + + +@dataclass(frozen=True) +class Source: + """Source built by the default factory; dataset fields are in `extra`, read-only. + + Hashable: `extra` (a MappingProxyType, which is unhashable) takes part in equality but not in the hash. + """ + + id: str + name: str + url: str + license: str + attribution: str + retrieved: date + adapter: bool = True + extra: Mapping[str, Any] = field(default_factory=lambda: MappingProxyType({}), hash=False) + + @classmethod + def from_entry(cls, entry: dict[str, Any]) -> "Source": + extra = {k: v for k, v in entry.items() if k not in GENERIC} + return cls(**{k: entry[k] for k in GENERIC}, extra=MappingProxyType(extra)) + + +S = TypeVar("S") + + +@dataclass(frozen=True) +class RegistrySchema(Generic[S]): + accepted_licenses: frozenset[str] + license_policy: str # names the decision behind the list, e.g. "ADR 0001" + fields: tuple[Field, ...] = () + checks: tuple[Callable[[Mapping[str, S]], Iterable[str]], ...] = () + factory: Callable[[dict[str, Any]], S] = Source.from_entry + + def __post_init__(self): + # A str is iterable: as a licence list it would turn the check into a substring test. + for name in ("accepted_licenses", "fields", "checks"): + if isinstance(getattr(self, name), (str, bytes)): + raise TypeError(f"{name} must be a collection, not a string") + object.__setattr__(self, "accepted_licenses", frozenset(self.accepted_licenses)) + object.__setattr__(self, "fields", tuple(self.fields)) + object.__setattr__(self, "checks", tuple(self.checks)) + for f in self.fields: + if not isinstance(f, Field): + raise TypeError(f"fields must contain Field, got {f!r}") + for c in self.checks: + if not callable(c): + raise TypeError(f"checks must contain callables, got {c!r}") + if not self.accepted_licenses: + raise ValueError("a registry schema needs a non-empty accepted_licenses list (ADR 0006)") + seen: set[str] = set() + for f in self.fields: + if f.name in GENERIC: + raise ValueError(f"dataset field '{f.name}' redefines a generic field") + if f.name in seen: + raise ValueError(f"dataset field '{f.name}' declared twice") + seen.add(f.name) + + +def load_registry(path: Path, schema: RegistrySchema[S]) -> dict[str, S]: + try: + data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except OSError as e: + raise RegistryError(f"cannot read {path}: {e}") from e + except yaml.YAMLError as e: + raise RegistryError(f"{path}: invalid YAML: {e}") from e + if data is None: + data = {} + if not isinstance(data, dict): + raise RegistryError(f"{path}: the root must be a mapping with the key 'sources'") + entries = data.get("sources", []) + if entries is None: + entries = [] + if not isinstance(entries, list): + raise RegistryError(f"{path}: 'sources' must be a list") + return parse_registry(entries, schema) + + +def _entry_errors(e: dict, label: str, schema: RegistrySchema) -> list[str]: + errors: list[str] = [] + required = list(REQUIRED) + [f.name for f in schema.fields if f.required] + missing = [name for name in required if name not in e] + if missing: + return [f"{label}: missing fields {missing}"] + known = set(GENERIC) | {f.name for f in schema.fields} + unknown = sorted(set(e) - known, key=str) + if unknown: + errors.append(f"{label}: unknown fields {unknown}") + for name in ("id", "name", "url", "attribution"): + if not isinstance(e[name], str) or not e[name].strip(): + errors.append(f"{label}: '{name}' empty") + if isinstance(e["id"], str) and e["id"].strip() and not ID_PATTERN.fullmatch(e["id"]): + errors.append(f"{label}: 'id' has an invalid format (use [a-z0-9]+(-[a-z0-9]+)*)") + if isinstance(e["url"], str) and e["url"].strip() and not is_http_url(e["url"]): + errors.append(f"{label}: 'url' must use the http or https scheme") + if not isinstance(e["license"], str) or e["license"] not in schema.accepted_licenses: + errors.append(f"{label}: license '{e['license']}' is not accepted ({schema.license_policy})") + if not isinstance(e["retrieved"], date) or isinstance(e["retrieved"], datetime): + errors.append(f"{label}: 'retrieved' must be a YYYY-MM-DD date, without time") + for name in OPTIONAL: + if name in e and not isinstance(e[name], bool): + errors.append(f"{label}: '{name}' must be true or false") + for f in schema.fields: + if f.check is not None and f.name in e: + problem = f.check(e[f.name]) + if problem: + errors.append(f"{label}: {problem}") + return errors + + +def parse_registry(entries: list, schema: RegistrySchema[S]) -> dict[str, S]: + """Validate every entry and build the sources; all problems are raised together in one RegistryError.""" + errors: list[str] = [] + sources: dict[str, S] = {} + for i, e in enumerate(entries): + if not isinstance(e, dict): + errors.append(f"#{i}: entry must be a mapping") + continue + label = str(e.get("id") or f"#{i}") + problems = _entry_errors(e, label, schema) + if problems: + errors.extend(problems) + continue + if e["id"] in sources: + errors.append(f"{label}: duplicate id") + continue + values = {name: e[name] for name in REQUIRED} + values.update({name: e.get(name, default) for name, default in OPTIONAL.items()}) + values.update({f.name: e.get(f.name, f.default) for f in schema.fields}) + sources[e["id"]] = schema.factory(values) + for check in schema.checks: + errors.extend(check(sources)) + if errors: + raise RegistryError("\n".join(errors)) + return sources diff --git a/src/datasettool/sources/spreadsheet.py b/src/datasettool/sources/spreadsheet.py new file mode 100644 index 0000000..b5d3cf8 --- /dev/null +++ b/src/datasettool/sources/spreadsheet.py @@ -0,0 +1,506 @@ +"""XLS and XLSX reader for source adapters (spec 004, R3; ADR 0005). + +The format is detected from the first bytes. XLSX is streamed with the standard library; +XLS is read with xlrd. Both return values of the same types: str, int, float, bool or None. + +Ported from BenchCommons/dataset-names@e7e8405 src/namescommons/spreadsheet.py, with the encoding +and expat-level DTD guards of BenchCommons/dataset-names@eaaa4d1 (PR #39). +""" + +import codecs +import io +import posixpath +import re +import xml.etree.ElementTree as ET +import xml.parsers.expat as expat +import zipfile +import zlib +from collections.abc import Iterable, Iterator +from pathlib import Path + +import xlrd + +from datasettool.sources.adapters import FormatError, parse_count + +_ZIP = b"PK\x03\x04" +_OLE2 = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + +_MAIN = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" +_REL_DOC = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}" +_REL_PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}" +_ROW, _C, _V, _IS, _T, _R, _SI, _SHEET, _SHEET_DATA = ( + _MAIN + n for n in ("row", "c", "v", "is", "t", "r", "si", "sheet", "sheetData") +) +_REF = re.compile(r"([A-Z]{1,3})([0-9]+)") +_MAX_COLUMNS = 16384 # XFD, Excel's last column +# Guard against disproportionate ZIPs: a member above 1 GiB, or above 1 MiB with a ratio above 100:1. +_MAX_MEMBER = 1 << 30 +_MAX_RATIO = 100 +_MIN_BYTES_RATIO = 1 << 20 +_INTEGER = re.compile(r"-?[0-9]+") +# Defence in depth (ADR 0005): an XML member with a DTD is refused before it reaches the parser, +# and the parser itself refuses one (_events). +_MAX_PROLOGUE = 1 << 16 +_SPACE = re.compile(r"[\s\ufeff]*") +_DECLARATION = re.compile(r"<\?xml(?=\s)(.*?)\?>", re.DOTALL) +_ENCODING = re.compile(r"""\sencoding\s*=\s*(?:"([^"]*)"|'([^']*)')""") +_ENCODING_OK = re.compile(r"utf-?(?:8|16)", re.IGNORECASE) +_BLOCK_SIZE = 1 << 16 + + +def _format(path: Path) -> str: + with open(path, "rb") as fh: + start = fh.read(8) + if start.startswith(_ZIP): + return "xlsx" + if start == _OLE2: + return "xls" + raise FormatError(f"{path.name}: not an XLS or XLSX spreadsheet (starts with {start!r})") + + +def sheet_names(path: str | Path) -> list[str]: + """Sheets in file order.""" + path = Path(path) + if _format(path) == "xls": + return _xls_sheet_names(path) + with _open_zip(path) as z: + return [name for name, _ in _xlsx_sheets(z, path)] + + +def rows(path: str | Path, sheet: str) -> Iterator[tuple[int, list]]: + """(Excel row number, values by column) of each non-empty row of the sheet.""" + path = Path(path) + if _format(path) == "xls": + yield from _xls_rows(path, sheet) + else: + yield from _xlsx_rows(path, sheet) + + +def cell_count(value: object, missing: Iterable[str] = (), field: str = "count") -> int | None: + """Count in a cell (R3.6). A marker from `missing` becomes None; anything else is FormatError.""" + markers = set(missing) + if value is None: + if "" in markers: + return None + raise FormatError(f"{field} empty; expected an integer or a known marker {sorted(markers)}") + if isinstance(value, bool): + raise FormatError(f"{field} {value!r} is not a number") + if isinstance(value, float): + if not value.is_integer(): + raise FormatError(f"{field} {value!r} is not an integer") + value = int(value) + if isinstance(value, int): + if value < 0: + raise FormatError(f"{field} {value!r} is negative") + return value + if isinstance(value, str): + return parse_count(value, markers, field) + raise FormatError(f"{field} {value!r} has unexpected type {type(value).__name__}") + + +def _drop_trailing_none(values: list) -> list: + while values and values[-1] is None: + values.pop() + return values + + +# XLSX --------------------------------------------------------------------------------------------- + +def _open_zip(path: Path) -> zipfile.ZipFile: + try: + z = zipfile.ZipFile(path) + except zipfile.BadZipFile as e: + raise FormatError(f"{path.name}: corrupt XLSX: {e}") from None + if "xl/workbook.xml" not in z.namelist(): + z.close() + raise FormatError(f"{path.name}: ZIP without xl/workbook.xml; not an XLSX") + return z + + +def _member(z: zipfile.ZipFile, name: str, path: Path): + """Open a ZIP member after checking its declared uncompressed size.""" + try: + info = z.getinfo(name) + except KeyError: + raise FormatError(f"{path.name}: member {name} missing from the ZIP") from None + size, compressed = info.file_size, max(info.compress_size, 1) + if size > _MAX_MEMBER: + raise FormatError(f"{path.name}: member {name} inflates to {size} bytes, above the limit of {_MAX_MEMBER}") + if size > _MIN_BYTES_RATIO and size / compressed > _MAX_RATIO: + raise FormatError(f"{path.name}: member {name} inflates to {size} bytes, " + f"compression ratio {size // compressed}:1 above {_MAX_RATIO}:1") + try: + fh = z.open(info) + except _CORRUPT as e: + raise FormatError(f"{path.name}: member {name} is corrupt: {e}") from None + return _NoDtd(fh, name, path) + + +# What zipfile raises on a damaged member while reading it: bad CRC, bad deflate data, truncation. +_CORRUPT = (zipfile.BadZipFile, zlib.error, EOFError) + + +class _NoDtd(io.RawIOBase): + """Stream of an XML member that refuses a DTD in the prologue, then passes the bytes to the parser. + + It reads at most 64 KiB, up to the first element tag, and hands that prefix to the parser before + the rest of the stream; streaming of large sheets is unchanged. + """ + + def __init__(self, fh, name: str, path: Path): + super().__init__() + self._fh = fh + self._where = f"{path.name}: member {name}" + prefix = b"" + while True: + try: + part = fh.read(4096) + except _CORRUPT as e: + fh.close() + raise FormatError(f"{self._where} is corrupt: {e}") from None + prefix += part + try: + decision = _prologue(prefix, f"{path.name}: member {name}") + except FormatError: + fh.close() + raise + if decision == "dtd": + fh.close() + raise FormatError(f"{path.name}: member {name} has a DTD (DOCTYPE or ENTITY); refused") + if decision == "root" or not part: + break + if len(prefix) >= _MAX_PROLOGUE: + fh.close() + raise FormatError(f"{path.name}: member {name} has a prologue above {_MAX_PROLOGUE} bytes without a root element") + self._prefix = memoryview(prefix) + + def readable(self) -> bool: + return True + + def readinto(self, buf) -> int: + if self._prefix: + n = min(len(buf), len(self._prefix)) + buf[:n] = self._prefix[:n] + self._prefix = self._prefix[n:] + return n + try: + return self._fh.readinto(buf) + except _CORRUPT as e: + raise FormatError(f"{self._where} is corrupt: {e}") from None + + def close(self) -> None: + self._fh.close() + super().close() + + +def _prefix_text(data: bytes, where: str) -> str: + """Decode the prefix as UTF-8 or UTF-16 (by BOM or NUL bytes); anything else is FormatError. + + expat accepts any declared encoding it can map (single-byte codecs such as ISO-8859-x, and others), + and a DOCTYPE written that way would slip past a scan that reads only UTF-8; so the rest is refused. + """ + if data[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff") or data[:4] in (b"<\x00\x00\x00", b"\x00\x00\x00<"): + raise FormatError(f"{where} is in UTF-32; only UTF-8 and UTF-16 are accepted") + if data[:2] in (b"\xff\xfe", b"\xfe\xff"): + codec = "utf-16" + elif data[:4] == b"<\x00?\x00": + codec = "utf-16-le" + elif data[:4] == b"\x00<\x00?": + codec = "utf-16-be" + elif b"\x00" in data[:4]: + raise FormatError(f"{where} starts with NUL bytes that are not UTF-16; only UTF-8 and UTF-16 are accepted") + else: + codec = "utf-8-sig" + try: + # final=False: the prefix may end in the middle of a multi-byte character. + return codecs.getincrementaldecoder(codec)("strict").decode(data, final=False) + except UnicodeDecodeError: + raise FormatError(f"{where} is not valid UTF-8 or UTF-16; only these encodings are accepted") from None + + +def _check_declaration(text: str, where: str) -> None: + """The declaration `` may only say UTF-8 or UTF-16 (or say nothing).""" + m = _DECLARATION.match(text) + if m and (e := _ENCODING.search(m.group(0))): + name = e.group(1) if e.group(1) is not None else e.group(2) + if not _ENCODING_OK.fullmatch(name): + raise FormatError(f"{where} declares the encoding {name!r}; only UTF-8 and UTF-16 are accepted") + + +def _prologue(data: bytes, where: str = "XML member") -> str: + """'dtd' if DOCTYPE/ENTITY comes before the root element, 'root' if the root has started, else 'more'. + + FormatError if the prefix is not UTF-8 or UTF-16, or if the XML declaration asks for another encoding. + """ + text = _prefix_text(data, where) + _check_declaration(text.lstrip("\ufeff"), where) + i = 0 + while True: + i = _SPACE.match(text, i).end() + if i >= len(text): + return "more" + if text.startswith("", i), 2 + elif text.startswith("", i), 3 + elif text[i:i + 9].upper() == " Iterator[tuple[str, ET.Element]]: + """Events ("start"/"end", element) of a member's XML, streamed, on an expat parser that refuses DTDs. + + Defence in depth, independent of the prologue pre-scan (ADR 0005): expat itself raises + FormatError when it sees DOCTYPE or ENTITY, in any encoding it accepts. The C `ET.XMLParser` + does not expose expat, so expat is used directly, with `ET.TreeBuilder`. + A syntax error becomes `ET.ParseError`, as with `iterparse`. + """ + def refuse(*_args): + raise FormatError(f"{path.name}: member {name} has a DTD (DOCTYPE or ENTITY); refused") + + builder = ET.TreeBuilder() + pending: list[tuple[str, ET.Element]] = [] + want_start, want_end = "start" in events, "end" in events + + def start(tag, attributes): + el = builder.start(tag, attributes) + if want_start: + pending.append(("start", el)) + + def end(tag): + el = builder.end(tag) + if want_end: + pending.append(("end", el)) + + parser = expat.ParserCreate(namespace_separator="}") + parser.buffer_text = True + parser.StartDoctypeDeclHandler = refuse + parser.EntityDeclHandler = refuse + parser.StartElementHandler = lambda tag, attrs: start(_tag(tag), {_tag(k): v for k, v in attrs.items()}) + parser.EndElementHandler = lambda tag: end(_tag(tag)) + parser.CharacterDataHandler = builder.data + try: + while True: + block = fh.read(_BLOCK_SIZE) + parser.Parse(block, not block) + yield from pending + pending.clear() + if not block: + break + except expat.ExpatError as e: + raise ET.ParseError(str(e)) from None + + +def _tag(name: str) -> str: + """expat's 'uri}local' becomes '{uri}local' (ElementTree's qualified name).""" + return "{" + name if "}" in name else name + + +def _root(fh, name: str, path: Path) -> ET.Element: + """Root element of a member's XML (`ET.parse(...).getroot()` with DTDs refused).""" + root = None + for _, el in _events(fh, name, path): + root = el + return root + + +def _xlsx_sheets(z: zipfile.ZipFile, path: Path) -> list[tuple[str, str]]: + """(name, ZIP member) of each sheet, in the order of xl/workbook.xml.""" + try: + with _member(z, "xl/workbook.xml", path) as fh: + book = _root(fh, "xl/workbook.xml", path) + with _member(z, "xl/_rels/workbook.xml.rels", path) as fh: + rels = _root(fh, "xl/_rels/workbook.xml.rels", path) + except ET.ParseError as e: + raise FormatError(f"{path.name}: unreadable XLSX workbook: {e}") from None + targets = {r.get("Id"): r.get("Target", "") for r in rels.iter(_REL_PKG + "Relationship")} + sheets = [] + for sheet in book.iter(_SHEET): + target = targets.get(sheet.get(_REL_DOC + "id")) + if target is None: + raise FormatError(f"{path.name}: sheet {sheet.get('name')!r} has no relationship in workbook.xml.rels") + member = target.lstrip("/") if target.startswith("/") else posixpath.normpath(posixpath.join("xl", target)) + sheets.append((sheet.get("name"), member)) + return sheets + + +def _text(element: ET.Element) -> str: + """Text of or : direct and those of runs; the phonetic reading is left out.""" + parts = [] + for child in element: + if child.tag == _T: + parts.append(child.text or "") + elif child.tag == _R: + parts.extend(t.text or "" for t in child.iter(_T)) + return "".join(parts) + + +def _shared_strings(z: zipfile.ZipFile, path: Path) -> list[str]: + if "xl/sharedStrings.xml" not in z.namelist(): + return [] + strings = [] + with _member(z, "xl/sharedStrings.xml", path) as fh: + try: + for _, el in _events(fh, "xl/sharedStrings.xml", path): + if el.tag == _SI: + strings.append(_text(el)) + el.clear() + except ET.ParseError as e: + raise FormatError(f"{path.name}: unreadable sharedStrings.xml: {e}") from None + return strings + + +def _column(ref: str, where: str) -> int: + """Column index (A = 0) of a reference such as 'C7'.""" + m = _REF.fullmatch(ref) + if not m: + raise FormatError(f"{where}: invalid cell reference {ref!r}") + n = 0 + for letter in m.group(1): + n = n * 26 + ord(letter) - ord("A") + 1 + if n > _MAX_COLUMNS: + raise FormatError(f"{where}, cell {ref}: column beyond XFD") + return n - 1 + + +def _number(text: str, where: str) -> int | float: + text = text.strip() + if _INTEGER.fullmatch(text): + return int(text) + try: + value = float(text) + except ValueError: + raise FormatError(f"{where}: invalid number {text!r}") from None + return int(value) if value.is_integer() else value + + +def _value(c: ET.Element, strings: list[str], where: str): + kind = c.get("t", "n") + v = c.find(_V) + if kind == "inlineStr": + block = c.find(_IS) + return None if block is None else _text(block) + if v is None or v.text is None: + return "" if kind == "str" and v is not None else None + if kind == "s": + try: + index = int(v.text) + except ValueError: + raise FormatError(f"{where}: shared string index {v.text!r} is not an integer") from None + if not 0 <= index < len(strings): + raise FormatError(f"{where}: shared string {index} outside the table of {len(strings)}") + return strings[index] + if kind == "str": + return v.text + if kind == "n": + return _number(v.text, where) + if kind == "b": + return v.text.strip() == "1" + if kind == "e": + raise FormatError(f"{where}: formula error {v.text!r}") + raise FormatError(f"{where}: unknown type {kind!r}") + + +def _xlsx_rows(path: Path, sheet: str) -> Iterator[tuple[int, list]]: + with _open_zip(path) as z: + sheets = _xlsx_sheets(z, path) + member = dict(sheets).get(sheet) + if member is None: + raise FormatError(f"{path.name}: sheet {sheet!r} does not exist; sheets: {[n for n, _ in sheets]}") + strings = _shared_strings(z, path) + prefix = f"{path.name}, sheet {sheet!r}" + if member not in z.namelist(): + raise FormatError(f"{path.name}: sheet {sheet!r} points to {member}, missing from the ZIP") + with _member(z, member, path) as fh: + number, parent = 0, None + try: + for event, el in _events(fh, member, path, ("start", "end")): + if event == "start": + if el.tag == _SHEET_DATA: + parent = el + continue + if el.tag != _ROW: + continue + r = el.get("r") + if r is None: + number += 1 + elif r.isdigit(): + number = int(r) + else: + raise FormatError(f"{prefix}: invalid row {r!r}") + values: list = [] + for c in el.iter(_C): + ref = c.get("r") + position = len(values) if ref is None else _column(ref, prefix) + values.extend([None] * (position - len(values) + 1)) + values[position] = _value(c, strings, f"{prefix}, cell {ref or position}") + # Remove the row just read from the tree: memory stays flat over the whole sheet. + el.clear() + if parent is not None: + parent.remove(el) + if _drop_trailing_none(values): + yield number, values + except ET.ParseError as e: + raise FormatError(f"{prefix}: unreadable XML: {e}") from None + + +# XLS ---------------------------------------------------------------------------------------------- + +def _xls_open(path: Path): + try: + return xlrd.open_workbook(path, on_demand=True) + except Exception as e: # xlrd raises XLRDError, CompDocError, AssertionError and others + raise FormatError(f"{path.name}: unreadable XLS: {type(e).__name__}: {e}") from None + + +def _xls_sheet_names(path: Path) -> list[str]: + book = _xls_open(path) + try: + return book.sheet_names() + finally: + book.release_resources() + + +def _xls_value(cell, where: str): + kind = cell.ctype + if kind in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + return None + if kind == xlrd.XL_CELL_TEXT: + return cell.value + if kind in (xlrd.XL_CELL_NUMBER, xlrd.XL_CELL_DATE): + value = float(cell.value) + return int(value) if value.is_integer() else value + if kind == xlrd.XL_CELL_BOOLEAN: + return bool(cell.value) + if kind == xlrd.XL_CELL_ERROR: + text = xlrd.error_text_from_code.get(cell.value, f"code {cell.value}") + raise FormatError(f"{where}: formula error {text!r}") + raise FormatError(f"{where}: unknown cell type {kind}") + + +def _xls_rows(path: Path, sheet: str) -> Iterator[tuple[int, list]]: + book = _xls_open(path) + try: + if sheet not in book.sheet_names(): + raise FormatError(f"{path.name}: sheet {sheet!r} does not exist; sheets: {book.sheet_names()}") + try: + ws = book.sheet_by_name(sheet) + except Exception as e: + raise FormatError(f"{path.name}, sheet {sheet!r}: unreadable XLS: {type(e).__name__}: {e}") from None + prefix = f"{path.name}, sheet {sheet!r}" + for i in range(ws.nrows): + values = [ + _xls_value(cell, f"{prefix}, cell {xlrd.cellname(i, j)}") + for j, cell in enumerate(ws.row(i)) + ] + if _drop_trailing_none(values): + yield i + 1, values + finally: + book.release_resources() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3a2843d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,120 @@ +"""Local HTTP server for download tests (spec 004, R7.3): tests never use the network. + +Ported from BenchCommons/dataset-names@e7e8405 tests/conftest.py (ServidorLocal). +""" + +import socket +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +_LOOPBACK = {"127.0.0.1", "::1", "localhost"} + + +class NetworkAccessError(RuntimeError): + """A test tried to reach a host other than the loopback (spec 004, R7.3).""" + + +def _host(address) -> str: + return address[0] if isinstance(address, tuple) else str(address) + + +@pytest.fixture(autouse=True) +def no_network(monkeypatch): + """Refuse every connection and name lookup outside the loopback, in every test.""" + connect, connect_ex, getaddrinfo = socket.socket.connect, socket.socket.connect_ex, socket.getaddrinfo + + def guard(address): + # str or bytes addresses are AF_UNIX sockets, local by definition. + if not isinstance(address, (str, bytes)) and _host(address) not in _LOOPBACK: + raise NetworkAccessError(f"network access attempted: {address!r}") + + def guarded_connect(self, address): + guard(address) + return connect(self, address) + + def guarded_connect_ex(self, address): + guard(address) + return connect_ex(self, address) + + def guarded_getaddrinfo(host, *args, **kwargs): + if host is not None and (host.decode() if isinstance(host, bytes) else host) not in _LOOPBACK: + raise NetworkAccessError(f"name lookup attempted: {host!r}") + return getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket.socket, "connect", guarded_connect) + monkeypatch.setattr(socket.socket, "connect_ex", guarded_connect_ex) + monkeypatch.setattr(socket, "getaddrinfo", guarded_getaddrinfo) + + +class LocalServer: + """HTTP server on 127.0.0.1 with responses programmable per path.""" + + def __init__(self): + self.files: dict[str, bytes] = {} + self.failures: dict[str, int] = {} # path -> how many 500 responses before serving + self.truncated: set[str] = set() # paths that announce twice their size and close + self.slow: dict[str, float] = {} # path -> seconds to wait before answering + self.refused: set[str] = set() # paths answering 403, like a server that blocks robots + self.content_types: dict[str, str] = {} # path -> Content-Type sent with the response + self.responses: dict[str, list[int]] = {} # path -> HTTP codes served in sequence before serving + self.content_length: dict[str, str] = {} # path -> literal Content-Length (for malformed values) + self.requests: list[tuple[str, str]] = [] # (path, User-Agent) + server = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + server.requests.append((self.path, self.headers.get("User-Agent", ""))) + if self.path in server.refused: + self.send_error(403) + return + if server.responses.get(self.path): + self.send_error(server.responses[self.path].pop(0)) + return + if server.failures.get(self.path, 0) > 0: + server.failures[self.path] -= 1 + self.send_error(500) + return + time.sleep(server.slow.get(self.path, 0)) + body = server.files.get(self.path) + if body is None: + self.send_error(404) + return + announced = len(body) * 2 if self.path in server.truncated else len(body) + self.send_response(200) + self.send_header("Content-Length", server.content_length.get(self.path, str(announced))) + if self.path in server.content_types: + self.send_header("Content-Type", server.content_types[self.path]) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + class Server(ThreadingHTTPServer): + def handle_error(self, request, client_address): + # A client that gives up (timeout) closes the connection; that is not a test failure. + if isinstance(sys.exc_info()[1], (BrokenPipeError, ConnectionResetError)): + return + super().handle_error(request, client_address) + + self.httpd = Server(("127.0.0.1", 0), Handler) + + def url(self, path: str) -> str: + return f"http://127.0.0.1:{self.httpd.server_port}{path}" + + def paths(self) -> list[str]: + return [path for path, _ in self.requests] + + +@pytest.fixture +def server(): + s = LocalServer() + thread = threading.Thread(target=s.httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + yield s + s.httpd.shutdown() + s.httpd.server_close() diff --git a/tests/example_adapters/__init__.py b/tests/example_adapters/__init__.py new file mode 100644 index 0000000..d272148 --- /dev/null +++ b/tests/example_adapters/__init__.py @@ -0,0 +1 @@ +"""Test-only adapter package (spec 004, R4.1).""" diff --git a/tests/example_adapters/example.py b/tests/example_adapters/example.py new file mode 100644 index 0000000..16e8aff --- /dev/null +++ b/tests/example_adapters/example.py @@ -0,0 +1,30 @@ +"""Test-only adapter, adapted from BenchCommons/dataset-names@e7e8405 tests/adaptadores/exemplo.py. + +It yields its own record type, as every dataset does (spec 004, R4.1). +""" + +import csv +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +from datasettool.sources.adapters import Resource + +SOURCE_ID = "example" +RESOURCES = [ + Resource("https://example.org/example/given.csv", "given.csv"), + Resource("https://example.org/example/family.csv", "family.csv"), +] + + +@dataclass(frozen=True) +class Row: + name: str + kind: str + + +def parse(files: dict[str, Path]) -> Iterator[Row]: + for filename, kind in (("given.csv", "given"), ("family.csv", "family")): + with open(files[filename], encoding="utf-8", newline="") as fh: + for line in csv.DictReader(fh): + yield Row(line["name"], kind) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..1bdd177 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,5 @@ +# Test fixtures + +| File | Origin | +|---|---| +| `spreadsheets/types.xls` | Synthetic XLS for the spreadsheet reader tests, byte-identical copy of `tests/fixtures/planilhas/tipos.xls` in [BenchCommons/dataset-names@e7e8405](https://github.com/BenchCommons/dataset-names/tree/e7e84054d193948e2e410b4903d3aeaa8b52b4ca/tests/fixtures/planilhas) (Apache-2.0). Sheets `Dados` and `Erros`. | diff --git a/tests/fixtures/spreadsheets/types.xls b/tests/fixtures/spreadsheets/types.xls new file mode 100644 index 0000000..184f65e Binary files /dev/null and b/tests/fixtures/spreadsheets/types.xls differ diff --git a/tests/names_registry.py b/tests/names_registry.py new file mode 100644 index 0000000..b5c30e9 --- /dev/null +++ b/tests/names_registry.py @@ -0,0 +1,85 @@ +"""dataset-names' registry rules expressed with the public registry API (spec 004, acceptance criterion 3). + +Mirrors BenchCommons/dataset-names@e7e8405 src/namescommons/sources.py. The only difference is the +country check: dataset-names uses pycountry, this test schema uses a fixed set of codes. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date + +from datasettool.sources.registry import ATTRIBUTION_ONLY_LICENSES, Field, RegistrySchema + +COUNTRIES = frozenset({"BR", "CA", "ES", "FR", "IE", "NO", "NZ", "PT", "US"}) +COUNT_TYPES = ("births", "population", "attested") + + +@dataclass(frozen=True) +class NamesSource: + id: str + name: str + url: str + license: str + attribution: str + country: str | None + count_type: str + primary: bool + suppression: int | None + retrieved: date + uniform_case: bool = False + adapter: bool = True + + +def _country(value): + if isinstance(value, bool): + return f"country read as boolean {value}; quote the code, e.g. \"NO\"" + if value is not None and not (isinstance(value, str) and value in COUNTRIES): + return f"country '{value}' is not ISO 3166-1 alpha-2" + return None + + +def _count_type(value): + return None if value in COUNT_TYPES else f"count_type '{value}' is invalid" + + +def _primary(value): + return None if isinstance(value, bool) else "'primary' must be true or false" + + +def _suppression(value): + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 1): + return "'suppression' must be a positive integer or empty" + return None + + +def _uniform_case(value): + return None if isinstance(value, bool) else "'uniform_case' must be true or false" + + +def one_primary(sources: Mapping[str, NamesSource]) -> list[str]: + """Exactly one primary per (country, count_type) with counts; attested is not checked.""" + groups: dict[tuple, list[NamesSource]] = {} + for s in sources.values(): + if s.count_type == "attested": + continue + groups.setdefault((s.country, s.count_type), []).append(s) + errors = [] + for (country, kind), group in sorted(groups.items(), key=lambda kv: (str(kv[0][0]), kv[0][1])): + if len(group) > 1 and sum(s.primary is True for s in group) != 1: + errors.append(f"{country}/{kind}: sources {sorted(s.id for s in group)}; exactly one must have primary: true") + return errors + + +SCHEMA = RegistrySchema( + accepted_licenses=ATTRIBUTION_ONLY_LICENSES | {"statcan-open-licence"}, + license_policy="ADR 0001", + fields=( + Field("country", _country), + Field("count_type", _count_type), + Field("primary", _primary), + Field("suppression", _suppression), + Field("uniform_case", _uniform_case, required=False, default=False), + ), + checks=(one_primary,), + factory=lambda entry: NamesSource(**entry), +) diff --git a/tests/test_adapters.py b/tests/test_adapters.py new file mode 100644 index 0000000..50e7c67 --- /dev/null +++ b/tests/test_adapters.py @@ -0,0 +1,197 @@ +"""Spec 004, R4: adapter base. + +Ported from BenchCommons/dataset-names@e7e8405 tests/test_adapters.py and +tests/test_contrato_adaptador.py (spec 002 R1, spec 003 R2.2 and R2.6 of dataset-names). +""" + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from datasettool.sources.adapters import ( + AdapterError, FormatError, Resource, SourceFiles, check_adapter, is_http_url, load_adapter, + module_name, parse_code, parse_count, +) + + +def _parse(files): + return iter(()) + + +def _adapter(**fields): + base = dict(SOURCE_ID="x", RESOURCES=[Resource("https://example.org/a.csv", "a.csv")], parse=_parse) + return SimpleNamespace(**{**base, **fields}) + + +def test_module_name(): + assert module_name("us-ssa-births") == "us_ssa_births" + + +def test_loads_the_example_adapter(): + mod = load_adapter("example", "example_adapters") + assert mod.SOURCE_ID == "example" + assert [r.filename for r in mod.RESOURCES] == ["given.csv", "family.csv"] + + +def test_dataset_defines_its_own_record_type(tmp_path): + mod = load_adapter("example", "example_adapters") + (tmp_path / "given.csv").write_text("name\nMARIA\n", encoding="utf-8") + (tmp_path / "family.csv").write_text("name\nSILVA\n", encoding="utf-8") + files = SourceFiles({r.filename: tmp_path / r.filename for r in mod.RESOURCES}) + assert list(mod.parse(files)) == [mod.Row("MARIA", "given"), mod.Row("SILVA", "family")] + + +def test_missing_adapter(): + with pytest.raises(AdapterError, match="example_adapters.missing_adapter"): + load_adapter("missing-adapter", "example_adapters") + + +def test_valid_adapter(): + check_adapter(_adapter(), "x") + + +@pytest.mark.parametrize("fields, fragment", [ + ({"SOURCE_ID": "y"}, "SOURCE_ID 'y' differs"), + ({"RESOURCES": []}, "RESOURCES"), + ({"RESOURCES": [("https://example.org/a.csv", "a.csv")]}, "RESOURCES"), + ({"RESOURCES": [Resource("ftp://example.org/a.csv", "a.csv")]}, "http or https"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "../a.csv")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a.csv.meta.json")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a.csv"), + Resource("https://example.org/b.csv", "a.csv")]}, "repeated"), + ({"parse": None}, "parse"), + ({"RESOURCES": [Resource("http://[::1/a.csv", "a.csv")]}, "'x': url 'http://\\[::1/a.csv' must use http or https"), + ({"RESOURCES": [Resource(" https://example.org/a.csv", "a.csv")]}, "http or https"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a.csv.part")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a\x00.csv")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a\n.csv")]}, "filename"), + ({"RESOURCES": [Resource("https://example.org/a.csv", "a\x7f.csv")]}, "filename"), +]) +def test_invalid_adapter(fields, fragment): + with pytest.raises(AdapterError, match=fragment): + check_adapter(_adapter(**fields), "x") + + +def test_adapter_without_attribute(): + with pytest.raises(AdapterError, match="SOURCE_ID"): + check_adapter(SimpleNamespace(RESOURCES=[], parse=_parse), "x") + + +# Optional functions of a dataset's own protocol (dataset-names: links) ------------------------------ + +def test_optional_function_absent_is_fine(): + check_adapter(_adapter(), "x", optional=("links",)) + + +def test_optional_function_present_must_be_callable(): + check_adapter(_adapter(links=_parse), "x", optional=("links",)) + with pytest.raises(AdapterError, match="'x': links must be a function"): + check_adapter(_adapter(links=None), "x", optional=("links",)) + + +def test_names_not_declared_optional_are_not_checked(): + check_adapter(_adapter(links=None), "x") + + +# URL rule shared with the registry ---------------------------------------------------------------- + +@pytest.mark.parametrize("url, expected", [ + ("https://example.org/a", True), ("HTTP://example.org/", True), ("ftp://example.org/a", False), + ("example.org/a", False), ("https://", False), ("http://[::1", False), (None, False), ("https://a b/", False), +]) +def test_is_http_url(url, expected): + assert is_http_url(url) is expected + + +# parse_count and parse_code (dataset-names spec 003, R2.2) ----------------------------------------- + +@pytest.mark.parametrize("text, expected", [("5", 5), (" 1532 ", 1532), ("0", 0), ("<5", None), ("..", None)]) +def test_parse_count(text, expected): + assert parse_count(text, {"<5", ".."}) == expected + + +@pytest.mark.parametrize("text", ["", "1.5", "-3", "x", "<3", "١٢"]) +def test_parse_count_refuses_what_it_does_not_know(text): + with pytest.raises(FormatError, match="count"): + parse_count(text, {"<5"}) + + +def test_parse_count_names_the_field_and_markers(): + with pytest.raises(FormatError, match=r"year 'x' is neither an integer nor a known marker \['<5'\]"): + parse_count("x", {"<5"}, field="year") + + +def test_parse_code_maps_and_refuses_unknown_code(): + sex = {"1": "M", "2": "F"} + assert parse_code(" 2 ", sex, "sex") == "F" + with pytest.raises(FormatError, match=r"sex 'M' unknown; expected one of \['1', '2'\]"): + parse_code("M", sex, "sex") + + +# Error messages (dataset-names spec 003, R2.6) ------------------------------------------------------ + +def test_file_outside_resources_lists_the_available_ones(): + files = SourceFiles({"a.csv": Path("a.csv")}) + with pytest.raises(AdapterError, match=r"parse asked for file 'b.csv', outside RESOURCES; available: \['a.csv'\]"): + files["b.csv"] + + +def test_missing_adapter_shows_the_expected_path(): + with pytest.raises(AdapterError) as e: + load_adapter("missing-adapter", "example_adapters") + message = str(e.value) + assert "adapter example_adapters.missing_adapter absent" in message + assert str(Path(__file__).parent / "example_adapters" / "missing_adapter.py") in message + + +def test_broken_adapter_is_not_reported_as_absent(tmp_path, monkeypatch): + package = tmp_path / "quebrados" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "fonte_x.py").write_text("import modulo_que_nao_existe\n", encoding="utf-8") + (package / "fonte_y.py").write_text("1/0\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + with pytest.raises(AdapterError, match="adapter quebrados.fonte_x broken: ModuleNotFoundError: .*modulo_que_nao_existe"): + load_adapter("fonte-x", "quebrados") + with pytest.raises(AdapterError, match="adapter quebrados.fonte_y broken: ZeroDivisionError"): + load_adapter("fonte-y", "quebrados") + finally: + for name in ("quebrados", "quebrados.fonte_x", "quebrados.fonte_y"): + sys.modules.pop(name, None) + + +def test_missing_adapter_package(): + with pytest.raises(AdapterError, match="package 'nao_ha_pacote' not found"): + load_adapter("x", "nao_ha_pacote") + + +def test_load_adapter_applies_optional_checks(tmp_path, monkeypatch): + package = tmp_path / "com_links" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "fonte.py").write_text( + "from datasettool.sources.adapters import Resource\n" + "SOURCE_ID = 'fonte'\nRESOURCES = [Resource('https://example.org/a.csv', 'a.csv')]\n" + "def parse(files):\n return iter(())\nlinks = 3\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + assert load_adapter("fonte", "com_links").SOURCE_ID == "fonte" + with pytest.raises(AdapterError, match="links must be a function"): + load_adapter("fonte", "com_links", optional=("links",)) + finally: + for name in ("com_links", "com_links.fonte"): + sys.modules.pop(name, None) + + +def test_optional_given_as_a_string_is_refused(): + # A str would be iterated character by character: "links" -> "l", "i", ... + with pytest.raises(TypeError, match="optional"): + check_adapter(_adapter(links=None), "x", optional="links") + with pytest.raises(TypeError, match="optional"): + load_adapter("example", "example_adapters", optional="links") diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..9aec790 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,284 @@ +"""Spec 004, R2: download cache, tested against a local HTTP server. + +Ported from BenchCommons/dataset-names@e7e8405 tests/test_cache.py (spec 002 R4, spec 003 R4, +spec 003e R4.1 of dataset-names). Test vectors unchanged; messages in English. +""" + +import hashlib +import json +from datetime import datetime + +import pytest + +import datasettool +from datasettool.sources.adapters import Resource +from datasettool.sources.cache import ( + DEFAULT_USER_AGENT, BlockedError, FetchError, cached_path, fetch_resource, make_user_agent, +) + +BODY = "nome,sexo,ano,quantidade\nMARIA,F,2020,100\n".encode() + + +def _resource(server, path="/a.csv", body=BODY): + server.files[path] = body + return Resource(server.url(path), path.lstrip("/")) + + +def test_downloads_and_writes_metadata(server, tmp_path): + dest = fetch_resource(tmp_path, "exemplo", _resource(server)) + assert dest == tmp_path / "exemplo" / "a.csv" == cached_path(tmp_path, "exemplo", "a.csv") + assert dest.read_bytes() == BODY + meta = json.loads((tmp_path / "exemplo" / "a.csv.meta.json").read_text(encoding="utf-8")) + assert set(meta) == {"url", "downloaded_at", "sha256", "size"} + assert meta["url"] == server.url("/a.csv") + assert meta["sha256"] == hashlib.sha256(BODY).hexdigest() + assert meta["size"] == len(BODY) + assert datetime.fromisoformat(meta["downloaded_at"]).tzinfo is not None + + +def test_default_user_agent_identifies_dataset_tool(server, tmp_path): + fetch_resource(tmp_path, "exemplo", _resource(server)) + agent = server.requests[0][1] + assert agent == DEFAULT_USER_AGENT + assert agent == f"tools-dataset/{datasettool.__version__} (+https://github.com/BenchCommons/tools-dataset)" + + +def test_user_agent_identifies_the_dataset_and_its_repository(server, tmp_path): + # dataset-names' header, byte for byte: namescommons/ (+). + agent = make_user_agent("namescommons", "0.1.0", "https://github.com/BenchCommons/dataset-names") + fetch_resource(tmp_path, "exemplo", _resource(server), user_agent=agent) + sent = server.requests[0][1] + assert sent == "namescommons/0.1.0 (+https://github.com/BenchCommons/dataset-names)" + assert sent.startswith("namescommons/") + assert "https://github.com/BenchCommons/dataset-names" in sent + + +def test_cached_file_is_not_downloaded_again(server, tmp_path): + r = _resource(server) + fetch_resource(tmp_path, "exemplo", r) + fetch_resource(tmp_path, "exemplo", r) + assert server.paths() == ["/a.csv"] + + +def test_refresh_downloads_again(server, tmp_path): + r = _resource(server) + fetch_resource(tmp_path, "exemplo", r) + server.files["/a.csv"] = b"novo\n" + dest = fetch_resource(tmp_path, "exemplo", r, refresh=True) + assert server.paths() == ["/a.csv", "/a.csv"] + assert dest.read_bytes() == b"novo\n" + meta = json.loads((tmp_path / "exemplo" / "a.csv.meta.json").read_text(encoding="utf-8")) + assert meta["size"] == 5 + + +def test_offline_uses_the_cache_without_network(server, tmp_path): + r = _resource(server) + fetch_resource(tmp_path, "exemplo", r) + assert fetch_resource(tmp_path, "exemplo", r, offline=True).read_bytes() == BODY + assert server.paths() == ["/a.csv"] + + +def test_offline_without_cache_fails_without_network(server, tmp_path): + with pytest.raises(FetchError, match="a.csv missing from the cache and --offline"): + fetch_resource(tmp_path, "exemplo", _resource(server), offline=True) + assert server.requests == [] + + +def test_offline_and_refresh_together_are_refused(server, tmp_path): + with pytest.raises(ValueError, match="incompatible"): + fetch_resource(tmp_path, "exemplo", _resource(server), offline=True, refresh=True) + + +def test_interrupted_download_does_not_stay_in_the_cache(server, tmp_path): + r = _resource(server) + server.truncated.add("/a.csv") + with pytest.raises(FetchError, match="incomplete"): + fetch_resource(tmp_path, "exemplo", r, backoff=0) + assert list((tmp_path / "exemplo").iterdir()) == [] + + +def test_interrupted_refresh_keeps_the_previous_version(server, tmp_path): + r = _resource(server) + fetch_resource(tmp_path, "exemplo", r) + server.files["/a.csv"] = b"versao nova que nao chega inteira\n" + server.truncated.add("/a.csv") + with pytest.raises(FetchError): + fetch_resource(tmp_path, "exemplo", r, refresh=True, backoff=0) + assert (tmp_path / "exemplo" / "a.csv").read_bytes() == BODY + assert sorted(p.name for p in (tmp_path / "exemplo").iterdir()) == ["a.csv", "a.csv.meta.json"] + + +def test_three_attempts_with_growing_wait(server, tmp_path): + r = _resource(server) + server.failures["/a.csv"] = 2 + waits = [] + dest = fetch_resource(tmp_path, "exemplo", r, backoff=1.0, sleep=waits.append) + assert dest.read_bytes() == BODY + assert waits == [1.0, 2.0] + assert server.paths() == ["/a.csv"] * 3 + + +def test_gives_up_after_three_attempts(server, tmp_path): + r = _resource(server) + server.failures["/a.csv"] = 5 + waits = [] + with pytest.raises(FetchError, match="after 3 attempt.*500"): + fetch_resource(tmp_path, "exemplo", r, backoff=1.0, sleep=waits.append) + assert waits == [1.0, 2.0] + assert len(server.requests) == 3 + assert not (tmp_path / "exemplo" / "a.csv").exists() + + +def test_default_backoff_is_two_seconds_doubling(server, tmp_path): + r = _resource(server) + server.failures["/a.csv"] = 5 + waits = [] + with pytest.raises(FetchError): + fetch_resource(tmp_path, "exemplo", r, sleep=waits.append) + assert waits == [2.0, 4.0] + + +def test_timeout(server, tmp_path): + r = _resource(server) + server.slow["/a.csv"] = 1.0 + with pytest.raises(FetchError, match="after 1 attempt"): + fetch_resource(tmp_path, "exemplo", r, timeout=0.2, attempts=1) + assert not (tmp_path / "exemplo" / "a.csv").exists() + + +def _failing(server, path, *codes): + server.responses[path] = list(codes) + return _resource(server, path) + + +@pytest.mark.parametrize("code", [400, 401, 403, 404, 410]) +def test_4xx_is_not_retried(server, tmp_path, code): + r = _failing(server, "/a.csv", code, code, code) + waits = [] + with pytest.raises(FetchError, match=f"after 1 attempt.*{code}"): + fetch_resource(tmp_path, "exemplo", r, backoff=1.0, sleep=waits.append) + assert len(server.requests) == 1 + assert waits == [] + + +@pytest.mark.parametrize("code", [408, 429, 500, 503]) +def test_408_429_and_5xx_are_retried(server, tmp_path, code): + r = _failing(server, "/a.csv", code, code) + assert fetch_resource(tmp_path, "exemplo", r, backoff=0).read_bytes() == BODY + assert len(server.requests) == 3 + + +def test_error_other_than_403_has_no_cache_hint(server, tmp_path): + r = _failing(server, "/a.csv", 404) + with pytest.raises(FetchError) as e: + fetch_resource(tmp_path, "exemplo", r, backoff=0) + assert "browser" not in str(e.value) and "--offline" not in str(e.value) + + +def test_403_has_the_cache_hint(server, tmp_path): + r = _failing(server, "/a.csv", 403) + with pytest.raises(FetchError, match="download the file with a browser") as e: + fetch_resource(tmp_path, "exemplo", r, backoff=0) + assert f"save it at {tmp_path / 'exemplo' / 'a.csv'} and rerun with --offline" in str(e.value) + + +def test_malformed_content_length_is_a_fetch_error(server, tmp_path): + r = _resource(server) + server.content_length["/a.csv"] = "abc" + with pytest.raises(FetchError, match="Content-Length 'abc' is invalid"): + fetch_resource(tmp_path, "exemplo", r, backoff=0, attempts=1) + assert list((tmp_path / "exemplo").iterdir()) == [] + + +@pytest.mark.parametrize("n", [0, -1]) +def test_attempts_below_one_are_refused(server, tmp_path, n): + with pytest.raises(ValueError, match="attempts"): + fetch_resource(tmp_path, "exemplo", _resource(server), attempts=n) + assert server.requests == [] + + +def test_failed_swap_on_refresh_never_leaves_metadata_describing_another_file(server, tmp_path, monkeypatch): + """The data file swap fails: the metadata next to it must never describe a file that is not there.""" + import os + + r = _resource(server) + fetch_resource(tmp_path, "exemplo", r) + old = (tmp_path / "exemplo" / "a.csv").read_bytes() + server.files["/a.csv"] = b"versao nova\n" + real = os.replace + + def fail_on_data_swap(src, dst): + if os.path.basename(dst) == "a.csv": + raise OSError("disk full") + return real(src, dst) + + monkeypatch.setattr("datasettool.sources.cache.os.replace", fail_on_data_swap) + with pytest.raises(FetchError, match="disk full"): + fetch_resource(tmp_path, "exemplo", r, refresh=True, attempts=1) + assert (tmp_path / "exemplo" / "a.csv").read_bytes() == old + meta = tmp_path / "exemplo" / "a.csv.meta.json" + assert not meta.exists() or json.loads(meta.read_text(encoding="utf-8"))["sha256"] == hashlib.sha256(old).hexdigest() + + +# A server answering 200 with an HTML block page instead of the file (dataset-names spec 003e, R4.1) -- + +CHALLENGE = b"\nPardon Our Interruption\n" + + +def test_html_page_instead_of_the_file_does_not_enter_the_cache(server, tmp_path): + resource = _resource(server, body=CHALLENGE) + server.content_types["/a.csv"] = "text/html; charset=UTF-8" + with pytest.raises(BlockedError) as e: + fetch_resource(tmp_path, "exemplo", resource, backoff=0) + message = str(e.value) + assert "HTML page (text/html) instead of a.csv" in message + assert server.url("/a.csv") in message + assert f"save it at {tmp_path / 'exemplo' / 'a.csv'} and rerun with --offline" in message + assert server.paths() == ["/a.csv"] + assert sorted(p.name for p in (tmp_path / "exemplo").iterdir()) == [] + + +def test_html_is_accepted_when_the_resource_is_html(server, tmp_path): + resource = _resource(server, path="/pagina.html", body=CHALLENGE) + server.content_types["/pagina.html"] = "Text/HTML" + assert fetch_resource(tmp_path, "exemplo", resource).read_bytes() == CHALLENGE + + +def test_csv_served_as_text_csv_is_still_accepted(server, tmp_path): + resource = _resource(server) + server.content_types["/a.csv"] = "text/csv; charset=utf-8" + assert fetch_resource(tmp_path, "exemplo", resource).read_bytes() == BODY + + +# Path traversal (review of PR #7): source ids and file names never leave the cache -------------- + +@pytest.mark.parametrize("source_id", ["..", "../x", "a/b", "", ".", "Us-Ssa", "us_ssa", "/etc"]) +def test_invalid_source_id_is_refused(tmp_path, source_id): + with pytest.raises(ValueError, match="source id"): + cached_path(tmp_path, source_id, "a.csv") + + +@pytest.mark.parametrize("filename", ["/etc/passwd", "../secret", "a/b.csv", "..", ".hidden", "a.csv.part", + "a.csv.meta.json", "", "a\\b.csv", "a\x00.csv"]) +def test_invalid_filename_is_refused(tmp_path, filename): + with pytest.raises(ValueError, match="file name"): + cached_path(tmp_path, "src", filename) + + +def test_offline_fetch_cannot_read_outside_the_cache(server, tmp_path): + cache = tmp_path / "cache" + cache.mkdir() + (tmp_path / "secret").write_text("x", encoding="utf-8") + with pytest.raises(ValueError): + fetch_resource(cache, "..", Resource(server.url("/secret"), "secret"), offline=True) + assert server.requests == [] + + +def test_symlinked_source_directory_leaving_the_cache_is_refused(tmp_path): + cache = tmp_path / "cache" + cache.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (cache / "src").symlink_to(outside) + with pytest.raises(ValueError, match="outside the cache"): + cached_path(cache, "src", "a.csv") diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..82acb32 --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,66 @@ +"""Spec 004, R6: file checksums and row counts. + +Ported from BenchCommons/dataset-names@e7e8405 tests/test_manifest.py and +tests/test_robustez.py (test_manifesto_conta_registros_csv_e_nao_linhas_fisicas). Test vectors unchanged. +""" + +import hashlib + +import pytest + +from datasettool.files import count_csv_records, describe_files, sha256_file + + +@pytest.mark.parametrize("content, expected", [ + ("", 0), + ("a,b\n", 0), + ("a,b\n1,2\n", 1), + ("a,b\n1,2\n\n", 1), + ("a,b\n1,2\n\n\n3,4\n", 2), + ('a,b\n"x\ny",2\n', 1), +]) +def test_csv_rows_exclude_header_and_blank_lines(tmp_path, content, expected): + path = tmp_path / "t.csv" + path.write_text(content, encoding="utf-8") + assert count_csv_records(path) == expected + + +def test_describe_files_counts_csv_records_not_physical_lines(tmp_path): + (tmp_path / "x.csv").write_text('name,nota\n"Ana","linha 1\nlinha 2"\nBia,ok\n', encoding="utf-8") + assert describe_files(tmp_path)["x.csv"]["rows"] == 2 + + +def test_sha256_file_reads_in_blocks(tmp_path): + data = b"x" * (3 * (1 << 20) + 7) + (tmp_path / "big.bin").write_bytes(data) + assert sha256_file(tmp_path / "big.bin") == hashlib.sha256(data).hexdigest() + + +def test_describe_files_maps_posix_paths_sorted_and_excludes_the_manifest(tmp_path): + (tmp_path / "by_country").mkdir() + (tmp_path / "by_country" / "PT.csv").write_text("name\nMaria\n", encoding="utf-8") + (tmp_path / "a.csv").write_text("name\n", encoding="utf-8") + (tmp_path / "manifest.json").write_text("{}", encoding="utf-8") + files = describe_files(tmp_path) + assert list(files) == ["a.csv", "by_country/PT.csv"] + assert files["by_country/PT.csv"] == {"sha256": hashlib.sha256(b"name\nMaria\n").hexdigest(), "rows": 1} + assert files["a.csv"]["rows"] == 0 + + +def test_file_without_a_counter_is_refused(tmp_path): + (tmp_path / "observations.parquet").write_bytes(b"PAR1") + with pytest.raises(ValueError, match="observations.parquet: no row counter for '.parquet'"): + describe_files(tmp_path) + + +def test_dataset_registers_its_own_counter(tmp_path): + (tmp_path / "observations.parquet").write_bytes(b"PAR1") + (tmp_path / "x.csv").write_text("a\n1\n", encoding="utf-8") + files = describe_files(tmp_path, counters={".csv": count_csv_records, ".parquet": lambda p: 42}) + assert files["observations.parquet"]["rows"] == 42 and files["x.csv"]["rows"] == 1 + + +def test_custom_exclusions(tmp_path): + (tmp_path / "SHA256SUMS").write_text("", encoding="utf-8") + (tmp_path / "x.csv").write_text("a\n", encoding="utf-8") + assert list(describe_files(tmp_path, exclude=("SHA256SUMS",))) == ["x.csv"] diff --git a/tests/test_no_network.py b/tests/test_no_network.py new file mode 100644 index 0000000..f00600e --- /dev/null +++ b/tests/test_no_network.py @@ -0,0 +1,29 @@ +"""Spec 004, R7.3: the autouse fixture in conftest.py refuses the network outside the loopback.""" + +import socket +import urllib.request + +import pytest + +from conftest import NetworkAccessError + + +def test_connect_outside_the_loopback_is_refused(): + with socket.socket() as s, pytest.raises(NetworkAccessError, match="93.184.215.14"): + s.connect(("93.184.215.14", 80)) + + +def test_name_lookup_is_refused(): + with pytest.raises(NetworkAccessError, match="example.org"): + socket.getaddrinfo("example.org", 443) + + +def test_urllib_to_a_public_host_is_refused(): + with pytest.raises(Exception) as e: + urllib.request.urlopen("https://example.org/", timeout=1) + assert "attempted" in str(e.value) + + +def test_loopback_server_still_works(server): + server.files["/a"] = b"ok" + assert urllib.request.urlopen(server.url("/a"), timeout=5).read() == b"ok" diff --git a/tests/test_package.py b/tests/test_package.py new file mode 100644 index 0000000..db28bcf --- /dev/null +++ b/tests/test_package.py @@ -0,0 +1,22 @@ +"""Spec 004, R1: package metadata and pinned runtime dependencies.""" + +import tomllib +from importlib.metadata import version +from pathlib import Path + +import datasettool + +PYPROJECT = tomllib.loads((Path(__file__).parent.parent / "pyproject.toml").read_text(encoding="utf-8")) + + +def test_version_comes_from_the_distribution(): + assert datasettool.__version__ == version("benchcommons-tools-dataset") == PYPROJECT["project"]["version"] + + +def test_runtime_dependencies_are_pinned(): + # ADR 0005: exactly these two, pinned; xlrd pin carried over from dataset-names ADR 0005. + assert sorted(PYPROJECT["project"]["dependencies"]) == ["pyyaml==6.0.3", "xlrd==2.0.2"] + + +def test_build_backend_is_pinned(): + assert PYPROJECT["build-system"]["requires"] == ["hatchling==1.32.4"] diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..115464e --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,381 @@ +"""Spec 004, R5: source registry and licence policy. + +Ported from BenchCommons/dataset-names@e7e8405 tests/test_sources.py and tests/test_sources_extensoes.py, +run through tests/names_registry.py, a registry schema built only from the public API +(acceptance criterion 3). Test vectors unchanged; messages in English. +""" + +from datetime import date, datetime + +import pytest + +from datasettool.sources.registry import ( + ATTRIBUTION_ONLY_LICENSES, Field, RegistryError, RegistrySchema, Source, load_registry, parse_registry, +) +from names_registry import SCHEMA + + +def parse_sources(entries): + return parse_registry(entries, SCHEMA) + + +def load_sources(path): + return load_registry(path, SCHEMA) + + +def source(**fields): + base = { + "id": "us-ssa-births", "name": "SSA Baby Names", "url": "https://www.ssa.gov/oact/babynames/", + "license": "public-domain", "attribution": "U.S. Social Security Administration", + "country": "US", "count_type": "births", "primary": True, "suppression": 5, + "retrieved": date(2026, 10, 1), + } + return {**base, **fields} + + +def test_valid_source(): + sources = parse_sources([source()]) + assert sources["us-ssa-births"].suppression == 5 + + +def test_loads_yaml(tmp_path): + p = tmp_path / "sources.yaml" + p.write_text( + "sources:\n" + " - id: wikidata\n" + " name: Wikidata\n" + " url: https://www.wikidata.org/\n" + " license: CC0-1.0\n" + " attribution: Wikidata\n" + " country: null\n" + " count_type: attested\n" + " primary: false\n" + " suppression: null\n" + " retrieved: 2026-10-01\n", + encoding="utf-8", + ) + assert load_sources(p)["wikidata"].retrieved == date(2026, 10, 1) + + +def test_file_without_sources(tmp_path): + p = tmp_path / "sources.yaml" + p.write_text("sources: []\n", encoding="utf-8") + assert load_sources(p) == {} + + +@pytest.mark.parametrize("license", ["CC-BY-SA-4.0", "ODbL-1.0", "CC-BY-NC-4.0", "desconhecida"]) +def test_incompatible_license_is_refused(license): + with pytest.raises(RegistryError, match="ADR 0001"): + parse_sources([source(license=license)]) + + +def test_statcan_open_licence_is_accepted_by_the_dataset_extension(): + # dataset-names ADR 0004 extends its ADR 0001 list; the dataset adds it to the shared starting set. + sources = parse_sources([source(license="statcan-open-licence")]) + assert sources["us-ssa-births"].license == "statcan-open-licence" + + +@pytest.mark.parametrize("fields, fragment", [ + ({"country": "XX"}, "ISO 3166-1"), + ({"country": "br"}, "ISO 3166-1"), + ({"count_type": "deaths"}, "count_type"), + ({"primary": "sim"}, "primary"), + ({"suppression": 0}, "suppression"), + ({"retrieved": "ontem"}, "retrieved"), + ({"url": ""}, "'url' empty"), + ({"extra": 1}, "unknown"), +]) +def test_invalid_field(fields, fragment): + with pytest.raises(RegistryError, match=fragment): + parse_sources([source(**fields)]) + + +@pytest.mark.parametrize("ident", ["../../etc", "Us-Ssa", "us_ssa", "-us", "us-", "us--ssa", "us ssa", "a/b"]) +def test_id_with_invalid_format(ident): + with pytest.raises(RegistryError, match="invalid format") as e: + parse_sources([source(id=ident)]) + assert ident in str(e.value) + + +def test_missing_field(): + entry = source() + del entry["license"] + with pytest.raises(RegistryError, match="missing"): + parse_sources([entry]) + + +def test_duplicate_id(): + with pytest.raises(RegistryError, match="duplicate"): + parse_sources([source(), source()]) + + +def test_two_sources_for_the_same_pair_need_one_primary(): + with pytest.raises(RegistryError, match="exactly one"): + parse_sources([source(id="a"), source(id="b")]) + with pytest.raises(RegistryError, match="exactly one"): + parse_sources([source(id="a", primary=False), source(id="b", primary=False)]) + assert set(parse_sources([source(id="a"), source(id="b", primary=False)])) == {"a", "b"} + + +def test_all_errors_are_listed(): + with pytest.raises(RegistryError) as e: + parse_sources([source(license="ODbL-1.0", country="XX")]) + assert "ADR 0001" in str(e.value) and "ISO 3166-1" in str(e.value) + + +def test_non_text_license_is_a_registry_error(): + with pytest.raises(RegistryError, match="ADR 0001"): + parse_sources([source(license=["CC0-1.0"])]) + + +def test_non_text_id_is_a_registry_error(): + with pytest.raises(RegistryError, match="'id' empty"): + parse_sources([source(id=["us-ssa-births"])]) + + +@pytest.mark.parametrize("fields, fragment", [ + ({"country": ["US"]}, "ISO 3166-1"), + ({"count_type": ["births"]}, "count_type"), +]) +def test_non_text_field_is_a_registry_error(fields, fragment): + with pytest.raises(RegistryError, match=fragment): + parse_sources([source(**fields)]) + + +def test_entry_that_is_not_a_mapping(): + with pytest.raises(RegistryError, match="#0: entry must be a mapping") as e: + parse_sources(["foo", None]) + assert "#1: entry must be a mapping" in str(e.value) + + +def test_missing_file(tmp_path): + with pytest.raises(RegistryError, match="cannot read"): + load_sources(tmp_path / "nao-existe.yaml") + + +def test_malformed_yaml(tmp_path): + p = tmp_path / "sources.yaml" + p.write_text("sources: [\n", encoding="utf-8") + with pytest.raises(RegistryError, match="invalid YAML"): + load_sources(p) + + +def test_root_that_is_not_a_mapping(tmp_path): + p = tmp_path / "sources.yaml" + p.write_text("- foo\n", encoding="utf-8") + with pytest.raises(RegistryError, match="the root must be a mapping"): + load_sources(p) + + +@pytest.mark.parametrize("content", ["sources: 5\n", "sources: texto\n"]) +def test_sources_that_is_not_a_list(tmp_path, content): + p = tmp_path / "sources.yaml" + p.write_text(content, encoding="utf-8") + with pytest.raises(RegistryError, match="'sources' must be a list"): + load_sources(p) + + +def test_attested_does_not_need_primary(): + """dataset-names spec 004, R5 (its issue #3): primary has no effect for attested and is not checked.""" + no_country = [source(id=i, country=None, count_type="attested", primary=False) for i in ("a", "b")] + assert set(parse_sources(no_country)) == {"a", "b"} + same_country = [source(id=i, count_type="attested", primary=True) for i in ("a", "b")] + assert set(parse_sources(same_country)) == {"a", "b"} + + +@pytest.mark.parametrize("kind", ["births", "population"]) +def test_counts_still_need_one_primary(kind): + with pytest.raises(RegistryError, match=f"US/{kind}: sources \\['a', 'b'\\]; exactly one"): + parse_sources([source(id=i, count_type=kind, primary=False) for i in ("a", "b")]) + + +def test_boolean_country_asks_for_quotes(): + with pytest.raises(RegistryError, match='country read as boolean False; quote the code, e.g. "NO"'): + parse_sources([source(country=False)]) + + +# Optional fields and the last refusals of dataset-names spec 002 R3 (test_sources_extensoes.py) ---- + +def ext_source(**fields): + base = { + "id": "pt-exemplo", "name": "Exemplo", "url": "https://example.org/nomes", + "license": "CC-BY-4.0", "attribution": "Exemplo", "country": "PT", + "count_type": "births", "primary": True, "suppression": None, + "retrieved": date(2026, 10, 1), + } + return {**base, **fields} + + +def test_optional_fields_have_defaults(): + s = parse_sources([ext_source()])["pt-exemplo"] + assert (s.uniform_case, s.adapter) == (False, True) + + +def test_optional_fields_declared(): + s = parse_sources([ext_source(uniform_case=True, adapter=False)])["pt-exemplo"] + assert (s.uniform_case, s.adapter) == (True, False) + + +@pytest.mark.parametrize("field", ["uniform_case", "adapter"]) +@pytest.mark.parametrize("value", ["sim", 1, None]) +def test_optional_field_not_boolean(field, value): + with pytest.raises(RegistryError, match=f"'{field}' must be true or false"): + parse_sources([ext_source(**{field: value})]) + + +def test_retrieved_with_time_is_refused(): + with pytest.raises(RegistryError, match="'retrieved' must be a YYYY-MM-DD date"): + parse_sources([ext_source(retrieved=datetime(2026, 10, 1, 12, 30))]) + + +def test_retrieved_with_time_in_yaml_is_refused(tmp_path): + p = tmp_path / "sources.yaml" + p.write_text( + "sources:\n" + " - id: pt-exemplo\n" + " name: Exemplo\n" + " url: https://example.org/nomes\n" + " license: CC-BY-4.0\n" + " attribution: Exemplo\n" + " country: PT\n" + " count_type: births\n" + " primary: true\n" + " suppression: null\n" + " retrieved: 2026-10-01T10:00:00\n", + encoding="utf-8", + ) + with pytest.raises(RegistryError, match="retrieved"): + load_sources(p) + + +@pytest.mark.parametrize("url", [ + "ftp://example.org/nomes", "example.org/nomes", "file:///tmp/nomes.csv", "https://", "mailto:a@b.c", + "http://[::1", " https://example.org/nomes", "https://example.org/nomes\n", "https://exam ple.org/", +]) +def test_url_without_http_scheme_is_refused(url): + with pytest.raises(RegistryError, match="http or https"): + parse_sources([ext_source(url=url)]) + + +@pytest.mark.parametrize("url", ["http://example.org/a.csv", "HTTPS://example.org/"]) +def test_http_and_https_urls_are_accepted(url): + assert parse_sources([ext_source(url=url)])["pt-exemplo"].url == url + + +def test_malformed_url_names_the_source_instead_of_raising_value_error(): + with pytest.raises(RegistryError, match="pt-exemplo: 'url' must use the http or https scheme"): + parse_sources([ext_source(url="http://[::1")]) + + +# Generic schema (spec 004, R5.2 to R5.5) ------------------------------------------------------------ + +def generic(**fields): + base = {"id": "wordnet", "name": "WordNet", "url": "https://wordnet.princeton.edu/", + "license": "CC-BY-4.0", "attribution": "Princeton University", "retrieved": date(2026, 10, 1)} + return {**base, **fields} + + +GENERIC = RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="ADR 0001 of dataset-commons") + + +def test_generic_schema_builds_the_default_source(): + s = parse_registry([generic()], GENERIC)["wordnet"] + assert s == Source("wordnet", "WordNet", "https://wordnet.princeton.edu/", "CC-BY-4.0", + "Princeton University", date(2026, 10, 1)) + assert s.adapter is True and dict(s.extra) == {} + + +def test_generic_schema_refuses_dataset_fields(): + with pytest.raises(RegistryError, match=r"wordnet: unknown fields \['country'\]"): + parse_registry([generic(country="US")], GENERIC) + + +def test_dataset_fields_go_to_extra_in_the_default_source(): + schema = RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", + fields=(Field("language"), Field("words", required=False, default=0))) + s = parse_registry([generic(language="en")], schema)["wordnet"] + assert dict(s.extra) == {"language": "en", "words": 0} + with pytest.raises(TypeError): + s.extra["language"] = "pt" + + +def test_licence_error_names_the_dataset_policy(): + with pytest.raises(RegistryError, match=r"wordnet: license 'ODbL-1.0' is not accepted \(ADR 0001 of dataset-commons\)"): + parse_registry([generic(license="ODbL-1.0")], GENERIC) + + +def test_attribution_only_licenses(): + assert ATTRIBUTION_ONLY_LICENSES == { + "public-domain", "CC0-1.0", "CC-BY-3.0", "CC-BY-4.0", "ODC-By-1.0", "OGL-UK-3.0", "etalab-2.0", + } + assert "statcan-open-licence" not in ATTRIBUTION_ONLY_LICENSES + + +def test_schema_without_accepted_licenses_is_an_error(): + with pytest.raises(ValueError, match="accepted_licenses"): + RegistrySchema(accepted_licenses=frozenset(), license_policy="x") + + +@pytest.mark.parametrize("name", ["id", "license", "retrieved", "adapter"]) +def test_dataset_field_cannot_redefine_a_generic_field(name): + with pytest.raises(ValueError, match=f"'{name}'"): + RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", fields=(Field(name),)) + + +def test_dataset_field_declared_twice_is_an_error(): + with pytest.raises(ValueError, match="'language'"): + RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", + fields=(Field("language"), Field("language"))) + + +def test_cross_checks_see_only_valid_entries_and_all_errors_are_reported(): + seen = [] + + def check(sources): + seen.append(sorted(sources)) + return ["cross-entry problem"] + + schema = RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", checks=(check,)) + with pytest.raises(RegistryError) as e: + parse_registry([generic(), generic(id="bad", license="ODbL-1.0")], schema) + assert seen == [["wordnet"]] + assert str(e.value).splitlines() == ["bad: license 'ODbL-1.0' is not accepted (x)", "cross-entry problem"] + + +# Collections passed as a single string (review of PR #7) ------------------------------------------- + +def test_license_list_given_as_a_string_is_refused(): + # A str would make the licence check a substring test: "BY" in "CC-BY-4.0". + with pytest.raises(TypeError, match="accepted_licenses"): + RegistrySchema(accepted_licenses="CC-BY-4.0", license_policy="x") + + +@pytest.mark.parametrize("name, value", [("fields", "country"), ("checks", "one_primary")]) +def test_fields_and_checks_given_as_a_string_are_refused(name, value): + with pytest.raises(TypeError, match=name): + RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", **{name: value}) + + +def test_schema_collections_are_coerced(): + schema = RegistrySchema(accepted_licenses=["CC-BY-4.0"], license_policy="x", fields=[Field("language")], + checks=[lambda sources: []]) + assert schema.accepted_licenses == frozenset({"CC-BY-4.0"}) + assert isinstance(schema.fields, tuple) and isinstance(schema.checks, tuple) + with pytest.raises(RegistryError, match="license 'BY' is not accepted"): + parse_registry([generic(license="BY", language="en")], schema) + + +def test_field_and_check_types_are_checked(): + with pytest.raises(TypeError, match="Field"): + RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", fields=("country",)) + with pytest.raises(TypeError, match="callable"): + RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", checks=(3,)) + + +def test_default_source_is_hashable(): + # extra is a read-only MappingProxyType (unhashable); it is left out of the hash, not of equality. + schema = RegistrySchema(accepted_licenses=ATTRIBUTION_ONLY_LICENSES, license_policy="x", fields=(Field("language"),)) + a = parse_registry([generic(language="en")], schema)["wordnet"] + b = parse_registry([generic(language="pt")], schema)["wordnet"] + assert len({a, a}) == 1 + assert a != b and len({a, b}) == 2 diff --git a/tests/test_spreadsheet.py b/tests/test_spreadsheet.py new file mode 100644 index 0000000..a7b07a3 --- /dev/null +++ b/tests/test_spreadsheet.py @@ -0,0 +1,541 @@ +"""Spec 004, R3: XLS and XLSX spreadsheet reader. + +Ported from BenchCommons/dataset-names@e7e8405 tests/test_spreadsheet.py (spec 003b R1 and ADR 0005 +of dataset-names). Test vectors unchanged; messages in English. The test on the INE adapter samples +stays in dataset-names. +""" + +import zipfile +from pathlib import Path + +import pytest + +from datasettool.sources.adapters import FormatError +from datasettool.sources.spreadsheet import cell_count, rows, sheet_names + +FIXTURES = Path(__file__).parent / "fixtures" +NS = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"' +NS_R = 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' +REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + + +def _xlsx(path: Path, sheets: list[tuple[str, str]], strings: list[str] = (), target="worksheets/sheet{n}.xml", + order=None) -> Path: + """Write a minimal XLSX. sheets: (name, sheetData XML); strings: XML of each .""" + order = order or list(range(1, len(sheets) + 1)) + entries = "".join(f'' + for n, (name, _) in zip(order, sheets)) + rels = "".join(f'' + for n in order) + rels += f'' + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("[Content_Types].xml", "") + z.writestr("xl/workbook.xml", f'{entries}') + z.writestr("xl/_rels/workbook.xml.rels", + f'{rels}') + z.writestr("xl/sharedStrings.xml", + f'' + "".join(f"{t}" for t in strings) + "") + for n, (_, data) in zip(order, sheets): + z.writestr(f"xl/worksheets/sheet{n}.xml", f'{data}') + return path + + +TYPES_XML = ( + '012' + '359093858.7' + 'IÑIGO1E3' + '' + 'JOANA D\'ARC1' + '0' + '4' +) +TYPES_STRINGS = ["Orden", "Nombre", "Frecuencia", "ANTONIO", ".."] +TYPES_EXPECTED = [ + (1, ["Orden", "Nombre", "Frecuencia"]), + (2, ["ANTONIO", 590938, 58.7]), + (3, ["IÑIGO", 1000]), + (5, ["JOANA D'ARC", True, None, False]), + (7, [None, None, ".."]), +] + + +# R3.1 -------------------------------------------------------------------------------------------- + +def test_sheet_names_in_file_order(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("Hombres", ""), ("Mujeres", ""), ("Notas", "")]) + assert sheet_names(f) == ["Hombres", "Mujeres", "Notas"] + + +def test_missing_sheet_lists_the_sheets(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("Hombres", ""), ("Mujeres", "")]) + with pytest.raises(FormatError, match=r"a.xlsx: sheet 'Varones' does not exist; sheets: \['Hombres', 'Mujeres'\]"): + list(rows(f, "Varones")) + + +def test_excel_row_number_and_column_from_the_reference(tmp_path): + data = ('1' + '3' + '27') + f = _xlsx(tmp_path / "a.xlsx", [("X", data)]) + assert list(rows(f, "X")) == [(1, [1]), (3, [None, None, 3]), (10, [None] * 26 + [27])] + + +# R3.2 -------------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("content", [ + b"Error 500", + b"", + b"Orden;Nombre;Frecuencia\n1;ANTONIO;590938\n", +]) +def test_content_that_is_not_a_spreadsheet(tmp_path, content): + f = tmp_path / "nombres.xlsx" + f.write_bytes(content) + with pytest.raises(FormatError, match="nombres.xlsx: not an XLS or XLSX spreadsheet"): + sheet_names(f) + with pytest.raises(FormatError, match="not an XLS or XLSX spreadsheet"): + list(rows(f, "Hombres")) + + +def test_zip_that_is_not_xlsx(tmp_path): + f = tmp_path / "a.xlsx" + with zipfile.ZipFile(f, "w") as z: + z.writestr("nomes.csv", "a,b\n") + with pytest.raises(FormatError, match="a.xlsx: ZIP without xl/workbook.xml; not an XLSX"): + sheet_names(f) + + +def test_format_from_content_not_extension(tmp_path): + f = _xlsx(tmp_path / "planilha.xls", [("X", '1')]) + assert list(rows(f, "X")) == [(1, [1])] + + +# R3.3 -------------------------------------------------------------------------------------------- + +def test_xlsx_cell_types(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("Dados", TYPES_XML)], TYPES_STRINGS) + got = list(rows(f, "Dados")) + assert got == TYPES_EXPECTED + assert [type(v) for v in got[1][1]] == [str, int, float] + assert type(got[2][1][1]) is int + + +def test_empty_rows_skipped_and_trailing_none_cut(tmp_path): + data = ('' + '2' + '3') + f = _xlsx(tmp_path / "a.xlsx", [("X", data)]) + assert list(rows(f, "X")) == [(2, ["", 2]), (3, [3])] + + +def test_formula_error_names_sheet_and_cell(tmp_path): + data = '1#N/A' + f = _xlsx(tmp_path / "a.xlsx", [("Hombres", data)]) + with pytest.raises(FormatError, match=r"a.xlsx, sheet 'Hombres', cell B2: formula error '#N/A'"): + list(rows(f, "Hombres")) + + +def test_unknown_cell_type(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", '2025-01-01')]) + with pytest.raises(FormatError, match="cell A1: unknown type 'd'"): + list(rows(f, "X")) + + +def test_invalid_number(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", 'doze')]) + with pytest.raises(FormatError, match="cell A1: invalid number 'doze'"): + list(rows(f, "X")) + + +# R3.4 -------------------------------------------------------------------------------------------- + +def test_shared_string_with_formatting_and_rph(tmp_path): + strings = [ + "MARIA BEGOÑA", + "東京トウキョウ", + ] + data = '01' + f = _xlsx(tmp_path / "a.xlsx", [("X", data)], strings) + assert list(rows(f, "X")) == [(1, ["MARIA BEGOÑA", "東京"])] + + +def test_shared_string_index_outside_the_table(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", '7')], ["a"]) + with pytest.raises(FormatError, match="cell A1: shared string 7 outside the table of 1"): + list(rows(f, "X")) + + +@pytest.mark.parametrize("target", ["worksheets/sheet{n}.xml", "/xl/worksheets/sheet{n}.xml"]) +def test_sheet_found_through_the_relationship_target(tmp_path, target): + sheets = [("Primeira", '2'), + ("Segunda", '1')] + f = _xlsx(tmp_path / "a.xlsx", sheets, target=target, order=[2, 1]) + assert sheet_names(f) == ["Primeira", "Segunda"] + assert list(rows(f, "Primeira")) == [(1, [2])] + assert list(rows(f, "Segunda")) == [(1, [1])] + + +# R3.6 -------------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("value, expected", [ + (590938, 590938), (0, 0), (" 20 ", 20), ("..", None), (None, None), +]) +def test_cell_count(value, expected): + assert cell_count(value, {"..", ""}) == expected + + +@pytest.mark.parametrize("value, fragment", [ + (20.5, "count 20.5 is not an integer"), + (-1, "count -1 is negative"), + (True, "count True is not a number"), + ("<20", r"count '<20' is neither an integer nor a known marker \['..'\]"), + (None, "count empty"), +]) +def test_cell_count_refuses(value, fragment): + with pytest.raises(FormatError, match=fragment): + cell_count(value, {".."}) + + +def test_cell_count_names_the_field(): + with pytest.raises(FormatError, match="Frecuencia 1.5 is not an integer"): + cell_count(1.5, field="Frecuencia") + + +# R3.3 and R3.5: XLS through xlrd ----------------------------------------------------------------- + +TYPES_XLS = FIXTURES / "spreadsheets" / "types.xls" + + +def test_xls_cell_types(): + assert sheet_names(TYPES_XLS) == ["Dados", "Erros"] + got = list(rows(TYPES_XLS, "Dados")) + assert got == TYPES_EXPECTED + assert [type(v) for v in got[1][1]] == [str, int, float] + assert [type(v) for v in got[3][1]] == [str, bool, type(None), bool] + + +def test_xls_formula_error(): + with pytest.raises(FormatError, match=r"types.xls, sheet 'Erros', cell B2: formula error '#N/A'"): + list(rows(TYPES_XLS, "Erros")) + + +def test_same_values_in_both_formats(tmp_path): + xlsx = _xlsx(tmp_path / "types.xlsx", [("Dados", TYPES_XML)], TYPES_STRINGS) + assert list(rows(xlsx, "Dados")) == list(rows(TYPES_XLS, "Dados")) + + +def test_missing_sheet_xls(): + with pytest.raises(FormatError, match=r"types.xls: sheet 'Notas' does not exist; sheets: \['Dados', 'Erros'\]"): + list(rows(TYPES_XLS, "Notas")) + + +def test_corrupt_xls(tmp_path): + f = tmp_path / "quebrado.xls" + f.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 600) + with pytest.raises(FormatError, match="quebrado.xls: unreadable XLS"): + sheet_names(f) + + +def test_xlrd_pinned(): + import tomllib + project = tomllib.loads((Path(__file__).parent.parent / "pyproject.toml").read_text(encoding="utf-8")) + assert "xlrd==2.0.2" in project["project"]["dependencies"] + + +# Malformed inputs and disproportionate ZIPs become FormatError (dataset-names PR #27 review) ------ + +def test_sheet_member_missing_from_the_zip(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", "")], target="worksheets/nao_existe{n}.xml") + with pytest.raises(FormatError, match=r"a.xlsx: sheet 'X' points to xl/worksheets/nao_existe1.xml, missing from the ZIP"): + list(rows(f, "X")) + + +def test_shared_string_index_not_an_integer(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", 'abc')], ["a"]) + with pytest.raises(FormatError, match="cell A1: shared string index 'abc' is not an integer"): + list(rows(f, "X")) + + +@pytest.mark.parametrize("ref, fragment", [ + ("XFE1", "cell XFE1: column beyond XFD"), + ("1A", "invalid cell reference '1A'"), + ("a1", "invalid cell reference 'a1'"), +]) +def test_invalid_cell_reference(tmp_path, ref, fragment): + f = _xlsx(tmp_path / "a.xlsx", [("X", f'1')]) + with pytest.raises(FormatError, match=fragment): + list(rows(f, "X")) + + +def test_invalid_row_number(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", '1')]) + with pytest.raises(FormatError, match="invalid row 'x'"): + list(rows(f, "X")) + + +def test_zip_with_disproportionate_compression_ratio(tmp_path): + padding = " " * 3_000_000 + f = _xlsx(tmp_path / "a.xlsx", [("X", f'1{padding}')]) + with pytest.raises(FormatError, match=r"a.xlsx: member xl/worksheets/sheet1.xml inflates to 3\d{6} bytes, .*ratio"): + list(rows(f, "X")) + + +def test_zip_with_member_too_large(tmp_path, monkeypatch): + import datasettool.sources.spreadsheet as spreadsheet + monkeypatch.setattr(spreadsheet, "_MAX_MEMBER", 100) + f = _xlsx(tmp_path / "a.xlsx", [("X", '1')]) + with pytest.raises(FormatError, match=r"a.xlsx: member xl/workbook.xml inflates to \d+ bytes, above the limit of 100"): + sheet_names(f) + + +def test_rows_read_leave_the_tree(tmp_path): + """Flat memory: a sheet with many rows does not accumulate elements already read.""" + import datasettool.sources.spreadsheet as spreadsheet + data = "".join(f'{i}' for i in range(1, 2001)) + f = _xlsx(tmp_path / "a.xlsx", [("X", data)]) + seen = [] + original = spreadsheet._events + + def spy(*args, **kwargs): + for event, el in original(*args, **kwargs): + yield event, el + if event == "end" and el.tag.endswith("sheetData"): + seen.append(len(el)) + + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(spreadsheet, "_events", spy) + try: + assert sum(1 for _ in rows(f, "X")) == 2000 + finally: + monkeypatch.undo() + assert seen == [0] + + +# Defence in depth against DTDs (dataset-names ADR 0005) ------------------------------------------ + +def _rewrite(src: Path, dest: Path, member: str, content: str) -> Path: + with zipfile.ZipFile(src) as zin, zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zout: + for info in zin.infolist(): + zout.writestr(info.filename, content if info.filename == member else zin.read(info.filename)) + return dest + + +BILLION_LAUGHS = ( + '' + '' + ']>' + f'&lol3;' +) +ROOTS = {"xl/workbook.xml": f"", "xl/_rels/workbook.xml.rels": "", + "xl/sharedStrings.xml": f"", "xl/worksheets/sheet1.xml": f""} +OPERATIONS = [ + ("xl/workbook.xml", lambda a: sheet_names(a)), + ("xl/_rels/workbook.xml.rels", lambda a: sheet_names(a)), + ("xl/sharedStrings.xml", lambda a: list(rows(a, "X"))), + ("xl/worksheets/sheet1.xml", lambda a: list(rows(a, "X"))), +] + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +@pytest.mark.parametrize("prolog", [ + ']>', + '', + '', +]) +def test_dtd_in_the_prologue_is_refused(tmp_path, member, operation, prolog): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + f = _rewrite(base, tmp_path / "a.xlsx", member, f'{prolog}{ROOTS[member]}') + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} has a DTD"): + operation(f) + + +def test_billion_laughs_fails_on_the_dtd_check(tmp_path): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + f = _rewrite(base, tmp_path / "a.xlsx", "xl/worksheets/sheet1.xml", BILLION_LAUGHS) + with pytest.raises(FormatError, match=r"a\.xlsx: member xl/worksheets/sheet1\.xml has a DTD"): + list(rows(f, "X")) + + +def test_doctype_inside_a_comment_is_not_a_dtd(tmp_path): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + content = f'7' + f = _rewrite(base, tmp_path / "a.xlsx", "xl/worksheets/sheet1.xml", content) + assert list(rows(f, "X")) == [(1, [7])] + + +def test_doctype_text_after_the_root_is_not_a_dtd(tmp_path): + f = _xlsx(tmp_path / "a.xlsx", [("X", '<!DOCTYPE')]) + assert list(rows(f, "X")) == [(1, ["1')]) + content = '' + f = _rewrite(base, tmp_path / "a.xlsx", "xl/worksheets/sheet1.xml", content) + with pytest.raises(FormatError, match=r"a\.xlsx: member xl/worksheets/sheet1\.xml has a prologue above 65536 bytes"): + list(rows(f, "X")) + + +def test_normal_xlsx_with_bom_is_still_read(tmp_path): + base = _xlsx(tmp_path / "base.xlsx", [("X", "")]) + content = '\n' + f'3' + f = _rewrite(base, tmp_path / "a.xlsx", "xl/worksheets/sheet1.xml", content) + assert list(rows(f, "X")) == [(1, [3])] + + +def test_300_thousand_rows_streamed_with_flat_memory(tmp_path): + import tracemalloc + n = 300_000 + data = "".join(f'{i}NOME{i}' + for i in range(1, n + 1)) + f = _xlsx(tmp_path / "a.xlsx", [("X", data)]) + del data + tracemalloc.start() + try: + total = last = 0 + for number, values in rows(f, "X"): + total += 1 + last = values[0] + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert (total, last) == (n, n) + assert peak < 5 * 1024 * 1024, f"peak of {peak} bytes: the sheet is not being streamed" + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +@pytest.mark.parametrize("prolog", [ + ']>', + ']>', +]) +def test_dtd_after_utf8_bom_is_refused(tmp_path, member, operation, prolog): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + f = _rewrite(base, tmp_path / "a.xlsx", member, f'{prolog}{ROOTS[member]}') + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} has a DTD"): + operation(f) + + +# DTD in encodings that escape a UTF-8 pre-scan (parser differential) -------------------------------- +# Ported from BenchCommons/dataset-names@eaaa4d1 (PR #39) tests/test_spreadsheet.py. + +_DTD = ']>' + + +def _rewrite_bytes(src: Path, dest: Path, member: str, data: bytes) -> Path: + with zipfile.ZipFile(src) as zin, zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zout: + for info in zin.infolist(): + zout.writestr(info.filename, data if info.filename == member else zin.read(info.filename)) + return dest + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +@pytest.mark.parametrize("declared, codec", [ + ("cp037", "cp037"), + ("utf-32", "utf-32"), + ("UTF-32LE", "utf-32-le"), + ("UTF-32BE", "utf-32-be"), + ("latin-1", "latin-1"), + ("iso-8859-1", "ascii"), + ("windows-1252", "cp1252"), + ("us-ascii", "ascii"), +]) +def test_dtd_in_a_non_utf_encoding_is_refused(tmp_path, member, operation, declared, codec): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + text = f'{_DTD}{ROOTS[member]}' + f = _rewrite_bytes(base, tmp_path / "a.xlsx", member, text.encode(codec)) + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} "): + operation(f) + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +@pytest.mark.parametrize("data", [ + '{dtd}{raiz}'.encode("cp037"), # no declaration: EBCDIC is not UTF-8 + '{dtd}{raiz}'.encode("utf-32-le"), + '{dtd}{raiz}'.encode("utf-32-be"), + b"\xff\xfe\x00\x00<\x00\x00\x00", + b"\x00\x00\xfe\xff\x00\x00\x00<", + b'\xe9{dtd}{raiz}', # invalid UTF-8 +], ids=["ebcdic", "utf32le", "utf32be", "bom-utf32le", "bom-utf32be", "invalid-utf8"]) +def test_prefix_that_is_neither_utf8_nor_utf16_is_refused(tmp_path, member, operation, data): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + f = _rewrite_bytes(base, tmp_path / "a.xlsx", member, data) + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} "): + operation(f) + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +@pytest.mark.parametrize("codec, declared", [ + ("utf-8", "UTF-8"), ("utf-8", "utf8"), ("utf-8", None), ("utf-16", "UTF-16"), ("utf-16", "utf16"), +]) +def test_dtd_in_utf8_and_utf16_with_a_valid_declaration_is_still_refused(tmp_path, member, operation, codec, declared): + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + enc = f' encoding="{declared}"' if declared else "" + text = f'{_DTD}{ROOTS[member]}' + f = _rewrite_bytes(base, tmp_path / "a.xlsx", member, text.encode(codec)) + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} has a DTD"): + operation(f) + + +@pytest.mark.parametrize("codec, declared", [ + ("utf-8", "UTF-8"), ("utf-8", "utf-8"), ("utf-16", "UTF-16"), ("utf-16-le", None)]) +def test_normal_xlsx_in_utf8_or_utf16_is_read(tmp_path, codec, declared): + base = _xlsx(tmp_path / "base.xlsx", [("X", "")]) + enc = f' encoding="{declared}"' if declared else "" + text = (f'' + 'IÑIGO') + f = _rewrite_bytes(base, tmp_path / "a.xlsx", "xl/worksheets/sheet1.xml", text.encode(codec)) + assert list(rows(f, "X")) == [(1, ["IÑIGO"])] + + +@pytest.mark.parametrize("member, operation", OPERATIONS) +def test_the_parser_refuses_a_dtd_even_without_the_prescan(tmp_path, monkeypatch, member, operation): + """Defence in depth: with the pre-scan switched off, the expat handlers still refuse DTDs.""" + import datasettool.sources.spreadsheet as spreadsheet + monkeypatch.setattr(spreadsheet, "_prologue", lambda data, where="": "root") + base = _xlsx(tmp_path / "base.xlsx", [("X", '1')]) + for dtd in (_DTD, BILLION_LAUGHS.split("')): + f = _rewrite(base, tmp_path / "a.xlsx", member, f'{dtd}{ROOTS[member]}') + with pytest.raises(FormatError, match=rf"a\.xlsx: member {member} has a DTD"): + operation(f) + + +# Corrupt ZIP members become FormatError (review of PR #7) ------------------------------------------ + +def _corrupt(path: Path, old: bytes, new: bytes) -> Path: + data = path.read_bytes() + assert data.count(old) == 1 + path.write_bytes(data.replace(old, new)) + return path + + +def _stored_xlsx(path: Path) -> Path: + src = _xlsx(path.with_name("deflated.xlsx"), [("X", '12345')]) + with zipfile.ZipFile(src) as zin, zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as zout: + for info in zin.infolist(): + zout.writestr(info.filename, zin.read(info.filename)) + return path + + +def test_bad_crc_in_a_sheet_is_a_format_error(tmp_path): + f = _corrupt(_stored_xlsx(tmp_path / "a.xlsx"), b"12345", b"54321") + with pytest.raises(FormatError, match=r"a\.xlsx: member xl/worksheets/sheet1\.xml is corrupt: .*CRC"): + list(rows(f, "X")) + + +def test_bad_crc_in_the_workbook_is_a_format_error(tmp_path): + f = _corrupt(_stored_xlsx(tmp_path / "a.xlsx"), b'sheetId="1"', b'sheetId="2"') + with pytest.raises(FormatError, match=r"a\.xlsx: member xl/workbook\.xml is corrupt"): + sheet_names(f) + + +def test_corrupt_deflate_stream_is_a_format_error(tmp_path): + padding = "".join(f'{i * 7919 % 100003}' for i in range(1, 3000)) + f = _xlsx(tmp_path / "a.xlsx", [("X", padding)]) + with zipfile.ZipFile(f) as z: + info = z.getinfo("xl/worksheets/sheet1.xml") + start = info.header_offset + 30 + len(info.filename.encode()) + len(info.extra) + data = bytearray(f.read_bytes()) + for k in range(start + 50, start + 250): + data[k] ^= 0xFF + f.write_bytes(bytes(data)) + with pytest.raises(FormatError, match=r"a\.xlsx: member xl/worksheets/sheet1\.xml is corrupt"): + list(rows(f, "X")) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..2c58e72 --- /dev/null +++ b/uv.lock @@ -0,0 +1,142 @@ +version = 1 +revision = 5 +requires-python = ">=3.12" + +[[package]] +name = "benchcommons-tools-dataset" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, + { name = "xlrd" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyyaml", specifier = "==6.0.3" }, + { name = "xlrd", specifier = "==2.0.2" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = "==9.1.1" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +]