|
| 1 | +"""Detect and resolve extra (non-core) artifacts shipped next to a :class:`CuratedContainer`. |
| 2 | +
|
| 3 | +Data Foundry persists six core files per container (dataset + dtypes + container metadata |
| 4 | ++ three pydantic-tagged metadata blobs) and optionally a test-dataset pair. Producers may |
| 5 | +also ship arbitrary sidecar files in the same directory — embedding caches, per-fold |
| 6 | +prediction archives, dataset documentation, etc. |
| 7 | +
|
| 8 | +Data Foundry does not interpret these files. It exposes just enough to discover them and |
| 9 | +resolve their path so the caller can load them with whatever library is appropriate |
| 10 | +(``pd.read_parquet``, ``json.load``, ``np.load``, …). |
| 11 | +
|
| 12 | +The toy container shipped with the package includes one such extra: ``toy_extra.parquet``. |
| 13 | +
|
| 14 | +Run:: |
| 15 | +
|
| 16 | + python examples/load_extra_files.py |
| 17 | + python examples/load_extra_files.py /path/to/warehouse/<name>/<uuid> |
| 18 | +""" |
| 19 | + |
| 20 | +from __future__ import annotations |
| 21 | + |
| 22 | +import sys |
| 23 | +from pathlib import Path |
| 24 | + |
| 25 | +import pandas as pd |
| 26 | +from data_foundry.curation_container import CuratedContainer |
| 27 | +from data_foundry.examples import get_toy_container_path |
| 28 | + |
| 29 | + |
| 30 | +def main(path: Path) -> None: |
| 31 | + """List the container's extra files and load the first one as a DataFrame.""" |
| 32 | + container = CuratedContainer.load(path) |
| 33 | + |
| 34 | + extras = container.list_extra_files() |
| 35 | + print(f"Extra files present in {path}:") |
| 36 | + for name in extras: |
| 37 | + print(f" - {name}") |
| 38 | + if not extras: |
| 39 | + print(" (none)") |
| 40 | + return |
| 41 | + |
| 42 | + target = extras[0] |
| 43 | + print(f"\nhas_extra_file({target!r}) -> {container.has_extra_file(target)}") |
| 44 | + print(f"extra_file_path({target!r}) -> {container.extra_file_path(target)}") |
| 45 | + |
| 46 | + print(f"has_extra_file('does_not_exist.bin') -> {container.has_extra_file('does_not_exist.bin')}") |
| 47 | + |
| 48 | + resolved = container.extra_file_path(target) |
| 49 | + if resolved.suffix == ".parquet": |
| 50 | + df = pd.read_parquet(resolved) |
| 51 | + print(f"\nLoaded {target} as DataFrame {df.shape}:") |
| 52 | + print(df.head().to_string()) |
| 53 | + |
| 54 | + |
| 55 | +if __name__ == "__main__": |
| 56 | + container_path = Path(sys.argv[1]) if len(sys.argv) > 1 else get_toy_container_path() |
| 57 | + main(container_path) |
0 commit comments