Skip to content

Commit 0447ea8

Browse files
add: new extra file example and in package data
1 parent fb46150 commit 0447ea8

2 files changed

Lines changed: 57 additions & 0 deletions

File tree

examples/load_extra_files.py

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"""Detect and resolve extra (non-core) artifacts shipped next to a :class:`CuratedContainer`.
2+
3+
Data Foundry persists six core files per container (dataset + dtypes + container metadata
4+
+ three pydantic-tagged metadata blobs) and optionally a test-dataset pair. Producers may
5+
also ship arbitrary sidecar files in the same directory — embedding caches, per-fold
6+
prediction archives, dataset documentation, etc.
7+
8+
Data Foundry does not interpret these files. It exposes just enough to discover them and
9+
resolve their path so the caller can load them with whatever library is appropriate
10+
(``pd.read_parquet``, ``json.load``, ``np.load``, …).
11+
12+
The toy container shipped with the package includes one such extra: ``toy_extra.parquet``.
13+
14+
Run::
15+
16+
python examples/load_extra_files.py
17+
python examples/load_extra_files.py /path/to/warehouse/<name>/<uuid>
18+
"""
19+
20+
from __future__ import annotations
21+
22+
import sys
23+
from pathlib import Path
24+
25+
import pandas as pd
26+
from data_foundry.curation_container import CuratedContainer
27+
from data_foundry.examples import get_toy_container_path
28+
29+
30+
def main(path: Path) -> None:
31+
"""List the container's extra files and load the first one as a DataFrame."""
32+
container = CuratedContainer.load(path)
33+
34+
extras = container.list_extra_files()
35+
print(f"Extra files present in {path}:")
36+
for name in extras:
37+
print(f" - {name}")
38+
if not extras:
39+
print(" (none)")
40+
return
41+
42+
target = extras[0]
43+
print(f"\nhas_extra_file({target!r}) -> {container.has_extra_file(target)}")
44+
print(f"extra_file_path({target!r}) -> {container.extra_file_path(target)}")
45+
46+
print(f"has_extra_file('does_not_exist.bin') -> {container.has_extra_file('does_not_exist.bin')}")
47+
48+
resolved = container.extra_file_path(target)
49+
if resolved.suffix == ".parquet":
50+
df = pd.read_parquet(resolved)
51+
print(f"\nLoaded {target} as DataFrame {df.shape}:")
52+
print(df.head().to_string())
53+
54+
55+
if __name__ == "__main__":
56+
container_path = Path(sys.argv[1]) if len(sys.argv) > 1 else get_toy_container_path()
57+
main(container_path)

0 commit comments

Comments
 (0)