I/O: Migrate from undatum to iterabledata

Author: amotl

cratedb_toolkit/io/mongodb/adapter.py

@@ -17,7 +17,7 @@
 from attrs import define, field
 from boltons.urlutils import URL
 from bson.raw_bson import RawBSONDocument
-from undatum.common.iterable import IterableData
+from iterable.helpers.detect import open_iterable
 
 from cratedb_toolkit.io.mongodb.model import DocumentDict
 from cratedb_toolkit.io.mongodb.util import batches
@@ -138,7 +138,7 @@ def query(self):
         if self._path.suffix in [".json", ".jsonl", ".ndjson"]:
             data = read_json(str(self._path))
         elif ".bson" in str(self._path):
-            data = IterableData(str(self._path), options={"format_in": "bson"}).iter()
+            data = open_iterable(str(self._path))
         else:
             raise ValueError(f"Unsupported file type: {self._path.suffix}")
         return batches(data, self.batch_size)

pyproject.toml

@@ -220,12 +220,12 @@ optional-dependencies.mcp = [
 optional-dependencies.mongodb = [
   "commons-codec[mongodb]>=0.0.22",
   "cratedb-toolkit[io,io-recipe]",
+  "iterabledata<2",
   "orjson>=3.3.1,<4",
   "pymongo>=3.10.1,<4.17",
   "pyorc<0.11; python_version<'3.10'",
   "python-bsonjs<0.8",
   "rich>=3.3.2,<16",
-  "undatum<1.2",
 ]
 optional-dependencies.nlsql = [
   "llama-index-llms-anthropic<0.12; python_version>='3.10'",
@@ -369,6 +369,7 @@ analysis.allowed-unresolved-imports = [
   "importlib_metadata.**",
   "importlib_resources.**",
   "ingestr.**",
+  "iterable.**",
   "jessiql.**",
   "kaggle.**",
   "kinesis.**",
@@ -378,7 +379,6 @@ analysis.allowed-unresolved-imports = [
   "pymongo.**",
   "sqlalchemy.**",
   "tikray.**",
-  "undatum.**",
 ]
 
 [tool.pytest]

tests/io/mongodb/conftest.py

@@ -14,7 +14,7 @@
 pytest.importorskip("bsonjs", reason="Skipping tests because bsonjs is not installed")
 pytest.importorskip("pymongo", reason="Skipping tests because pymongo is not installed")
 pytest.importorskip("rich", reason="Skipping tests because rich is not installed")
-pytest.importorskip("undatum", reason="Skipping tests because undatum is not installed")
+pytest.importorskip("iterable", reason="Skipping tests because iterabledata is not installed")
 
 
 # Define databases to be deleted before running each test case.