feat(memstore): mem/db scriptable API + entity stringify (Benerator parity)

datamimic_ce/clients/mongodb_client.py

@@ -17,6 +17,7 @@
 from datamimic_ce.connection_config.mongodb_connection_config import MongoDBConnectionConfig
 from datamimic_ce.constants.attribute_constants import META_SELECTOR, META_TARGET_ENTITY, META_TYPE
 from datamimic_ce.data_sources.data_source_pagination import DataSourcePagination
+from datamimic_ce.domains.domain_core.base_entity import stringify_if_entity
 
 
 class MongoDBClient(DatabaseClient):
@@ -30,6 +31,7 @@ def _to_bson(value: Any) -> Any:
         DATAMIMIC ``type="decimal"`` field round-trips through mongo."""
         if isinstance(value, Decimal):
             return Decimal128(value)
+        value = stringify_if_entity(value)  # whole entity bound to a field -> str(entity.to_dict())
         if isinstance(value, Mapping):
             return {k: MongoDBClient._to_bson(v) for k, v in value.items()}
         if isinstance(value, list):

datamimic_ce/clients/rdbms_client.py

@@ -18,6 +18,7 @@
 from datamimic_ce.config import settings
 from datamimic_ce.connection_config.rdbms_connection_config import RdbmsConnectionConfig
 from datamimic_ce.data_sources.data_source_pagination import DataSourcePagination
+from datamimic_ce.domains.domain_core.base_entity import stringify_if_entity
 from datamimic_ce.logger import logger
 
 # SQLAlchemy create_engine kwargs DATAMIMIC forwards (pooling/behavior). Anything else in the
@@ -413,6 +414,11 @@ def insert(self, table_name: str, data_list: list):
             return
 
         data_list = self._apply_global_json_config(data_list)
+        # A whole entity bound into a scalar column (Benerator's toString() idiom, e.g.
+        # <key script="person"> into a varchar field) has no driver-level binding otherwise -
+        # confirmed this raises hard today (sqlite3.ProgrammingError: type not supported), so
+        # this only turns a crash into a correct write, never changes behavior for what works now.
+        data_list = [{k: stringify_if_entity(v) for k, v in row.items()} for row in data_list]
 
         with engine.begin() as connection:
             try:

datamimic_ce/contexts/setup_context.py

@@ -446,6 +446,12 @@ def add_client(self, client_id: str, client: Client):
         :return:
         """
         self._clients[client_id] = client
+        # Also bind by id into the script namespace (Benerator parity: <execute>/<variable script=>
+        # can reference a declared <database>/<mongodb> id directly, e.g. `db.something()`) - both
+        # eval_namespace (copies self._namespace wholesale) and evaluate_python_expression's scope
+        # building read from this same dict, so this covers both script-evaluation paths regardless
+        # of statement order.
+        self._namespace[client_id] = client
 
     def get_client_by_id(self, client_id: str):
         """

datamimic_ce/domains/domain_core/base_entity.py

@@ -54,3 +54,12 @@ def __getattr__(self, name: str) -> Any:
     def to_dict(self) -> dict[str, Any]:
         """Convert the entity to a dictionary."""
         raise NotImplementedError("Subclasses must implement this method.")
+
+
+def stringify_if_entity(value: Any) -> Any:
+    """A whole entity bound into a scalar column (e.g. `<key script="person">` into a varchar/
+    string field, Benerator's toString() idiom) has no sane driver-level representation - both the
+    RDBMS and Mongo write paths call this first. `BaseEntity` defines no `__str__`, so a bare
+    `str(value)` would write Python's default `<...Person object at 0x...>` (a non-deterministic
+    memory address); `.to_dict()` is the one meaningful representation every entity provides."""
+    return str(value.to_dict()) if isinstance(value, BaseEntity) else value

datamimic_ce/exporters/memstore.py

@@ -64,3 +64,24 @@ def consume(self, product: tuple):
         name = product[0]
         data = product[1]
         self._storage[name] = self._storage.get(name, []) + data
+
+    def sumEntityColumn(self, product_type: str, column: str):
+        """Sum a numeric column across all rows of one type (Benerator parity). Values are
+        coerced via float() - memstore rows sourced from CSV carry strings, not numbers."""
+        total = sum((float(row[column]) for row in self._storage.get(product_type, [])), 0.0)
+        return int(total) if total.is_integer() else total
+
+    def entityCount(self, product_type: str) -> int:
+        """Alias of get_data_len_by_type (Benerator's camelCase naming)."""
+        return self.get_data_len_by_type(product_type)
+
+    def removeNotExistingIds(self, product_type: str, id_col: str, ref_type: str, client) -> None:
+        """Keep only the rows of `product_type` whose `id_col` value exists in `ref_type` as read
+        from an RDBMS `client` (Benerator parity: an inner-join filter). Mutates the stored rows in
+        place - no return value, matching Benerator's imperative "remove" semantics. Both sides of
+        the id comparison are string-coerced: `client`'s column is DB-typed (e.g. int), memstore
+        rows sourced from CSV carry strings for the same logical id."""
+        existing = {str(row[0]) for row in client.get_random_rows_by_columns(ref_type, [id_col])}
+        self._storage[product_type] = [
+            r for r in self._storage.get(product_type, []) if str(r.get(id_col)) in existing
+        ]

datamimic_ce/tasks/memstore_task.py

@@ -19,3 +19,7 @@ def statement(self) -> MemstoreStatement:
 
     def execute(self, ctx: Context):
         ctx.root.memstore_manager.add_memstore(self.statement.id)
+        # Bind by id into the script namespace (Benerator parity: <execute>/<variable script=>
+        # can reference `mem` directly, e.g. `mem.sumEntityColumn(...)`) - mirrors add_client's
+        # binding for <database>/<mongodb>.
+        ctx.root.namespace[self.statement.id] = ctx.root.memstore_manager.get_memstore(self.statement.id)

tests_ce/external_service_tests/test_entity_stringify_mongo/conf/environment.env.properties

@@ -0,0 +1,31 @@
+#
+#  Copyright (c) 2024 Rapiddweller Asia Co., Ltd.
+#  All rights reserved.
+#
+#  This software and related documentation are provided under a license
+#  agreement containing restrictions on use and disclosure and are
+#  protected by intellectual property laws. Except as expressly permitted
+#  in your license agreement or allowed by law, you may not use, copy,
+#  reproduce, translate, broadcast, modify, license, transmit, distribute,
+#  exhibit, perform, publish, or display any part, in any form, or by any means.
+#
+#  This software is the confidential and proprietary information of
+#  Rapiddweller Asia Co., Ltd. ("Confidential Information"). You shall not
+#  disclose such Confidential Information and shall use it only in accordance
+#  with the terms of the license agreement you entered into with Rapiddweller Asia Co., Ltd.
+#
+#
+
+postgres.db.host=postgres
+postgres.db.port=5432
+postgres.db.database=rd-datamimic
+postgres.db.user=datamimic
+postgres.db.password=datamimic
+postgres.db.schema=simple
+postgres.db.dbms=postgresql
+
+mongodb.mongo.host=mongo
+mongodb.mongo.port=27017
+mongodb.mongo.database=datamimic
+mongodb.mongo.user=datamimic
+mongodb.mongo.password=datamimic

tests_ce/external_service_tests/test_entity_stringify_mongo/conf/local.env.properties

@@ -0,0 +1,31 @@
+#
+#  Copyright (c) 2024 Rapiddweller Asia Co., Ltd.
+#  All rights reserved.
+#
+#  This software and related documentation are provided under a license
+#  agreement containing restrictions on use and disclosure and are
+#  protected by intellectual property laws. Except as expressly permitted
+#  in your license agreement or allowed by law, you may not use, copy,
+#  reproduce, translate, broadcast, modify, license, transmit, distribute,
+#  exhibit, perform, publish, or display any part, in any form, or by any means.
+#
+#  This software is the confidential and proprietary information of
+#  Rapiddweller Asia Co., Ltd. ("Confidential Information"). You shall not
+#  disclose such Confidential Information and shall use it only in accordance
+#  with the terms of the license agreement you entered into with Rapiddweller Asia Co., Ltd.
+#
+
+postgres.db.host=localhost
+postgres.db.port=45432
+postgres.db.database=rd-datamimic
+postgres.db.user=rdadm
+postgres.db.password=rdtest!
+postgres.db.schema=simple
+postgres.db.dbms=postgresql
+
+mongodb.mongo.host=localhost
+mongodb.mongo.port=47017
+mongodb.mongo.database=datamimic
+mongodb.mongo.user=datamimic
+mongodb.mongo.password=datamimic
+mongodb.mongo.authSource=admin

tests_ce/external_service_tests/test_entity_stringify_mongo/test_entity_stringify_mongo.py

@@ -0,0 +1,26 @@
+# DATAMIMIC
+# Copyright (c) 2023-2025 Rapiddweller Asia Co., Ltd.
+# This software is licensed under the MIT License.
+# See LICENSE file for the full text of the license.
+# For questions and support, contact: info@rapiddweller.com
+
+"""Binding a whole entity (<key script="person">) into a scalar Mongo field - Benerator parity
+(Benerator's toString()). Confirmed empirically before this fix: bson.errors.InvalidDocument (a
+bare Person can't be BSON-encoded) - so this is a pure improvement, not a behavior change with
+regression risk."""
+
+from pathlib import Path
+
+from datamimic_ce.data_mimic_test import DataMimicTest
+
+_dir = Path(__file__).resolve().parent
+
+
+def test_entity_stringify_mongo():
+    engine = DataMimicTest(_dir, "test_entity_stringify_mongo.xml", capture_test_result=True)
+    engine.test_with_timer()
+    rows = engine.capture_result()["check"]
+    assert len(rows) == 1
+    name = rows[0]["name"]
+    assert isinstance(name, str)
+    assert name.startswith("{") and "given_name" in name  # str(person.to_dict()), not a repr

tests_ce/external_service_tests/test_entity_stringify_mongo/test_entity_stringify_mongo.xml

@@ -0,0 +1,10 @@
+<setup>
+    <mongodb id="mongodb"/>
+    <!-- clean slate: earlier runs leave documents behind otherwise -->
+    <generate name="t_cleanup" source="mongodb" selector="find: 't', filter: {}" target="mongodb.delete"/>
+    <generate name="t" count="1" target="mongodb">
+        <variable name="person" entity="Person"/>
+        <key name="name" script="person"/>
+    </generate>
+    <generate name="check" source="mongodb" selector="find: 't', filter: {}" target=""/>
+</setup>

tests_ce/integration_tests/test_entity_stringify/setup_rdbms.sql

@@ -0,0 +1 @@
+CREATE TABLE t (id INTEGER, name VARCHAR(2000));

tests_ce/integration_tests/test_entity_stringify/test_entity_stringify.py

@@ -0,0 +1,46 @@
+# DATAMIMIC
+# Copyright (c) 2023-2025 Rapiddweller Asia Co., Ltd.
+# This software is licensed under the MIT License.
+# See LICENSE file for the full text of the license.
+# For questions and support, contact: info@rapiddweller.com
+
+"""Binding a whole entity (<key script="person">) into a scalar RDBMS column - Benerator parity
+(Benerator's toString()). Confirmed empirically before this fix: sqlite3.ProgrammingError,
+'type Person is not supported' - so this is a pure improvement, not a behavior change with
+regression risk (nothing succeeds today for this input)."""
+
+import shutil
+import sqlite3
+from pathlib import Path
+
+from datamimic_ce.data_mimic_test import DataMimicTest
+
+_dir = Path(__file__).resolve().parent
+
+
+def test_entity_stringify_rdbms():
+    """capture_result() reflects the generated record BEFORE export-time normalization (the
+    stringify guard lives in RdbmsClient.insert) - read the actual written row back from sqlite
+    to verify what really landed in the column."""
+    db_dir = _dir / "db"
+    repo_root_db_dir = _dir.parents[2] / "db"
+    for d in (db_dir, repo_root_db_dir):
+        shutil.rmtree(d, ignore_errors=True)
+    try:
+        engine = DataMimicTest(_dir, "test_entity_stringify_rdbms.xml", capture_test_result=True)
+        engine.test_with_timer()
+
+        db_files = list(repo_root_db_dir.glob("*.sqlite")) if repo_root_db_dir.is_dir() else []
+        db_files += list(db_dir.glob("*.sqlite")) if db_dir.is_dir() else []
+        assert len(db_files) == 1, f"expected exactly one sqlite file, found {db_files}"
+        conn = sqlite3.connect(db_files[0])
+        try:
+            name = conn.execute("SELECT name FROM t").fetchone()[0]
+        finally:
+            conn.close()
+
+        assert isinstance(name, str)
+        assert name.startswith("{") and "given_name" in name  # str(person.to_dict()), not a repr
+    finally:
+        for d in (db_dir, repo_root_db_dir):
+            shutil.rmtree(d, ignore_errors=True)

tests_ce/integration_tests/test_entity_stringify/test_entity_stringify_rdbms.xml

@@ -0,0 +1,9 @@
+<setup>
+    <database id="db" dbms="sqlite" database="stringify_rdbms.sqlite"/>
+    <execute uri="setup_rdbms.sql" target="db"/>
+    <generate name="t" count="1" target="db">
+        <variable name="person" entity="Person"/>
+        <key name="id" generator="IncrementGenerator"/>
+        <key name="name" script="person"/>
+    </generate>
+</setup>

tests_ce/integration_tests/test_memstore_api/data/test_file.wgt.ent.csv

@@ -0,0 +1,4 @@
+test_id|flag|count
+1|true|5
+2|false|3
+3|true|7

tests_ce/integration_tests/test_memstore_api/setup_ref.sql

@@ -0,0 +1,2 @@
+CREATE TABLE ref (id INTEGER);
+INSERT INTO ref (id) VALUES (1), (3);

tests_ce/integration_tests/test_memstore_api/test_memstore_api.py

@@ -0,0 +1,73 @@
+# DATAMIMIC
+# Copyright (c) 2023-2025 Rapiddweller Asia Co., Ltd.
+# This software is licensed under the MIT License.
+# See LICENSE file for the full text of the license.
+# For questions and support, contact: info@rapiddweller.com
+
+"""Memstore API extensions unblocking memstore/memstore.ben.xml (Benerator parity)."""
+
+import shutil
+from pathlib import Path
+
+from datamimic_ce.data_mimic_test import DataMimicTest
+
+_dir = Path(__file__).resolve().parent
+
+
+def test_wgt_ent_csv_reads_via_iterate():
+    """'.wgt.ent.csv' despite the name is a normal headered CSV (test_id|flag|count), NOT the
+    headerless value|weight '.wgt.csv' format - already readable via <iterate> today, verified
+    directly rather than assumed. This is a regression test, not a fix."""
+    engine = DataMimicTest(_dir, "test_wgt_ent_csv_iterate.xml", capture_test_result=True)
+    engine.test_with_timer()
+    rows = engine.capture_result()["test_file"]
+    assert len(rows) == 3
+    assert {r["test_id"] for r in rows} == {"1", "2", "3"}
+    assert {r["flag"] for r in rows} == {"true", "false"}
+
+
+def test_memstore_accessible_by_id_before_and_after_execute():
+    """mem is bound into the script namespace at <memstore> registration time, not only inside
+    <execute> - so a <key script="mem...."> resolves it whether it runs BEFORE or AFTER any
+    <execute> statement in the descriptor (proves this isn't riding eval_namespace's diff-
+    writeback side effect, which would only work in the "after" ordering)."""
+    engine = DataMimicTest(_dir, "test_memstore_variable_script_access.xml", capture_test_result=True)
+    engine.test_with_timer()
+    result = engine.capture_result()
+    assert result["before_execute"][0]["count"] == 5
+    assert result["after_execute"][0]["count"] == 5
+
+
+def test_memstore_sum_feeds_a_subsequent_count():
+    """mem.sumEntityColumn() inside <execute>, its result driving count="{totalCount}" on a later
+    <generate>, and mem.entityCount() confirming it - the exact pipeline
+    memstore/memstore.ben.xml uses. values="5,3,7" picks randomly per row (not one-of-each), so
+    only the total's range is asserted, not an exact value - the point is internal consistency
+    across the whole sum -> count -> generate -> recount pipeline."""
+    engine = DataMimicTest(_dir, "test_memstore_sum_and_count.xml", capture_test_result=True)
+    engine.test_with_timer()
+    row = engine.capture_result()["result"][0]
+    assert 9 <= row["totalCount"] <= 21  # 3 rows, each in {3,5,7}
+    assert row["teCount"] == row["totalCount"]
+
+
+def test_memstore_remove_not_existing_ids():
+    """mem.removeNotExistingIds('t','id','ref',db) inside <execute> - an inner-join filter
+    against a real RDBMS table. ref has ids {1,3}; mem has ids {1,2,3} - only 2 should be
+    removed."""
+    output_dir = _dir / "output"
+    # sqlite relative paths resolve against the process CWD, not the descriptor dir - clean up
+    # both the test's own dir (the documented convention elsewhere in this suite) and the repo
+    # root (where it actually lands when pytest's CWD is the repo root).
+    repo_root_db_dir = _dir.parents[2] / "db"
+    local_db_dir = _dir / "db"
+    for d in (output_dir, repo_root_db_dir, local_db_dir):
+        shutil.rmtree(d, ignore_errors=True)
+    try:
+        engine = DataMimicTest(_dir, "test_memstore_remove_not_existing_ids.xml", capture_test_result=True)
+        engine.test_with_timer()
+        row = engine.capture_result()["result"][0]
+        assert row["remaining_ids"] == [1, 3]
+    finally:
+        for d in (output_dir, repo_root_db_dir, local_db_dir):
+            shutil.rmtree(d, ignore_errors=True)

tests_ce/integration_tests/test_memstore_api/test_memstore_remove_not_existing_ids.xml

@@ -0,0 +1,15 @@
+<setup>
+    <database id="db" dbms="sqlite" database="output/remove_not_existing_ids.sqlite"/>
+    <execute uri="setup_ref.sql" target="db"/>
+    <memstore id="mem"/>
+
+    <generate name="t" count="3" target="mem">
+        <key name="id" generator="IncrementGenerator"/>
+    </generate>
+
+    <execute>mem.removeNotExistingIds('t', 'id', 'ref', db)</execute>
+
+    <generate name="result" count="1" target="JSON">
+        <array name="remaining_ids" script="sorted(r['id'] for r in mem.get_all_data_by_type('t'))"/>
+    </generate>
+</setup>

tests_ce/integration_tests/test_memstore_api/test_memstore_sum_and_count.xml

@@ -0,0 +1,17 @@
+<setup>
+    <memstore id="mem"/>
+    <generate name="test_file" count="3" target="mem">
+        <key name="count" values="5,3,7"/>
+    </generate>
+
+    <execute>totalCount = mem.sumEntityColumn('test_file', 'count')</execute>
+
+    <generate name="te" count="{totalCount}" target="mem">
+        <key name="v" generator="IncrementGenerator"/>
+    </generate>
+
+    <generate name="result" count="1" target="JSON">
+        <key name="totalCount" script="totalCount"/>
+        <key name="teCount" script="mem.entityCount('te')"/>
+    </generate>
+</setup>

tests_ce/integration_tests/test_memstore_api/test_memstore_variable_script_access.xml

@@ -0,0 +1,19 @@
+<setup>
+    <memstore id="mem"/>
+    <generate name="rows" count="5" target="mem">
+        <key name="v" generator="IncrementGenerator"/>
+    </generate>
+
+    <!-- mem accessed via <key script> BEFORE any <execute> ran - proves the binding does not
+         depend on eval_namespace's diff-writeback side effect (order-independent). -->
+    <generate name="before_execute" count="1" target="JSON">
+        <key name="count" script="mem.entityCount('rows')"/>
+    </generate>
+
+    <execute>marker = 1</execute>
+
+    <!-- mem accessed via <key script> AFTER an <execute> ran. -->
+    <generate name="after_execute" count="1" target="JSON">
+        <key name="count" script="mem.entityCount('rows')"/>
+    </generate>
+</setup>

tests_ce/integration_tests/test_memstore_api/test_wgt_ent_csv_iterate.xml

@@ -0,0 +1,4 @@
+<setup>
+    <memstore id="mem"/>
+    <iterate name="test_file" source="data/test_file.wgt.ent.csv" separator="|" target="mem,ConsoleExporter"/>
+</setup>