From a82936c60bbdbd011a5733314df4390f4e85a1f0 Mon Sep 17 00:00:00 2001 From: marcoSanti Date: Fri, 25 Sep 2026 11:49:53 +0200 Subject: [PATCH 1/3] Extended configuration options --- bindings/python_bindings.cpp | 8 +++++++- capiocl/configuration.h | 7 +++++-- capiocl/engine.h | 3 +++ capiocl/parser.h | 22 +++++++++++++++----- src/Engine.cpp | 8 ++++++++ src/Parser.cpp | 31 +++++++++++++++++++++++----- src/configuration.cpp | 4 ++++ src/parsers/v1.1.cpp | 35 +++++++++++++++++++------------- src/parsers/v1.cpp | 19 +++++++++-------- tests/cpp/test_configuration.hpp | 20 ++++++++++++++++++ 10 files changed, 121 insertions(+), 36 deletions(-) diff --git a/bindings/python_bindings.cpp b/bindings/python_bindings.cpp index fe96f2e..71095b4 100644 --- a/bindings/python_bindings.cpp +++ b/bindings/python_bindings.cpp @@ -119,7 +119,13 @@ PYBIND11_MODULE(_py_capio_cl, m) { .def(py::self == py::self); py::class_(m, "Parser", "The CAPIO-CL Parser component.") - .def_static("parse", &capiocl::parser::Parser::parse, py::arg("source"), + .def_static("parse", + [](const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { + return capiocl::parser::Parser::parse(source, resolve_prefix, + store_only_in_memory); + }, + py::arg("source"), py::arg("resolve_prefix") = "", py::arg("store_only_in_memory") = false) .def("__str__", [](const capiocl::parser::Parser &e) { diff --git a/capiocl/configuration.h b/capiocl/configuration.h index db3c4b5..f9a29fb 100644 --- a/capiocl/configuration.h +++ b/capiocl/configuration.h @@ -55,7 +55,10 @@ class capiocl::configuration::CapioClConfiguration { void set(const ConfigurationEntry &entry); public: - ~CapioClConfiguration() = default; + CapioClConfiguration() = default; + + /// @brief Build a configuration from an external generated unordered map of key values + explicit CapioClConfiguration(std::unordered_map config); /** * Load a configuration from a TOML file @@ -106,4 +109,4 @@ class capiocl::configuration::CapioClConfigurationException final : public std:: [[nodiscard]] const char *what() const noexcept override { return message.c_str(); } }; -#endif \ No newline at end of file +#endif diff --git a/capiocl/engine.h b/capiocl/engine.h index f759d1b..bc5b4e5 100644 --- a/capiocl/engine.h +++ b/capiocl/engine.h @@ -92,6 +92,9 @@ class Engine final { ///@brief Configuration imported from CAPIO-CL config TOML file configuration::CapioClConfiguration configuration; + /// @brief Configure the monitor backend from the currently loaded configuration + void configureMonitorBackends(); + /// @brief Monitor instance to check runtime information of CAPIO-CL files monitor::Monitor monitor; diff --git a/capiocl/parser.h b/capiocl/parser.h index 6ebafee..b0c29e4 100644 --- a/capiocl/parser.h +++ b/capiocl/parser.h @@ -37,11 +37,14 @@ class Parser final { * @param resolve_prefix Prefix to prepend to path if found to be relative * @param store_only_in_memory Flag to set to returned instance of Engine if required to * store all files in memory + * @param config An externally provided configuration to be used within the CAPIO-CL engine + * instance that will be started up * @return Parsed Engine. */ static engine::Engine *parse_v1(const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, - bool store_only_in_memory); + bool store_only_in_memory, + const configuration::CapioClConfiguration *config); /** * Parser for the V1.1 Specification of the CAPIO-CL language @@ -49,11 +52,14 @@ class Parser final { * @param resolve_prefix Prefix to prepend to path if found to be relative * @param store_only_in_memory Flag to set to returned instance of Engine if required to * store all files in memory + * @param config An externally provided configuration to be used within the + * CAPIO-CL engine instance that will be started up * @return Parsed Engine. */ static engine::Engine *parse_v1_1(const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, - bool store_only_in_memory); + bool store_only_in_memory, + const configuration::CapioClConfiguration *config); }; /** @@ -74,6 +80,9 @@ class Parser final { const std::filesystem::path &prefix); public: + /** Build an engine using only CAPIO-CL runtime configuration parameters. */ + static engine::Engine *parse(const configuration::CapioClConfiguration &config); + /** * Validate a CAPIO-CL configuration file according to the JSON schema of the language * @param doc The loaded CAPIO-CL configuration file @@ -87,13 +96,16 @@ class Parser final { * @param source Input CAPIO-CL Json configuration File * @param resolve_prefix If paths are found to be relative, they are appended to this path * @param store_only_in_memory Set to true to set all files to be stored in memory + * @param config An externally provided configuration to be used within the CAPIO-CL engine + * instance that will be started up * @return Engine instance with the information provided by the config file * @throw ParserException */ static engine::Engine *parse(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix = "", - bool store_only_in_memory = false); + const std::filesystem::path &resolve_prefix = "", + bool store_only_in_memory = false, + const configuration::CapioClConfiguration *config = nullptr); }; } // namespace capiocl::parser -#endif // CAPIO_CL_PARSER_H \ No newline at end of file +#endif // CAPIO_CL_PARSER_H diff --git a/src/Engine.cpp b/src/Engine.cpp index 3fa0739..d56cf43 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -195,6 +195,7 @@ capiocl::engine::Engine::Engine(const configuration::CapioClConfiguration &confi CAPIO_CL_DEFAULT_WF_NAME); LOG("selected workflow name=%s source=environment node=%s", workflow_name.c_str(), node_name.c_str()); + configureMonitorBackends(); } void capiocl::engine::Engine::_newFile(const std::filesystem::path &path) const { @@ -1129,6 +1130,12 @@ void capiocl::engine::Engine::loadConfiguration(const std::string &path) { configuration.load(path); LOG("loaded engine configuration path=%s", path.c_str()); + configureMonitorBackends(); +} + +void capiocl::engine::Engine::configureMonitorBackends() { + START_LOG(calf_current_tid(), "call()"); + std::string multicast_monitor_enabled, fs_monitor_enabled; configuration.getParameter("monitor.mcast.enabled", &multicast_monitor_enabled, "false"); @@ -1156,6 +1163,7 @@ void capiocl::engine::Engine::loadConfiguration(const std::string &path) { CALF_PRINT_COLOR(CALF_CLI_LEVEL_WARNING, "Skipping registration of FileSystemMonitor"); } } + void capiocl::engine::Engine::useDefaultConfiguration() { START_LOG(calf_current_tid(), "call()"); configuration.loadDefaults(); diff --git a/src/Parser.cpp b/src/Parser.cpp index 5af41a7..3f4a237 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -52,9 +52,30 @@ void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, const cha } } -capiocl::engine::Engine *capiocl::parser::Parser::parse(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix, - bool store_only_in_memory) { +capiocl::engine::Engine * +capiocl::parser::Parser::parse(const configuration::CapioClConfiguration &config) { + std::string source, resolve_prefix, store_only_in_memory, dynamic_api_enabled; + config.getParameter("config_path", &source, ""); + config.getParameter("resolve_path", &resolve_prefix, ""); + config.getParameter("store_all_in_memory", &store_only_in_memory, "false"); + config.getParameter("dynamic_api.enabled", &dynamic_api_enabled, "false"); + + auto *engine = source.empty() + ? new engine::Engine(config) + : parse(source, resolve_prefix, store_only_in_memory == "true", &config); + + if (source.empty() && store_only_in_memory == "true") { + engine->setAllStoreInMemory(); + } + if (dynamic_api_enabled == "true") { + engine->startApiServer(); + } + return engine; +} + +capiocl::engine::Engine *capiocl::parser::Parser::parse( + const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, + bool store_only_in_memory, const configuration::CapioClConfiguration *config) { START_LOG(calf_current_tid(), "call()"); LOG("parse requested source=%s resolve_prefix=%s memory_only=%d", source.string().c_str(), resolve_prefix.string().c_str(), static_cast(store_only_in_memory)); @@ -90,9 +111,9 @@ capiocl::engine::Engine *capiocl::parser::Parser::parse(const std::filesystem::p capio_cl_release.c_str()); if (capio_cl_release == CAPIO_CL_VERSION::V1) { - return available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory); + return available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory, config); } else if (capio_cl_release == CAPIO_CL_VERSION::V1_1) { - return available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory); + return available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory, config); } else { LOG("unsupported specification version=%s source=%s", capio_cl_release.c_str(), source.string().c_str()); diff --git a/src/configuration.cpp b/src/configuration.cpp index 426d5c8..6e960e2 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -6,6 +6,10 @@ #include "capiocl/configuration.h" #include "toml++/toml.hpp" +capiocl::configuration::CapioClConfiguration::CapioClConfiguration( + std::unordered_map config) + : config(std::move(config)) {} + void load_config_to_memory(const toml::table &tbl, std::unordered_map &map, const std::string &prefix = "") { diff --git a/src/parsers/v1.1.cpp b/src/parsers/v1.1.cpp index b03093f..ab65c66 100644 --- a/src/parsers/v1.1.cpp +++ b/src/parsers/v1.1.cpp @@ -7,13 +7,14 @@ #include "capiocl/engine.h" #include "capiocl/parser.h" -capiocl::engine::Engine * -capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix, - bool store_only_in_memory) { +capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1_1( + const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, + bool store_only_in_memory, const configuration::CapioClConfiguration *config) { + START_LOG(calf_current_tid(), "call()"); + std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - auto engine = new engine::Engine(false); + auto engine = config == nullptr ? new engine::Engine(false) : new engine::Engine(*config); // ---- Load JSON ---- std::ifstream file(source); @@ -22,20 +23,26 @@ capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::pa validate_json(doc, schema_v1_1); // ---- workflow name ---- - workflow_name = doc["name"].as(); - engine->setWorkflowName(workflow_name); + if (config == nullptr) { + workflow_name = doc["name"].as(); + engine->setWorkflowName(workflow_name); + } else { + workflow_name = engine->getWorkflowName(); + } UPDATE_CALF_WORKFLOW_NAME(workflow_name); CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Parsing configuration for workflow: %s", workflow_name.c_str()); // ---- CAPIO-CL TOML CONFIGURATION ---- - if (doc.contains("configuration")) { - auto toml_config_path = doc["configuration"].as(); - CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Using TOML configuration file: %s", - toml_config_path.c_str()); - engine->loadConfiguration(toml_config_path); - } else { - engine->useDefaultConfiguration(); + if (config == nullptr) { + if (doc.contains("configuration")) { + auto toml_config_path = doc["configuration"].as(); + CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Using TOML configuration file: %s", + toml_config_path.c_str()); + engine->loadConfiguration(toml_config_path); + } else { + engine->useDefaultConfiguration(); + } } // ---- IO_Graph ---- diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index bc845a1..141f517 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -7,15 +7,12 @@ #include "capiocl/engine.h" #include "capiocl/parser.h" -capiocl::engine::Engine * -capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix, - bool store_only_in_memory) { +capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1( + const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, + bool store_only_in_memory, const configuration::CapioClConfiguration *config) { START_LOG(calf_current_tid(), "call()"); std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - auto engine = new engine::Engine(true); - - engine->useDefaultConfiguration(); + auto engine = config == nullptr ? new engine::Engine(true) : new engine::Engine(*config); // ---- Load JSON ---- std::ifstream file(source); @@ -24,8 +21,12 @@ capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path validate_json(doc, schema_v1); // ---- workflow name ---- - workflow_name = doc["name"].as(); - engine->setWorkflowName(workflow_name); + if (config == nullptr) { + workflow_name = doc["name"].as(); + engine->setWorkflowName(workflow_name); + } else { + workflow_name = engine->getWorkflowName(); + } UPDATE_CALF_WORKFLOW_NAME(workflow_name); CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Parsing configuration for workflow: %s", workflow_name.c_str()); diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp index fc4384c..0d3de19 100644 --- a/tests/cpp/test_configuration.hpp +++ b/tests/cpp/test_configuration.hpp @@ -36,6 +36,26 @@ TEST(CONFIGURATION_SUITE_NAME, TestGetParameter) { std::invalid_argument); } +TEST(CONFIGURATION_SUITE_NAME, TestConstructFromMap) { + capiocl::configuration::CapioClConfiguration config( + std::unordered_map{{"workflow_name", "test-workflow"}}); + std::string workflow_name; + config.getParameter("workflow_name", &workflow_name, "fallback"); + EXPECT_EQ(workflow_name, "test-workflow"); + EXPECT_EQ(capiocl::engine::Engine(config).getWorkflowName(), "test-workflow"); +} + +TEST(CONFIGURATION_SUITE_NAME, TestParseFromConfiguration) { + capiocl::configuration::CapioClConfiguration config( + std::unordered_map{ + {"workflow_name", "configured-workflow"}, + {"config_path", "/tmp/capio_cl_jsons/V1.1/test24.json"}, + {"monitor.mcast.enabled", "false"}, + {"monitor.filesystem.enabled", "false"}}); + std::unique_ptr engine(capiocl::parser::Parser::parse(config)); + EXPECT_EQ(engine->getWorkflowName(), "configured-workflow"); +} + TEST(CONFIGURATION_SUITE_NAME, TestFailureParsingTOML) { capiocl::configuration::CapioClConfiguration config; EXPECT_THROW(config.load("/tmp/capio_cl_tomls/sample0.toml"), From 2c1762191ed2d8c794c66d4aa94b9ea1e0c9dc3c Mon Sep 17 00:00:00 2001 From: marcoSanti Date: Fri, 25 Sep 2026 15:06:15 +0200 Subject: [PATCH 2/3] Cleanup --- README.md | 103 ++++++++++++++++++++++- bindings/python_bindings.cpp | 23 +++-- capiocl/configuration.h | 2 + capiocl/engine.h | 6 +- capiocl/monitor.h | 3 + capiocl/parser.h | 15 ---- doxygen/configuration.md | 27 +++--- doxygen/semantics.md | 5 +- doxygen/syntax_v1_1.md | 3 +- src/Engine.cpp | 29 +++---- src/Parser.cpp | 58 +++++++------ src/api.cpp | 8 +- src/configuration.cpp | 1 + src/defaults.cpp | 23 ++--- src/monitors/FileSystem.cpp | 24 ++++-- src/monitors/Multicast.cpp | 10 +-- src/parsers/v1.1.cpp | 10 +-- src/parsers/v1.cpp | 10 +-- tests/cpp/main.cpp | 10 +++ tests/cpp/test_configuration.hpp | 31 ++++--- tests/cpp/test_engine.hpp | 6 -- tests/cpp/test_exceptions.hpp | 18 ++-- tests/cpp/test_runtime_commit.hpp | 90 ++++++++++++-------- tests/cpp/test_serialize_deserialize.hpp | 16 ++-- tests/python/test_parser_serializer.py | 39 +++++++-- tests/tomls/runtime_fs_only.toml | 4 +- tests/tomls/runtime_mixed_monitors.toml | 8 +- tests/tomls/runtime_multicast_only.toml | 8 +- tests/tomls/sample1.toml | 12 ++- tests/tomls/sample2.toml | 6 +- tests/tomls/sample3.toml | 2 +- 31 files changed, 387 insertions(+), 223 deletions(-) diff --git a/README.md b/README.md index a8e6e61..6e76660 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,108 @@ This assumes that all build dependencies not fetched by cmake are available. --- +## Runtime TOML Configuration + +Runtime behavior is configured with a TOML file loaded into `CapioClConfiguration`. This is separate from the JSON +coordination-language document: the TOML file selects the JSON document, monitor backends, metadata storage, and dynamic +API settings. + +### Complete example + +```toml +# Workflow and optional JSON coordination-language document +[capiocl] +workflow_name = "my-workflow" +config_path = "workflow.json" +resolve_path = "/data/run-42" +store_all_in_memory = false + +[capiocl.dynamic_api] +enabled = false +ip = "224.224.224.3" +port = 11223 + +[capiocl.monitor.filesystem] +enabled = true +metadata_dir = "/shared/trusted/run-42" + +[capiocl.monitor.mcast] +enabled = false +delay_ms = 300 + +[capiocl.monitor.mcast.commit] +ip = "224.224.224.1" +port = 12345 + +[capiocl.monitor.mcast.homenode] +ip = "224.224.224.2" +port = 12345 +``` + +### Options + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `capiocl.workflow_name` | string | JSON `name`, or `CAPIO` without JSON | Overrides the workflow name. | +| `capiocl.config_path` | string/path | empty | JSON CAPIO-CL document to parse. An empty value creates a runtime-only engine. | +| `capiocl.resolve_path` | string/path | empty | Prefix applied to relative paths in the JSON document. | +| `capiocl.store_all_in_memory` | boolean | `false` | Marks every parsed data path for in-memory storage. | +| `capiocl.dynamic_api.enabled` | boolean | `false` | Starts the dynamic configuration API. | +| `capiocl.dynamic_api.ip` | string | `224.224.224.3` | Multicast address used by the dynamic API. | +| `capiocl.dynamic_api.port` | integer | `11223` | UDP port used by the dynamic API. | +| `capiocl.monitor.filesystem.enabled` | boolean | see below | Enables filesystem commit and home-node tokens. | +| `capiocl.monitor.filesystem.metadata_dir` | string/path | empty | Trusted metadata root for persistent counted `ON_CLOSE` state. | +| `capiocl.monitor.mcast.enabled` | boolean | see below | Enables multicast commit and home-node propagation. | +| `capiocl.monitor.mcast.delay_ms` | integer | `300` | Delay before multicast status operations, in milliseconds. | +| `capiocl.monitor.mcast.commit.ip` | string | `224.224.224.1` | Multicast group for commit state. | +| `capiocl.monitor.mcast.commit.port` | integer | `12345` | UDP port for commit state. | +| `capiocl.monitor.mcast.homenode.ip` | string | `224.224.224.2` | Multicast group for home-node state. | +| `capiocl.monitor.mcast.homenode.port` | integer | `12345` | UDP port for home-node state. | + +Every CAPIO-CL option is under the top-level `capiocl` table. Other top-level tables may coexist in the same TOML file +and are ignored by CAPIO-CL. When parsing a user-provided configuration, omitted +`capiocl.monitor.filesystem.enabled` and `capiocl.monitor.mcast.enabled` values are `false`. `Engine()` and +`CapioClConfiguration.loadDefaults()` use the built-in configuration, which enables both monitors. Set both values +explicitly in deployed TOML files to avoid ambiguity. + +TOML booleans must be unquoted `true` or `false`, and ports/delays must be integers. Relative `capiocl.config_path`, +`capiocl.resolve_path`, and `capiocl.monitor.filesystem.metadata_dir` values are interpreted from the process working +directory. Unknown keys are retained but ignored by CAPIO-CL. + +### Filesystem metadata + +`capiocl.monitor.filesystem.metadata_dir` is required only when an `ON_CLOSE` rule commits after more than one close. It +must name a trusted, non-attacker-writable directory unique to the workflow run. CAPIO-CL creates and owns a `capiocl` +subdirectory beneath it. Multi-process and multi-node producers must share that directory through storage providing +coherent atomic exclusive file creation, rename, and unlink. Ordinary commit and home-node token operations do not +require this option. + +### Loading configuration + +```cpp +#include "capiocl/configuration.h" +#include "capiocl/parser.h" + +using capiocl::configuration::CapioClConfiguration; + +CapioClConfiguration config; +config.load("runtime.toml"); +std::unique_ptr engine(capiocl::parser::Parser::parse(config)); +``` + +```python +import py_capio_cl + +config = py_capio_cl.CapioClConfiguration() +config.load("runtime.toml") +engine = py_capio_cl.Parser.parse(config) +``` + +Configuration can also be constructed from a string map/dictionary. Values in that form must use their flattened keys +and string representations, for example `{"capiocl.monitor.filesystem.enabled": "true"}`. + +--- + ## API Snapshot A simplified example of CAPIO-CL usage in C++: @@ -210,4 +312,3 @@ Serializer.dump(engine, "my_workflow", "my_workflow.json") | Name | Role | Contact | |---------------------------------|----------|------------------------------------------------------------------------------------------------------------------| | **Alberto Riccardo Martinelli** | Designer | [email](mailto:albertoriccardo.martinelli@unito.it) \| [Homepage](https://alpha.di.unito.it/alberto-martinelli/) | - diff --git a/bindings/python_bindings.cpp b/bindings/python_bindings.cpp index 71095b4..1855cbd 100644 --- a/bindings/python_bindings.cpp +++ b/bindings/python_bindings.cpp @@ -6,6 +6,7 @@ #include #include "capiocl.hpp" +#include "capiocl/configuration.h" #include "capiocl/engine.h" #include "capiocl/monitor.h" #include "capiocl/parser.h" @@ -18,6 +19,8 @@ PYBIND11_MODULE(_py_capio_cl, m) { "CAPIO-CL: Cross Application Programmable I/O - Coordination Language python bindings."; py::register_exception(m, "ParserException"); + py::register_exception( + m, "CapioClConfigurationException"); py::register_exception(m, "SerializerException"); py::register_exception(m, "MonitorException"); @@ -39,9 +42,16 @@ PYBIND11_MODULE(_py_capio_cl, m) { py::module_ VERSION = m.def_submodule("VERSION", "CAPIO-CL version"); VERSION.attr("V1") = py::str(capiocl::CAPIO_CL_VERSION::V1); + py::class_(m, "CapioClConfiguration") + .def(py::init<>()) + .def(py::init>(), py::arg("values")) + .def("load", &capiocl::configuration::CapioClConfiguration::load, py::arg("path")) + .def("loadDefaults", &capiocl::configuration::CapioClConfiguration::loadDefaults); + py::class_( m, "Engine", "The main CAPIO-CL engine for managing data communication and I/O operations.") .def(py::init<>()) + .def(py::init(), py::arg("config")) .def("newFile", &capiocl::engine::Engine::newFile, py::arg("filename")) .def("print", &capiocl::engine::Engine::print) .def("contains", &capiocl::engine::Engine::contains, py::arg("path")) @@ -119,14 +129,11 @@ PYBIND11_MODULE(_py_capio_cl, m) { .def(py::self == py::self); py::class_(m, "Parser", "The CAPIO-CL Parser component.") - .def_static("parse", - [](const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { - return capiocl::parser::Parser::parse(source, resolve_prefix, - store_only_in_memory); - }, - py::arg("source"), - py::arg("resolve_prefix") = "", py::arg("store_only_in_memory") = false) + .def_static( + "parse", + py::overload_cast( + &capiocl::parser::Parser::parse), + py::arg("config")) .def("__str__", [](const capiocl::parser::Parser &e) { return "(&e)) + ">"; diff --git a/capiocl/configuration.h b/capiocl/configuration.h index f9a29fb..cb9a4f0 100644 --- a/capiocl/configuration.h +++ b/capiocl/configuration.h @@ -29,6 +29,8 @@ struct capiocl::configuration::defaults { static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_PORT; /// @brief Enable File system monitor by default static ConfigurationEntry DEFAULT_MONITOR_FS_ENABLED; + /// @brief Directory used for persistent filesystem monitor metadata + static ConfigurationEntry DEFAULT_MONITOR_FS_METADATA_DIR; /// @brief IP multicast address for receiving and sending changes in the CapioCL configuration static ConfigurationEntry DEFAULT_API_MULTICAST_IP; /// @brief IP multicast port for receiving and sending changes in the CapioCL configuration diff --git a/capiocl/engine.h b/capiocl/engine.h index bc5b4e5..9168d5c 100644 --- a/capiocl/engine.h +++ b/capiocl/engine.h @@ -435,9 +435,9 @@ class Engine final { /** * @brief Record one qualified logical close for an ON_CLOSE path. - * Filesystem-backed counts greater than one require CAPIO_METADATA_DIR to identify a trusted, - * unique-per-run metadata directory with coherent atomic exclusive create/unlink. Multicast - * counts are volatile and eventual. + * Filesystem-backed counts greater than one require + * capiocl.monitor.filesystem.metadata_dir to identify a trusted, unique-per-run directory with + * coherent atomic exclusive create/unlink. Multicast counts are volatile and eventual. * @return true when the path is effectively committed after this call. */ bool increaseCloseCount(const std::filesystem::path &path) const; diff --git a/capiocl/monitor.h b/capiocl/monitor.h index 2bab45f..bdd195e 100644 --- a/capiocl/monitor.h +++ b/capiocl/monitor.h @@ -261,6 +261,7 @@ class MulticastMonitor final : public MonitorInterface { class FileSystemMonitor final : public MonitorInterface { typedef enum { COMMIT, HOME_NODE } CAPIO_CL_COMMIT_TOKEN_TYPES; + std::filesystem::path metadata_root; /** * @brief Compute the token filename used to represent the commit state of the given file. @@ -299,6 +300,8 @@ class FileSystemMonitor final : public MonitorInterface { */ FileSystemMonitor(); + explicit FileSystemMonitor(const configuration::CapioClConfiguration &config); + /** * @brief Destructor for FileSystemMonitor. */ diff --git a/capiocl/parser.h b/capiocl/parser.h index b0c29e4..2921385 100644 --- a/capiocl/parser.h +++ b/capiocl/parser.h @@ -90,21 +90,6 @@ class Parser final { */ static void validate_json(const jsoncons::json &doc, const char *str_schema); - /** - * @brief Perform the parsing of the capio_server configuration file - * - * @param source Input CAPIO-CL Json configuration File - * @param resolve_prefix If paths are found to be relative, they are appended to this path - * @param store_only_in_memory Set to true to set all files to be stored in memory - * @param config An externally provided configuration to be used within the CAPIO-CL engine - * instance that will be started up - * @return Engine instance with the information provided by the config file - * @throw ParserException - */ - static engine::Engine *parse(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix = "", - bool store_only_in_memory = false, - const configuration::CapioClConfiguration *config = nullptr); }; } // namespace capiocl::parser diff --git a/doxygen/configuration.md b/doxygen/configuration.md index dd929c6..f72b628 100644 --- a/doxygen/configuration.md +++ b/doxygen/configuration.md @@ -14,19 +14,19 @@ If no configuration file is provided, CAPIO-CL falls back to its built-in defaul ## TOML Configuration Structure -The configuration file uses a structured namespace under the top-level table `monitor.mcast`. +All CAPIO-CL options use the top-level `capiocl` table, so unrelated application settings can coexist in the same file. Available configuration parameters: | Key | Type | Default | Description | |-------------------------------|---------|-----------------|------------------------------------------------------------------------------------------------------------------------------------| -| `monitor.filesystem.enabled` | boolean | `false` | Enable FileSystem commit monitor | -| `monitor.mcast.enabled` | boolean | `false` | Enable Multicast commit monitor | -| `monitor.mcast.commit.ip` | string | `224.224.224.1` | Multicast IP address used for commit messages | -| `monitor.mcast.commit.port` | integer | `12345` | UDP port for commit messages | -| `monitor.mcast.delay_ms` | integer | `300` | Artificial delay (in milliseconds) inserted before sending multicast messages. Useful for debugging or simulating slower networks. | -| `monitor.mcast.homenode.ip` | string | `224.224.224.2` | IP address of the home node for monitoring operations | -| `monitor.mcast.homenode.port` | integer | `12345` | Port associated with the home node monitoring endpoint | +| `capiocl.monitor.filesystem.enabled` | boolean | `false` | Enable FileSystem commit monitor | +| `capiocl.monitor.mcast.enabled` | boolean | `false` | Enable Multicast commit monitor | +| `capiocl.monitor.mcast.commit.ip` | string | `224.224.224.1` | Multicast IP address used for commit messages | +| `capiocl.monitor.mcast.commit.port` | integer | `12345` | UDP port for commit messages | +| `capiocl.monitor.mcast.delay_ms` | integer | `300` | Artificial delay (in milliseconds) inserted before sending multicast messages. Useful for debugging or simulating slower networks. | +| `capiocl.monitor.mcast.homenode.ip` | string | `224.224.224.2` | IP address of the home node for monitoring operations | +| `capiocl.monitor.mcast.homenode.port` | integer | `12345` | Port associated with the home node monitoring endpoint | --- @@ -36,9 +36,10 @@ Below is a complete example of a `config.toml` file: # Example CAPIO-CL TOML configuration - monitor.filesystem.enabled = true + [capiocl.monitor.filesystem] + enabled = true - [monitor.mcast] + [capiocl.monitor.mcast] enabled = true # Multicast settings for commit messages @@ -56,12 +57,12 @@ Below is a complete example of a `config.toml` file: ## How CAPIO-CL Uses These Settings -### `commit.ip` and `commit.port` +### `capiocl.monitor.mcast.commit.ip` and `capiocl.monitor.mcast.commit.port` These fields define where CAPIO-CL sends **commit broadcast messages**. Commit messages are used for consistency coordination across distributed nodes. -### `delay_ms` +### `capiocl.monitor.mcast.delay_ms` A small configurable delay may help with: @@ -71,7 +72,7 @@ A small configurable delay may help with: A value of `0` means no delay. -### `homenode.ip` and `homenode.port` +### `capiocl.monitor.mcast.homenode.ip` and `capiocl.monitor.mcast.homenode.port` These define the **central monitoring endpoint** (the “home node”). CAPIO-CL uses this endpoint to coordinate monitoring metadata and cluster-wide communication. diff --git a/doxygen/semantics.md b/doxygen/semantics.md index ab03254..1a9b924 100644 --- a/doxygen/semantics.md +++ b/doxygen/semantics.md @@ -26,8 +26,9 @@ commit behaviors: invokes a close operation on a given file, signaling that all I/O operations on that file are completed. It is also allowed to consider a file committed after `n` close operations are performed. Counts of zero or one commit on the first close; larger counts require a counter-capable filesystem or multicast monitor. Filesystem counters require - `CAPIO_METADATA_DIR` to name a trusted, non-attacker-writable directory unique to the workflow run. CAPIO-CL owns its - `capiocl` subtree. Multi-process and multi-node producers must share this metadata directory through a distributed + `capiocl.monitor.filesystem.metadata_dir` in the TOML runtime configuration to name a trusted, non-attacker-writable + directory unique to the workflow run. CAPIO-CL owns its `capiocl` subtree. Multi-process and multi-node producers must + share this metadata directory through a distributed filesystem providing coherent atomic exclusive file creation and unlink. Transient locks use a private subtree separate from counters. Counter replacement is atomic, but CAPIO-CL does not claim power-loss durability for the latest close on distributed storage. diff --git a/doxygen/syntax_v1_1.md b/doxygen/syntax_v1_1.md index 3dd520b..3710d79 100644 --- a/doxygen/syntax_v1_1.md +++ b/doxygen/syntax_v1_1.md @@ -62,11 +62,10 @@ parameters, refer to **configuration.md**. **engine-config.toml:** ``` - [monitor.mcast] + [capiocl.monitor.mcast] commit.ip = "224.224.224.1" commit.port = 12345 delay_ms = 300 homenode.ip = "224.224.224.2" homenode.port = 12345 ``` - diff --git a/src/Engine.cpp b/src/Engine.cpp index d56cf43..4e27886 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -168,15 +168,9 @@ capiocl::engine::Engine::Engine(const bool use_default_settings) { gethostname(node_name.data(), node_name.size()); node_name.resize(std::strlen(node_name.c_str())); - if (const char *_wf_name = std::getenv("WORKFLOW_NAME"); _wf_name != nullptr) { - this->workflow_name = _wf_name; - LOG("selected workflow name=%s source=environment node=%s", workflow_name.c_str(), - node_name.c_str()); - } else { - this->workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - LOG("selected workflow name=%s source=default node=%s", workflow_name.c_str(), - node_name.c_str()); - } + this->workflow_name = CAPIO_CL_DEFAULT_WF_NAME; + LOG("selected workflow name=%s source=default node=%s", workflow_name.c_str(), + node_name.c_str()); if (use_default_settings) { this->useDefaultConfiguration(); @@ -191,7 +185,7 @@ capiocl::engine::Engine::Engine(const configuration::CapioClConfiguration &confi gethostname(node_name.data(), node_name.size()); node_name.resize(std::strlen(node_name.c_str())); - this->configuration.getParameter("workflow_name", &this->workflow_name, + this->configuration.getParameter("capiocl.workflow_name", &this->workflow_name, CAPIO_CL_DEFAULT_WF_NAME); LOG("selected workflow name=%s source=environment node=%s", workflow_name.c_str(), node_name.c_str()); @@ -1138,9 +1132,11 @@ void capiocl::engine::Engine::configureMonitorBackends() { std::string multicast_monitor_enabled, fs_monitor_enabled; - configuration.getParameter("monitor.mcast.enabled", &multicast_monitor_enabled, "false"); + configuration.getParameter("capiocl.monitor.mcast.enabled", &multicast_monitor_enabled, + "false"); - LOG("configuration key=monitor.mcast.enabled value=%s", multicast_monitor_enabled.c_str()); + LOG("configuration key=capiocl.monitor.mcast.enabled value=%s", + multicast_monitor_enabled.c_str()); if (multicast_monitor_enabled == "true") { monitor.registerMonitorBackend(new monitor::MulticastMonitor(configuration)); @@ -1150,13 +1146,14 @@ void capiocl::engine::Engine::configureMonitorBackends() { CALF_PRINT_COLOR(CALF_CLI_LEVEL_WARNING, "Skipping registration of MulticastMonitor"); } - configuration.getParameter("monitor.filesystem.enabled", &fs_monitor_enabled, "false"); + configuration.getParameter("capiocl.monitor.filesystem.enabled", &fs_monitor_enabled, + "false"); - LOG("configuration fallback key=monitor.filesystem.enabled value=%s", + LOG("configuration fallback key=capiocl.monitor.filesystem.enabled value=%s", fs_monitor_enabled.c_str()); if (fs_monitor_enabled == "true") { - monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor(configuration)); LOG("registered monitor backend=filesystem"); } else { LOG("skipped monitor backend=filesystem reason=disabled"); @@ -1170,7 +1167,7 @@ void capiocl::engine::Engine::useDefaultConfiguration() { // TODO: add a vector with registered instances of backends to avoid multiple instantiations monitor.registerMonitorBackend(new monitor::MulticastMonitor(configuration)); - monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor(configuration)); LOG("loaded default configuration monitor_backends=multicast,filesystem"); } diff --git a/src/Parser.cpp b/src/Parser.cpp index 3f4a237..6305098 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -54,38 +54,32 @@ void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, const cha capiocl::engine::Engine * capiocl::parser::Parser::parse(const configuration::CapioClConfiguration &config) { - std::string source, resolve_prefix, store_only_in_memory, dynamic_api_enabled; - config.getParameter("config_path", &source, ""); - config.getParameter("resolve_path", &resolve_prefix, ""); - config.getParameter("store_all_in_memory", &store_only_in_memory, "false"); - config.getParameter("dynamic_api.enabled", &dynamic_api_enabled, "false"); - auto *engine = source.empty() - ? new engine::Engine(config) - : parse(source, resolve_prefix, store_only_in_memory == "true", &config); + START_LOG(calf_current_tid(), "call()"); - if (source.empty() && store_only_in_memory == "true") { - engine->setAllStoreInMemory(); - } - if (dynamic_api_enabled == "true") { - engine->startApiServer(); - } - return engine; -} + std::string source, resolve_prefix, store_only_in_memory, dynamic_api_enabled; + config.getParameter("capiocl.config_path", &source, ""); + config.getParameter("capiocl.resolve_path", &resolve_prefix, ""); + config.getParameter("capiocl.store_all_in_memory", &store_only_in_memory, "false"); + config.getParameter("capiocl.dynamic_api.enabled", &dynamic_api_enabled, "false"); + + LOG("parse requested source=%s resolve_prefix=%s memory_only=%s", source.c_str(), + resolve_prefix.c_str(), store_only_in_memory.c_str()); -capiocl::engine::Engine *capiocl::parser::Parser::parse( - const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, - bool store_only_in_memory, const configuration::CapioClConfiguration *config) { - START_LOG(calf_current_tid(), "call()"); - LOG("parse requested source=%s resolve_prefix=%s memory_only=%d", source.string().c_str(), - resolve_prefix.string().c_str(), static_cast(store_only_in_memory)); if (source.empty()) { - throw ParserException("Empty source file name!"); + auto *engine = new engine::Engine(config); + if (store_only_in_memory == "true") { + engine->setAllStoreInMemory(); + } + if (dynamic_api_enabled == "true") { + engine->startApiServer(); + } + return engine; } std::ifstream file(source); if (!file.is_open()) { - LOG("failed to open parse source=%s", source.string().c_str()); + LOG("failed to open parse source=%s", source.c_str()); throw ParserException("Failed to open file!"); } std::string capio_cl_release; @@ -94,7 +88,7 @@ capiocl::engine::Engine *capiocl::parser::Parser::parse( try { doc = jsoncons::json::parse(file); } catch (const jsoncons::json_exception &e) { - LOG("failed to decode JSON source=%s error=%s", source.string().c_str(), e.what()); + LOG("failed to decode JSON source=%s error=%s", source.c_str(), e.what()); throw; } if (!doc.contains("version")) { @@ -110,13 +104,21 @@ capiocl::engine::Engine *capiocl::parser::Parser::parse( CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Parsing CAPIO-CL config file for version: %s", capio_cl_release.c_str()); + engine::Engine *engine; if (capio_cl_release == CAPIO_CL_VERSION::V1) { - return available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory, config); + engine = available_parsers::parse_v1(source, resolve_prefix, + store_only_in_memory == "true", &config); } else if (capio_cl_release == CAPIO_CL_VERSION::V1_1) { - return available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory, config); + engine = available_parsers::parse_v1_1(source, resolve_prefix, + store_only_in_memory == "true", &config); } else { LOG("unsupported specification version=%s source=%s", capio_cl_release.c_str(), - source.string().c_str()); + source.c_str()); throw ParserException("Invalid CAPIO-CL specification version!"); } + + if (dynamic_api_enabled == "true") { + engine->startApiServer(); + } + return engine; } diff --git a/src/api.cpp b/src/api.cpp index da6c860..0857439 100644 --- a/src/api.cpp +++ b/src/api.cpp @@ -127,16 +127,16 @@ capiocl::api::CapioClApiServer::CapioClApiServer(engine::Engine *engine, std::string address; int port; - config.getParameter("dynamic_api.ip", &address, + config.getParameter("capiocl.dynamic_api.ip", &address, configuration::defaults::DEFAULT_API_MULTICAST_IP.v); // GCOVR_EXCL_LINE - LOG("API configuration key=dynamic_api.ip value=%s", address.c_str()); + LOG("API configuration key=capiocl.dynamic_api.ip value=%s", address.c_str()); config.getParameter( - "dynamic_api.port", &port, + "capiocl.dynamic_api.port", &port, std::stoi(configuration::defaults::DEFAULT_API_MULTICAST_PORT.v)); // GCOVR_EXCL_LINE - LOG("API configuration key=dynamic_api.port value=%d", port); + LOG("API configuration key=capiocl.dynamic_api.port value=%d", port); _webApiThread = std::thread(server, address, port, engine, &_terminate); diff --git a/src/configuration.cpp b/src/configuration.cpp index 6e960e2..636385f 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -48,6 +48,7 @@ void capiocl::configuration::CapioClConfiguration::loadDefaults() { this->set(defaults::DEFAULT_MONITOR_HOMENODE_PORT); this->set(defaults::DEFAULT_MONITOR_MCAST_DELAY); this->set(defaults::DEFAULT_MONITOR_FS_ENABLED); + this->set(defaults::DEFAULT_MONITOR_FS_METADATA_DIR); this->set(defaults::DEFAULT_MONITOR_MCAST_ENABLED); this->set(defaults::DEFAULT_API_MULTICAST_PORT); this->set(defaults::DEFAULT_API_MULTICAST_IP); diff --git a/src/defaults.cpp b/src/defaults.cpp index f9c00b2..18b3b7d 100644 --- a/src/defaults.cpp +++ b/src/defaults.cpp @@ -1,28 +1,31 @@ #include "capiocl/configuration.h" ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_IP{ - "monitor.mcast.commit.ip", "224.224.224.1"}; + "capiocl.monitor.mcast.commit.ip", "224.224.224.1"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_PORT{ - "monitor.mcast.commit.port", "12345"}; + "capiocl.monitor.mcast.commit.port", "12345"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_DELAY{ - "monitor.mcast.delay_ms", "300"}; + "capiocl.monitor.mcast.delay_ms", "300"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_IP{ - "monitor.mcast.homenode.ip", "224.224.224.2"}; + "capiocl.monitor.mcast.homenode.ip", "224.224.224.2"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_PORT{ - "monitor.mcast.homenode.port", "12345"}; + "capiocl.monitor.mcast.homenode.port", "12345"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_ENABLED{ - "monitor.mcast.enabled", "true"}; + "capiocl.monitor.mcast.enabled", "true"}; ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_FS_ENABLED{ - "monitor.filesystem.enabled", "true"}; + "capiocl.monitor.filesystem.enabled", "true"}; -ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_IP{"dynamic_api.ip", +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_FS_METADATA_DIR{ + "capiocl.monitor.filesystem.metadata_dir", ""}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_IP{"capiocl.dynamic_api.ip", "224.224.224.3"}; -ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_PORT{"dynamic_api.port", - "11223"}; \ No newline at end of file +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_PORT{"capiocl.dynamic_api.port", + "11223"}; diff --git a/src/monitors/FileSystem.cpp b/src/monitors/FileSystem.cpp index f5c03cb..7b1cf26 100644 --- a/src/monitors/FileSystem.cpp +++ b/src/monitors/FileSystem.cpp @@ -23,14 +23,14 @@ struct CloseMetadataPaths final { std::filesystem::path lock; }; -CloseMetadataPaths close_metadata_paths(const std::filesystem::path &path) { - const char *configured = std::getenv("CAPIO_METADATA_DIR"); - if (configured == nullptr || configured[0] == '\0') { +CloseMetadataPaths close_metadata_paths(const std::filesystem::path &path, + const std::filesystem::path &metadata_root) { + if (metadata_root.empty()) { throw capiocl::monitor::MonitorException( - "Counted ON_CLOSE requires CAPIO_METADATA_DIR to name a trusted, unique workflow " - "metadata directory"); + "Counted ON_CLOSE requires capiocl.monitor.filesystem.metadata_dir to name a trusted, " + "unique workflow metadata directory"); } - const auto root = std::filesystem::absolute(configured).lexically_normal() / "capiocl"; + const auto root = std::filesystem::absolute(metadata_root).lexically_normal() / "capiocl"; const auto normalized = std::filesystem::absolute(path).lexically_normal().generic_string(); static constexpr char digits[] = "0123456789abcdef"; std::string encoded; @@ -214,8 +214,16 @@ void capiocl::monitor::FileSystemMonitor::generate_commit_token(const std::files LOG("created commit token=%s", token_name.string().c_str()); } -capiocl::monitor::FileSystemMonitor::FileSystemMonitor() { +capiocl::monitor::FileSystemMonitor::FileSystemMonitor() + : FileSystemMonitor(configuration::CapioClConfiguration{}) {} + +capiocl::monitor::FileSystemMonitor::FileSystemMonitor( + const configuration::CapioClConfiguration &config) { START_LOG(calf_current_tid(), "call()"); + std::string configured_metadata_root; + config.getParameter("capiocl.monitor.filesystem.metadata_dir", &configured_metadata_root, + configuration::defaults::DEFAULT_MONITOR_FS_METADATA_DIR.v); + metadata_root = configured_metadata_root; gethostname(_hostname, sizeof(_hostname)); _hostname[sizeof(_hostname) - 1] = '\0'; LOG("filesystem monitor initialized hostname=%s", _hostname); @@ -238,7 +246,7 @@ capiocl::monitor::FileSystemMonitor::increaseCloseCount(const std::filesystem::p if (threshold <= 1) { throw std::invalid_argument("Persistent ON_CLOSE threshold must be greater than one"); } - const auto metadata = close_metadata_paths(path); + const auto metadata = close_metadata_paths(path, metadata_root); FileLock lock(metadata.lock); long value = read_counter(metadata.counter); diff --git a/src/monitors/Multicast.cpp b/src/monitors/Multicast.cpp index 303f99b..044c374 100644 --- a/src/monitors/Multicast.cpp +++ b/src/monitors/Multicast.cpp @@ -343,19 +343,19 @@ void capiocl::monitor::MulticastMonitor::_send_message(const std::string &ip_add capiocl::monitor::MulticastMonitor::MulticastMonitor( const configuration::CapioClConfiguration &config) { START_LOG(calf_current_tid(), "call()"); - config.getParameter("monitor.mcast.commit.ip", &MULTICAST_COMMIT_ADDR, + config.getParameter("capiocl.monitor.mcast.commit.ip", &MULTICAST_COMMIT_ADDR, configuration::defaults::DEFAULT_MONITOR_MCAST_IP.v); - config.getParameter("monitor.mcast.commit.port", &MULTICAST_COMMIT_PORT, + config.getParameter("capiocl.monitor.mcast.commit.port", &MULTICAST_COMMIT_PORT, std::stoi(configuration::defaults::DEFAULT_MONITOR_MCAST_PORT.v)); - config.getParameter("monitor.mcast.homenode.ip", &MULTICAST_HOME_NODE_ADDR, + config.getParameter("capiocl.monitor.mcast.homenode.ip", &MULTICAST_HOME_NODE_ADDR, configuration::defaults::DEFAULT_MONITOR_HOMENODE_IP.v); - config.getParameter("monitor.mcast.homenode.port", &MULTICAST_HOME_NODE_PORT, + config.getParameter("capiocl.monitor.mcast.homenode.port", &MULTICAST_HOME_NODE_PORT, std::stoi(configuration::defaults::DEFAULT_MONITOR_HOMENODE_PORT.v)); - config.getParameter("monitor.mcast.delay_ms", &MULTICAST_DELAY_MILLIS, + config.getParameter("capiocl.monitor.mcast.delay_ms", &MULTICAST_DELAY_MILLIS, std::stoi(configuration::defaults::DEFAULT_MONITOR_MCAST_DELAY.v)); gethostname(_hostname, sizeof(_hostname)); _hostname[sizeof(_hostname) - 1] = '\0'; diff --git a/src/parsers/v1.1.cpp b/src/parsers/v1.1.cpp index ab65c66..cc0ecb5 100644 --- a/src/parsers/v1.1.cpp +++ b/src/parsers/v1.1.cpp @@ -13,7 +13,6 @@ capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1_1( START_LOG(calf_current_tid(), "call()"); - std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto engine = config == nullptr ? new engine::Engine(false) : new engine::Engine(*config); // ---- Load JSON ---- @@ -23,12 +22,11 @@ capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1_1( validate_json(doc, schema_v1_1); // ---- workflow name ---- - if (config == nullptr) { - workflow_name = doc["name"].as(); - engine->setWorkflowName(workflow_name); - } else { - workflow_name = engine->getWorkflowName(); + std::string workflow_name = doc["name"].as(); + if (config != nullptr) { + config->getParameter("capiocl.workflow_name", &workflow_name, workflow_name); } + engine->setWorkflowName(workflow_name); UPDATE_CALF_WORKFLOW_NAME(workflow_name); CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Parsing configuration for workflow: %s", workflow_name.c_str()); diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index 141f517..37dd6e9 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -11,7 +11,6 @@ capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1( const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, bool store_only_in_memory, const configuration::CapioClConfiguration *config) { START_LOG(calf_current_tid(), "call()"); - std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto engine = config == nullptr ? new engine::Engine(true) : new engine::Engine(*config); // ---- Load JSON ---- @@ -21,12 +20,11 @@ capiocl::engine::Engine *capiocl::parser::Parser::available_parsers::parse_v1( validate_json(doc, schema_v1); // ---- workflow name ---- - if (config == nullptr) { - workflow_name = doc["name"].as(); - engine->setWorkflowName(workflow_name); - } else { - workflow_name = engine->getWorkflowName(); + std::string workflow_name = doc["name"].as(); + if (config != nullptr) { + config->getParameter("capiocl.workflow_name", &workflow_name, workflow_name); } + engine->setWorkflowName(workflow_name); UPDATE_CALF_WORKFLOW_NAME(workflow_name); CALF_PRINT_COLOR(CALF_CLI_LEVEL_INFO, "Parsing configuration for workflow: %s", workflow_name.c_str()); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index adb80a5..2f05583 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -20,6 +20,16 @@ template std::string demangled_name(const T &obj) { #include "capiocl/parser.h" #include "capiocl/serializer.h" +capiocl::engine::Engine *parseConfiguration( + const std::filesystem::path &source, const std::filesystem::path &resolve_prefix = "", + bool store_only_in_memory = false) { + return capiocl::parser::Parser::parse(capiocl::configuration::CapioClConfiguration({ + {"capiocl.config_path", source.string()}, + {"capiocl.resolve_path", resolve_prefix.string()}, + {"capiocl.store_all_in_memory", store_only_in_memory ? "true" : "false"}, + })); +} + #include "test_apis.hpp" #include "test_configuration.hpp" #include "test_engine.hpp" diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp index 0d3de19..7dd5ad7 100644 --- a/tests/cpp/test_configuration.hpp +++ b/tests/cpp/test_configuration.hpp @@ -21,26 +21,28 @@ TEST(CONFIGURATION_SUITE_NAME, TestGetParameter) { config.load("/tmp/capio_cl_tomls/sample1.toml"); int int_value; - config.getParameter("monitor.mcast.delay_ms", &int_value, -1); + config.getParameter("capiocl.monitor.mcast.delay_ms", &int_value, -1); EXPECT_EQ(int_value, 300); config.getParameter("not.a.valid.key", &int_value, 42); EXPECT_EQ(int_value, 42); std::string string_value; - config.getParameter("monitor.mcast.commit.ip", &string_value, "fallback"); + config.getParameter("capiocl.monitor.mcast.commit.ip", &string_value, "fallback"); EXPECT_EQ(string_value, "224.224.224.3"); config.getParameter("not.a.valid.key", &string_value, "fallback"); EXPECT_EQ(string_value, "fallback"); + config.getParameter("capiocl.monitor.filesystem.metadata_dir", &string_value, "fallback"); + EXPECT_EQ(string_value, "/tmp/capiocl-metadata"); - EXPECT_THROW(config.getParameter("workflow_name", &int_value, 0), + EXPECT_THROW(config.getParameter("capiocl.workflow_name", &int_value, 0), std::invalid_argument); } TEST(CONFIGURATION_SUITE_NAME, TestConstructFromMap) { capiocl::configuration::CapioClConfiguration config( - std::unordered_map{{"workflow_name", "test-workflow"}}); + std::unordered_map{{"capiocl.workflow_name", "test-workflow"}}); std::string workflow_name; - config.getParameter("workflow_name", &workflow_name, "fallback"); + config.getParameter("capiocl.workflow_name", &workflow_name, "fallback"); EXPECT_EQ(workflow_name, "test-workflow"); EXPECT_EQ(capiocl::engine::Engine(config).getWorkflowName(), "test-workflow"); } @@ -48,14 +50,21 @@ TEST(CONFIGURATION_SUITE_NAME, TestConstructFromMap) { TEST(CONFIGURATION_SUITE_NAME, TestParseFromConfiguration) { capiocl::configuration::CapioClConfiguration config( std::unordered_map{ - {"workflow_name", "configured-workflow"}, - {"config_path", "/tmp/capio_cl_jsons/V1.1/test24.json"}, - {"monitor.mcast.enabled", "false"}, - {"monitor.filesystem.enabled", "false"}}); + {"capiocl.workflow_name", "configured-workflow"}, + {"capiocl.config_path", "/tmp/capio_cl_jsons/V1.1/test24.json"}, + {"capiocl.monitor.mcast.enabled", "false"}, + {"capiocl.monitor.filesystem.enabled", "false"}}); std::unique_ptr engine(capiocl::parser::Parser::parse(config)); EXPECT_EQ(engine->getWorkflowName(), "configured-workflow"); } +TEST(CONFIGURATION_SUITE_NAME, TestParseRuntimeOnlyConfiguration) { + capiocl::configuration::CapioClConfiguration config( + std::unordered_map{{"capiocl.workflow_name", "runtime-only"}}); + std::unique_ptr engine(capiocl::parser::Parser::parse(config)); + EXPECT_EQ(engine->getWorkflowName(), "runtime-only"); +} + TEST(CONFIGURATION_SUITE_NAME, TestFailureParsingTOML) { capiocl::configuration::CapioClConfiguration config; EXPECT_THROW(config.load("/tmp/capio_cl_tomls/sample0.toml"), @@ -74,11 +83,11 @@ TEST(CONFIGURATION_SUITE_NAME, testNoBackendLoadedWithExplicitNoLoadOption) { config.load("/tmp/capio_cl_tomls/sample3.toml"); std::string value; - config.getParameter("monitor.mcast.enabled", &value, "true"); + config.getParameter("capiocl.monitor.mcast.enabled", &value, "true"); EXPECT_TRUE("false" == value); value = ""; - config.getParameter("monitor.filesystem.enabled", &value, "true"); + config.getParameter("capiocl.monitor.filesystem.enabled", &value, "true"); EXPECT_TRUE("false" == value); } diff --git a/tests/cpp/test_engine.hpp b/tests/cpp/test_engine.hpp index bd8b071..d78f7cc 100644 --- a/tests/cpp/test_engine.hpp +++ b/tests/cpp/test_engine.hpp @@ -16,12 +16,6 @@ TEST(ENGINE_SUITE_NAME, testSetGetWfName) { EXPECT_TRUE(engine.getWorkflowName() == "test"); } -TEST(ENGINE_SUITE_NAME, testSetGetWfNameFromEnv) { - setenv("WORKFLOW_NAME", "my_custom_wf_name", 1); - capiocl::engine::Engine engine; - EXPECT_TRUE(engine.getWorkflowName() == "my_custom_wf_name"); -} - TEST(ENGINE_SUITE_NAME, testInstantiationFromConfiguration) { capiocl::configuration::CapioClConfiguration config; capiocl::engine::Engine default_engine(config); diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index db1ea16..ebaa38a 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -5,12 +5,9 @@ #include "capiocl/serializer.h" TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { - try { - capiocl::parser::Parser::parse(""); - } catch (const capiocl::parser::ParserException &e) { - EXPECT_TRUE(demangled_name(e) == "capiocl::parser::ParserException"); - EXPECT_GT(strlen(e.what()), 0); - } + const capiocl::parser::ParserException parser_error("test"); + EXPECT_TRUE(demangled_name(parser_error) == "capiocl::parser::ParserException"); + EXPECT_GT(strlen(parser_error.what()), 0); try { const auto engine = capiocl::engine::Engine(); @@ -24,7 +21,7 @@ TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { TEST(EXCEPTION_SUITE_NAME, testFailedDump) { for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; - auto engine = capiocl::parser::Parser::parse(source, "/tmp"); + auto engine = parseConfiguration(source, "/tmp"); EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "/"), capiocl::serializer::SerializerException); @@ -34,7 +31,7 @@ TEST(EXCEPTION_SUITE_NAME, testFailedDump) { TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; - auto engine = capiocl::parser::Parser::parse(source, "/tmp"); + auto engine = parseConfiguration(source, "/tmp"); EXPECT_THROW( capiocl::serializer::Serializer::dump(*engine, "test.json", false, "1234.5678"), @@ -48,7 +45,6 @@ TEST(EXCEPTION_SUITE_NAME, testParserException) { std::cout << "Loading jsons from " << JSON_DIR << std::endl; std::vector test_filenames = { - "", "ANonExistingFile", "test1.json", "test2.json", @@ -80,7 +76,7 @@ TEST(EXCEPTION_SUITE_NAME, testParserException) { const auto test_file_path = test.empty() ? test : JSON_DIR / ("V" + version) / test; std::cout << "Testing on file " << test_file_path << std::endl; - EXPECT_THROW(capiocl::parser::Parser::parse(test_file_path), + EXPECT_THROW(parseConfiguration(test_file_path), capiocl::parser::ParserException); } } @@ -114,4 +110,4 @@ TEST(EXCEPTION_SUITE_NAME, testWrongFireRule) { EXPECT_TRUE(exception_caught); } -#endif // CAPIO_CL_EXCEPTIONS_HPP \ No newline at end of file +#endif // CAPIO_CL_EXCEPTIONS_HPP diff --git a/tests/cpp/test_runtime_commit.hpp b/tests/cpp/test_runtime_commit.hpp index e96dd7f..e90ea81 100644 --- a/tests/cpp/test_runtime_commit.hpp +++ b/tests/cpp/test_runtime_commit.hpp @@ -15,8 +15,6 @@ namespace { class RuntimeTestScope final { static std::atomic sequence; - bool had_metadata_root = false; - std::string previous_metadata_root; static std::string hexEncode(const std::string &value) { static constexpr char digits[] = "0123456789abcdef"; @@ -32,28 +30,43 @@ class RuntimeTestScope final { std::filesystem::path root; std::filesystem::path data; std::filesystem::path metadata; + std::filesystem::path runtime_configuration; RuntimeTestScope() : root(std::filesystem::temp_directory_path() / ("capiocl-runtime-" + std::to_string(getpid()) + "-" + std::to_string(sequence++))), - data(root / "data"), metadata(root / "metadata") { - if (const char *current = std::getenv("CAPIO_METADATA_DIR")) { - had_metadata_root = true; - previous_metadata_root = current; - } + data(root / "data"), metadata(root / "metadata"), + runtime_configuration(root / "runtime.toml") { std::filesystem::remove_all(root); std::filesystem::create_directories(data); - setenv("CAPIO_METADATA_DIR", metadata.c_str(), 1); + std::ofstream(runtime_configuration) + << "[capiocl.monitor.filesystem]\n" + << "enabled = true\n" + << "metadata_dir = \"" << metadata.string() << "\"\n\n" + << "[capiocl.monitor.mcast]\n" + << "enabled = false\n"; } - ~RuntimeTestScope() { - if (had_metadata_root) { - setenv("CAPIO_METADATA_DIR", previous_metadata_root.c_str(), 1); - } else { - unsetenv("CAPIO_METADATA_DIR"); - } - std::filesystem::remove_all(root); + ~RuntimeTestScope() { std::filesystem::remove_all(root); } + + [[nodiscard]] capiocl::configuration::CapioClConfiguration configuration() const { + return capiocl::configuration::CapioClConfiguration({ + {"capiocl.monitor.filesystem.metadata_dir", metadata.string()}, + }); + } + + [[nodiscard]] capiocl::configuration::CapioClConfiguration mixedConfiguration() const { + return capiocl::configuration::CapioClConfiguration({ + {"capiocl.monitor.filesystem.enabled", "true"}, + {"capiocl.monitor.filesystem.metadata_dir", metadata.string()}, + {"capiocl.monitor.mcast.enabled", "true"}, + {"capiocl.monitor.mcast.delay_ms", "50"}, + {"capiocl.monitor.mcast.commit.ip", "224.224.226.12"}, + {"capiocl.monitor.mcast.commit.port", "26103"}, + {"capiocl.monitor.mcast.homenode.ip", "224.224.226.13"}, + {"capiocl.monitor.mcast.homenode.port", "26104"}, + }); } [[nodiscard]] std::filesystem::path file(const std::string &name) const { @@ -88,6 +101,10 @@ void useFilesystemMonitor(capiocl::engine::Engine &engine) { engine.loadConfiguration("/tmp/capio_cl_tomls/runtime_fs_only.toml"); } +void useFilesystemMonitor(capiocl::engine::Engine &engine, const RuntimeTestScope &scope) { + engine.loadConfiguration(scope.runtime_configuration.string()); +} + void useMulticastMonitor(capiocl::engine::Engine &engine) { engine.loadConfiguration("/tmp/capio_cl_tomls/runtime_multicast_only.toml"); } @@ -125,7 +142,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onClosePlainAndOneCommitOnFirstClose) { for (const long threshold : {0L, 1L}) { const auto path = scope.file("plain-" + std::to_string(threshold)); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, threshold); EXPECT_TRUE(engine.increaseCloseCount(path)); @@ -136,7 +153,6 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onClosePlainAndOneCommitOnFirstClose) { TEST(RUNTIME_COMMIT_SUITE_NAME, countedCloseRequiresConfiguredMetadataRoot) { RuntimeTestScope scope; - unsetenv("CAPIO_METADATA_DIR"); const auto counted = scope.file("missing-metadata"); capiocl::engine::Engine engine(false); useFilesystemMonitor(engine); @@ -144,9 +160,10 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, countedCloseRequiresConfiguredMetadataRoot) { try { engine.increaseCloseCount(counted); - FAIL() << "counted ON_CLOSE accepted a missing CAPIO_METADATA_DIR"; + FAIL() << "counted ON_CLOSE accepted a missing capiocl.monitor.filesystem.metadata_dir"; } catch (const capiocl::monitor::MonitorException &error) { - EXPECT_NE(std::string(error.what()).find("CAPIO_METADATA_DIR"), std::string::npos); + EXPECT_NE(std::string(error.what()).find("capiocl.monitor.filesystem.metadata_dir"), + std::string::npos); } EXPECT_FALSE(engine.isCommitted(counted)); EXPECT_FALSE(std::filesystem::exists(counted.parent_path() / @@ -162,7 +179,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onCloseThresholdTwoAndNonClose) { RuntimeTestScope scope; const auto path = scope.file("threshold-two"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 2); EXPECT_FALSE(engine.increaseCloseCount(path)); @@ -184,14 +201,14 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onCloseCountPersistsAcrossEngines) { const auto path = scope.file("persistent"); { capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 3); EXPECT_FALSE(engine.increaseCloseCount(path)); EXPECT_FALSE(engine.increaseCloseCount(path)); } { capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 3); EXPECT_TRUE(engine.increaseCloseCount(path)); } @@ -201,7 +218,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onCloseIsThreadSafe) { RuntimeTestScope scope; const auto path = scope.file("threads"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 16); std::vector threads; for (int i = 0; i < 16; ++i) { @@ -315,8 +332,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, multicastConcurrentClosesReconcile) { TEST(RUNTIME_COMMIT_SUITE_NAME, mixedBackendsDoNotDoubleCount) { RuntimeTestScope scope; const auto path = scope.file("mixed-backends"); - capiocl::engine::Engine engine(false); - engine.loadConfiguration("/tmp/capio_cl_tomls/runtime_mixed_monitors.toml"); + capiocl::engine::Engine engine(scope.mixedConfiguration()); configureClose(engine, path, 2); EXPECT_FALSE(engine.increaseCloseCount(path)); @@ -340,7 +356,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, simultaneousFirstCloseIsProcessSafe) { } try { capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 2); engine.increaseCloseCount(path); _exit(0); @@ -361,7 +377,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, simultaneousFirstCloseIsProcessSafe) { } capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 2); EXPECT_TRUE(engine.isCommitted(path)); EXPECT_FALSE(std::filesystem::exists(scope.closeLock(path))); @@ -400,7 +416,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, concurrentExplicitCommitWinsBelowThreshold) { close(ready[0]); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 10); std::atomic started{false}; bool result = false; @@ -429,7 +445,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, relativeAndAbsolutePathsShareRuntimeIdentity) { const auto absolute = scope.file("alias"); const auto relative = std::filesystem::relative(absolute, std::filesystem::current_path()); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, relative, 2); EXPECT_FALSE(engine.increaseCloseCount(absolute)); @@ -442,7 +458,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onCloseRejectsMalformedAndUnsupportedCounters) { RuntimeTestScope scope; const auto malformed = scope.file("malformed"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, malformed, 2); const auto counter = scope.closeMetadata(malformed, "count"); std::filesystem::create_directories(counter.parent_path()); @@ -462,7 +478,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onCloseRejectsMalformedAndUnsupportedCounters) { TEST(RUNTIME_COMMIT_SUITE_NAME, fileSystemCloseMetadataRejectsSymlinks) { RuntimeTestScope scope; - const capiocl::monitor::FileSystemMonitor monitor; + const capiocl::monitor::FileSystemMonitor monitor(scope.configuration()); const auto target = scope.root / "target"; std::ofstream(target) << "unchanged\n"; const auto expect_target_unchanged = [&] { @@ -504,7 +520,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, fileSystemCloseMetadataRejectsSymlinks) { TEST(RUNTIME_COMMIT_SUITE_NAME, fileSystemCloseCounterRejectsTrailingDataAndOverflow) { RuntimeTestScope scope; - const capiocl::monitor::FileSystemMonitor monitor; + const capiocl::monitor::FileSystemMonitor monitor(scope.configuration()); const auto trailing_path = scope.file("trailing"); const auto trailing = scope.closeMetadata(trailing_path, "count"); @@ -544,7 +560,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, multicastOversizedOutboundCommitIsLocallyRecorde TEST(RUNTIME_COMMIT_SUITE_NAME, earlyEmptyCloseAndWildcardDependencyRemainUncommitted) { RuntimeTestScope scope; capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); EXPECT_FALSE(engine.increaseCloseCount({})); const auto path = scope.file("wildcard-dependency"); @@ -563,7 +579,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, lockAcquisitionErrorsDoNotSpin) { ASSERT_EQ(chmod(lock.parent_path().c_str(), 0500), 0); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureClose(engine, path, 2); EXPECT_THROW(engine.increaseCloseCount(path), capiocl::monitor::MonitorException); @@ -578,7 +594,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onFileChainAndFanIn) { const auto c = scope.file("c"); const auto d = scope.file("d"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureFile(engine, a, {b, c}); configureFile(engine, b, {d}); @@ -596,7 +612,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onFileEmptyAndCyclesRemainUncommitted) { const auto a = scope.file("a"); const auto b = scope.file("b"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureFile(engine, empty, {}); configureFile(engine, self, {self}); configureFile(engine, a, {b}); @@ -613,7 +629,7 @@ TEST(RUNTIME_COMMIT_SUITE_NAME, onFileCycleResolvesFromRawSeed) { const auto a = scope.file("a"); const auto b = scope.file("b"); capiocl::engine::Engine engine(false); - useFilesystemMonitor(engine); + useFilesystemMonitor(engine, scope); configureFile(engine, a, {b}); configureFile(engine, b, {a}); diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index aec709b..cb059a4 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -47,12 +47,12 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1) { capiocl::serializer::Serializer::dump(engine, path, false, _cl_version); std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + auto new_engine = parseConfiguration(path, resolve); EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); EXPECT_TRUE(engine == *new_engine); - auto new_engine1 = capiocl::parser::Parser::parse(path, resolve, true); + auto new_engine1 = parseConfiguration(path, resolve, true); EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); std::filesystem::remove(path); @@ -78,7 +78,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1NcloseNfiles) capiocl::serializer::Serializer::dump(engine, path, false, _cl_version); std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + auto new_engine = parseConfiguration(path, resolve); EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); EXPECT_TRUE(engine == *new_engine); @@ -113,7 +113,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1FileDeps) { capiocl::serializer::Serializer::dump(engine, path, false, _cl_version); std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + auto new_engine = parseConfiguration(path, resolve); EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); EXPECT_TRUE(engine == *new_engine); @@ -141,7 +141,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeCommitOnCloseCountNoCommitRu capiocl::serializer::Serializer::dump(engine, path, false, _cl_version); std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + auto new_engine = parseConfiguration(path, resolve); EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); EXPECT_FALSE(engine == *new_engine); @@ -177,7 +177,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testCompressedSerializationUsesLongestPre } capiocl::serializer::Serializer::dump(engine, config_path, true, _cl_version); - auto compressed = capiocl::parser::Parser::parse(config_path, ""); + auto compressed = parseConfiguration(config_path); const auto paths = compressed->getPaths(); EXPECT_EQ(paths.size(), 2); @@ -238,7 +238,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsolute) { for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + "/test0.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + auto engine = parseConfiguration(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); EXPECT_TRUE(engine->contains("/tmp/file1")); @@ -252,7 +252,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSection) { for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + "/test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + auto engine = parseConfiguration(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); EXPECT_TRUE(engine->contains("/tmp/file1")); diff --git a/tests/python/test_parser_serializer.py b/tests/python/test_parser_serializer.py index 6493392..8093406 100644 --- a/tests/python/test_parser_serializer.py +++ b/tests/python/test_parser_serializer.py @@ -4,6 +4,18 @@ import py_capio_cl +def parse(config_path, resolve_path="", store_all_in_memory=False): + return py_capio_cl.Parser.parse( + py_capio_cl.CapioClConfiguration( + { + "capiocl.config_path": str(config_path), + "capiocl.resolve_path": str(resolve_path), + "capiocl.store_all_in_memory": "true" if store_all_in_memory else "false", + } + ) + ) + + def test_serialize_parse_py_capio_cl_v1(tmp_path): path = tmp_path / "config.json" workflow_name = "demo" @@ -55,12 +67,12 @@ def test_serialize_parse_py_capio_cl_v1(tmp_path): py_capio_cl.serialize(engine, path) # Parse back - new_engine = py_capio_cl.Parser.parse(path) + new_engine = parse(path) assert new_engine.getWorkflowName() == workflow_name # Parse with memory flag - new_engine1 = py_capio_cl.Parser.parse(path, store_only_in_memory=True) + new_engine1 = parse(path, store_all_in_memory=True) assert len(new_engine1.getFileToStoreInMemory()) == engine.size() # cleanup @@ -70,16 +82,33 @@ def test_serialize_parse_py_capio_cl_v1(tmp_path): def test_parser_resolve_absolute(): json_path = "/tmp/capio_cl_jsons/V1.0/test0.json" - engine = py_capio_cl.Parser.parse(str(json_path), "/tmp") + engine = parse(json_path, "/tmp") assert engine.getWorkflowName() == "test" for f in ["/tmp/file", "/tmp/file1", "/tmp/file2", "/tmp/file3"]: assert engine.contains(f) +def test_parser_configuration_binding(): + config = py_capio_cl.CapioClConfiguration( + { + "capiocl.workflow_name": "configured-workflow", + "capiocl.config_path": "/tmp/capio_cl_jsons/V1.0/test0.json", + "capiocl.resolve_path": "/tmp", + } + ) + engine = py_capio_cl.Parser.parse(config) + assert engine.getWorkflowName() == "configured-workflow" + assert engine.contains("/tmp/file") + + runtime_engine = py_capio_cl.Parser.parse( + py_capio_cl.CapioClConfiguration({"capiocl.workflow_name": "runtime-only"}) + ) + assert runtime_engine.getWorkflowName() == "runtime-only" + + def test_parser_exception(): json_dir = Path("/tmp/capio_cl_jsons") test_filenames = [ - "", "ANonExistingFile", *(str(json_dir / f"V1_test{i}.json") for i in range(1, 24)), ] @@ -87,7 +116,7 @@ def test_parser_exception(): for test_file in test_filenames: caught = False try: - wf_name, engine = py_capio_cl.Parser.parse(test_file) + wf_name, engine = parse(test_file) except Exception as e: caught = True typename = type(e).__name__ diff --git a/tests/tomls/runtime_fs_only.toml b/tests/tomls/runtime_fs_only.toml index e9be950..45ea344 100644 --- a/tests/tomls/runtime_fs_only.toml +++ b/tests/tomls/runtime_fs_only.toml @@ -1,5 +1,5 @@ -[monitor.filesystem] +[capiocl.monitor.filesystem] enabled = true -[monitor.mcast] +[capiocl.monitor.mcast] enabled = false diff --git a/tests/tomls/runtime_mixed_monitors.toml b/tests/tomls/runtime_mixed_monitors.toml index 9580427..a0ddb91 100644 --- a/tests/tomls/runtime_mixed_monitors.toml +++ b/tests/tomls/runtime_mixed_monitors.toml @@ -1,14 +1,14 @@ -[monitor.filesystem] +[capiocl.monitor.filesystem] enabled = true -[monitor.mcast] +[capiocl.monitor.mcast] delay_ms = 50 enabled = true -[monitor.mcast.commit] +[capiocl.monitor.mcast.commit] ip = "224.224.226.12" port = 26103 -[monitor.mcast.homenode] +[capiocl.monitor.mcast.homenode] ip = "224.224.226.13" port = 26104 diff --git a/tests/tomls/runtime_multicast_only.toml b/tests/tomls/runtime_multicast_only.toml index 589d9ee..e639895 100644 --- a/tests/tomls/runtime_multicast_only.toml +++ b/tests/tomls/runtime_multicast_only.toml @@ -1,14 +1,14 @@ -[monitor.filesystem] +[capiocl.monitor.filesystem] enabled = false -[monitor.mcast] +[capiocl.monitor.mcast] delay_ms = 50 enabled = true -[monitor.mcast.commit] +[capiocl.monitor.mcast.commit] ip = "224.224.226.10" port = 26101 -[monitor.mcast.homenode] +[capiocl.monitor.mcast.homenode] ip = "224.224.226.11" port = 26102 diff --git a/tests/tomls/sample1.toml b/tests/tomls/sample1.toml index 4c35543..8dfcac6 100644 --- a/tests/tomls/sample1.toml +++ b/tests/tomls/sample1.toml @@ -1,16 +1,20 @@ +workflow_name = "external_workflow" + +[capiocl] workflow_name = "configured_workflow" -[monitor.filesystem] +[capiocl.monitor.filesystem] enabled = true +metadata_dir = "/tmp/capiocl-metadata" -[monitor.mcast] +[capiocl.monitor.mcast] delay_ms = 300 enabled = true -[monitor.mcast.commit] +[capiocl.monitor.mcast.commit] ip = "224.224.224.3" port = 11223 -[monitor.mcast.homenode] +[capiocl.monitor.mcast.homenode] ip = "224.224.224.4" port = 22334 diff --git a/tests/tomls/sample2.toml b/tests/tomls/sample2.toml index b8e486f..c82a1c0 100644 --- a/tests/tomls/sample2.toml +++ b/tests/tomls/sample2.toml @@ -1,10 +1,10 @@ -[monitor.mcast] +[capiocl.monitor.mcast] delay_ms = 300 -[monitor.mcast.commit] +[capiocl.monitor.mcast.commit] ip = "224.224.224.3" port = 11223 -[monitor.mcast.homenode] +[capiocl.monitor.mcast.homenode] ip = "224.224.224.4" port = 22334 diff --git a/tests/tomls/sample3.toml b/tests/tomls/sample3.toml index ee66795..93b6deb 100644 --- a/tests/tomls/sample3.toml +++ b/tests/tomls/sample3.toml @@ -1,3 +1,3 @@ -[monitor] +[capiocl.monitor] filesystem.enabled = false mcast.enabled = false From f0540c98a8982101d7d6d4ede27007292f9377ad Mon Sep 17 00:00:00 2001 From: marcoSanti Date: Fri, 25 Sep 2026 15:18:55 +0200 Subject: [PATCH 3/3] Cleanup --- bindings/python_bindings.cpp | 9 ++++----- capiocl/monitor.h | 4 ++++ capiocl/parser.h | 1 - src/Engine.cpp | 3 +-- src/Parser.cpp | 4 ++-- src/defaults.cpp | 8 ++++---- src/monitors/FileSystem.cpp | 2 +- tests/cpp/main.cpp | 6 +++--- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/bindings/python_bindings.cpp b/bindings/python_bindings.cpp index 1855cbd..5cccdbe 100644 --- a/bindings/python_bindings.cpp +++ b/bindings/python_bindings.cpp @@ -129,11 +129,10 @@ PYBIND11_MODULE(_py_capio_cl, m) { .def(py::self == py::self); py::class_(m, "Parser", "The CAPIO-CL Parser component.") - .def_static( - "parse", - py::overload_cast( - &capiocl::parser::Parser::parse), - py::arg("config")) + .def_static("parse", + py::overload_cast( + &capiocl::parser::Parser::parse), + py::arg("config")) .def("__str__", [](const capiocl::parser::Parser &e) { return "(&e)) + ">"; diff --git a/capiocl/monitor.h b/capiocl/monitor.h index bdd195e..4cec684 100644 --- a/capiocl/monitor.h +++ b/capiocl/monitor.h @@ -300,6 +300,10 @@ class FileSystemMonitor final : public MonitorInterface { */ FileSystemMonitor(); + /** + * @brief Construct a filesystem-based monitor using runtime configuration. + * @param config Configuration containing the filesystem metadata directory. + */ explicit FileSystemMonitor(const configuration::CapioClConfiguration &config); /** diff --git a/capiocl/parser.h b/capiocl/parser.h index 2921385..2748264 100644 --- a/capiocl/parser.h +++ b/capiocl/parser.h @@ -89,7 +89,6 @@ class Parser final { * @param str_schema Raw JSON schema to use */ static void validate_json(const jsoncons::json &doc, const char *str_schema); - }; } // namespace capiocl::parser diff --git a/src/Engine.cpp b/src/Engine.cpp index 4e27886..d9993b4 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -1146,8 +1146,7 @@ void capiocl::engine::Engine::configureMonitorBackends() { CALF_PRINT_COLOR(CALF_CLI_LEVEL_WARNING, "Skipping registration of MulticastMonitor"); } - configuration.getParameter("capiocl.monitor.filesystem.enabled", &fs_monitor_enabled, - "false"); + configuration.getParameter("capiocl.monitor.filesystem.enabled", &fs_monitor_enabled, "false"); LOG("configuration fallback key=capiocl.monitor.filesystem.enabled value=%s", fs_monitor_enabled.c_str()); diff --git a/src/Parser.cpp b/src/Parser.cpp index 6305098..0224b36 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -106,8 +106,8 @@ capiocl::parser::Parser::parse(const configuration::CapioClConfiguration &config engine::Engine *engine; if (capio_cl_release == CAPIO_CL_VERSION::V1) { - engine = available_parsers::parse_v1(source, resolve_prefix, - store_only_in_memory == "true", &config); + engine = available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory == "true", + &config); } else if (capio_cl_release == CAPIO_CL_VERSION::V1_1) { engine = available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory == "true", &config); diff --git a/src/defaults.cpp b/src/defaults.cpp index 18b3b7d..6ec78f2 100644 --- a/src/defaults.cpp +++ b/src/defaults.cpp @@ -24,8 +24,8 @@ ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_FS_ENABLED{ ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_FS_METADATA_DIR{ "capiocl.monitor.filesystem.metadata_dir", ""}; -ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_IP{"capiocl.dynamic_api.ip", - "224.224.224.3"}; +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_IP{ + "capiocl.dynamic_api.ip", "224.224.224.3"}; -ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_PORT{"capiocl.dynamic_api.port", - "11223"}; +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_API_MULTICAST_PORT{ + "capiocl.dynamic_api.port", "11223"}; diff --git a/src/monitors/FileSystem.cpp b/src/monitors/FileSystem.cpp index 7b1cf26..f140a29 100644 --- a/src/monitors/FileSystem.cpp +++ b/src/monitors/FileSystem.cpp @@ -30,7 +30,7 @@ CloseMetadataPaths close_metadata_paths(const std::filesystem::path &path, "Counted ON_CLOSE requires capiocl.monitor.filesystem.metadata_dir to name a trusted, " "unique workflow metadata directory"); } - const auto root = std::filesystem::absolute(metadata_root).lexically_normal() / "capiocl"; + const auto root = std::filesystem::absolute(metadata_root).lexically_normal() / "capiocl"; const auto normalized = std::filesystem::absolute(path).lexically_normal().generic_string(); static constexpr char digits[] = "0123456789abcdef"; std::string encoded; diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 2f05583..970cf7e 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -20,9 +20,9 @@ template std::string demangled_name(const T &obj) { #include "capiocl/parser.h" #include "capiocl/serializer.h" -capiocl::engine::Engine *parseConfiguration( - const std::filesystem::path &source, const std::filesystem::path &resolve_prefix = "", - bool store_only_in_memory = false) { +capiocl::engine::Engine *parseConfiguration(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix = "", + bool store_only_in_memory = false) { return capiocl::parser::Parser::parse(capiocl::configuration::CapioClConfiguration({ {"capiocl.config_path", source.string()}, {"capiocl.resolve_path", resolve_prefix.string()},