From b0e0fd15dbc73c196d174b24df7029e208d46163 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Mon, 5 May 2025 14:27:03 +0200 Subject: [PATCH 01/14] Remove one of the two functions used to read json files. --- .../data_request_api/content/dreq_content.py | 46 +++++++++---------- .../content/dump_transformation.py | 6 +-- .../tests/test_data_request.py | 10 ++-- .../data_request_api/tests/test_optimize.py | 4 +- .../tests/test_vocabulary_server.py | 4 +- .../data_request_api/utilities/tools.py | 5 -- .../check_consolidation_and_transformation.py | 2 +- 7 files changed, 36 insertions(+), 41 deletions(-) diff --git a/data_request_api/data_request_api/content/dreq_content.py b/data_request_api/data_request_api/content/dreq_content.py index fa0ab5cc..5cf662a7 100644 --- a/data_request_api/data_request_api/content/dreq_content.py +++ b/data_request_api/data_request_api/content/dreq_content.py @@ -18,6 +18,7 @@ from data_request_api.content.mapping_table import mapping_table from data_request_api.utilities.decorators import append_kwargs_from_config from data_request_api.utilities.logger import get_logger # noqa +from data_request_api.utilities.tools import read_json_file # Suppress pooch info output pooch.get_logger().setLevel("WARNING") @@ -588,28 +589,16 @@ def load(version="latest_stable", **kwargs): _dreq_content_loaded["json_path"] = json_path - with open(json_path) as f: - consolidate_error = ( - "Consolidation mapping is not supported for raw exports of versions < v1.2." - " Set 'export' to \"release\" (recommended), or set 'consolidate' to True" - " or set 'force_consolidate' to True to force consolidation regardless." - ) - consolidate_warning = ( - "Consolidation mapping is not supported for raw exports of versions < v1.2." " Forcing it regardless ..." - ) - if "consolidate" in kwargs: - if kwargs["consolidate"]: - if "export" in kwargs and kwargs["export"] == "raw": - if _parse_version(version) < _parse_version("v1.2") and version != "dev": - if "force_consolidate" in kwargs and kwargs["force_consolidate"]: - logger.warning(consolidate_warning) - else: - logger.error(consolidate_error) - raise ValueError(consolidate_error) - return ce.map_data(json.load(f), mapping_table, next(iter(version_dict.keys())), **kwargs) - else: - return json.load(f) - else: + consolidate_error = ( + "Consolidation mapping is not supported for raw exports of versions < v1.2." + " Set 'export' to \"release\" (recommended), or set 'consolidate' to True" + " or set 'force_consolidate' to True to force consolidation regardless." + ) + consolidate_warning = ( + "Consolidation mapping is not supported for raw exports of versions < v1.2." " Forcing it regardless ..." + ) + if "consolidate" in kwargs: + if kwargs["consolidate"]: if "export" in kwargs and kwargs["export"] == "raw": if _parse_version(version) < _parse_version("v1.2") and version != "dev": if "force_consolidate" in kwargs and kwargs["force_consolidate"]: @@ -617,4 +606,15 @@ def load(version="latest_stable", **kwargs): else: logger.error(consolidate_error) raise ValueError(consolidate_error) - return ce.map_data(json.load(f), mapping_table, next(iter(version_dict.keys())), **kwargs) + return ce.map_data(read_json_file(json_path), mapping_table, next(iter(version_dict.keys())), **kwargs) + else: + return read_json_file(json_path) + else: + if "export" in kwargs and kwargs["export"] == "raw": + if _parse_version(version) < _parse_version("v1.2") and version != "dev": + if "force_consolidate" in kwargs and kwargs["force_consolidate"]: + logger.warning(consolidate_warning) + else: + logger.error(consolidate_error) + raise ValueError(consolidate_error) + return ce.map_data(read_json_file(json_path), mapping_table, next(iter(version_dict.keys())), **kwargs) diff --git a/data_request_api/data_request_api/content/dump_transformation.py b/data_request_api/data_request_api/content/dump_transformation.py index bcd8ba6d..1511f709 100644 --- a/data_request_api/data_request_api/content/dump_transformation.py +++ b/data_request_api/data_request_api/content/dump_transformation.py @@ -16,7 +16,7 @@ from data_request_api.utilities.decorators import append_kwargs_from_config from data_request_api.utilities.logger import get_logger, change_log_level, change_log_file -from data_request_api.utilities.tools import read_json_input_file_content, write_json_output_file_content +from data_request_api.utilities.tools import write_json_output_file_content, read_json_file from data_request_api.content import dreq_content as dc @@ -85,7 +85,7 @@ def update_dict(elt_1, elt_2): rep[elt] = value return rep - transform = read_json_input_file_content(os.sep.join([os.path.dirname(os.path.abspath(__file__)), "transform.json"])) + transform = read_json_file(os.sep.join([os.path.dirname(os.path.abspath(__file__)), "transform.json"])) common = transform.pop("common", dict()) if version not in ["default", ]: common = update_dict(common["default"], common.get(version, dict())) @@ -500,7 +500,7 @@ def get_transformed_content(version="latest_stable", export="release", consolida help="Template to be used for output files") parser.add_argument("--version", default="unknown", help="Version of the data used") args = parser.parse_args() - content = read_json_input_file_content(args.input_file) + content = read_json_file(args.input_file) data_request, vocabulary_server = transform_content(content, args.version) write_json_output_file_content("_".join(["DR", args.output_files_template]), data_request) write_json_output_file_content("_".join(["VS", args.output_files_template]), vocabulary_server) diff --git a/data_request_api/data_request_api/tests/test_data_request.py b/data_request_api/data_request_api/tests/test_data_request.py index 6dddfe10..a25fa7f8 100644 --- a/data_request_api/data_request_api/tests/test_data_request.py +++ b/data_request_api/data_request_api/tests/test_data_request.py @@ -12,7 +12,7 @@ import unittest -from data_request_api.utilities.tools import read_json_input_file_content +from data_request_api.utilities.tools import read_json_file from data_request_api.query.data_request import DRObjects, ExperimentsGroup, VariablesGroup, Opportunity, \ DataRequest, version from data_request_api.query.vocabulary_server import VocabularyServer, ConstantValueObj @@ -653,12 +653,12 @@ def test_filter_on_request(self): class TestDataRequest(unittest.TestCase): def setUp(self): self.vs_file = filepath("one_base_VS_output.json") - self.vs_dict = read_json_input_file_content(self.vs_file) + self.vs_dict = read_json_file(self.vs_file) self.vs = VocabularyServer.from_input(self.vs_file) self.input_database_file = filepath("one_base_DR_output.json") - self.input_database = read_json_input_file_content(self.input_database_file) + self.input_database = read_json_file(self.input_database_file) self.complete_input_file = filepath("one_base_input.json") - self.complete_input = read_json_input_file_content(self.complete_input_file) + self.complete_input = read_json_file(self.complete_input_file) self.DR_dump = filepath("one_base_DR_dump.txt") def test_init(self): @@ -877,7 +877,7 @@ def setUp(self): self.vs_file = filepath("one_base_VS_output.json") self.vs = VocabularyServer.from_input(self.vs_file) self.input_database_file = filepath("one_base_DR_output.json") - self.input_database = read_json_input_file_content(self.input_database_file) + self.input_database = read_json_file(self.input_database_file) self.dr = DataRequest(input_database=self.input_database, VS=self.vs) self.exp_export = filepath("experiments_export.txt") self.exp_expgrp_summmary = filepath("exp_expgrp_summary.txt") diff --git a/data_request_api/data_request_api/tests/test_optimize.py b/data_request_api/data_request_api/tests/test_optimize.py index 2a1ea8a6..395bf393 100644 --- a/data_request_api/data_request_api/tests/test_optimize.py +++ b/data_request_api/data_request_api/tests/test_optimize.py @@ -18,7 +18,7 @@ from data_request_api.query.data_request import DataRequest from data_request_api.content.dreq_content import _dreq_res -from data_request_api.utilities.tools import read_json_input_file_content +from data_request_api.utilities.tools import read_json_file from data_request_api.content.dump_transformation import correct_dictionaries, transform_content_inner, \ get_transformed_content, get_transform_settings from data_request_api.tests import filepath @@ -60,7 +60,7 @@ def setUp(self): self.vs_dict = content["VS_input"] self.input_database = content["DR_input"] self.single = f"{_dreq_res}/{self.version}/dreq_{export_version}_export.json" - self.single_content = read_json_input_file_content(self.single) + self.single_content = read_json_file(self.single) self.single_format = correct_dictionaries(self.single_content) @unittest.skip diff --git a/data_request_api/data_request_api/tests/test_vocabulary_server.py b/data_request_api/data_request_api/tests/test_vocabulary_server.py index 431ecdac..4d73ba95 100644 --- a/data_request_api/data_request_api/tests/test_vocabulary_server.py +++ b/data_request_api/data_request_api/tests/test_vocabulary_server.py @@ -9,7 +9,7 @@ import copy import unittest -from data_request_api.utilities.tools import read_json_input_file_content +from data_request_api.utilities.tools import read_json_file from data_request_api.query.vocabulary_server import VocabularyServer, is_link_id_or_value, build_link_from_id, \ to_plural, to_singular from data_request_api.tests import filepath @@ -50,7 +50,7 @@ def test_to_plural(self): class TestVocabularyServer(unittest.TestCase): def setUp(self): self.vs_file = filepath("one_base_VS_output.json") - self.vs_content = read_json_input_file_content(self.vs_file) + self.vs_content = read_json_file(self.vs_file) self.vs_content_infinite_loop = copy.deepcopy(self.vs_content) self.vs_content_infinite_loop["cell_methods"]["CellMethods::am-tm"]["structure_title"] = "link::default_483" diff --git a/data_request_api/data_request_api/utilities/tools.py b/data_request_api/data_request_api/utilities/tools.py index db6a7fbc..66a5924a 100644 --- a/data_request_api/data_request_api/utilities/tools.py +++ b/data_request_api/data_request_api/utilities/tools.py @@ -24,11 +24,6 @@ def read_json_file(filename): return content -def read_json_input_file_content(filename): - content = read_json_file(filename) - return content - - def write_json_output_file_content(filename, content, **kwargs): logger = get_logger() logger.debug(f"Writing file {filename}.") diff --git a/scripts/check_consolidation_and_transformation.py b/scripts/check_consolidation_and_transformation.py index 97ef6f8e..968d3139 100644 --- a/scripts/check_consolidation_and_transformation.py +++ b/scripts/check_consolidation_and_transformation.py @@ -12,7 +12,7 @@ get_logger, ) from data_request_api.utilities.tools import ( - read_json_input_file_content, + read_json_file, write_json_output_file_content, ) From 2a9d26dff0ea37f4dea0ecbf07175fff1cad8ebc Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Thu, 15 May 2025 11:18:37 +0200 Subject: [PATCH 02/14] Add region to DR and remove it from VS part. --- .../content/dump_transformation.py | 3 +- .../data_request_api/query/data_request.py | 18 +++++++--- .../data_request_api/query/filtering.json | 6 ++-- .../test_datasets/one_base_DR_output.json | 33 ++++++++++++------- .../test_datasets/one_base_VS_output.json | 1 - .../several_bases_DR_output.json | 30 +++++++++++------ 6 files changed, 62 insertions(+), 29 deletions(-) diff --git a/data_request_api/data_request_api/content/dump_transformation.py b/data_request_api/data_request_api/content/dump_transformation.py index 1511f709..c57e1372 100644 --- a/data_request_api/data_request_api/content/dump_transformation.py +++ b/data_request_api/data_request_api/content/dump_transformation.py @@ -395,7 +395,8 @@ def split_content_one_base(content): ("mips", list, list())], "variable_groups": [("variables", list, list()), ("mips", list, list()), - ("priority_level", (str, type(None)), None)], + ("priority_level", (str, type(None)), None), + ("region", (str, type(None)), None)], "experiment_groups": [("experiments", list, list()), ] } if isinstance(content, dict): diff --git a/data_request_api/data_request_api/query/data_request.py b/data_request_api/data_request_api/query/data_request.py index 432ea34d..c84a3e21 100644 --- a/data_request_api/data_request_api/query/data_request.py +++ b/data_request_api/data_request_api/query/data_request.py @@ -272,7 +272,7 @@ def filter_on_request(self, request_value): class VariablesGroup(DRObjects): def __init__(self, id, dr, DR_type="variable_groups", - structure=dict(variables=list(), mips=list(), priority_level="High"), **attributes): + structure=dict(variables=list(), mips=list(), priority_level="High", region=None), **attributes): super().__init__(id=id, dr=dr, DR_type=DR_type, structure=structure, **attributes) def check(self): @@ -282,9 +282,10 @@ def check(self): logger.critical(f"No variable defined for {self.DR_type} id {self.id}") @classmethod - def from_input(cls, dr, id, variables=list(), mips=list(), priority_level="High", **kwargs): + def from_input(cls, dr, id, variables=list(), mips=list(), priority_level="High", region=None, **kwargs): return super().from_input(DR_type="variable_groups", dr=dr, id=id, elements=kwargs, - structure=dict(variables=variables, mips=mips, priority_level=priority_level)) + structure=dict(variables=variables, mips=mips, priority_level=priority_level, + region=None)) def count(self): """ @@ -293,6 +294,13 @@ def count(self): """ return len(self.get_variables()) + def get_region(self): + """ + Return the region of the variable group. + :return: DRObject: the region associated with the variable group. + """ + return self.structure["region"] + def get_variables(self): """ Return the list of Variables linked to the VariablesGroup. @@ -332,6 +340,8 @@ def filter_on_request(self, request_value): found = request_value in self.get_variables() elif request_type in ["mips", ]: found = request_value in self.get_mips() + elif request_type in ["regions", ]: + found = request_value == self.get_region() elif request_type in ["max_priority_levels", ]: priority = self.dr.find_element("priority_level", self.get_priority_level().id) req_priority = self.dr.find_element("priority_level", request_value.id) @@ -457,7 +467,7 @@ def filter_on_request(self, request_value): list_to_check=self.get_variable_groups()) elif request_type in ["variables", "priority_levels", "cmip6_tables_identifiers", "temporal_shapes", "spatial_shapes", "structure_titles", "physical_parameters", "modelling_realms", "esm-bcvs", - "cf_standard_names", "cell_methods", "cell_measures", "max_priority_levels"]: + "cf_standard_names", "cell_methods", "cell_measures", "max_priority_levels", "regions"]: found = self.filter_on_request_list(request_values=request_value, list_to_check=self.get_variable_groups()) elif request_type in ["experiments", ]: diff --git a/data_request_api/data_request_api/query/filtering.json b/data_request_api/data_request_api/query/filtering.json index cb8e6b74..8c639d3a 100644 --- a/data_request_api/data_request_api/query/filtering.json +++ b/data_request_api/data_request_api/query/filtering.json @@ -17,7 +17,8 @@ "mips": ["opportunities", "variable_groups"], "time_slices": ["opportunities"], "experiment_groups": ["opportunities"], - "variable_groups": ["opportunities"] + "variable_groups": ["opportunities"], + "region": ["variable_groups"] }, "filtering": { "default": ["opportunities"], @@ -36,6 +37,7 @@ "variables": ["opportunities", "variable_groups"], "priority_levels": ["opportunities", "variable_groups"], "max_priority_levels": ["opportunities", "variable_groups"], - "mips": ["opportunities", "variable_groups"] + "mips": ["opportunities", "variable_groups"], + "region": ["opportunities", "variable_groups"] } } \ No newline at end of file diff --git a/data_request_api/data_request_api/tests/test_datasets/one_base_DR_output.json b/data_request_api/data_request_api/tests/test_datasets/one_base_DR_output.json index 4be45a12..13402fdf 100644 --- a/data_request_api/data_request_api/tests/test_datasets/one_base_DR_output.json +++ b/data_request_api/data_request_api/tests/test_datasets/one_base_DR_output.json @@ -155,7 +155,8 @@ "variables": [ "link::baa71c7c-e5dd-11e5-8482-ac72891c3257", "link::babb20b4-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_570": { "mips": [ @@ -172,7 +173,8 @@ "link::83bbfc6e-7f07-11ef-9308-b1dd71e64bec", "link::83bbfc6f-7f07-11ef-9308-b1dd71e64bec", "link::baa4e07e-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_571": { "mips": [ @@ -196,7 +198,8 @@ "link::baa72514-e5dd-11e5-8482-ac72891c3257", "link::c9180bae-c5e8-11e6-84e6-5404a60d96b5", "link::c9181982-c5e8-11e6-84e6-5404a60d96b5" - ] + ], + "region": null }, "default_572": { "mips": [ @@ -216,7 +219,8 @@ "link::83bbfb69-7f07-11ef-9308-b1dd71e64bec", "link::baa50c2a-e5dd-11e5-8482-ac72891c3257", "link::default_580" - ] + ], + "region": null }, "default_573": { "mips": [], @@ -248,7 +252,8 @@ "link::babbbbe6-e5dd-11e5-8482-ac72891c3257", "link::babbd25c-e5dd-11e5-8482-ac72891c3257", "link::babd0e56-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_574": { "mips": [], @@ -268,7 +273,8 @@ "link::bab742c8-e5dd-11e5-8482-ac72891c3257", "link::bab9bd00-e5dd-11e5-8482-ac72891c3257", "link::f2fad86e-c38d-11e6-abc1-1b922e5e1118" - ] + ], + "region": null }, "default_575": { "mips": [], @@ -356,7 +362,8 @@ "link::babd0906-e5dd-11e5-8482-ac72891c3257", "link::babd9ace-e5dd-11e5-8482-ac72891c3257", "link::d241a6d2-4a9f-11e6-b84e-ac72891c3257" - ] + ], + "region": null }, "default_576": { "mips": [], @@ -372,7 +379,8 @@ "link::bab91b20-e5dd-11e5-8482-ac72891c3257", "link::babb5db8-e5dd-11e5-8482-ac72891c3257", "link::babbdec8-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_577": { "mips": [], @@ -387,7 +395,8 @@ "link::83bbfbca-7f07-11ef-9308-b1dd71e64bec", "link::8baebea6-4a5b-11e6-9cd2-ac72891c3257", "link::bab2f9d4-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": "Southern Ocean and Antarctic (south of 30\u00b0)" }, "default_578": { "mips": [ @@ -412,7 +421,8 @@ "link::bab65138-e5dd-11e5-8482-ac72891c3257", "link::bab91b20-e5dd-11e5-8482-ac72891c3257", "link::babb12ae-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_579": { "mips": [ @@ -435,7 +445,8 @@ "link::baa58b1e-e5dd-11e5-8482-ac72891c3257", "link::babb20b4-e5dd-11e5-8482-ac72891c3257", "link::d243ba76-4a9f-11e6-b84e-ac72891c3257" - ] + ], + "region": null } }, "version": "test" diff --git a/data_request_api/data_request_api/tests/test_datasets/one_base_VS_output.json b/data_request_api/data_request_api/tests/test_datasets/one_base_VS_output.json index 03918330..a428092b 100644 --- a/data_request_api/data_request_api/tests/test_datasets/one_base_VS_output.json +++ b/data_request_api/data_request_api/tests/test_datasets/one_base_VS_output.json @@ -4660,7 +4660,6 @@ "default_577": { "justification": "Minimal set of variables needed as input for physics-based microclimate models, which are essential for ecophysiological models of organism heat and water balance.", "name": "biodiv_microclim_hourly", - "region": "Southern Ocean and Antarctic (south of 30\u00b0)", "title": "Minimal set of variables for physics-based microclimate models as input for ecophysiological models." }, "default_578": { diff --git a/data_request_api/data_request_api/tests/test_datasets/several_bases_DR_output.json b/data_request_api/data_request_api/tests/test_datasets/several_bases_DR_output.json index 79453998..219783c5 100644 --- a/data_request_api/data_request_api/tests/test_datasets/several_bases_DR_output.json +++ b/data_request_api/data_request_api/tests/test_datasets/several_bases_DR_output.json @@ -132,7 +132,8 @@ "link::83bbfb6e-7f07-11ef-9308-b1dd71e64bec", "link::baa71c7c-e5dd-11e5-8482-ac72891c3257", "link::babb20b4-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_533": { "mips": [ @@ -149,7 +150,8 @@ "link::83bbfc6e-7f07-11ef-9308-b1dd71e64bec", "link::83bbfc6f-7f07-11ef-9308-b1dd71e64bec", "link::baa4e07e-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_534": { "mips": [ @@ -173,7 +175,8 @@ "link::baa72514-e5dd-11e5-8482-ac72891c3257", "link::c9180bae-c5e8-11e6-84e6-5404a60d96b5", "link::c9181982-c5e8-11e6-84e6-5404a60d96b5" - ] + ], + "region": null }, "default_535": { "mips": [ @@ -193,7 +196,8 @@ "link::527f5ccc-8c97-11ef-944e-41a8eb05f654", "link::83bbfb69-7f07-11ef-9308-b1dd71e64bec", "link::baa50c2a-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_536": { "mips": [], @@ -225,7 +229,8 @@ "link::babbbbe6-e5dd-11e5-8482-ac72891c3257", "link::babbd25c-e5dd-11e5-8482-ac72891c3257", "link::babd0e56-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_537": { "mips": [], @@ -245,7 +250,8 @@ "link::bab742c8-e5dd-11e5-8482-ac72891c3257", "link::bab9bd00-e5dd-11e5-8482-ac72891c3257", "link::f2fad86e-c38d-11e6-abc1-1b922e5e1118" - ] + ], + "region": null }, "default_538": { "mips": [], @@ -333,7 +339,8 @@ "link::babd0906-e5dd-11e5-8482-ac72891c3257", "link::babd9ace-e5dd-11e5-8482-ac72891c3257", "link::d241a6d2-4a9f-11e6-b84e-ac72891c3257" - ] + ], + "region": null }, "default_539": { "mips": [], @@ -349,7 +356,8 @@ "link::bab91b20-e5dd-11e5-8482-ac72891c3257", "link::babb5db8-e5dd-11e5-8482-ac72891c3257", "link::babbdec8-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_540": { "mips": [ @@ -374,7 +382,8 @@ "link::bab65138-e5dd-11e5-8482-ac72891c3257", "link::bab91b20-e5dd-11e5-8482-ac72891c3257", "link::babb12ae-e5dd-11e5-8482-ac72891c3257" - ] + ], + "region": null }, "default_541": { "mips": [ @@ -397,7 +406,8 @@ "link::baa58b1e-e5dd-11e5-8482-ac72891c3257", "link::babb20b4-e5dd-11e5-8482-ac72891c3257", "link::d243ba76-4a9f-11e6-b84e-ac72891c3257" - ] + ], + "region": null } }, "version": "test" From 4f0cbcae7145c35effffb27fe0f75c2afb747fd3 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Tue, 13 May 2025 12:49:16 +0200 Subject: [PATCH 03/14] Remove unused elements --- .../data_request_api/query/filtering.json | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/data_request_api/data_request_api/query/filtering.json b/data_request_api/data_request_api/query/filtering.json index 8c639d3a..fb37fafb 100644 --- a/data_request_api/data_request_api/query/filtering.json +++ b/data_request_api/data_request_api/query/filtering.json @@ -19,25 +19,5 @@ "experiment_groups": ["opportunities"], "variable_groups": ["opportunities"], "region": ["variable_groups"] - }, - "filtering": { - "default": ["opportunities"], - "experiments": ["opportunities", "experiment_groups"], - "cmip6_tables_identifiers": ["opportunities", "variable_groups", "variables"], - "temporal_shapes": ["opportunities", "variable_groups", "variables"], - "spatial_shapes": ["opportunities", "variable_groups", "variables"], - "structures": ["opportunities", "variable_groups", "variables"], - "structure_titles": ["opportunities", "variable_groups", "variables"], - "physical_parameters": ["opportunities", "variable_groups", "variables"], - "modelling_realms": ["opportunities", "variable_groups", "variables"], - "esm-bcvs": ["opportunities", "variable_groups", "variables"], - "cf_standard_names": ["opportunities", "variable_groups", "variables"], - "cell_methods": ["opportunities", "variable_groups", "variables"], - "cell_measures": ["opportunities", "variable_groups", "variables"], - "variables": ["opportunities", "variable_groups"], - "priority_levels": ["opportunities", "variable_groups"], - "max_priority_levels": ["opportunities", "variable_groups"], - "mips": ["opportunities", "variable_groups"], - "region": ["opportunities", "variable_groups"] } } \ No newline at end of file From 5f3dcc400fc7b68f4515d0cabed1cacbb6cbb000 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Tue, 13 May 2025 16:18:15 +0200 Subject: [PATCH 04/14] Use pytest-cov instead of coverage to have correct coverage report due to subprocesses use --- env.yml | 1 + pyproject.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/env.yml b/env.yml index 69de3fa1..f358bb8f 100644 --- a/env.yml +++ b/env.yml @@ -4,6 +4,7 @@ channels: dependencies: - bs4 - coverage + - pytest-cov - openpyxl - pooch - pytest diff --git a/pyproject.toml b/pyproject.toml index 9d404c57..bdae6968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ write_to = "data_request_api/data_request_api/version.py" # updated path for ve [tool.pytest.ini_options] cache_dir = "data_request_api/data_request_api/tests/.pytest_cache" +addopts = " --cov --cov-append --cov-branch" [tool.coverage.run] branch = true @@ -65,6 +66,7 @@ source = [ "scripts" ] data_file = "data_request_api/data_request_api/tests/.coverage" +sigterm = true [tool.coverage.report] fail_under = 50 From 38401bcc6f93d616ffa9afe8a4a87b0fd4c0502b Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Thu, 15 May 2025 16:01:27 +0200 Subject: [PATCH 05/14] Add a test (ok alone, not with others) --- .../data_request_api/tests/test_scripts.py | 391 ++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 data_request_api/data_request_api/tests/test_scripts.py diff --git a/data_request_api/data_request_api/tests/test_scripts.py b/data_request_api/data_request_api/tests/test_scripts.py new file mode 100644 index 00000000..1852f301 --- /dev/null +++ b/data_request_api/data_request_api/tests/test_scripts.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Test scripts +""" +import json +import os +import subprocess +import sys +from pathlib import Path + +import data_request_api.content.dreq_content as dc +import pytest +import yaml + + +@pytest.fixture(scope="class") +def monkeyclass(): + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="class") +def temp_config_file(tmp_path_factory, monkeyclass): + temp_dir = tmp_path_factory.mktemp("data") + config_file = temp_dir / ".CMIP7_data_request_api_config" + monkeyclass.setenv("CMIP7_DR_API_CONFIGFILE", str(config_file)) + # Provide the test with the config file + try: + yield config_file + finally: + config_file.unlink(missing_ok=True) + + +@pytest.fixture(scope="class") +def export(request): + # "consolidate" or "no consolidate" + return request.param + + +@pytest.mark.parametrize( + "export", + ["raw", "release"], + indirect=True, + scope="class", +) +class TestWorkflowV10: + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, request): + # Initialize config and load v1.0 content version + self.temp_config_file = request.getfixturevalue("temp_config_file") + self.export = request.getfixturevalue("export") + self.version = "v1.0" + with open(self.temp_config_file, "w") as fh: + config = { + "export": self.export, + "consolidate": False, + "cache_dir": str(self.temp_config_file.parent), + } + yaml.dump(config, fh) + dc.load(self.version) + + def test_database_transformation(self, temp_config_file, export): + DRfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="DR", export=export) + DRfile.unlink(missing_ok=True) + VSfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="VS", export=export) + VSfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/database_transformation.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(DRfile) and os.path.getsize(DRfile) > 0 + assert os.path.exists(VSfile) and os.path.getsize(VSfile) > 0 + + def test_workflow_example_2(self, temp_config_file, export): + output_files = [temp_config_file.parent / out for out in ["op_per_th.csv", "var_per_op.csv","var_per_spsh.csv", + "var_per_op_filtered.csv", + "var_per_op_regrouped_filtered.csv", + "var_per_exp_regrouped_filtered.csv", + "var_per_exp_filtered.csv", "exp_per_op.csv", + "exp_per_op_regrouped_filtered.csv", + "exp_per_op_filtered.csv", "op.csv"]] + for output_file in output_files: + output_file.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/workflow_example_2.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + for output_file in output_files: + assert os.path.exists(output_file) and os.path.getsize(output_file) > 0 + + def test_check_variables_attributes(self, temp_config_file, export): + checkfile = temp_config_file.parent / self.version / "check_attributes_{}.json".format(export) + checkfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/check_variables_attributes.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(checkfile) and os.path.getsize(checkfile) > 0 + + +@pytest.mark.parametrize( + "export", + ["release", ], + indirect=True, + scope="class", +) +class TestWorkflowV11: + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, request): + # Initialize config and load v1.0 content version + self.temp_config_file = request.getfixturevalue("temp_config_file") + self.export = request.getfixturevalue("export") + self.version = "v1.1" + with open(self.temp_config_file, "w") as fh: + config = { + "export": self.export, + "consolidate": False, + "cache_dir": str(self.temp_config_file.parent), + } + yaml.dump(config, fh) + dc.load(self.version) + + def test_database_transformation(self, temp_config_file, export): + DRfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="DR", export=export) + DRfile.unlink(missing_ok=True) + VSfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="VS", export=export) + VSfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/database_transformation.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(DRfile) and os.path.getsize(DRfile) > 0 + assert os.path.exists(VSfile) and os.path.getsize(VSfile) > 0 + + def test_workflow_example_2(self, temp_config_file, export): + output_files = [temp_config_file.parent / out for out in ["op_per_th.csv", "var_per_op.csv","var_per_spsh.csv", + "var_per_op_filtered.csv", + "var_per_op_regrouped_filtered.csv", + "var_per_exp_regrouped_filtered.csv", + "var_per_exp_filtered.csv", "exp_per_op.csv", + "exp_per_op_regrouped_filtered.csv", + "exp_per_op_filtered.csv", "op.csv"]] + for output_file in output_files: + output_file.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/workflow_example_2.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + for output_file in output_files: + assert os.path.exists(output_file) and os.path.getsize(output_file) > 0 + + def test_check_variables_attributes(self, temp_config_file, export): + checkfile = temp_config_file.parent / self.version / "check_attributes_{}.json".format(export) + checkfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/check_variables_attributes.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(checkfile) and os.path.getsize(checkfile) > 0 + + +@pytest.mark.parametrize( + "export", + ["raw", "release"], + indirect=True, + scope="class", +) +class TestWorkflowV12: + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, request): + # Initialize config and load v1.0 content version + self.temp_config_file = request.getfixturevalue("temp_config_file") + self.export = request.getfixturevalue("export") + self.version = "v1.2" + with open(self.temp_config_file, "w") as fh: + config = { + "export": self.export, + "consolidate": False, + "cache_dir": str(self.temp_config_file.parent), + } + yaml.dump(config, fh) + dc.load(self.version) + + def test_database_transformation(self, temp_config_file, export): + DRfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="DR", export=export) + DRfile.unlink(missing_ok=True) + VSfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="VS", export=export) + VSfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/database_transformation.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(DRfile) and os.path.getsize(DRfile) > 0 + assert os.path.exists(VSfile) and os.path.getsize(VSfile) > 0 + + def test_workflow_example_2(self, temp_config_file, export): + output_files = [temp_config_file.parent / out for out in ["op_per_th.csv", "var_per_op.csv","var_per_spsh.csv", + "var_per_op_filtered.csv", + "var_per_op_regrouped_filtered.csv", + "var_per_exp_regrouped_filtered.csv", + "var_per_exp_filtered.csv", "exp_per_op.csv", + "exp_per_op_regrouped_filtered.csv", + "exp_per_op_filtered.csv", "op.csv"]] + for output_file in output_files: + output_file.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/workflow_example_2.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + for output_file in output_files: + assert os.path.exists(output_file) and os.path.getsize(output_file) > 0 + + def test_check_variables_attributes(self, temp_config_file, export): + checkfile = temp_config_file.parent / self.version / "check_attributes_{}.json".format(export) + checkfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/check_variables_attributes.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(checkfile) and os.path.getsize(checkfile) > 0 + + +@pytest.mark.parametrize( + "export", + ["raw", "release"], + indirect=True, + scope="class", +) +class TestWorkflowV121: + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, request): + # Initialize config and load v1.0 content version + self.temp_config_file = request.getfixturevalue("temp_config_file") + self.export = request.getfixturevalue("export") + self.version = "v1.2.1" + with open(self.temp_config_file, "w") as fh: + config = { + "export": self.export, + "consolidate": False, + "cache_dir": str(self.temp_config_file.parent), + } + yaml.dump(config, fh) + dc.load(self.version) + + def test_database_transformation(self, temp_config_file, export): + DRfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="DR", export=export) + DRfile.unlink(missing_ok=True) + VSfile = temp_config_file.parent / self.version / "{base}_{export}_content.json".format(base="VS", export=export) + VSfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/database_transformation.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(DRfile) and os.path.getsize(DRfile) > 0 + assert os.path.exists(VSfile) and os.path.getsize(VSfile) > 0 + + def test_workflow_example_2(self, temp_config_file, export): + output_files = [temp_config_file.parent / out for out in ["op_per_th.csv", "var_per_op.csv","var_per_spsh.csv", + "var_per_op_filtered.csv", + "var_per_op_regrouped_filtered.csv", + "var_per_exp_regrouped_filtered.csv", + "var_per_exp_filtered.csv", "exp_per_op.csv", + "exp_per_op_regrouped_filtered.csv", + "exp_per_op_filtered.csv", "op.csv"]] + for output_file in output_files: + output_file.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/workflow_example_2.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + for output_file in output_files: + assert os.path.exists(output_file) and os.path.getsize(output_file) > 0 + + def test_check_variables_attributes(self, temp_config_file, export): + checkfile = temp_config_file.parent / self.version / "check_attributes_{}.json".format(export) + checkfile.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "scripts/check_variables_attributes.py", + "--output_dir", + str(temp_config_file.parent), + "--version", + self.version, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(checkfile) and os.path.getsize(checkfile) > 0 From 2c5dbd572f5250a371f0311650e969b65984ce83 Mon Sep 17 00:00:00 2001 From: sol1105 Date: Mon, 19 May 2025 19:40:44 +0200 Subject: [PATCH 06/14] config.py: Added reload function and debug statements to load_config --- .../data_request_api/utilities/config.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/data_request_api/data_request_api/utilities/config.py b/data_request_api/data_request_api/utilities/config.py index 8d8186f8..995f4839 100644 --- a/data_request_api/data_request_api/utilities/config.py +++ b/data_request_api/data_request_api/utilities/config.py @@ -1,10 +1,12 @@ #!/usr/bin/env python -from importlib.metadata import version, PackageNotFoundError import os +from importlib.metadata import PackageNotFoundError, version from pathlib import Path + import requests import yaml +from data_request_api.utilities.logger import get_logger # noqa # Config file location in the user's home directory PACKAGE_NAME = "CMIP7_data_request_api" @@ -71,8 +73,11 @@ def _sanity_check(key, value): ) -def load_config() -> dict: +def load_config(reload=False) -> dict: """Load the configuration file, creating it if necessary. + Args: + reload (bool): Reload config from disk and do not load from cache. + Default is False. Returns: dict: The configuration data. @@ -85,7 +90,12 @@ def load_config() -> dict: ValueError: If the value is not within the valid values for the key. """ global CONFIG + logger = get_logger() + + if reload: + CONFIG = {} if CONFIG == {}: + logger.debug(f"Loading config file: {CONFIG_FILE}") try: with open(CONFIG_FILE) as f: CONFIG = yaml.safe_load(f) @@ -99,20 +109,19 @@ def load_config() -> dict: yaml.dump(DEFAULT_CONFIG, f) CONFIG = DEFAULT_CONFIG.copy() elif not isinstance(CONFIG, dict): - raise TypeError( - f"Config file ('{CONFIG_FILE}') must contain a dictionary" - ) + raise TypeError(f"Config file ('{CONFIG_FILE}') must contain a dictionary") # Sanity test for allowed types and values for key, value in CONFIG.items(): _sanity_check(key, value) # Ensure all required keys are present and update config file if necessary - missing_keys = { - k: v for k, v in DEFAULT_CONFIG.items() if k not in CONFIG - } + missing_keys = {k: v for k, v in DEFAULT_CONFIG.items() if k not in CONFIG} for key, value in missing_keys.items(): update_config(key, value) + logger.debug(f"Loaded config from file: {CONFIG}") + else: + logger.debug(f"Loaded config from cache: {CONFIG}") return CONFIG @@ -169,7 +178,7 @@ def check_api_version(): try: response = requests.get(f"https://pypi.org/pypi/{PACKAGE_NAME}/json", timeout=5) response.raise_for_status() - latest_version = response.json()['info']['version'] + latest_version = response.json()["info"]["version"] except requests.RequestException as e: print(f"Error checking PyPI: {e}") return @@ -183,7 +192,7 @@ def check_api_version(): msg += f" pip install --upgrade {PACKAGE_NAME}\n" msg += "To turn off this warning:\n" msg += " CMIP7_data_request_api_config check_api_version false" - msg = '\n' + msg + '\n' + msg = "\n" + msg + "\n" # Add color to the warning message color_code = "\033[91m" From a74d2775e38b387cbc8080e7a8b83f5bd6c48fd5 Mon Sep 17 00:00:00 2001 From: sol1105 Date: Mon, 19 May 2025 19:42:33 +0200 Subject: [PATCH 07/14] dreq_content.py: Added debug statement to retrieve --- data_request_api/data_request_api/content/dreq_content.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_request_api/data_request_api/content/dreq_content.py b/data_request_api/data_request_api/content/dreq_content.py index 0579aa3c..d956d5c3 100644 --- a/data_request_api/data_request_api/content/dreq_content.py +++ b/data_request_api/data_request_api/content/dreq_content.py @@ -476,6 +476,7 @@ def retrieve(version="latest_stable", **kwargs): # Store the path to the dreq.json in the json_paths dictionary json_paths[version] = json_path + logger.debug(f"'{version}' stored under '{json_path}'") # Capture no correct export found for cached versions (offline mode) if not json_paths or json_paths == {}: From 4aa592a5a436ad4c3bdb8063b9325dd77a4980a2 Mon Sep 17 00:00:00 2001 From: sol1105 Date: Mon, 19 May 2025 19:44:23 +0200 Subject: [PATCH 08/14] test_cli.py: Revert os.chdir after test, set neutral state at beginning of each test --- .../data_request_api/tests/test_cli.py | 418 ++++++++++-------- 1 file changed, 231 insertions(+), 187 deletions(-) diff --git a/data_request_api/data_request_api/tests/test_cli.py b/data_request_api/data_request_api/tests/test_cli.py index 51605342..a5d3107a 100644 --- a/data_request_api/data_request_api/tests/test_cli.py +++ b/data_request_api/data_request_api/tests/test_cli.py @@ -5,6 +5,7 @@ from pathlib import Path import data_request_api.content.dreq_content as dc +import data_request_api.utilities.config as dreqcfg import pytest import yaml @@ -51,6 +52,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load("v1.2") def test_export_dreq_lists_json(self, temp_config_file, consolidate): @@ -71,7 +76,9 @@ def test_export_dreq_lists_json(self, temp_config_file, consolidate): assert result.returncode == 0 assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - def test_export_dreq_lists_json_with_opportunities_file(self, temp_config_file, consolidate): + def test_export_dreq_lists_json_with_opportunities_file( + self, temp_config_file, consolidate + ): # Test that the script creates an opportunities file template opportunities_file = temp_config_file.parent / "opportunities.json" opportunities_file.unlink(missing_ok=True) @@ -91,7 +98,10 @@ def test_export_dreq_lists_json_with_opportunities_file(self, temp_config_file, text=True, ) assert result.returncode == 0 - assert os.path.exists(opportunities_file) and os.path.getsize(opportunities_file) > 0 + assert ( + os.path.exists(opportunities_file) + and os.path.getsize(opportunities_file) > 0 + ) assert not os.path.exists(ofile) or os.path.getsize(ofile) == 0 # Test that it now applies the opportunities settings from opportunities_file @@ -111,7 +121,9 @@ def test_export_dreq_lists_json_with_opportunities_file(self, temp_config_file, assert result.returncode == 0 assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - def test_export_dreq_lists_json_with_invalid_opportunities_file(self, temp_config_file, consolidate): + def test_export_dreq_lists_json_with_invalid_opportunities_file( + self, temp_config_file, consolidate + ): # Test that the script raises an error with an invalid opportunities file opportunities_file = temp_config_file.parent / "invalid_opportunities.json" opportunities_file.unlink(missing_ok=True) @@ -170,6 +182,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load("v1.2") def test_get_variables_metadata(self, temp_config_file, consolidate): @@ -190,7 +206,9 @@ def test_get_variables_metadata(self, temp_config_file, consolidate): assert result.returncode == 0 assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - def test_get_variables_metadata_with_compound_names(self, temp_config_file, consolidate): + def test_get_variables_metadata_with_compound_names( + self, temp_config_file, consolidate + ): ofile = temp_config_file.parent / "test2.json" ofile.unlink(missing_ok=True) result = subprocess.run( @@ -211,7 +229,9 @@ def test_get_variables_metadata_with_compound_names(self, temp_config_file, cons assert result.returncode == 0 assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - def test_get_variables_metadata_with_cmor_tables(self, temp_config_file, consolidate): + def test_get_variables_metadata_with_cmor_tables( + self, temp_config_file, consolidate + ): ofile = temp_config_file.parent / "test3.json" ofile.unlink(missing_ok=True) result = subprocess.run( @@ -232,7 +252,9 @@ def test_get_variables_metadata_with_cmor_tables(self, temp_config_file, consoli assert result.returncode == 0 assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - def test_get_variables_metadata_with_cmor_variables(self, temp_config_file, consolidate): + def test_get_variables_metadata_with_cmor_variables( + self, temp_config_file, consolidate + ): ofile = temp_config_file.parent / "test4.json" ofile.unlink(missing_ok=True) result = subprocess.run( @@ -288,124 +310,138 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load("v1.2") dc.load("v1.2.1") def test_compare_variables(self, temp_config_file, consolidate): - os.chdir(temp_config_file.parent) - ofileA = temp_config_file.parent / "testA.json" - ofileB = temp_config_file.parent / "testB.json" - ofile_vars = temp_config_file.parent / "diffs_by_variable.json" - ofile_attr = temp_config_file.parent / "diffs_by_attribute.json" - ofile_missing = temp_config_file.parent / "missing_variables.json" - attr_file = temp_config_file.parent / "attributes.yaml" + prev_cwd = os.getcwd() + try: + os.chdir(temp_config_file.parent) + ofileA = temp_config_file.parent / "testA.json" + ofileB = temp_config_file.parent / "testB.json" + ofile_vars = temp_config_file.parent / "diffs_by_variable.json" + ofile_attr = temp_config_file.parent / "diffs_by_attribute.json" + ofile_missing = temp_config_file.parent / "missing_variables.json" + attr_file = temp_config_file.parent / "attributes.yaml" - # Part 1 - Standard comparison - ofile_vars.unlink(missing_ok=True) - ofile_attr.unlink(missing_ok=True) - ofile_missing.unlink(missing_ok=True) - ofileA.unlink(missing_ok=True) - ofileB.unlink(missing_ok=True) - attr_file.unlink(missing_ok=True) - # Create Variable List A - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.get_variables_metadata", - "v1.2", - "-o", - ofileA, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofileA) and os.path.getsize(ofileA) > 0 - # Create Variable List B - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.get_variables_metadata", - "v1.2.1", - "-o", - ofileB, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofileB) and os.path.getsize(ofileB) > 0 - # Actual comparison - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.compare_variables", - ofileA, - ofileB, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 - assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 - assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 - assert os.path.exists(attr_file) and os.path.getsize(attr_file) > 0 + # Part 1 - Standard comparison + ofile_vars.unlink(missing_ok=True) + ofile_attr.unlink(missing_ok=True) + ofile_missing.unlink(missing_ok=True) + ofileA.unlink(missing_ok=True) + ofileB.unlink(missing_ok=True) + attr_file.unlink(missing_ok=True) + # Create Variable List A + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.get_variables_metadata", + "v1.2", + "-o", + ofileA, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofileA) and os.path.getsize(ofileA) > 0 + # Create Variable List B + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.get_variables_metadata", + "v1.2.1", + "-o", + ofileB, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofileB) and os.path.getsize(ofileB) > 0 + # Actual comparison + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.compare_variables", + ofileA, + ofileB, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 + assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 + assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 + assert os.path.exists(attr_file) and os.path.getsize(attr_file) > 0 - # Part 2 - Provide attribute file - ofile_vars.unlink(missing_ok=True) - ofile_attr.unlink(missing_ok=True) - ofile_missing.unlink(missing_ok=True) - # Write custom attributes file - cattr_file = temp_config_file.parent / "custom_attrs.yaml" - cattr_file.unlink(missing_ok=True) - config = { - "compare_attributes": ["standard_name", "units", "cell_methods"], - "repos": { - "cmip6": { - "url": "https://github.com/PCMDI/cmip6-cmor-tables", - } - }, - } - with open(cattr_file, "w") as f: - yaml.dump(config, f, default_flow_style=False) - # Actual comparison - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.compare_variables", - ofileA, - ofileB, - "-c", - cattr_file, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 - assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 - assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 - assert os.path.getsize(cattr_file) < os.path.getsize(attr_file) + # Part 2 - Provide attribute file + ofile_vars.unlink(missing_ok=True) + ofile_attr.unlink(missing_ok=True) + ofile_missing.unlink(missing_ok=True) + # Write custom attributes file + cattr_file = temp_config_file.parent / "custom_attrs.yaml" + cattr_file.unlink(missing_ok=True) + config = { + "compare_attributes": ["standard_name", "units", "cell_methods"], + "repos": { + "cmip6": { + "url": "https://github.com/PCMDI/cmip6-cmor-tables", + } + }, + } + with open(cattr_file, "w") as f: + yaml.dump(config, f, default_flow_style=False) + # Actual comparison + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.compare_variables", + ofileA, + ofileB, + "-c", + cattr_file, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 + assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 + assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 + assert os.path.getsize(cattr_file) < os.path.getsize(attr_file) - # Part 3 - Compare with CMIP6 - ofile_vars.unlink(missing_ok=True) - ofile_attr.unlink(missing_ok=True) - ofile_missing.unlink(missing_ok=True) - # Actual comparison - result = subprocess.run( - [sys.executable, "-m", "data_request_api.command_line.compare_variables", ofileB, "cmip6"], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 - assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 - assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 + # Part 3 - Compare with CMIP6 + ofile_vars.unlink(missing_ok=True) + ofile_attr.unlink(missing_ok=True) + ofile_missing.unlink(missing_ok=True) + # Actual comparison + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.compare_variables", + ofileB, + "cmip6", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofile_missing) and os.path.getsize(ofile_missing) > 0 + assert os.path.exists(ofile_vars) and os.path.getsize(ofile_vars) > 0 + assert os.path.exists(ofile_attr) and os.path.getsize(ofile_attr) > 0 + finally: + os.chdir(prev_cwd) @pytest.mark.parametrize( @@ -426,76 +462,84 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load("v1.2") def test_estimate_dreq_volume(self, temp_config_file, consolidate): - os.chdir(temp_config_file.parent) - ofile = temp_config_file.parent / "test1.json" - sizecfg = temp_config_file.parent / "size.yaml" - ofile.unlink(missing_ok=True) - sizecfg.unlink(missing_ok=True) - # Part 1 - Create size.yaml - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.estimate_dreq_volume", - "v1.2", - "-o", - ofile, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert not os.path.exists(ofile) or os.path.getsize(ofile) == 0 - assert os.path.exists(sizecfg) and os.path.getsize(sizecfg) > 0 - # Part 2 - Actual volume estimate - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.estimate_dreq_volume", - "v1.2", - "-o", - ofile, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - assert os.path.exists(sizecfg) and os.path.getsize(sizecfg) > 0 - # Part 3 - Custom size.yaml - ofile.unlink(missing_ok=True) - csizecfg = temp_config_file.parent / "custom_size.yaml" - csizecfg.unlink(missing_ok=True) - # Read default size.yaml - with open(sizecfg) as fh: - config = yaml.safe_load(fh) - sizecfg.unlink(missing_ok=True) - # Update config - config["longitude"] = 720 - config["latitude"] = 360 - # Write custom size.yaml - with open(csizecfg, "w") as fh: - yaml.dump(config, fh) - # Actual volume estimate - result = subprocess.run( - [ - sys.executable, - "-m", - "data_request_api.command_line.estimate_dreq_volume", - "v1.2", - "-o", - ofile, - "-c", - csizecfg, - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 - assert os.path.exists(csizecfg) and os.path.getsize(csizecfg) > 0 - assert not os.path.exists(sizecfg) or os.path.getsize(sizecfg) == 0 + prev_cwd = os.getcwd() + try: + os.chdir(temp_config_file.parent) + ofile = temp_config_file.parent / "test1.json" + sizecfg = temp_config_file.parent / "size.yaml" + ofile.unlink(missing_ok=True) + sizecfg.unlink(missing_ok=True) + # Part 1 - Create size.yaml + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.estimate_dreq_volume", + "v1.2", + "-o", + ofile, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert not os.path.exists(ofile) or os.path.getsize(ofile) == 0 + assert os.path.exists(sizecfg) and os.path.getsize(sizecfg) > 0 + # Part 2 - Actual volume estimate + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.estimate_dreq_volume", + "v1.2", + "-o", + ofile, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 + assert os.path.exists(sizecfg) and os.path.getsize(sizecfg) > 0 + # Part 3 - Custom size.yaml + ofile.unlink(missing_ok=True) + csizecfg = temp_config_file.parent / "custom_size.yaml" + csizecfg.unlink(missing_ok=True) + # Read default size.yaml + with open(sizecfg) as fh: + config = yaml.safe_load(fh) + sizecfg.unlink(missing_ok=True) + # Update config + config["longitude"] = 720 + config["latitude"] = 360 + # Write custom size.yaml + with open(csizecfg, "w") as fh: + yaml.dump(config, fh) + # Actual volume estimate + result = subprocess.run( + [ + sys.executable, + "-m", + "data_request_api.command_line.estimate_dreq_volume", + "v1.2", + "-o", + ofile, + "-c", + csizecfg, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert os.path.exists(ofile) and os.path.getsize(ofile) > 0 + assert os.path.exists(csizecfg) and os.path.getsize(csizecfg) > 0 + assert not os.path.exists(sizecfg) or os.path.getsize(sizecfg) == 0 + finally: + os.chdir(prev_cwd) From ab5f2cd6ffea3f5dd37ef27f8dfd695916933763 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Tue, 20 May 2025 17:59:39 +0200 Subject: [PATCH 09/14] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1f8a8f9e..ce8af842 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pooch pytest pyyaml requests +pytest-cov From 8d7e2b1496c6042e82eb71dff7c258d29aa1c385 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Tue, 20 May 2025 18:00:11 +0200 Subject: [PATCH 10/14] Update scripts test following @sol1105 --- .../data_request_api/tests/test_scripts.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/data_request_api/data_request_api/tests/test_scripts.py b/data_request_api/data_request_api/tests/test_scripts.py index 1852f301..fa907461 100644 --- a/data_request_api/data_request_api/tests/test_scripts.py +++ b/data_request_api/data_request_api/tests/test_scripts.py @@ -11,6 +11,7 @@ from pathlib import Path import data_request_api.content.dreq_content as dc +import data_request_api.utilities.config as dreqcfg import pytest import yaml @@ -59,6 +60,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load(self.version) def test_database_transformation(self, temp_config_file, export): @@ -147,6 +152,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load(self.version) def test_database_transformation(self, temp_config_file, export): @@ -235,6 +244,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load(self.version) def test_database_transformation(self, temp_config_file, export): @@ -323,6 +336,10 @@ def setup_method(self, request): "cache_dir": str(self.temp_config_file.parent), } yaml.dump(config, fh) + dc._dreq_res = self.temp_config_file.parent + dc.versions = {"tags": [], "branches": []} + dreqcfg.CONFIG_FILE = self.temp_config_file + dreqcfg.CONFIG = {} # alternatively: dreqcfg.load_config(reload=True) dc.load(self.version) def test_database_transformation(self, temp_config_file, export): From 699ce188f06a5ba9a3f446b23fd94ddce9a97856 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Wed, 21 May 2025 14:36:13 +0200 Subject: [PATCH 11/14] Bugfixes for Windows --- .../data_request_api/tests/test_scripts.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/data_request_api/data_request_api/tests/test_scripts.py b/data_request_api/data_request_api/tests/test_scripts.py index fa907461..48eaa377 100644 --- a/data_request_api/data_request_api/tests/test_scripts.py +++ b/data_request_api/data_request_api/tests/test_scripts.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """ Test scripts """ @@ -74,7 +71,7 @@ def test_database_transformation(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/database_transformation.py", + os.sep.join(["scripts", "database_transformation.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -100,7 +97,7 @@ def test_workflow_example_2(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/workflow_example_2.py", + os.sep.join(["scripts", "workflow_example_2.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -119,7 +116,7 @@ def test_check_variables_attributes(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/check_variables_attributes.py", + os.sep.join(["scripts", "check_variables_attributes.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -166,7 +163,7 @@ def test_database_transformation(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/database_transformation.py", + os.sep.join(["scripts", "database_transformation.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -192,7 +189,7 @@ def test_workflow_example_2(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/workflow_example_2.py", + os.sep.join(["scripts", "workflow_example_2.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -211,7 +208,7 @@ def test_check_variables_attributes(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/check_variables_attributes.py", + os.sep.join(["scripts", "check_variables_attributes.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -258,7 +255,7 @@ def test_database_transformation(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/database_transformation.py", + os.sep.join(["scripts", "database_transformation.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -284,7 +281,7 @@ def test_workflow_example_2(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/workflow_example_2.py", + os.sep.join(["scripts", "workflow_example_2.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -303,7 +300,7 @@ def test_check_variables_attributes(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/check_variables_attributes.py", + os.sep.join(["scripts", "check_variables_attributes.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -350,7 +347,7 @@ def test_database_transformation(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/database_transformation.py", + os.sep.join(["scripts", "database_transformation.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -376,7 +373,7 @@ def test_workflow_example_2(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/workflow_example_2.py", + os.sep.join(["scripts", "workflow_example_2.py"]), "--output_dir", str(temp_config_file.parent), "--version", @@ -395,7 +392,7 @@ def test_check_variables_attributes(self, temp_config_file, export): result = subprocess.run( [ sys.executable, - "scripts/check_variables_attributes.py", + os.sep.join(["scripts", "check_variables_attributes.py"]), "--output_dir", str(temp_config_file.parent), "--version", From 7174e44298f7808e0919300e5a9585e120b3df6b Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Wed, 21 May 2025 14:55:33 +0200 Subject: [PATCH 12/14] Remove one test --- data_request_api/data_request_api/tests/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_request_api/data_request_api/tests/test_scripts.py b/data_request_api/data_request_api/tests/test_scripts.py index 48eaa377..bee8524c 100644 --- a/data_request_api/data_request_api/tests/test_scripts.py +++ b/data_request_api/data_request_api/tests/test_scripts.py @@ -129,6 +129,7 @@ def test_check_variables_attributes(self, temp_config_file, export): assert os.path.exists(checkfile) and os.path.getsize(checkfile) > 0 +@pytest.mark.skip @pytest.mark.parametrize( "export", ["release", ], From 2e8d5a9cf26a74d67605fb63a5dc86be369bbccc Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Wed, 21 May 2025 16:25:58 +0200 Subject: [PATCH 13/14] New change to test pytest-cov computation --- .github/workflows/pip_install.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index d20de893..00e4a588 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -35,7 +35,7 @@ jobs: - name: Run tests run: | - pytest --maxfail=1 --disable-warnings -q || exit 1 # Fail this step if tests fail + pytest --cov --cov-append --cov-branch --maxfail=1 --disable-warnings -q || exit 1 # Fail this step if tests fail continue-on-error: false # Allow the job to continue even if tests fail - name: Output result diff --git a/pyproject.toml b/pyproject.toml index bdae6968..fc0101f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.0", "coverage", "pytest", "setuptools_scm"] +requires = ["setuptools>=61.0", "coverage", "pytest", "setuptools_scm", "pytest-cov"] build-backend = "setuptools.build_meta" [project] From fd9a35d9d1dd332ac2d32bf138d174e8ec1ed1c7 Mon Sep 17 00:00:00 2001 From: rigoudyg Date: Thu, 22 May 2025 10:21:57 +0200 Subject: [PATCH 14/14] Revert pip install changes --- .github/workflows/pip_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_install.yml b/.github/workflows/pip_install.yml index 00e4a588..d20de893 100644 --- a/.github/workflows/pip_install.yml +++ b/.github/workflows/pip_install.yml @@ -35,7 +35,7 @@ jobs: - name: Run tests run: | - pytest --cov --cov-append --cov-branch --maxfail=1 --disable-warnings -q || exit 1 # Fail this step if tests fail + pytest --maxfail=1 --disable-warnings -q || exit 1 # Fail this step if tests fail continue-on-error: false # Allow the job to continue even if tests fail - name: Output result