From 4363be03bcfc789cfb0f9eee9d0120f36e656e6d Mon Sep 17 00:00:00 2001 From: tylo Date: Thu, 20 Nov 2025 19:49:30 +0100 Subject: [PATCH 1/4] Bug fixing: 1. create_dreq_tables_for_request() has **kwarg argument, but it was not passed down to _get_base_dict(), so the args where not updated. Fixed. 2. create_dreq_tables_for_variables() did not have **kwargs, but also calls _get_base_dict() which should have used it. Fixed. 3. A number of dict keys are used internally with old names *after* the dict was updated to use new keys. Changed the keys to the new names. --- .../data_request_api/query/dreq_query.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/data_request_api/data_request_api/query/dreq_query.py b/data_request_api/data_request_api/query/dreq_query.py index 33b6d95e..dc49500f 100644 --- a/data_request_api/data_request_api/query/dreq_query.py +++ b/data_request_api/data_request_api/query/dreq_query.py @@ -154,8 +154,7 @@ def create_dreq_tables_for_request(content, dreq_version, **kwargs): ------- Dict 'base' whose keys are table names and values are DreqTable objects. ''' - base, content_type = _get_base_dict(content, dreq_version, purpose='request') - # base, content_type = _get_base_dict(content, dreq_version) + base, content_type = _get_base_dict(content, dreq_version, purpose='request', **kwargs) # Config defaults CONFIG = {'consolidate': True} @@ -226,7 +225,7 @@ def create_dreq_tables_for_request(content, dreq_version, **kwargs): return base -def create_dreq_tables_for_variables(content, dreq_version): +def create_dreq_tables_for_variables(content, dreq_version, **kwargs): ''' For the "data" part of the data request content (Variables, Cell Methods etc), render airtable export content as DreqTable objects. @@ -234,7 +233,7 @@ def create_dreq_tables_for_variables(content, dreq_version): For the "request" part of the data request, the corresponding function is create_dreq_tables_for_request(). ''' - base, content_type = _get_base_dict(content, dreq_version, purpose='variables') + base, content_type = _get_base_dict(content, dreq_version, purpose='variables', **kwargs) # Create objects representing data request tables table_id2name = get_table_id2name(base) @@ -778,7 +777,7 @@ def get_variables_metadata(content, dreq_version, if hasattr(var, 'cell_methods'): assert len(var.cell_methods) == 1 link = var.cell_methods[0] - cm = dreq_tables['cell methods'].get_record(link) + cm = dreq_tables['Cell Methods'].get_record(link) cell_methods = cm.cell_methods if hasattr(cm, 'brand_id'): area_label_dd = cm.brand_id @@ -806,7 +805,7 @@ def get_variables_metadata(content, dreq_version, # Get the 'Spatial Shape' record, which contains info about dimensions assert len(var.spatial_shape) == 1 link = var.spatial_shape[0] - spatial_shape = dreq_tables['spatial shape'].get_record(link) + spatial_shape = dreq_tables['Spatial Shape'].get_record(link) if hasattr(spatial_shape, 'dimensions'): for link in spatial_shape.dimensions: dimension = dreq_tables['coordinates and dimensions'].get_record(link) @@ -822,7 +821,7 @@ def get_variables_metadata(content, dreq_version, dims_list.append(dimension.name) # Add temporal dimensions link = var.temporal_shape[0] - temporal_shape = dreq_tables['temporal shape'].get_record(link) + temporal_shape = dreq_tables['Temporal Shape'].get_record(link) # dims_list.append(temporal_shape.name) # An example of temporal_shape.name is 'time-point', but the equivalent dimensions list # entry for this is 'time1'. @@ -855,7 +854,7 @@ def get_variables_metadata(content, dreq_version, # Get physical parameter record and use its name as out_name link = var.physical_parameter[0] - phys_param = dreq_tables['physical parameters'].get_record(link) + phys_param = dreq_tables['Physical Parameters'].get_record(link) if hasattr(phys_param, 'variablerootdd'): # variableRootDD (aka "root name") is available in DR v1.2.2 onward out_name = phys_param.variablerootdd @@ -896,7 +895,7 @@ def get_variables_metadata(content, dreq_version, cell_measures = '' if hasattr(var, 'cell_measures'): - cell_measures = [dreq_tables['cell measures'].get_record(link).name for link in var.cell_measures] + cell_measures = [dreq_tables['Cell Measures'].get_record(link).name for link in var.cell_measures] positive = '' if hasattr(var, 'positive_direction'): @@ -1063,7 +1062,7 @@ def get_dimension_sizes(dreq_tables): 'spatial shape': base['Spatial Shape'], } ''' - dim_names = [dimension.name for dimension in dreq_tables['coordinates and dimensions'].records.values()] + dim_names = [dimension.name for dimension in dreq_tables['Coordinates and Dimensions'].records.values()] assert len(set(dim_names)) == len(dim_names) dim_names.sort(key=str.lower) # Initialize dict having names of all dimensions in the data request (to ensure we don't miss any). @@ -1072,7 +1071,7 @@ def get_dimension_sizes(dreq_tables): dim_sizes = OrderedDict({dim: set() for dim in dim_names}) # Determine dimension sizes based on their records in the Coordinates & Dimensions table. - for dimension in dreq_tables['coordinates and dimensions'].records.values(): + for dimension in dreq_tables['Coordinates and Dimensions'].records.values(): dim = dimension.name if hasattr(dimension, 'grid_class'): # Get size based on what type of grid this dimension is labelled as. @@ -1109,11 +1108,11 @@ def get_dimension_sizes(dreq_tables): # Determine dimension sizes where possible by looking in the Spatial Shape table records. # This is an extra consistency check on the results from dimensions, but it doesn't seem to change # the results (as tested on dreq v1.2 content). - for spatial_shape in dreq_tables['spatial shape'].records.values(): + for spatial_shape in dreq_tables['Spatial Shape'].records.values(): if hasattr(spatial_shape, 'dimensions'): # Follow links from Spatial Shape to dimensions, if they exist for link in spatial_shape.dimensions: - dimension = dreq_tables['coordinates and dimensions'].get_record(link) + dimension = dreq_tables['Coordinates and Dimensions'].get_record(link) dim = dimension.name if hasattr(dimension, 'axis_flag') and dimension.axis_flag == 'Z': dim_sizes[dim].add(spatial_shape.number_of_levels) From f5ea797f2cd3513f3a41cbed9779580a88e7c8fb Mon Sep 17 00:00:00 2001 From: tylo Date: Thu, 20 Nov 2025 21:34:19 +0100 Subject: [PATCH 2/4] Reverted most of the key mapping changes, only needed to enable the out-commented .update() on dreq_tables in get_dimensions(). --- .../data_request_api/query/dreq_query.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/data_request_api/data_request_api/query/dreq_query.py b/data_request_api/data_request_api/query/dreq_query.py index dc49500f..c4943011 100644 --- a/data_request_api/data_request_api/query/dreq_query.py +++ b/data_request_api/data_request_api/query/dreq_query.py @@ -777,7 +777,7 @@ def get_variables_metadata(content, dreq_version, if hasattr(var, 'cell_methods'): assert len(var.cell_methods) == 1 link = var.cell_methods[0] - cm = dreq_tables['Cell Methods'].get_record(link) + cm = dreq_tables['cell methods'].get_record(link) cell_methods = cm.cell_methods if hasattr(cm, 'brand_id'): area_label_dd = cm.brand_id @@ -805,7 +805,7 @@ def get_variables_metadata(content, dreq_version, # Get the 'Spatial Shape' record, which contains info about dimensions assert len(var.spatial_shape) == 1 link = var.spatial_shape[0] - spatial_shape = dreq_tables['Spatial Shape'].get_record(link) + spatial_shape = dreq_tables['spatial shape'].get_record(link) if hasattr(spatial_shape, 'dimensions'): for link in spatial_shape.dimensions: dimension = dreq_tables['coordinates and dimensions'].get_record(link) @@ -821,7 +821,7 @@ def get_variables_metadata(content, dreq_version, dims_list.append(dimension.name) # Add temporal dimensions link = var.temporal_shape[0] - temporal_shape = dreq_tables['Temporal Shape'].get_record(link) + temporal_shape = dreq_tables['temporal shape'].get_record(link) # dims_list.append(temporal_shape.name) # An example of temporal_shape.name is 'time-point', but the equivalent dimensions list # entry for this is 'time1'. @@ -854,7 +854,7 @@ def get_variables_metadata(content, dreq_version, # Get physical parameter record and use its name as out_name link = var.physical_parameter[0] - phys_param = dreq_tables['Physical Parameters'].get_record(link) + phys_param = dreq_tables['physical parameters'].get_record(link) if hasattr(phys_param, 'variablerootdd'): # variableRootDD (aka "root name") is available in DR v1.2.2 onward out_name = phys_param.variablerootdd @@ -895,7 +895,7 @@ def get_variables_metadata(content, dreq_version, cell_measures = '' if hasattr(var, 'cell_measures'): - cell_measures = [dreq_tables['Cell Measures'].get_record(link).name for link in var.cell_measures] + cell_measures = [dreq_tables['cell measures'].get_record(link).name for link in var.cell_measures] positive = '' if hasattr(var, 'positive_direction'): @@ -1057,12 +1057,12 @@ def get_dimension_sizes(dreq_tables): ---------- dreq_tables: dict Dict values are DreqTable objects for the required tables, e.g.: - dreq_tables = { - 'coordinates and dimensions': base['Coordinates and Dimensions'], - 'spatial shape': base['Spatial Shape'], - } ''' - dim_names = [dimension.name for dimension in dreq_tables['Coordinates and Dimensions'].records.values()] + dreq_tables.update({ + 'coordinates and dimensions': base['Coordinates and Dimensions'], + 'spatial shape': base['Spatial Shape'], + }) + dim_names = [dimension.name for dimension in dreq_tables['coordinates and dimensions'].records.values()] assert len(set(dim_names)) == len(dim_names) dim_names.sort(key=str.lower) # Initialize dict having names of all dimensions in the data request (to ensure we don't miss any). @@ -1071,7 +1071,7 @@ def get_dimension_sizes(dreq_tables): dim_sizes = OrderedDict({dim: set() for dim in dim_names}) # Determine dimension sizes based on their records in the Coordinates & Dimensions table. - for dimension in dreq_tables['Coordinates and Dimensions'].records.values(): + for dimension in dreq_tables['coordinates and dimensions'].records.values(): dim = dimension.name if hasattr(dimension, 'grid_class'): # Get size based on what type of grid this dimension is labelled as. @@ -1108,11 +1108,11 @@ def get_dimension_sizes(dreq_tables): # Determine dimension sizes where possible by looking in the Spatial Shape table records. # This is an extra consistency check on the results from dimensions, but it doesn't seem to change # the results (as tested on dreq v1.2 content). - for spatial_shape in dreq_tables['Spatial Shape'].records.values(): + for spatial_shape in dreq_tables['spatial shape'].records.values(): if hasattr(spatial_shape, 'dimensions'): # Follow links from Spatial Shape to dimensions, if they exist for link in spatial_shape.dimensions: - dimension = dreq_tables['Coordinates and Dimensions'].get_record(link) + dimension = dreq_tables['coordinates and dimensions'].get_record(link) dim = dimension.name if hasattr(dimension, 'axis_flag') and dimension.axis_flag == 'Z': dim_sizes[dim].add(spatial_shape.number_of_levels) From cf62e3eff5f56f3b8c4ad353f3f3ab21ab63127d Mon Sep 17 00:00:00 2001 From: tylo Date: Thu, 20 Nov 2025 21:55:03 +0100 Subject: [PATCH 3/4] Fixed wrong update of wrong variable from the commented code. --- data_request_api/data_request_api/query/dreq_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_request_api/data_request_api/query/dreq_query.py b/data_request_api/data_request_api/query/dreq_query.py index c4943011..387f86e1 100644 --- a/data_request_api/data_request_api/query/dreq_query.py +++ b/data_request_api/data_request_api/query/dreq_query.py @@ -1059,8 +1059,8 @@ def get_dimension_sizes(dreq_tables): Dict values are DreqTable objects for the required tables, e.g.: ''' dreq_tables.update({ - 'coordinates and dimensions': base['Coordinates and Dimensions'], - 'spatial shape': base['Spatial Shape'], + 'coordinates and dimensions': dreq_tables['Coordinates and Dimensions'], + 'spatial shape': dreq_tables['Spatial Shape'], }) dim_names = [dimension.name for dimension in dreq_tables['coordinates and dimensions'].records.values()] assert len(set(dim_names)) == len(dim_names) From 71b9a2c3cb5baee3844f5847c990c7972ac76572 Mon Sep 17 00:00:00 2001 From: tylo Date: Thu, 20 Nov 2025 22:12:59 +0100 Subject: [PATCH 4/4] Proper fix instead of .update() --- data_request_api/data_request_api/query/dreq_query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data_request_api/data_request_api/query/dreq_query.py b/data_request_api/data_request_api/query/dreq_query.py index 387f86e1..6e10881b 100644 --- a/data_request_api/data_request_api/query/dreq_query.py +++ b/data_request_api/data_request_api/query/dreq_query.py @@ -1058,10 +1058,10 @@ def get_dimension_sizes(dreq_tables): dreq_tables: dict Dict values are DreqTable objects for the required tables, e.g.: ''' - dreq_tables.update({ - 'coordinates and dimensions': dreq_tables['Coordinates and Dimensions'], - 'spatial shape': dreq_tables['Spatial Shape'], - }) + if 'Coordinates and Dimensions' in dreq_tables: + dreq_tables['coordinates and dimensions'] = dreq_tables['Coordinates and Dimensions'] + if 'Spatial Shape' in dreq_tables: + dreq_tables['spatial shape'] = dreq_tables['Spatial Shape'] dim_names = [dimension.name for dimension in dreq_tables['coordinates and dimensions'].records.values()] assert len(set(dim_names)) == len(dim_names) dim_names.sort(key=str.lower)