Add csv file output

Author: rigoudyg

dr2xml/Xwrite.py

@@ -409,7 +409,7 @@ def is_singleton(sdim):
 
 
 def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_per_table, actually_written_vars,
-                        context, enddate=None, attributes=[]):
+                        csv_file_content, context, enddate=None, attributes=[]):
     """
     Write XIOS file_def.
     """
@@ -470,6 +470,7 @@ def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_p
         write_xios_file_def_for_svars_list(hgrid=hgrid, xml_file_definition=xml_file_definition, dummies=dummies,
                                            skipped_vars_per_table=skipped_vars_per_table,
                                            actually_written_vars=actually_written_vars,
+                                           csv_file_content=csv_file_content,
                                            attributes=attributes, **file_dict)
     grid_defs = get_config_variable("grid_defs")
     # Add cfsites if needed
@@ -544,7 +545,7 @@ def write_xios_file_def_for_svars_list(vars_list, hgrid, xml_file_definition, fr
                                        split_start_offset, split_end_offset, split_last_date, grid_description,
                                        grid_label, grid_resolution, table, skipped_vars_per_table, dummies,
                                        target_hgrid_id, zgrid_id, source_grid, region, cell_measures, cell_methods,
-                                       actually_written_vars, attributes=list(), debug=list()):
+                                       actually_written_vars, csv_file_content, attributes=list(), debug=list()):
     logger = get_logger()
     internal_dict = get_settings_values("internal")
     context = internal_dict["context"]
@@ -589,6 +590,8 @@ def write_xios_file_def_for_svars_list(vars_list, hgrid, xml_file_definition, fr
             xml_file.append(end_field)
             actually_written_vars.append((svar.ref_var, svar.long_name, svar.stdname, " ".join(svar.modeling_realm),
                                           svar.mipTable, svar.region, svar.frequency, svar.Priority, svar.spatial_shp))
+            csv_file_content.append(("-".join(svar.modeling_realm), svar.official_label, svar.home, svar.frequency,
+                                     svar.label, svar.region, svar.Priority))
         # Add content to xml_file to out
         if found_begin_A:
             # create a field_def entry for surface pressure

dr2xml/__init__.py

@@ -699,6 +699,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     # --------------------------------------------------------------------
     skipped_vars_per_table = OrderedDict()
     actually_written_vars = list()
+    csv_file_content = list()
     svars_per_table = select_variables_to_be_processed()
     #
     # --------------------------------------------------------------------
@@ -724,7 +725,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     os.chdir(dirname)
     filename = dirname + "dr2xml_%s.xml" % context
     write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_per_table, actually_written_vars,
-                        context, enddate, attributes)
+                        csv_file_content, context, enddate, attributes)
     logger.info("\nfile_def written as %s" % filename)
 
     #
@@ -734,7 +735,8 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     # mpmoine_petitplus:generate_file_defs: pour sortir des stats sur ce que l'on sort reelement
     # SS - non : gros plus
     print_some_stats(context, svars_per_table, skipped_vars_per_table,
-                     actually_written_vars, internal_settings["print_stats_per_var_label"])
+                     actually_written_vars, csv_file_content, extended=internal_settings["print_stats_per_var_label"],
+                     csv_file=internal_settings["create_csv_file"])
 
     warn = OrderedDict()
     for warning, label, table in get_config_variable("cell_method_warnings", to_change=True):

dr2xml/dr_interface/definition.py

@@ -201,7 +201,7 @@ def __init__(self, type=False, modeling_realm=list(), grids=[""], label=None, mi
                  cell_methods=None, cell_measures=None, spatial_shp=None, temporal_shp=None, experiment=None,
                  Priority=1, mip_era=False, prec="float", missing=1.e+20, cmvar=None, ref_var=None, mip=None,
                  sdims=dict(), comments=None, coordinates=None, cm=False, id=None, flag_meanings=None, flag_values=None,
-                 region=None, official_label=None, extravar=None, extra_name=None, **kwargs):
+                 region=None, official_label=None, extravar=None, extra_name=None, home=False, **kwargs):
         self.type = type
         self.modeling_realm = modeling_realm
         self.set_modeling_realms = set()
@@ -246,6 +246,7 @@ def __init__(self, type=False, modeling_realm=list(), grids=[""], label=None, mi
         self.region = region
         self.official_label = official_label
         self.extra_name = extra_name
+        self.home = home
         super(SimpleCMORVar, self).__init__(**kwargs)
 
     def set_attributes(self, **kwargs):

dr2xml/infos.py

@@ -6,14 +6,16 @@
 """
 from __future__ import print_function, division, absolute_import, unicode_literals
 
+import csv
 from collections import OrderedDict, defaultdict
 
 from utilities.logger import get_logger
 
 
 # mpmoine_petitplus: nouvelle fonction print_some_stats (plus d'info sur les skipped_vars, nbre de vars / (shape,freq) )
 # SS - non : gros plus
-def print_some_stats(context, svars_per_table, skipped_vars_per_table, actually_written_vars, extended=False):
+def print_some_stats(context, svars_per_table, skipped_vars_per_table, actually_written_vars, csv_file_content,
+                     extended=False, csv_file=False):
     """
 
     :param context:
@@ -116,5 +118,31 @@ def print_some_stats(context, svars_per_table, skipped_vars_per_table, actually_
                     logger.warning(14 * " " + "Warning: several standard names are available:")
                     for stdname in sn:
                         logger.warning(18 * " " + "- %s" % stdname)
+        if csv_file:
+            table_titles = ["Modeling realms", "Variable", "Home Data Request", "Frequency", "Temporal structure",
+                            "Vertical structure", "Horizontal Structure", "Mask", "Region", "Physical parameter",
+                            "Priority"]
+            csv_content = list()
+            for (realms, official_label, home, frequency, label, region, Priority) in csv_file_content:
+                official_label = official_label.split("_")
+                (tmp_struct, vert_struct, horz_struct, mask) = official_label[1].split("-")
+                csv_content.append({
+                    "Modeling realms": realms,
+                    "Variable": official_label[0],
+                    "Home Data Request": home,
+                    "Frequency": frequency,
+                    "Temporal structure": tmp_struct,
+                    "Vertical structure": vert_struct,
+                    "Horizontal Structure": horz_struct,
+                    "Mask": mask,
+                    "Region": region,
+                    "Physical parameter": label,
+                    "Priority": Priority})
+            with open(csv_file % context, "w", newline="") as csv_fic:
+                writer = csv.DictWriter(csv_fic, fieldnames=table_titles)
+                writer.writeheader()
+                writer.writerows(csv_content)
+
+            logger.info("\n\nSome Statistics in csv file will be available in %s..." % csv_file)
 
         return True

dr2xml/projects/dr2xml.json

@@ -322,6 +322,17 @@
         }
       ]
     },
+    "create_csv_file": {
+      "help": "Pattern containing context if a csv file should be produced, else False.",
+      "values": [
+        {
+          "type": "value",
+          "origin": "laboratory",
+          "keys": ["create_csv_file"]
+        },
+        false
+      ]
+    },
     "debug_parsing": {
       "help": "In order to identify which xml files generates a problem, you can use this flag.",
       "values": [

dr2xml/vars_interface/cmor.py

@@ -23,7 +23,7 @@
 
 def read_home_var_cmor(line_split, mips, expid):
     home_var = read_home_var(line_split, home_attrs)
-    home_var.set_attributes(label_with_area=home_var.label)
+    home_var.set_attributes(label_with_area=home_var.label, home=True)
     home_var = fill_homevar(home_var)
     if check_homevar(home_var, mips, expid):
         return home_var

dr2xml/vars_interface/dev.py

@@ -29,7 +29,8 @@ def read_home_var_dev(line_split, mips, expid):
         else:
             cell_methods = tcmName2tcmValue[home_var.temporal_shp]
         home_var.set_attributes(label_with_area=home_var.label, mip_era="DEV", mipVarLabel=home_var.label,
-                                cell_methods=cell_methods, label_without_psuffix=home_var.label, cell_measures="")
+                                cell_methods=cell_methods, label_without_psuffix=home_var.label, cell_measures="",
+                                home=True)
         home_var = fill_homevar(home_var)
         if check_homevar(home_var, mips, expid):
             return home_var

dr2xml/vars_interface/extra.py

@@ -52,6 +52,7 @@ def read_home_var_extra(line_split, expid, mips, path_extra_tables=None, extra_v
             if var_found.official_label is None:
                 var_found.set_attributes(official_label="_".join([var_found.label, var_found.mipTable]))
             var_found = [var_found, ]
+
     if home_var.mip in ["ANY", ] or home_var.mip in mips:
         if get_settings_values("internal", "select_on_expt") and home_var.experiment not in ["ANY", ]:
             if expid in home_var.experiment:
@@ -173,7 +174,7 @@ def read_extra_table(path, table):
                 label_without_psuffix=remove_p_suffix(extra_var, multi_plev_suffixes | add_multi_plevs_suffixes,
                                                       single_plev_suffixes | add_single_plevs_suffixes,
                                                       realms=["atmos", "aerosol", "atmosChem"]))
-            extra_var.set_attributes(ref_var=extra_var.label_without_psuffix)
+            extra_var.set_attributes(ref_var=extra_var.label_without_psuffix, home=True)
             extravars.append(extra_var)
     logger.info("For extra table %s (which has %d variables):" % (table, len(extravars)))
     logger.info("\tVariables which dim was found in extra coordinates table:\n%s" %

dr2xml/vars_interface/perso.py

@@ -20,7 +20,7 @@ def read_home_var_perso(line_split, mips, expid):
     home_var = read_home_var(line_split, home_attrs)
     home_var.set_attributes(label_with_area=home_var.label, mip_era="PERSO", label_without_psuffix=home_var.label,
                             cell_measures="",
-                            cell_methods=tcmName2tcmValue[home_var.temporal_shp])
+                            cell_methods=tcmName2tcmValue[home_var.temporal_shp], home=True)
     home_var = fill_homevar(home_var)
     if check_homevar(home_var, mips, expid):
         return home_var