2878 extraction update

.github/workflows/compilation.yml

@@ -100,6 +100,18 @@ jobs:
         module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
         module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
         pytest -n 4 --f90=nvfortran --compile --compileopencl src/psyclone/tests
+    - name: Libraries compilation - gfortran
+      run: |
+        . .runner_venv/bin/activate
+        make -C lib/extract clean
+        module load gcc/${GFORTRAN_VERSION}
+        make -C lib/extract test
+    - name: Libraries compilation - nvfortran
+      run: |
+        . .runner_venv/bin/activate
+        make -C lib/extract clean
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        F90=nvfortran make -C lib/extract test
     - name: Examples with compilation - gfortran
       run: |
         . .runner_venv/bin/activate

changelog

@@ -1,3 +1,5 @@
+   63) PR #2907 for #2878. Improves Extraction library output stats. 
+
    62) PR #3058 for #3059. Update linkcheck parameters, fix links in docs
    and READMEs, update mentions of nan_check with value_range_check.
 

doc/user_guide/psyke.rst

@@ -544,10 +544,10 @@ Therefore, compilation for a created driver, e.g. the one created in
 
    $ gfortran -g -O0 driver-main-update.F90 -o driver-main-update
    $ ./driver-main-update
-       Variable      max_abs      max_rel      l2_diff       l2_cos    identical    #rel<1E-9    #rel<1E-6    #rel<1E-3
-           cell .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .1000000E+01 .0000000E+00 .0000000E+00 .0000000E+00
-    field1_data .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .5390000E+03 .0000000E+00 .0000000E+00 .0000000E+00
-     dummy_var1 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .1000000E+01 .0000000E+00 .0000000E+00 .0000000E+00
+     Variable        count    identical    #rel<1E-9    #rel<1E-6    #rel<1E-3   #rel>=1E-3      max_abs      max_rel      l2_diff       l2_cos
+         cell            1            1            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
+  field1_data          539          539            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
+   dummy_var1            1            1            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
 
 (see :ref:`driver_summary_statistics` for details about the statistics`).
 Note that the Makefile in the example will actually provide additional include
@@ -588,10 +588,10 @@ by changing the compilation options, or compiler version. Example output:
 
 .. code-block:: output
 
-       Variable      max_abs      max_rel      l2_diff       l2_cos    identical    #rel<1E-9    #rel<1E-6    #rel<1E-3
-           cell .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .1000000E+01 .0000000E+00 .0000000E+00 .0000000E+00
-    field1_data .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .5390000E+03 .0000000E+00 .0000000E+00 .0000000E+00
-     dummy_var1 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01 .1000000E+01 .0000000E+00 .0000000E+00 .0000000E+00
+     Variable        count    identical    #rel<1E-9    #rel<1E-6    #rel<1E-3   #rel>=1E-3      max_abs      max_rel      l2_diff       l2_cos
+         cell            1            1            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
+  field1_data          539          539            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
+   dummy_var1            1            1            0            0            0            0 .0000000E+00 .0000000E+00 .0000000E+00 .1000000E+01
 
 The columns from left to right are:
 
@@ -607,28 +607,33 @@ The columns from left to right are:
 .. only:: latex or has_dvipng
 
   * The variable name.
+  * The number of elements for this variable (i.e. 1 for scalar).
+  * How many values are identical.
+  * How many values have a relative error of less than 10\ :sup:`-9` but are not identical. Note that
+    single precision variables typically do not have enough significant digits to have an error of 10\ :sup:`-9`.
+  * How many values have a relative error of less than 10\ :sup:`-6` but more than 10\ :sup:`-9`.
+  * How many values have a relative error of less than 10\ :sup:`-3` but more than 10\ :sup:`-6`.
+  * How many values have a relative error of more than 10\ :sup:`-3`.
   * The maximum absolute error of all elements.
   * The maximum relative error of all elements. If an element has the value
     0, the relative error for this element is considered to be 1.0.
   * The L2 difference: :math:`\sqrt{\sum{(original-new)^2}}`.
   * The cosine of the angle between the two vectors: :math:`\frac{\sum{original*new}}{\sqrt{\sum{original*original}}*\sqrt{\sum{new*new}}}`.
-  * How many values are identical.
-  * How many values have a relative error of less than 10\ :sup:`-9` but are not identical.
-  * How many values have a relative error of less than 10\ :sup:`-6` but more than 10\ :sup:`-9`.
-  * How many values have a relative error of less than 10\ :sup:`-3` but more than 10\ :sup:`-6`.
 
 .. only:: html and not has_dvipng
 
   * The variable name.
+  * The number of elements for this variable (i.e. 1 for scalar).
+  * How many values are identical.
+  * How many values have a relative error of less than 10\ :sup:`-9` but are not identical. Note that
+    single precision variables typically do not have enough significant digits to have an error of 10\ :sup:`-9`.
+  * How many values have a relative error of less than 10\ :sup:`-6` but more than 10\ :sup:`-9`.
+  * How many values have a relative error of less than 10\ :sup:`-3` but more than 10\ :sup:`-6`.
   * The maximum absolute error of all elements.
   * The maximum relative error of all elements. If an element has the value
     0, the relative error for this element is considered to be 1.0.
   * The L2 difference: `sqrt(sum((original-new)`\ :sup:`2` `))`.
   * The cosine of the angle between the two vectors: `sum(original*new)/(sqrt(sum(original*original))*sqrt(sum(new*new)))`.
-  * How many values are identical.
-  * How many values have a relative error of less than 10\ :sup:`-9` but are not identical.
-  * How many values have a relative error of less than 10\ :sup:`-6` but more than 10\ :sup:`-9`.
-  * How many values have a relative error of less than 10\ :sup:`-3` but more than 10\ :sup:`-6`.
 
 .. note:: The usefulness of the columns printed is still being evaluated. Early
     indications are that the cosine of the angle between the two vectors,

lib/extract/.gitignore

@@ -1,2 +1,3 @@
 # The leading / means only ignore the file in this directory
 /compare_variables_mod.F90
+test_compare

lib/extract/Makefile

@@ -49,19 +49,35 @@ PSYDATA_LIB_DIR ?= ./..
 
 # These arguments are responsible for creating the Fortran
 # `compare_variables_mod` file based on the jinja template. It will support
-# the listed data types and scalar and 2-dimensional arrays. Note that
-# domain-specific implementation will create their own version with different
-# number of dimensions if required.
-PROCESS_ARGS = -types=char,int,long,logical,real,double -dims=2
+# the listed data types and scalar and 2-dimensional arrays.
+# The type and n-arity of the comparable type may need to be adjusted to
+# match the capabilities of the target application using the arguments below.
+
+PROCESS_ARGS ?= -types=char,int,long,logical,real,double -dims=1,2,3,4
 PROCESS = $$($(PSYDATA_LIB_DIR)/get_python.sh) $(PSYDATA_LIB_DIR)/process.py
 
-.PHONY: all compare
+.PHONY: all compare test
 
 compare: compare_variables_mod.o
 
+test_compare: compare_variables_mod.o test_compare.o
+	$(F90) $(F90FLAGS) $^ -o $@
+
+make test: test_compare
+	# The first 6 fields of the output contain the counts of various
+	# errors. Only check these counts, not the computed floating point
+	# differences (which are the next 4 columns), since they might
+	# slightly vary between compilers and hardware platforms.
+	./test_compare | grep "a_dbl    *15 *1 *2 *3 *4 *5" > /dev/null
+	./test_compare | grep "a_single *10 *1 *0 *2 *3 *4" > /dev/null
+	./test_compare | grep "a_int    *15 *1 *2 *3 *4 *5" > /dev/null
+
 %.o: %.F90
 	$(F90) -c $(F90FLAGS) $<
 
+%.o: %.f90
+	$(F90) -c $(F90FLAGS) $<
+
 # ------------------------------------------------
 compare_variables_mod.F90:      compare_variables_mod.jinja Makefile
 	$(PROCESS) $(PROCESS_ARGS) $< > compare_variables_mod.F90
@@ -74,4 +90,5 @@ all: compare
 
 clean:
 	rm -f compare_variables_mod.mod compare_variables_mod.o \
-		  compare_variables_mod.F90
+		  compare_variables_mod.F90 test_compare.mod test_compare.o \
+		  test_compare