From d04d16473c70a72b99b87fba2cf7b1a4e4586fe8 Mon Sep 17 00:00:00 2001 From: sit23 Date: Tue, 30 Apr 2024 11:05:58 +0100 Subject: [PATCH 1/5] Adding codebase method to set compilation to single precision. This uses a combination of Iscas in-built macros with overload_c4 etc, whilst maintaining other default flags. --- src/extra/python/isca/codebase.py | 8 ++++++++ src/extra/python/isca/templates/compile.sh | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/extra/python/isca/codebase.py b/src/extra/python/isca/codebase.py index ada901b27..22b46b459 100644 --- a/src/extra/python/isca/codebase.py +++ b/src/extra/python/isca/codebase.py @@ -120,6 +120,7 @@ def __init__(self, repo=None, commit=None, directory=None, storedir=P(GFDL_WORK, # read path names from the default file self.path_names = [] self.compile_flags = [] # users can append to this to add additional compiler options + self.precision_compile_flags = ['-DOVERLOAD_C8'] #Default is to use double precision. User can change to single precision using cb.use_single_precision() before compile step @property def code_is_available(self): @@ -255,6 +256,7 @@ def compile(self, debug=False, optimisation=None): # compile_flags.append('-O%d' % optimisation) compile_flags.extend(self.compile_flags) + compile_flags.extend(self.precision_compile_flags) compile_flags_str = ' '.join(compile_flags) # get path_names from the directory @@ -282,6 +284,12 @@ def compile(self, debug=False, optimisation=None): self.log.info('Compilation complete.') + def use_single_precision(self): + self.log.info('Going to use single_precision') + self.precision_compile_flags = ['-DOVERLOAD_C4', '-DOVERLOAD_R4'] + self.executable_name = self.executable_name.strip('.x')+'_single.x' + self.builddir = P(self.workdir, 'build', self.executable_name.split('.')[0]) + class IscaCodeBase(CodeBase): diff --git a/src/extra/python/isca/templates/compile.sh b/src/extra/python/isca/templates/compile.sh index d20377492..759cd1f5f 100755 --- a/src/extra/python/isca/templates/compile.sh +++ b/src/extra/python/isca/templates/compile.sh @@ -53,13 +53,13 @@ if [ $debug == True ]; then echo "Compiling in debug mode" # execute mkmf to create makefile -cppDefs="-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DINTERNAL_FILE_NML -DOVERLOAD_C8 {{compile_flags}}" +cppDefs="-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DINTERNAL_FILE_NM {{compile_flags}}" $mkmf -a $sourcedir -t $template_debug -p $executable -c "$cppDefs" $pathnames $sourcedir/shared/include $sourcedir/shared/mpp/include else # execute mkmf to create makefile -cppDefs="-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DINTERNAL_FILE_NML -DOVERLOAD_C8 {{compile_flags}}" +cppDefs="-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DINTERNAL_FILE_NM {{compile_flags}}" $mkmf -a $sourcedir -t $template -p $executable -c "$cppDefs" $pathnames $sourcedir/shared/include $sourcedir/shared/mpp/include fi From 5751b16929975ca9503611fd4ad249d274b07e50 Mon Sep 17 00:00:00 2001 From: sit23 Date: Tue, 30 Apr 2024 11:16:51 +0100 Subject: [PATCH 2/5] Adding single-precision test case for Held-suarez. --- .../held_suarez_test_case_single.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 exp/test_cases/held_suarez/held_suarez_test_case_single.py diff --git a/exp/test_cases/held_suarez/held_suarez_test_case_single.py b/exp/test_cases/held_suarez/held_suarez_test_case_single.py new file mode 100644 index 000000000..d815ce4ad --- /dev/null +++ b/exp/test_cases/held_suarez/held_suarez_test_case_single.py @@ -0,0 +1,110 @@ +import numpy as np + +from isca import DryCodeBase, DiagTable, Experiment, Namelist, GFDL_BASE + +NCORES = 16 +RESOLUTION = 'T42', 25 # T42 horizontal resolution, 25 levels in pressure + +# a CodeBase can be a directory on the computer, +# useful for iterative development +cb = DryCodeBase.from_directory(GFDL_BASE) + +# or it can point to a specific git repo and commit id. +# This method should ensure future, independent, reproducibility of results. +# cb = DryCodeBase.from_repo(repo='https://github.com/isca/isca', commit='isca1.1') + +# compilation depends on computer specific settings. The $GFDL_ENV +# environment variable is used to determine which `$GFDL_BASE/src/extra/env` file +# is used to load the correct compilers. The env file is always loaded from +# $GFDL_BASE and not the checked out git repo. +cb.use_single_precision() + +cb.compile() # compile the source code to working directory $GFDL_WORK/codebase + +# create an Experiment object to handle the configuration of model parameters +# and output diagnostics + +exp_name = 'held_suarez_default_single_precision' +exp = Experiment(exp_name, codebase=cb) + +#Tell model how to write diagnostics +diag = DiagTable() +diag.add_file('atmos_monthly', 30, 'days', time_units='days') + +#Tell model which diagnostics to write +diag.add_field('dynamics', 'ps', time_avg=True) +diag.add_field('dynamics', 'bk') +diag.add_field('dynamics', 'pk') +diag.add_field('dynamics', 'ucomp', time_avg=True) +diag.add_field('dynamics', 'vcomp', time_avg=True) +diag.add_field('dynamics', 'temp', time_avg=True) +diag.add_field('dynamics', 'vor', time_avg=True) +diag.add_field('dynamics', 'div', time_avg=True) + +exp.diag_table = diag + +# define namelist values as python dictionary +# wrapped as a namelist object. +namelist = Namelist({ + 'main_nml': { + 'dt_atmos': 600, + 'days': 30, + 'calendar': 'thirty_day', + 'current_date': [2000,1,1,0,0,0] + }, + + 'atmosphere_nml': { + 'idealized_moist_model': False # False for Newtonian Cooling. True for Isca/Frierson + }, + + 'spectral_dynamics_nml': { + 'damping_order' : 4, # default: 2 + 'water_correction_limit' : 200.e2, # default: 0 + 'reference_sea_level_press': 1.0e5, # default: 101325 + 'valid_range_t' : [100., 800.], # default: (100, 500) + 'initial_sphum' : 0.0, # default: 0 + 'vert_coord_option' : 'uneven_sigma', # default: 'even_sigma' + 'scale_heights': 6.0, + 'exponent': 7.5, + 'surf_res': 0.5 + }, + + # configure the relaxation profile + 'hs_forcing_nml': { + 't_zero': 315., # temperature at reference pressure at equator (default 315K) + 't_strat': 200., # stratosphere temperature (default 200K) + 'delh': 60., # equator-pole temp gradient (default 60K) + 'delv': 10., # lapse rate (default 10K) + 'eps': 0., # stratospheric latitudinal variation (default 0K) + 'sigma_b': 0.7, # boundary layer friction height (default p/ps = sigma = 0.7) + + # negative sign is a flag indicating that the units are days + 'ka': -40., # Constant Newtonian cooling timescale (default 40 days) + 'ks': -4., # Boundary layer dependent cooling timescale (default 4 days) + 'kf': -1., # BL momentum frictional timescale (default 1 days) + + 'do_conserve_energy': True, # convert dissipated momentum into heat (default True) + }, + + 'diag_manager_nml': { + 'mix_snapshot_average_fields': False + }, + + 'fms_nml': { + 'domains_stack_size': 600000 # default: 0 + }, + + 'fms_io_nml': { + 'threading_write': 'single', # default: multi + 'fileset_write': 'single', # default: multi + } +}) + +exp.namelist = namelist +exp.set_resolution(*RESOLUTION) + +#Lets do a run! +if __name__ == '__main__': + exp.run(1, num_cores=NCORES, use_restart=False) + for i in range(2, 13): + exp.run(i, num_cores=NCORES) # use the restart i-1 by default \ No newline at end of file From ec5992a66b5a914f0e755def8b0d04e35dd506fa Mon Sep 17 00:00:00 2001 From: sit23 Date: Tue, 30 Apr 2024 11:50:57 +0100 Subject: [PATCH 3/5] When compiling with alternative compile options a few errors were encountered with the simple cloud fortran files, which had missed a few file-exists imports from fms-mod. Also lcl.f90 explicitly defined variables as double precision, which caused problems when variables were inputted to its functions as single. Now changed to real so that compiler chooses the representation of real. --- .../cloud_simple/cloud_cover_diags.F90 | 2 +- src/atmos_param/cloud_simple/cloud_simple.F90 | 2 +- .../cloud_simple/large_scale_cloud.F90 | 2 +- src/atmos_param/cloud_simple/lcl.F90 | 150 +++++++++--------- .../cloud_simple/marine_strat_cloud.F90 | 2 +- 5 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/atmos_param/cloud_simple/cloud_cover_diags.F90 b/src/atmos_param/cloud_simple/cloud_cover_diags.F90 index aea5b337d..197db9639 100644 --- a/src/atmos_param/cloud_simple/cloud_cover_diags.F90 +++ b/src/atmos_param/cloud_simple/cloud_cover_diags.F90 @@ -7,7 +7,7 @@ module cloud_cover_diags_mod #endif use fms_mod, only: stdlog, FATAL, WARNING, NOTE, error_mesg, & - uppercase, check_nml_error + uppercase, check_nml_error, file_exist use time_manager_mod, only: time_type use diag_manager_mod, only: register_diag_field, send_data diff --git a/src/atmos_param/cloud_simple/cloud_simple.F90 b/src/atmos_param/cloud_simple/cloud_simple.F90 index 2e65dfaa6..6aa521726 100644 --- a/src/atmos_param/cloud_simple/cloud_simple.F90 +++ b/src/atmos_param/cloud_simple/cloud_simple.F90 @@ -7,7 +7,7 @@ module cloud_simple_mod #endif use fms_mod, only: stdlog, FATAL, WARNING, NOTE, error_mesg, & - uppercase, check_nml_error + uppercase, check_nml_error, file_exist use time_manager_mod, only: time_type use sat_vapor_pres_mod, only: compute_qs, lookup_es use diag_manager_mod, only: register_diag_field, send_data diff --git a/src/atmos_param/cloud_simple/large_scale_cloud.F90 b/src/atmos_param/cloud_simple/large_scale_cloud.F90 index c26a78f4b..7e13c13c9 100644 --- a/src/atmos_param/cloud_simple/large_scale_cloud.F90 +++ b/src/atmos_param/cloud_simple/large_scale_cloud.F90 @@ -7,7 +7,7 @@ module large_scale_cloud_mod #endif use fms_mod, only: stdlog, FATAL, WARNING, NOTE, error_mesg, & - uppercase, check_nml_error + uppercase, check_nml_error, file_exist use time_manager_mod, only: time_type use sat_vapor_pres_mod, only: compute_qs, lookup_es use diag_manager_mod, only: register_diag_field, send_data diff --git a/src/atmos_param/cloud_simple/lcl.F90 b/src/atmos_param/cloud_simple/lcl.F90 index 7205bed09..0d8260789 100644 --- a/src/atmos_param/cloud_simple/lcl.F90 +++ b/src/atmos_param/cloud_simple/lcl.F90 @@ -38,17 +38,17 @@ function lcl(p,T,rh,rhl,rhs,return_ldl,return_min_lcl_ldl) implicit none ! In - double precision, intent(in) :: p, T - double precision, intent(in), optional :: rh, rhl, rhs + real, intent(in) :: p, T + real, intent(in), optional :: rh, rhl, rhs logical, intent(in), optional :: return_ldl, return_min_lcl_ldl ! Out - double precision :: lcl + real :: lcl logical :: liquid2, solid2 integer(kind=4) :: error - double precision :: Ttrip, ptrip, E0v, E0s, ggr, rgasa, & + real :: Ttrip, ptrip, E0v, E0s, ggr, rgasa, & rgasv, cva, cvv, cvl, cvs, cpa, cpv, al, bl, cl, as, bs, cs, & pv, qv, cpm, rgasm, rh2, rhl2, rhs2, ldl integer :: rh_counter @@ -167,8 +167,8 @@ function pvstarl(T) implicit none - double precision :: T ! In - double precision :: pvstarl ! Out + real :: T ! In + real :: pvstarl ! Out pvstarl = ptrip * (T/Ttrip)**((cpv-cvl)/rgasv) * & exp( (E0v - (cvv-cvl)*Ttrip) / rgasv * (1./Ttrip - 1./T) ) @@ -180,8 +180,8 @@ function pvstars(T) implicit none - double precision :: T ! In - double precision :: pvstars ! Out + real :: T ! In + real :: pvstars ! Out pvstars = ptrip * (T/Ttrip)**((cpv-cvs)/rgasv) * & exp( (E0v + E0s - (cvv-cvs)*Ttrip) / rgasv * (1./Ttrip - 1./T) ) @@ -234,7 +234,7 @@ function bisect ( xx, nb, ner, l ) ! ! Parameters: ! -! Input, double precision :: XX, the argument. +! Input, real :: XX, the argument. ! ! Input, integer ( kind = 4 ) NB, indicates the branch of the W function. ! 0, the upper branch; @@ -248,15 +248,15 @@ function bisect ( xx, nb, ner, l ) ! 1, XX represents the offset of the argument from -exp(-1). ! not 1, XX is the actual argument. ! -! Output, double precision :: BISECT, the value of W(X), as determined +! Output, real :: BISECT, the value of W(X), as determined ! implicit none - double precision :: bisect -! double precision :: crude - double precision :: d - double precision :: f - double precision :: fd + real :: bisect +! real :: crude + real :: d + real :: f + real :: fd integer ( kind = 4 ) i integer ( kind = 4 ) l integer ( kind = 4 ) n0 @@ -264,12 +264,12 @@ function bisect ( xx, nb, ner, l ) integer ( kind = 4 ) nb integer ( kind = 4 ), save :: nbits = 0 integer ( kind = 4 ) ner - double precision :: r - double precision :: test - double precision :: tol - double precision :: u - double precision :: x - double precision :: xx + real :: r + real :: test + real :: tol + real :: u + real :: x + real :: xx bisect = 0.0D+00 ner = 0 @@ -414,34 +414,34 @@ function crude ( xx, nb ) ! ! Parameters: ! -! Input, double precision :: XX, the argument. +! Input, real :: XX, the argument. ! ! Input, integer ( kind = 4 ) NB, indicates the desired branch. ! * 0, the upper branch; ! * nonzero, the lower branch. ! -! Output, double precision :: CRUDE, the crude approximation to W at XX. +! Output, real :: CRUDE, the crude approximation to W at XX. ! implicit none - double precision :: an2 - double precision :: c13 - double precision :: crude - double precision :: em - double precision :: em2 - double precision :: em9 - double precision :: eta + real :: an2 + real :: c13 + real :: crude + real :: em + real :: em2 + real :: em9 + real :: eta integer ( kind = 4 ) init integer ( kind = 4 ) nb - double precision :: reta - double precision :: s2 - double precision :: s21 - double precision :: s22 - double precision :: s23 - double precision :: t - double precision :: ts - double precision :: xx - double precision :: zl + real :: reta + real :: s2 + real :: s21 + real :: s22 + real :: s23 + real :: t + real :: ts + real :: xx + real :: zl save c13 save em @@ -525,7 +525,7 @@ subroutine nbits_compute ( nbits ) ! function should be calculated. ! ! Most machines use a 24-bit matissa for single precision and -! 53-56 bits for double precision ::. The IEEE standard is 53 +! 53-56 bits for real ::. The IEEE standard is 53 ! bits. The Fujitsu VP2200 uses 56 bits. Long word length ! machines vary, e.g., the Cray X/MP has a 48-bit mantissa for ! single precision. @@ -559,10 +559,10 @@ subroutine nbits_compute ( nbits ) ! implicit none - double precision :: b + real :: b integer ( kind = 4 ) i integer ( kind = 4 ) nbits - double precision :: v + real :: v nbits = 0 @@ -701,7 +701,7 @@ function wapr ( x, nb, nerror, l ) ! ! Parameters: ! -! Input, double precision :: X, the argument. +! Input, real :: X, the argument. ! ! Input, integer ( kind = 4 ) NB, indicates the desired branch. ! * 0, the upper branch; @@ -715,23 +715,23 @@ function wapr ( x, nb, nerror, l ) ! * 1, X is actually the offset from -(exp-1), so compute W(X-exp(-1)). ! * not 1, X is the argument; compute W(X); ! -! Output, double precision :: WAPR, the approximate value of W(X). +! Output, real :: WAPR, the approximate value of W(X). ! implicit none - double precision :: an2 - double precision :: an3 - double precision :: an4 - double precision :: an5 - double precision :: an6 - double precision :: c13 - double precision :: c23 - double precision :: d12 - double precision :: delx - double precision :: em - double precision :: em2 - double precision :: em9 - double precision :: eta + real :: an2 + real :: an3 + real :: an4 + real :: an5 + real :: an6 + real :: c13 + real :: c23 + real :: d12 + real :: delx + real :: em + real :: em2 + real :: em9 + real :: eta integer ( kind = 4 ) i integer ( kind = 4 ) init integer ( kind = 4 ) l @@ -740,24 +740,24 @@ function wapr ( x, nb, nerror, l ) integer ( kind = 4 ) nbits integer ( kind = 4 ) nerror integer ( kind = 4 ) niter - double precision :: reta - double precision :: s2 - double precision :: s21 - double precision :: s22 - double precision :: s23 - double precision :: t - double precision :: tb - double precision :: tb2 - double precision :: temp - double precision :: temp2 - double precision :: ts - double precision :: wapr - double precision :: x - double precision :: x0 - double precision :: x1 - double precision :: xx - double precision :: zl - double precision :: zn + real :: reta + real :: s2 + real :: s21 + real :: s22 + real :: s23 + real :: t + real :: tb + real :: tb2 + real :: temp + real :: temp2 + real :: ts + real :: wapr + real :: x + real :: x0 + real :: x1 + real :: xx + real :: zl + real :: zn save an3 save an4 diff --git a/src/atmos_param/cloud_simple/marine_strat_cloud.F90 b/src/atmos_param/cloud_simple/marine_strat_cloud.F90 index 007af2e5f..de65f6516 100644 --- a/src/atmos_param/cloud_simple/marine_strat_cloud.F90 +++ b/src/atmos_param/cloud_simple/marine_strat_cloud.F90 @@ -7,7 +7,7 @@ module marine_strat_cloud_mod #endif use fms_mod, only: stdlog, FATAL, WARNING, NOTE, error_mesg, & - uppercase, check_nml_error + uppercase, check_nml_error, file_exist use time_manager_mod, only: time_type use sat_vapor_pres_mod, only: compute_qs, lookup_es use diag_manager_mod, only: register_diag_field, send_data From 8fa14bbfb8da3986b301f5066acbda4e62dd7f9f Mon Sep 17 00:00:00 2001 From: sit23 Date: Tue, 30 Apr 2024 12:24:07 +0100 Subject: [PATCH 4/5] Realised that -r8 option needs to be in mkmf template file for double precision, but should not be there for single precision. Do not currently have a dynamic way of editing that mkmf template file, so added feature to choose different mkmf template file for single precision. Format is just same name as current mkmf template file with _simple added at the end. --- src/extra/python/isca/codebase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/extra/python/isca/codebase.py b/src/extra/python/isca/codebase.py index 22b46b459..6b86872e0 100644 --- a/src/extra/python/isca/codebase.py +++ b/src/extra/python/isca/codebase.py @@ -121,6 +121,7 @@ def __init__(self, repo=None, commit=None, directory=None, storedir=P(GFDL_WORK, self.path_names = [] self.compile_flags = [] # users can append to this to add additional compiler options self.precision_compile_flags = ['-DOVERLOAD_C8'] #Default is to use double precision. User can change to single precision using cb.use_single_precision() before compile step + self.env_suffix = '' @property def code_is_available(self): @@ -271,7 +272,7 @@ def compile(self, debug=False, optimisation=None): 'srcdir': self.srcdir, 'workdir': self.workdir, 'compile_flags': compile_flags_str, - 'env_source': env, + 'env_source': env+self.env_suffix, 'path_names': path_names_str, 'executable_name': self.executable_name, 'run_idb': debug, @@ -289,6 +290,7 @@ def use_single_precision(self): self.precision_compile_flags = ['-DOVERLOAD_C4', '-DOVERLOAD_R4'] self.executable_name = self.executable_name.strip('.x')+'_single.x' self.builddir = P(self.workdir, 'build', self.executable_name.split('.')[0]) + self.env_suffix = '_single' #needs a seperate env file as flags are different From 03d555aa0741100f9c7fa0d94f92ca467e052512 Mon Sep 17 00:00:00 2001 From: sit23 Date: Tue, 30 Apr 2024 12:44:11 +0100 Subject: [PATCH 5/5] Needs new env file to point to new mkmf file. --- .../templates/mkmf.template.maths2_single | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 src/extra/python/isca/templates/mkmf.template.maths2_single diff --git a/src/extra/python/isca/templates/mkmf.template.maths2_single b/src/extra/python/isca/templates/mkmf.template.maths2_single new file mode 100755 index 000000000..8b80014da --- /dev/null +++ b/src/extra/python/isca/templates/mkmf.template.maths2_single @@ -0,0 +1,28 @@ +# template for the Intel fortran compiler +# typical use with mkmf +# mkmf -t template.ifc -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include +CPPFLAGS = `nc-config --cflags` +NETCDF_LIBS = `nc-config --libs` + +# FFLAGS: +# -fpp: Use the fortran preprocessor +# -stack_temps: Put temporary runtime arrays on the stack, not heap. +# -safe_cray_ptr: Cray pointers don't alias other variables. +# -ftz: Denormal numbers are flushed to zero. +# -assume byterecl: Specifies the units for the OPEN statement as bytes. +# -shared-intel: Load intel libraries dynamically +# -i4: 4 byte integers +# -r8: 8 byte reals +# -g: Generate symbolic debugging info in code +# -O2: Level 2 speed optimisations +# -diag-disable 6843: +# This suppresses the warning: `warning #6843: A dummy argument with an explicit INTENT(OUT) declaration is not given an explicit value.` of which +# there are a lot of instances in the GFDL codebase. +FFLAGS = $(CPPFLAGS) -fpp -stack_temps -safe_cray_ptr -ftz -assume byterecl -shared-intel -i4 -g -O2 -diag-disable 6843 -mcmodel large +#FFLAGS = $(CPPFLAGS) -fltconsistency -stack_temps -safe_cray_ptr -ftz -shared-intel -assume byterecl -g -O0 -i4 -r8 -check -warn -warn noerrors -debug variable_locations -inline_debug_info -traceback +FC = $(F90) +LD = $(F90) $(NETCDF_LIBS) +#CC = mpicc + +LDFLAGS = -lnetcdff -lnetcdf -lmpi -shared-intel -lhdf5_hl -lhdf5 -lm -lz -lsz -lbz2 -lxml2 -lcurl +CFLAGS = -D__IFC