diff --git a/ci/environment-py3.12_ncar-derecho.yml b/ci/environment-py3.12_ncar-derecho.yml new file mode 100644 index 000000000..2c67f179d --- /dev/null +++ b/ci/environment-py3.12_ncar-derecho.yml @@ -0,0 +1,16 @@ +name: isca_env +channels: +- conda-forge +dependencies: +- dask=2024.1.0 +- f90nml=1.4.4 +- ipykernel=6.29.0 +- jinja2=3.1.3 +- numpy=1.26.3 +- pandas=2.2.0 +- python=3.12.1 +- pip=23.3.2 +- pytest=7.4.4 +- sh=2.0.6 +- tqdm=4.66.1 +- xarray=2024.1.1 diff --git a/exp/site_specific/ncar/README b/exp/site_specific/ncar/README deleted file mode 100644 index 8f62038eb..000000000 --- a/exp/site_specific/ncar/README +++ /dev/null @@ -1,59 +0,0 @@ -Guildline to start running test case in Cheyenne Machine --- July 15, 2018 Wanying Kang -1. download Isca -========================================= -git clone https://github.com/ExeClim/Isca [IscaCodeDir] -cd [IscaCodeDir] -========================================= - -2. install python package necessary following the Cheyenne guideline: https://www2.cisl.ucar.edu/resources/computational-systems/cheyenne/software/python. The following are the key steps, and detail please refer to cheyenne official site. -========================================= -cd [IscaCodeDir]/src/extra/python -ncar_pylib -c 20180128 [personal_package_dir] -cd [personal_package_dir]/bin -pip install -r requirements.txt -========================================= - -Each time before using, type the following to activate. Suggest to add the following line to ~/.bashrc -========================================= -source [personal_package_dir]/bin/activate.csh (tcsh) -OR -source [personal_package_dir]/bin/activate (bash) -========================================= - -3. install Isca -pip install -e . - -4. define environment variables. Please add the following to ~/.bashrc. - -========================================= -# python package location -export python_package_lib_path=[personal_package_dir]/bin/activate -# directory of the Isca source code -export GFDL_BASE=[IscaCodeDir] -# "environment" configuration for emps-gv4 -export GFDL_ENV=cheyenne -# temporary working directory used in running the model, you may change to other places, but need to change the code accordingly as well. -export GFDL_WORK=/glade/scratch/${USER}/isca_rundir/ -# directory for storing model output, you may change to other places, but need to change the code accordingly. -export GFDL_DATA=/glade/scratch/${USER}/isca_out/ -export ACCOUNT = [YourAccount] -========================================= - -5. to run the model -========================================= -cp [IscaCodeDir]/exp/test_cases/cheyenne_run/build_isca.* [SomePlaceAsYouWant] -cd [SomePlaceAsYouWant] -./build_isca.sh [CaseName] [NCORES] [TODO] -========================================= - for [TODO]: you can combine the following keywords - clean: clean up the [CaseName] workdir and rundir - compile: set up experiments using the settings in build_isca.py, and compile the F90s. Copy exec and run script to the [builddir] folder. - run: submit the job to queue. - - - - -For any questions regarding the porting and running on Cheyenne machine, you may send me an email: wanyingkang@g.harvard.edu. - - diff --git a/exp/site_specific/ncar/Readme.md b/exp/site_specific/ncar/Readme.md new file mode 100644 index 000000000..35e3ab9aa --- /dev/null +++ b/exp/site_specific/ncar/Readme.md @@ -0,0 +1,53 @@ +Quick start guide for running Isca on NSF NCAR's Derecho HPC Machine +-- March 2025, Rory Kelly (rory@ucar.edu) +1. Download the Isca code + ```bash + git clone https://github.com/ExeClim/Isca + ``` + +2. Set up the environment + ```bash + cd Isca + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + + mamba env create -f ci/environment-py3.12_ncar-derecho.yml + conda activate isca_env + + cd src/extra/python + pip install -e . + ``` +3. Add Isca settings to `~/.bashrc` (modify paths as desired). After adding these settings you'll + need to source your ~/.bashrc file, or export the settings in the current shell as well. + ```bash + export GFDL_ENV=ncar-derecho + export GFDL_MKMF_TEMPLATE=ncar-derecho-intel + export GFDL_BASE=/glade/work/$USER/Isca + export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work + export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data + ``` +4. Build and Run a Test Case on a batch node + ```bash + cd $GFDL_BASE/exp/test_cases/held_suarez + qsub isca-test.sh + ``` + Example batch script `isca-test.sh` (you'll need to add you account code to the `-A` argument) + ```bash + #!/bin/bash + #PBS -l walltime=00:30:00 + #PBS -l select=1:ncpus=128:mpiprocs=128 + #PBS -q main + #PBS -A [ACCOUNT CODE] + #PBS -N Isca-test + #PBS -k eod + #PBS -j oe + #PBS -o Isca-test.out + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + conda activate isca_env + + cd $GFDL_BASE/exp/test_cases/held_suarez + python held_suarez_test_case.py + ``` diff --git a/exp/site_specific/ncar/build_isca.py b/exp/site_specific/ncar/build_isca.py deleted file mode 100644 index 796858626..000000000 --- a/exp/site_specific/ncar/build_isca.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import glob -import numpy as np -import shutil -import string -import sys -from isca import IscaCodeBase, DiagTable, Experiment, Namelist, GFDL_BASE - -casename = sys.argv[1] -print('building case:'+casename) -builddir = '/glade/u/home/'+os.environ["USER"]+'/Isca_caseroot/'+casename -print('in:'+builddir) -#executable_name=casename+'.x' -executable_name='isca.x' -NCORES= int(sys.argv[2]) -print('Using '+str(NCORES)+' cores') -RESOLUTION = 'T42', 40 -todo = sys.argv[3] - -#Step 1: Soft link the original code. Point the python script to the location of the code (the bash environment variable GFDL_BASE is that directory) -if 'compile' in todo: - shutil.rmtree(builddir, ignore_errors=True) - os.makedirs(builddir) - if not os.path.exists(builddir+'/Isca'): - os.symlink(os.environ['GFDL_BASE'],builddir+'/Isca') - -objdir=builddir.replace('/','_')+'_Isca' -if 'clean' in todo: - if os.path.exists(os.environ['GFDL_WORK']+'/codebase/'+objdir): - print('clean up '+os.environ['GFDL_WORK']+'/codebase/'+objdir) - shutil.rmtree(os.environ['GFDL_WORK']+'/codebase/'+objdir) - if os.path.exists(os.environ['GFDL_WORK']+'/experiments/'+casename): - print('clean up '+os.environ['GFDL_WORK']+'/experiments/'+casename) - shutil.rmtree(os.environ['GFDL_WORK']+'/experiments/'+casename) - if os.path.exists(builddir): - print('clean up '+builddir) - shutil.rmtree(builddir) - -#cb = IscaCodeBase.from_directory(builddir+'/Isca',storedir=builddir) # for saving obj files in Isca_caseroot/ -cb = IscaCodeBase.from_directory(builddir+'/Isca') - -if 'reinit' in todo: - if os.path.exists(os.environ['GFDL_DATA']+casename): - shutil.rmtree(os.environ['GFDL_DATA']+casename) - -# srcmod: -if 'compile' in todo: - if os.path.exists(builddir+'/srcmod'): - os.removedirs(builddir+'/srcmod') - if os.path.exists('./isca_srcmod'): - shutil.copytree('./isca_srcmod', builddir+'/srcmod') - extra_pathnames_tmp = os.listdir(builddir+'/srcmod/') - extra_pathnames = [builddir+'/srcmod/' + s for s in extra_pathnames_tmp] - print('Including the following modified CODEs:') - print(extra_pathnames) - for s in extra_pathnames_tmp[:]: - ss = os.path.splitext(s)[0] - for sss in glob.glob(os.environ["GFDL_WORK"]+"/codebase/"+objdir+"/build/isca/"+ss+"*"): - os.remove(sss) - -#Step 2. Provide the necessary inputs for the model to run: -inputfiles = [os.path.join(GFDL_BASE,'input/rrtm_input_files/ozone_1990.nc')] -# copy/link qflux file to builddir -qflux_file_name = 'merlis_schneider_30_16' -if not os.path.exists(builddir+'/'+qflux_file_name+'.nc'): - shutil.copy(os.environ['GFDL_BASE']+'/ictp-isca-workshop-2018/experiments/earth_projects/project_5_qfluxes/'+qflux_file_name+'.nc',builddir+'/'+qflux_file_name+'.nc') - -#Step 3. Define the diagnostics we want to be output from the model -diag = DiagTable() -diag.add_file('atmos_monthly', 30, 'days', time_units='days') -diag.add_file('atmos_daily', 1, 'days', time_units='days') -#Tell model which diagnostics to write to those files -diag.add_field('dynamics', 'ps', time_avg=True) -diag.add_field('dynamics', 'bk') -diag.add_field('dynamics', 'pk') -diag.add_field('dynamics', 'zsurf') -diag.add_field('atmosphere', 'precipitation', time_avg=True) -diag.add_field('mixed_layer', 't_surf', time_avg=True) -diag.add_field('dynamics', 'sphum', time_avg=True) -diag.add_field('dynamics', 'ucomp', time_avg=True) -diag.add_field('dynamics', 'vcomp', time_avg=True) -diag.add_field('dynamics', 'temp', time_avg=True) -diag.add_field('dynamics', 'vor', time_avg=True) -diag.add_field('dynamics', 'div', time_avg=True) -diag.add_field('dynamics', 'height', time_avg=True) -diag.add_field('dynamics', 'omega', time_avg=True) -diag.add_field('mixed_layer', 'flux_oceanq', time_avg=True) -diag.add_field('mixed_layer', 'flux_t', time_avg=True) -diag.add_field('mixed_layer', 'flux_lhe', time_avg=True) - -#Step 4. Define the namelist options, which will get passed to the fortran to configure the model. -runlen=30 -todoappend='' -if 'debug' in todo: - runlen=1 - todoappend='debug' -namelist = Namelist({ - 'main_nml':{ - 'days' : runlen, - 'hours' : 0, - 'minutes': 0, - 'seconds': 0, - 'dt_atmos':720, - 'current_date' : [1,1,1,0,0,0], - 'calendar' : 'thirty_day' - }, - - 'idealized_moist_phys_nml': { - 'do_damping': True, - 'turb':True, - 'mixed_layer_bc':True, - 'do_virtual' :False, - 'do_simple': True, - 'roughness_mom':3.21e-05, - 'roughness_heat':3.21e-05, - 'roughness_moist':3.21e-05, - 'two_stream_gray': False, #Use RRTM, not grey radiation: - 'do_rrtm_radiation':True, - 'convection_scheme': 'FULL_BETTS_MILLER' #Use the full Betts-miller convection scheme - }, - - 'vert_turb_driver_nml': { - 'do_mellor_yamada': False, # default: True - 'do_diffusivity': True, # default: False - 'do_simple': True, # default: False - 'constant_gust': 0.0, # default: 1.0 - 'use_tau': False - }, - - 'diffusivity_nml': { - 'do_entrain':False, - 'do_simple': True, - }, - - 'surface_flux_nml': { - 'use_virtual_temp': False, - 'do_simple': True, - 'old_dtaudv': True - }, - - 'atmosphere_nml': { - 'idealized_moist_model': True - }, - - #Use a large mixed-layer depth, and the Albedo of the CTRL case in Jucker & Gerber, 2017 - 'mixed_layer_nml': { - 'tconst' : 285., - 'prescribe_initial_dist':True, - 'evaporation':True, - 'albedo_value': 0.25, #set albedo value - 'depth': 10., - }, - - 'betts_miller_nml': { - 'rhbm': .7 , - 'do_simp': False, - 'do_shallower': True, - }, - - 'lscale_cond_nml': { - 'do_simple':True, - 'do_evap':True - }, - - 'sat_vapor_pres_nml': { - 'do_simple':True - }, - - 'damping_driver_nml': { - 'do_rayleigh': True, - 'trayfric': -0.5, # neg. value: time in *days* - 'sponge_pbottom': 150., #Setting the lower pressure boundary for the model sponge layer in Pa. - 'do_conserve_energy': True, - }, - - 'rrtm_radiation_nml': { - 'do_read_ozone':True, - 'ozone_file':'ozone_1990', - 'solr_cnst': 1360., #s set solar constant to 1360, rather than default of 1368.22 - 'dt_rad': 4320, #Use 4320 as RRTM radiation timestep - }, - - # FMS Framework configuration - 'diag_manager_nml': { - 'mix_snapshot_average_fields': False # time avg fields are labelled with time in middle of window - }, - - 'fms_nml': { - 'domains_stack_size': 600000 # default: 0 - }, - - 'fms_io_nml': { - 'threading_write': 'single', # default: multi - 'fileset_write': 'single', # default: multi - }, - - 'spectral_dynamics_nml': { - 'damping_order': 4, - 'water_correction_limit': 200.e2, - 'reference_sea_level_press':1.0e5, - 'valid_range_t':[100.,800.], - 'initial_sphum':[2.e-6], - 'vert_coord_option':'uneven_sigma', - 'surf_res':0.2, #Parameter that sets the vertical distribution of sigma levels - 'scale_heights' : 11.0, - 'exponent':7.0, - 'robert_coeff':0.03 - }, - -}) - -#Step 5. Compile the fortran code -if 'compile' in todo: - cb.compile(extra_pathnames=extra_pathnames, executable_name=executable_name) - if os.path.exists(builddir+'/run_isca.py'): - os.remove(builddir+'/run_isca.py') - shutil.copy(os.path.realpath(__file__),builddir+'/run_isca.py') - if os.path.exists(builddir+'/run.sub'): - os.remove(builddir+'/run.sub') - runsub_template=os.environ['GFDL_BASE']+'/src/extra/python/isca/templates/run.sub' - with open(runsub_template, 'r') as file : - filedata = file.read() - filedata = filedata.replace('_CASENAME_', casename) - filedata = filedata.replace('_NCORES_', str(NCORES)) - filedata = filedata.replace('_TODO_', 'run'+todoappend) - with open(builddir+'/run.sub', 'w') as file : - file.write(filedata) - -#Step 6. Run the fortran code -if 'run' in todo: - exp = Experiment(casename, codebase=cb) - exp.clear_rundir() - - exp.diag_table = diag - exp.namelist = namelist.copy() - - if qflux_file_name!='off': - inputfiles.append(os.path.join(builddir,qflux_file_name+'.nc')) - exp.namelist['mixed_layer_nml']['load_qflux'] = True - exp.namelist['mixed_layer_nml']['time_varying_qflux'] = False - exp.namelist['mixed_layer_nml']['qflux_file_name'] = qflux_file_name - else: - exp.namelist['mixed_layer_nml']['load_qflux'] = False - - exp.inputfiles = inputfiles - exp.set_resolution(*RESOLUTION) - exp.run(1, use_restart=False, num_cores=NCORES) - for i in range(2,20): - exp.run(i, num_cores=NCORES) diff --git a/exp/site_specific/ncar/build_isca.sh b/exp/site_specific/ncar/build_isca.sh deleted file mode 100755 index 8a699bd1b..000000000 --- a/exp/site_specific/ncar/build_isca.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -f -#PBS -A UHAR0008 -#PBS -N fixqflux -#PBS -q regular -#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1 -#PBS -l walltime=12:00:00 -#PBS -j oe -#PBS -S /bin/csh -V -#PBS -M kangwanying1992@gmail.com -#PBS -m abe -############# -# environment: -source ${GFDL_BASE}/src/extra/env/${GFDL_ENV} - -# passing info -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` - -export casename=${1:-"test"} -export ncores=${2:-"16"} -export todo=${3:-"compile"} -echo casename=$casename, ncores=$ncores, todo=$todo - -# running model -python build_isca.py $casename $ncores $todo - diff --git a/exp/site_specific/ncar/isca-test.sh b/exp/site_specific/ncar/isca-test.sh new file mode 100644 index 000000000..6516c58e3 --- /dev/null +++ b/exp/site_specific/ncar/isca-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This script is an example used to build and run the held suarez test case +# on the NSF NCAR Derecho Supercomputer. Following the previous steps in the +# README is required for this script to work correctly. You must substitute +# the [ACCOUNT CODE] with your own account code before submitting the job. +#PBS -l walltime=00:30:00 +#PBS -l select=1:ncpus=128:mpiprocs=128 +#PBS -q main +#PBS -A [ACCOUNT CODE] +#PBS -N Isca-test +#PBS -k eod +#PBS -j oe +#PBS -o Isca-test.out + +module --force purge +module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest +conda activate isca_env + +cd $GFDL_BASE/exp/test_cases/held_suarez +python held_suarez_test_case.py diff --git a/src/extra/env/ncar-derecho b/src/extra/env/ncar-derecho new file mode 100644 index 000000000..56bdaa7dd --- /dev/null +++ b/src/extra/env/ncar-derecho @@ -0,0 +1,5 @@ +echo loadmodules +source /glade/u/home/${USER}/.bashrc +module --force purge +module load ncarenv/23.09 craype/2.7.31 intel/2023.2.1 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest +conda activate isca_env diff --git a/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel b/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel new file mode 100644 index 000000000..2e4f84f53 --- /dev/null +++ b/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel @@ -0,0 +1,22 @@ +# template for the Intel 2023.2.1 compiler suite on NCAR's derecho machine +# typical use with mkmf +# mkmf -t template.ifc -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include + +# FFLAGS: +# -fpp: Use the fortran preprocessor +# -stack_temps: Put temporary runtime arrays on the stack, not heap. +# -safe_cray_ptr: Cray pointers don't alias other variables. +# -ftz: Denormal numbers are flushed to zero. +# -assume byterecl: Specifies the units for the OPEN statement as bytes. +# -shared-intel: Load intel libraries dynamically +# -i4: 4 byte integers +# -r8: 8 byte reals +# -g: Generate symbolic debugging info in code +# -O2: Level 2 speed optimisations +# -diag-disable 6843: +# This suppresses the warning: `warning #6843: A dummy argument with an explicit INTENT(OUT) declaration is not given an explicit value.` of which +# there are a lot of instances in the GFDL codebase. +FFLAGS = -fpp -stack-temps -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -g -O2 -diag-disable 6843 -mcmodel large +CFLAGS = -DHAVE_GETTID -fPIC -g -O2 +LD = ifort +LDFLAGS = -lnetcdf -lnetcdff -lm diff --git a/src/shared/mosaic/mosaic_util.h b/src/shared/mosaic/mosaic_util.h index a719c65c0..8ee075bdc 100644 --- a/src/shared/mosaic/mosaic_util.h +++ b/src/shared/mosaic/mosaic_util.h @@ -79,6 +79,7 @@ void addEnd(struct Node *list, double x, double y, double z, int intersect, doub int getInbound( struct Node node ); int length(struct Node *list); +int isHeadNode(struct Node *list, struct Node nodeIn); struct Node *getNextNode(struct Node *list); struct Node *getNode(struct Node *list, struct Node inNode); int sameNode(struct Node node1, struct Node node2); diff --git a/src/shared/mpp/affinity.c b/src/shared/mpp/affinity.c index f0eb78149..aace1ebed 100644 --- a/src/shared/mpp/affinity.c +++ b/src/shared/mpp/affinity.c @@ -30,10 +30,14 @@ #include #include +#ifdef HAVE_GETTID +/* will use gettid() from unistd.h */ +#else static pid_t gettid(void) { return syscall(__NR_gettid); } +#endif /* * Returns this thread's CPU affinity, if bound to a single core,