From 4b31f24315f1f5428feb7e547a4cf2f36f12921e Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Fri, 14 Mar 2025 09:38:43 -0600 Subject: [PATCH 1/7] Initial mods for the NCAR Derecho system using the Intel compiler. Tested under the environment --- module --force purge module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest --- - Added a function declaration for isHeadNode in mosaic_util.h - Added #ifdef HAVE_GETTID block around gittid function in affinity.c to use system version where available - Added mkmf template, conda env, and environment with settings appropriate for Derecho --- --- ci/environment-py3.12_ncar-derecho.yml | 16 ++++++++++++++ src/extra/env/ncar-derecho | 5 +++++ .../mkmf.template.ncar-derecho-intel | 22 +++++++++++++++++++ src/shared/mosaic/mosaic_util.h | 1 + src/shared/mpp/affinity.c | 4 ++++ 5 files changed, 48 insertions(+) create mode 100644 ci/environment-py3.12_ncar-derecho.yml create mode 100644 src/extra/env/ncar-derecho create mode 100644 src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel diff --git a/ci/environment-py3.12_ncar-derecho.yml b/ci/environment-py3.12_ncar-derecho.yml new file mode 100644 index 000000000..2c67f179d --- /dev/null +++ b/ci/environment-py3.12_ncar-derecho.yml @@ -0,0 +1,16 @@ +name: isca_env +channels: +- conda-forge +dependencies: +- dask=2024.1.0 +- f90nml=1.4.4 +- ipykernel=6.29.0 +- jinja2=3.1.3 +- numpy=1.26.3 +- pandas=2.2.0 +- python=3.12.1 +- pip=23.3.2 +- pytest=7.4.4 +- sh=2.0.6 +- tqdm=4.66.1 +- xarray=2024.1.1 diff --git a/src/extra/env/ncar-derecho b/src/extra/env/ncar-derecho new file mode 100644 index 000000000..56bdaa7dd --- /dev/null +++ b/src/extra/env/ncar-derecho @@ -0,0 +1,5 @@ +echo loadmodules +source /glade/u/home/${USER}/.bashrc +module --force purge +module load ncarenv/23.09 craype/2.7.31 intel/2023.2.1 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest +conda activate isca_env diff --git a/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel b/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel new file mode 100644 index 000000000..2e4f84f53 --- /dev/null +++ b/src/extra/python/isca/templates/mkmf.template.ncar-derecho-intel @@ -0,0 +1,22 @@ +# template for the Intel 2023.2.1 compiler suite on NCAR's derecho machine +# typical use with mkmf +# mkmf -t template.ifc -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include + +# FFLAGS: +# -fpp: Use the fortran preprocessor +# -stack_temps: Put temporary runtime arrays on the stack, not heap. +# -safe_cray_ptr: Cray pointers don't alias other variables. +# -ftz: Denormal numbers are flushed to zero. +# -assume byterecl: Specifies the units for the OPEN statement as bytes. +# -shared-intel: Load intel libraries dynamically +# -i4: 4 byte integers +# -r8: 8 byte reals +# -g: Generate symbolic debugging info in code +# -O2: Level 2 speed optimisations +# -diag-disable 6843: +# This suppresses the warning: `warning #6843: A dummy argument with an explicit INTENT(OUT) declaration is not given an explicit value.` of which +# there are a lot of instances in the GFDL codebase. +FFLAGS = -fpp -stack-temps -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -g -O2 -diag-disable 6843 -mcmodel large +CFLAGS = -DHAVE_GETTID -fPIC -g -O2 +LD = ifort +LDFLAGS = -lnetcdf -lnetcdff -lm diff --git a/src/shared/mosaic/mosaic_util.h b/src/shared/mosaic/mosaic_util.h index a719c65c0..8ee075bdc 100644 --- a/src/shared/mosaic/mosaic_util.h +++ b/src/shared/mosaic/mosaic_util.h @@ -79,6 +79,7 @@ void addEnd(struct Node *list, double x, double y, double z, int intersect, doub int getInbound( struct Node node ); int length(struct Node *list); +int isHeadNode(struct Node *list, struct Node nodeIn); struct Node *getNextNode(struct Node *list); struct Node *getNode(struct Node *list, struct Node inNode); int sameNode(struct Node node1, struct Node node2); diff --git a/src/shared/mpp/affinity.c b/src/shared/mpp/affinity.c index f0eb78149..aace1ebed 100644 --- a/src/shared/mpp/affinity.c +++ b/src/shared/mpp/affinity.c @@ -30,10 +30,14 @@ #include #include +#ifdef HAVE_GETTID +/* will use gettid() from unistd.h */ +#else static pid_t gettid(void) { return syscall(__NR_gettid); } +#endif /* * Returns this thread's CPU affinity, if bound to a single core, From 528d28c0cc300252f489d022e1b93e124290abba Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 00:06:13 -0600 Subject: [PATCH 2/7] Update ReadMe.md Added instructions on how to build and run on Derecho to the README --- ReadMe.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/ReadMe.md b/ReadMe.md index 46dbc6359..bdd1868bf 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -27,6 +27,67 @@ alt="twitter IscaModel">

+## This is a temporary fork targeting NSF NCAR's Derecho System + +This fork of the Isca repository is intended to assist users in building +and running Isca on the Derecho Supercomputer at NSF NCAR. Once the modifications +in this fork are pulled into the main Isca repository this repo will be archived +and eventually removed. + +## Procedure for building and testing Isca on Derecho + +1. Check out the code + ```bash + git clone https://github.com/roryck/Isca-derecho.git + ``` +2. Set up the environment + ```bash + cd Isca-derecho + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + + mamba env create -f ci/environment-py3.12_ncar-derecho.yml + conda activate isca_env + + cd src/extra/python + pip install -e . + ``` +3. Add Isca settings to ~/.bashrc + ```bash + export GFDL_ENV=ncar-derecho + export GFDL_MKMF_TEMPLATE=ncar-derecho-intel + export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data + export GFDL_BASE=/glade/derecho/scratch/$USER/tickets/rc-31217/Isca-derecho + export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work + ``` +12. Build and Run a Test Case on a batch node + ```bash + cd $GFDL_BASE/exp/test_cases/held_suarez + qsub isca-test.sh + ``` + Example batch script isca-test.sh + ``` + #!/bin/bash + #PBS -l walltime=00:30:00 + #PBS -l select=1:ncpus=128:mpiprocs=128 + #PBS -q main + #PBS -A [ACCOUNT CODE] + #PBS -N Isca-test + #PBS -k eod + #PBS -j oe + #PBS -o Isca-test.out + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + conda activate isca_env + + cd $GFDL_BASE/exp/test_cases/held_suarez + python held_suarez_test_case.py + ``` + +## Main Isca README + Isca is a framework for the idealized modelling of the global circulation of planetary atmospheres at varying levels of complexity and realism. The framework is an outgrowth of models from GFDL designed for Earth's atmosphere, From 5a0e950e6487165c162712dee5ef067b51129048 Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 00:12:05 -0600 Subject: [PATCH 3/7] Update ReadMe.md simplification and formatting --- ReadMe.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ReadMe.md b/ReadMe.md index bdd1868bf..ae7e4eb49 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -53,21 +53,21 @@ and eventually removed. cd src/extra/python pip install -e . ``` -3. Add Isca settings to ~/.bashrc +3. Add Isca settings to `~/.bashrc` (modify paths as desired) ```bash export GFDL_ENV=ncar-derecho export GFDL_MKMF_TEMPLATE=ncar-derecho-intel - export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data - export GFDL_BASE=/glade/derecho/scratch/$USER/tickets/rc-31217/Isca-derecho + export GFDL_BASE=/glade/work/$USER/Isca-derecho export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work + export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data ``` 12. Build and Run a Test Case on a batch node ```bash cd $GFDL_BASE/exp/test_cases/held_suarez qsub isca-test.sh ``` - Example batch script isca-test.sh - ``` + Example batch script `isca-test.sh` (you'll need to add you account code to the `-A` argument) + ```bash #!/bin/bash #PBS -l walltime=00:30:00 #PBS -l select=1:ncpus=128:mpiprocs=128 From 9643874435b9b477e44b335dfb239c0d62bbbcb9 Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 10:38:29 -0600 Subject: [PATCH 4/7] Update ReadMe.md minor clarification --- ReadMe.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ReadMe.md b/ReadMe.md index ae7e4eb49..99f3aa6a1 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -53,7 +53,8 @@ and eventually removed. cd src/extra/python pip install -e . ``` -3. Add Isca settings to `~/.bashrc` (modify paths as desired) +3. Add Isca settings to `~/.bashrc` (modify paths as desired). After adding these settings you'll + need to source your ~/.bashrc file, or export the settings in the current shell as well. ```bash export GFDL_ENV=ncar-derecho export GFDL_MKMF_TEMPLATE=ncar-derecho-intel From 0753de140980cd65b17d82f3d108a8795a0f03fa Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 11:49:00 -0600 Subject: [PATCH 5/7] Added build infor for Derecho into the site specific README. Restored original Isca README. Added test script to run and build the HS test case for Derecho. --- ReadMe.md | 62 ------- exp/site_specific/ncar/README | 112 ++++++------ exp/site_specific/ncar/build_isca.py | 249 --------------------------- exp/site_specific/ncar/build_isca.sh | 27 --- exp/site_specific/ncar/isca-test.sh | 20 +++ 5 files changed, 73 insertions(+), 397 deletions(-) delete mode 100644 exp/site_specific/ncar/build_isca.py delete mode 100755 exp/site_specific/ncar/build_isca.sh create mode 100644 exp/site_specific/ncar/isca-test.sh diff --git a/ReadMe.md b/ReadMe.md index 99f3aa6a1..46dbc6359 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -27,68 +27,6 @@ alt="twitter IscaModel">

-## This is a temporary fork targeting NSF NCAR's Derecho System - -This fork of the Isca repository is intended to assist users in building -and running Isca on the Derecho Supercomputer at NSF NCAR. Once the modifications -in this fork are pulled into the main Isca repository this repo will be archived -and eventually removed. - -## Procedure for building and testing Isca on Derecho - -1. Check out the code - ```bash - git clone https://github.com/roryck/Isca-derecho.git - ``` -2. Set up the environment - ```bash - cd Isca-derecho - - module --force purge - module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest - - mamba env create -f ci/environment-py3.12_ncar-derecho.yml - conda activate isca_env - - cd src/extra/python - pip install -e . - ``` -3. Add Isca settings to `~/.bashrc` (modify paths as desired). After adding these settings you'll - need to source your ~/.bashrc file, or export the settings in the current shell as well. - ```bash - export GFDL_ENV=ncar-derecho - export GFDL_MKMF_TEMPLATE=ncar-derecho-intel - export GFDL_BASE=/glade/work/$USER/Isca-derecho - export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work - export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data - ``` -12. Build and Run a Test Case on a batch node - ```bash - cd $GFDL_BASE/exp/test_cases/held_suarez - qsub isca-test.sh - ``` - Example batch script `isca-test.sh` (you'll need to add you account code to the `-A` argument) - ```bash - #!/bin/bash - #PBS -l walltime=00:30:00 - #PBS -l select=1:ncpus=128:mpiprocs=128 - #PBS -q main - #PBS -A [ACCOUNT CODE] - #PBS -N Isca-test - #PBS -k eod - #PBS -j oe - #PBS -o Isca-test.out - - module --force purge - module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest - conda activate isca_env - - cd $GFDL_BASE/exp/test_cases/held_suarez - python held_suarez_test_case.py - ``` - -## Main Isca README - Isca is a framework for the idealized modelling of the global circulation of planetary atmospheres at varying levels of complexity and realism. The framework is an outgrowth of models from GFDL designed for Earth's atmosphere, diff --git a/exp/site_specific/ncar/README b/exp/site_specific/ncar/README index 8f62038eb..7f03f2152 100644 --- a/exp/site_specific/ncar/README +++ b/exp/site_specific/ncar/README @@ -1,59 +1,53 @@ -Guildline to start running test case in Cheyenne Machine --- July 15, 2018 Wanying Kang -1. download Isca -========================================= -git clone https://github.com/ExeClim/Isca [IscaCodeDir] -cd [IscaCodeDir] -========================================= - -2. install python package necessary following the Cheyenne guideline: https://www2.cisl.ucar.edu/resources/computational-systems/cheyenne/software/python. The following are the key steps, and detail please refer to cheyenne official site. -========================================= -cd [IscaCodeDir]/src/extra/python -ncar_pylib -c 20180128 [personal_package_dir] -cd [personal_package_dir]/bin -pip install -r requirements.txt -========================================= - -Each time before using, type the following to activate. Suggest to add the following line to ~/.bashrc -========================================= -source [personal_package_dir]/bin/activate.csh (tcsh) -OR -source [personal_package_dir]/bin/activate (bash) -========================================= - -3. install Isca -pip install -e . - -4. define environment variables. Please add the following to ~/.bashrc. - -========================================= -# python package location -export python_package_lib_path=[personal_package_dir]/bin/activate -# directory of the Isca source code -export GFDL_BASE=[IscaCodeDir] -# "environment" configuration for emps-gv4 -export GFDL_ENV=cheyenne -# temporary working directory used in running the model, you may change to other places, but need to change the code accordingly as well. -export GFDL_WORK=/glade/scratch/${USER}/isca_rundir/ -# directory for storing model output, you may change to other places, but need to change the code accordingly. -export GFDL_DATA=/glade/scratch/${USER}/isca_out/ -export ACCOUNT = [YourAccount] -========================================= - -5. to run the model -========================================= -cp [IscaCodeDir]/exp/test_cases/cheyenne_run/build_isca.* [SomePlaceAsYouWant] -cd [SomePlaceAsYouWant] -./build_isca.sh [CaseName] [NCORES] [TODO] -========================================= - for [TODO]: you can combine the following keywords - clean: clean up the [CaseName] workdir and rundir - compile: set up experiments using the settings in build_isca.py, and compile the F90s. Copy exec and run script to the [builddir] folder. - run: submit the job to queue. - - - - -For any questions regarding the porting and running on Cheyenne machine, you may send me an email: wanyingkang@g.harvard.edu. - - +Quick start guide for running Isca on NSF NCAR's Derecho HPC Machine +-- March 2025, Rory Kelly (rory@ucar.edu) +1. Download the Isca code + ```bash + git clone https://github.com/ExeClim/Isca + ``` + +2. Set up the environment + ```bash + cd Isca + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + + mamba env create -f ci/environment-py3.12_ncar-derecho.yml + conda activate isca_env + + cd src/extra/python + pip install -e . + ``` +3. Add Isca settings to `~/.bashrc` (modify paths as desired). After adding these settings you'll + need to source your ~/.bashrc file, or export the settings in the current shell as well. + ```bash + export GFDL_ENV=ncar-derecho + export GFDL_MKMF_TEMPLATE=ncar-derecho-intel + export GFDL_BASE=/glade/work/$USER/Isca-derecho + export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work + export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data + ``` +4. Build and Run a Test Case on a batch node + ```bash + cd $GFDL_BASE/exp/test_cases/held_suarez + qsub isca-test.sh + ``` + Example batch script `isca-test.sh` (you'll need to add you account code to the `-A` argument) + ```bash + #!/bin/bash + #PBS -l walltime=00:30:00 + #PBS -l select=1:ncpus=128:mpiprocs=128 + #PBS -q main + #PBS -A [ACCOUNT CODE] + #PBS -N Isca-test + #PBS -k eod + #PBS -j oe + #PBS -o Isca-test.out + + module --force purge + module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest + conda activate isca_env + + cd $GFDL_BASE/exp/test_cases/held_suarez + python held_suarez_test_case.py + ``` diff --git a/exp/site_specific/ncar/build_isca.py b/exp/site_specific/ncar/build_isca.py deleted file mode 100644 index 796858626..000000000 --- a/exp/site_specific/ncar/build_isca.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import glob -import numpy as np -import shutil -import string -import sys -from isca import IscaCodeBase, DiagTable, Experiment, Namelist, GFDL_BASE - -casename = sys.argv[1] -print('building case:'+casename) -builddir = '/glade/u/home/'+os.environ["USER"]+'/Isca_caseroot/'+casename -print('in:'+builddir) -#executable_name=casename+'.x' -executable_name='isca.x' -NCORES= int(sys.argv[2]) -print('Using '+str(NCORES)+' cores') -RESOLUTION = 'T42', 40 -todo = sys.argv[3] - -#Step 1: Soft link the original code. Point the python script to the location of the code (the bash environment variable GFDL_BASE is that directory) -if 'compile' in todo: - shutil.rmtree(builddir, ignore_errors=True) - os.makedirs(builddir) - if not os.path.exists(builddir+'/Isca'): - os.symlink(os.environ['GFDL_BASE'],builddir+'/Isca') - -objdir=builddir.replace('/','_')+'_Isca' -if 'clean' in todo: - if os.path.exists(os.environ['GFDL_WORK']+'/codebase/'+objdir): - print('clean up '+os.environ['GFDL_WORK']+'/codebase/'+objdir) - shutil.rmtree(os.environ['GFDL_WORK']+'/codebase/'+objdir) - if os.path.exists(os.environ['GFDL_WORK']+'/experiments/'+casename): - print('clean up '+os.environ['GFDL_WORK']+'/experiments/'+casename) - shutil.rmtree(os.environ['GFDL_WORK']+'/experiments/'+casename) - if os.path.exists(builddir): - print('clean up '+builddir) - shutil.rmtree(builddir) - -#cb = IscaCodeBase.from_directory(builddir+'/Isca',storedir=builddir) # for saving obj files in Isca_caseroot/ -cb = IscaCodeBase.from_directory(builddir+'/Isca') - -if 'reinit' in todo: - if os.path.exists(os.environ['GFDL_DATA']+casename): - shutil.rmtree(os.environ['GFDL_DATA']+casename) - -# srcmod: -if 'compile' in todo: - if os.path.exists(builddir+'/srcmod'): - os.removedirs(builddir+'/srcmod') - if os.path.exists('./isca_srcmod'): - shutil.copytree('./isca_srcmod', builddir+'/srcmod') - extra_pathnames_tmp = os.listdir(builddir+'/srcmod/') - extra_pathnames = [builddir+'/srcmod/' + s for s in extra_pathnames_tmp] - print('Including the following modified CODEs:') - print(extra_pathnames) - for s in extra_pathnames_tmp[:]: - ss = os.path.splitext(s)[0] - for sss in glob.glob(os.environ["GFDL_WORK"]+"/codebase/"+objdir+"/build/isca/"+ss+"*"): - os.remove(sss) - -#Step 2. Provide the necessary inputs for the model to run: -inputfiles = [os.path.join(GFDL_BASE,'input/rrtm_input_files/ozone_1990.nc')] -# copy/link qflux file to builddir -qflux_file_name = 'merlis_schneider_30_16' -if not os.path.exists(builddir+'/'+qflux_file_name+'.nc'): - shutil.copy(os.environ['GFDL_BASE']+'/ictp-isca-workshop-2018/experiments/earth_projects/project_5_qfluxes/'+qflux_file_name+'.nc',builddir+'/'+qflux_file_name+'.nc') - -#Step 3. Define the diagnostics we want to be output from the model -diag = DiagTable() -diag.add_file('atmos_monthly', 30, 'days', time_units='days') -diag.add_file('atmos_daily', 1, 'days', time_units='days') -#Tell model which diagnostics to write to those files -diag.add_field('dynamics', 'ps', time_avg=True) -diag.add_field('dynamics', 'bk') -diag.add_field('dynamics', 'pk') -diag.add_field('dynamics', 'zsurf') -diag.add_field('atmosphere', 'precipitation', time_avg=True) -diag.add_field('mixed_layer', 't_surf', time_avg=True) -diag.add_field('dynamics', 'sphum', time_avg=True) -diag.add_field('dynamics', 'ucomp', time_avg=True) -diag.add_field('dynamics', 'vcomp', time_avg=True) -diag.add_field('dynamics', 'temp', time_avg=True) -diag.add_field('dynamics', 'vor', time_avg=True) -diag.add_field('dynamics', 'div', time_avg=True) -diag.add_field('dynamics', 'height', time_avg=True) -diag.add_field('dynamics', 'omega', time_avg=True) -diag.add_field('mixed_layer', 'flux_oceanq', time_avg=True) -diag.add_field('mixed_layer', 'flux_t', time_avg=True) -diag.add_field('mixed_layer', 'flux_lhe', time_avg=True) - -#Step 4. Define the namelist options, which will get passed to the fortran to configure the model. -runlen=30 -todoappend='' -if 'debug' in todo: - runlen=1 - todoappend='debug' -namelist = Namelist({ - 'main_nml':{ - 'days' : runlen, - 'hours' : 0, - 'minutes': 0, - 'seconds': 0, - 'dt_atmos':720, - 'current_date' : [1,1,1,0,0,0], - 'calendar' : 'thirty_day' - }, - - 'idealized_moist_phys_nml': { - 'do_damping': True, - 'turb':True, - 'mixed_layer_bc':True, - 'do_virtual' :False, - 'do_simple': True, - 'roughness_mom':3.21e-05, - 'roughness_heat':3.21e-05, - 'roughness_moist':3.21e-05, - 'two_stream_gray': False, #Use RRTM, not grey radiation: - 'do_rrtm_radiation':True, - 'convection_scheme': 'FULL_BETTS_MILLER' #Use the full Betts-miller convection scheme - }, - - 'vert_turb_driver_nml': { - 'do_mellor_yamada': False, # default: True - 'do_diffusivity': True, # default: False - 'do_simple': True, # default: False - 'constant_gust': 0.0, # default: 1.0 - 'use_tau': False - }, - - 'diffusivity_nml': { - 'do_entrain':False, - 'do_simple': True, - }, - - 'surface_flux_nml': { - 'use_virtual_temp': False, - 'do_simple': True, - 'old_dtaudv': True - }, - - 'atmosphere_nml': { - 'idealized_moist_model': True - }, - - #Use a large mixed-layer depth, and the Albedo of the CTRL case in Jucker & Gerber, 2017 - 'mixed_layer_nml': { - 'tconst' : 285., - 'prescribe_initial_dist':True, - 'evaporation':True, - 'albedo_value': 0.25, #set albedo value - 'depth': 10., - }, - - 'betts_miller_nml': { - 'rhbm': .7 , - 'do_simp': False, - 'do_shallower': True, - }, - - 'lscale_cond_nml': { - 'do_simple':True, - 'do_evap':True - }, - - 'sat_vapor_pres_nml': { - 'do_simple':True - }, - - 'damping_driver_nml': { - 'do_rayleigh': True, - 'trayfric': -0.5, # neg. value: time in *days* - 'sponge_pbottom': 150., #Setting the lower pressure boundary for the model sponge layer in Pa. - 'do_conserve_energy': True, - }, - - 'rrtm_radiation_nml': { - 'do_read_ozone':True, - 'ozone_file':'ozone_1990', - 'solr_cnst': 1360., #s set solar constant to 1360, rather than default of 1368.22 - 'dt_rad': 4320, #Use 4320 as RRTM radiation timestep - }, - - # FMS Framework configuration - 'diag_manager_nml': { - 'mix_snapshot_average_fields': False # time avg fields are labelled with time in middle of window - }, - - 'fms_nml': { - 'domains_stack_size': 600000 # default: 0 - }, - - 'fms_io_nml': { - 'threading_write': 'single', # default: multi - 'fileset_write': 'single', # default: multi - }, - - 'spectral_dynamics_nml': { - 'damping_order': 4, - 'water_correction_limit': 200.e2, - 'reference_sea_level_press':1.0e5, - 'valid_range_t':[100.,800.], - 'initial_sphum':[2.e-6], - 'vert_coord_option':'uneven_sigma', - 'surf_res':0.2, #Parameter that sets the vertical distribution of sigma levels - 'scale_heights' : 11.0, - 'exponent':7.0, - 'robert_coeff':0.03 - }, - -}) - -#Step 5. Compile the fortran code -if 'compile' in todo: - cb.compile(extra_pathnames=extra_pathnames, executable_name=executable_name) - if os.path.exists(builddir+'/run_isca.py'): - os.remove(builddir+'/run_isca.py') - shutil.copy(os.path.realpath(__file__),builddir+'/run_isca.py') - if os.path.exists(builddir+'/run.sub'): - os.remove(builddir+'/run.sub') - runsub_template=os.environ['GFDL_BASE']+'/src/extra/python/isca/templates/run.sub' - with open(runsub_template, 'r') as file : - filedata = file.read() - filedata = filedata.replace('_CASENAME_', casename) - filedata = filedata.replace('_NCORES_', str(NCORES)) - filedata = filedata.replace('_TODO_', 'run'+todoappend) - with open(builddir+'/run.sub', 'w') as file : - file.write(filedata) - -#Step 6. Run the fortran code -if 'run' in todo: - exp = Experiment(casename, codebase=cb) - exp.clear_rundir() - - exp.diag_table = diag - exp.namelist = namelist.copy() - - if qflux_file_name!='off': - inputfiles.append(os.path.join(builddir,qflux_file_name+'.nc')) - exp.namelist['mixed_layer_nml']['load_qflux'] = True - exp.namelist['mixed_layer_nml']['time_varying_qflux'] = False - exp.namelist['mixed_layer_nml']['qflux_file_name'] = qflux_file_name - else: - exp.namelist['mixed_layer_nml']['load_qflux'] = False - - exp.inputfiles = inputfiles - exp.set_resolution(*RESOLUTION) - exp.run(1, use_restart=False, num_cores=NCORES) - for i in range(2,20): - exp.run(i, num_cores=NCORES) diff --git a/exp/site_specific/ncar/build_isca.sh b/exp/site_specific/ncar/build_isca.sh deleted file mode 100755 index 8a699bd1b..000000000 --- a/exp/site_specific/ncar/build_isca.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -f -#PBS -A UHAR0008 -#PBS -N fixqflux -#PBS -q regular -#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1 -#PBS -l walltime=12:00:00 -#PBS -j oe -#PBS -S /bin/csh -V -#PBS -M kangwanying1992@gmail.com -#PBS -m abe -############# -# environment: -source ${GFDL_BASE}/src/extra/env/${GFDL_ENV} - -# passing info -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` - -export casename=${1:-"test"} -export ncores=${2:-"16"} -export todo=${3:-"compile"} -echo casename=$casename, ncores=$ncores, todo=$todo - -# running model -python build_isca.py $casename $ncores $todo - diff --git a/exp/site_specific/ncar/isca-test.sh b/exp/site_specific/ncar/isca-test.sh new file mode 100644 index 000000000..6516c58e3 --- /dev/null +++ b/exp/site_specific/ncar/isca-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This script is an example used to build and run the held suarez test case +# on the NSF NCAR Derecho Supercomputer. Following the previous steps in the +# README is required for this script to work correctly. You must substitute +# the [ACCOUNT CODE] with your own account code before submitting the job. +#PBS -l walltime=00:30:00 +#PBS -l select=1:ncpus=128:mpiprocs=128 +#PBS -q main +#PBS -A [ACCOUNT CODE] +#PBS -N Isca-test +#PBS -k eod +#PBS -j oe +#PBS -o Isca-test.out + +module --force purge +module load ncarenv/23.09 intel/2023.2.1 craype/2.7.31 cray-mpich/8.1.27 ncarcompilers/1.0.0 hdf5/1.12.2 netcdf/4.9.2 conda/latest +conda activate isca_env + +cd $GFDL_BASE/exp/test_cases/held_suarez +python held_suarez_test_case.py From c191380d70d0222176701f9bef482e0b6553c434 Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 11:53:31 -0600 Subject: [PATCH 6/7] changed Derecho Readme format --- exp/site_specific/ncar/{README => Readme.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename exp/site_specific/ncar/{README => Readme.md} (100%) diff --git a/exp/site_specific/ncar/README b/exp/site_specific/ncar/Readme.md similarity index 100% rename from exp/site_specific/ncar/README rename to exp/site_specific/ncar/Readme.md From 9380c45bbe2d1e4976158e6887901ceb9333591f Mon Sep 17 00:00:00 2001 From: Rory Kelly Date: Tue, 18 Mar 2025 13:05:56 -0600 Subject: [PATCH 7/7] Update Readme.md typo fix --- exp/site_specific/ncar/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exp/site_specific/ncar/Readme.md b/exp/site_specific/ncar/Readme.md index 7f03f2152..35e3ab9aa 100644 --- a/exp/site_specific/ncar/Readme.md +++ b/exp/site_specific/ncar/Readme.md @@ -23,7 +23,7 @@ Quick start guide for running Isca on NSF NCAR's Derecho HPC Machine ```bash export GFDL_ENV=ncar-derecho export GFDL_MKMF_TEMPLATE=ncar-derecho-intel - export GFDL_BASE=/glade/work/$USER/Isca-derecho + export GFDL_BASE=/glade/work/$USER/Isca export GFDL_WORK=/glade/derecho/scratch/$USER/isca_work export GFDL_DATA=/glade/derecho/scratch/$USER/isca_data ```