Add new CAAR optimization to HOMME with a build option to enable it

components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/caar/opt/shell_commands

@@ -0,0 +1 @@
+./xmlchange --append SCREAM_CMAKE_OPTIONS='HOMMEXX_ENABLE_CAAR_OPT ON'

components/homme/cmake/machineFiles/pm-cpu-bfb.cmake

@@ -125,6 +125,7 @@ SET(Kokkos_ENABLE_CUDA_LAMBDA OFF CACHE BOOL "")
 SET(Kokkos_ARCH_AMPERE80 OFF CACHE BOOL "")
 SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "")
+SET(HOMMEXX_ENABLE_CAAR_OPT OFF CACHE BOOL "")
 
 SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "")
 

components/homme/cmake/machineFiles/pm-cpu.cmake

@@ -71,6 +71,7 @@ SET(Kokkos_ENABLE_CUDA_LAMBDA OFF CACHE BOOL "")
 SET(Kokkos_ARCH_AMPERE80 OFF CACHE BOOL "")
 SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "")
+SET(HOMMEXX_ENABLE_CAAR_OPT OFF CACHE BOOL "")
 
 SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "")
 

components/homme/cmake/machineFiles/pm-gpu-bfb.cmake

@@ -127,6 +127,7 @@ SET(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "")
 SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "")
+SET(HOMMEXX_ENABLE_CAAR_OPT OFF CACHE BOOL "")
 
 SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "")
 
@@ -139,5 +140,5 @@ SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "")
 SET(USE_NUM_PROCS 4 CACHE STRING "") # only default
 
 SET(USE_MPIEXEC "srun" CACHE STRING "")
-SET(USE_MPI_OPTIONS "-K --cpu-bind=cores" CACHE STRING "")
+SET(USE_MPI_OPTIONS "-C gpu -K --cpu-bind=cores" CACHE STRING "")
 SET(HOMME_TESTING_TIMELIMIT 1800 CACHE STRING "")

components/homme/cmake/machineFiles/pm-gpu.cmake

@@ -76,6 +76,7 @@ SET(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "")
 SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 #SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "")
+SET(HOMMEXX_ENABLE_CAAR_OPT OFF CACHE BOOL "")
 
 SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "")
 
@@ -88,5 +89,5 @@ SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "")
 SET(USE_NUM_PROCS 4 CACHE STRING "") # only default
 
 SET(USE_MPIEXEC "srun" CACHE STRING "")
-SET(USE_MPI_OPTIONS "-K --cpu-bind=cores" CACHE STRING "")
+SET(USE_MPI_OPTIONS "-C gpu -K --cpu-bind=cores" CACHE STRING "")
 SET(HOMME_TESTING_TIMELIMIT 1800 CACHE STRING "")

components/homme/src/preqx_kokkos/cxx/CaarFunctorImpl-caar-opt.hpp

@@ -0,0 +1,3 @@
+// For preqx_kokkos, the caar-opt path is not implemented.
+// Fall back to the standard CaarFunctorImpl.
+#include "CaarFunctorImpl.hpp"

components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp

@@ -297,7 +297,7 @@ class HyperviscosityFunctorImpl
   SphereOperators       m_sphere_ops;
 
   // Policies
-#ifndef NDEBUG
+#if defined(KOKKOS_ENABLE_CUDA) && !defined(NDEBUG)
   template<typename Tag>
   using TeamPolicyType = Kokkos::TeamPolicy<ExecSpace,Kokkos::LaunchBounds<512,1>,Tag>;
 #else

components/homme/src/share/cxx/CaarFunctor.cpp

@@ -5,7 +5,12 @@
  *******************************************************************************/
 
 #include "CaarFunctor.hpp"
+#include "Hommexx_config.h"
+#ifdef HOMMEXX_ENABLE_CAAR_OPT
+#include "CaarFunctorImpl-caar-opt.hpp"
+#else
 #include "CaarFunctorImpl.hpp"
+#endif
 #include "Context.hpp"
 #include "Elements.hpp"
 #include "ErrorDefs.hpp"

components/homme/src/share/cxx/Hommexx_config.h.in

@@ -24,6 +24,9 @@
 // Whether the debug parts of cxx code should be compiled or not
 #cmakedefine HOMMEXX_DEBUG
 
+// Enable CAAR Optimization
+#cmakedefine HOMMEXX_ENABLE_CAAR_OPT
+
 // Whether the MPI operations have to be performed directly on the device
 #cmakedefine01 HOMMEXX_MPI_ON_DEVICE