Skip to content

Commit 2a3cd75

Browse files
ajnonakaajnonaka
authored
Merge pull request #195 from jmsexton03/fix_cuda_ci
Update Cuda CI based on amrex and ERF
2 parents 5f03726 + ebc7ed8 commit 2a3cd75

34 files changed

Lines changed: 515 additions & 325 deletions

‎.github/problem-matchers/gcc.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"problemMatcher": [
3+
{
4+
"owner": "gcc-problem-matcher",
5+
"pattern": [
6+
{
7+
"regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error|note):\\s+(.*)$",
8+
"file": 1,
9+
"line": 2,
10+
"column": 3,
11+
"severity": 4,
12+
"message": 5
13+
}
14+
]
15+
}
16+
]
17+
}

‎.github/workflows/cuda.yml‎

Lines changed: 182 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -33,53 +33,202 @@ jobs:
3333
# -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON
3434
# make -j 2
3535

36-
# Build libamrex and all tests with CUDA 11.0.2
36+
# Build libamrex and all tests with CUDA 12.0
37+
3738
tests-cuda11:
38-
name: CUDA@11.2 GNU@9.3.0 C++17 Release [tests]
39-
runs-on: ubuntu-20.04
39+
name: CUDA@12.0 GNU C++17 Release [${{ matrix.app }}] DIM=${{ matrix.dim }}
40+
runs-on: ubuntu-22.04
41+
strategy:
42+
fail-fast: false
43+
matrix:
44+
include:
45+
# 3D tests for main applications
46+
- app: compressible
47+
dim: 3
48+
artifact_name: compressible
49+
- app: compressible_stag
50+
dim: 3
51+
artifact_name: compressible_stag
52+
- app: immersedIons
53+
dim: 3
54+
artifact_name: immersedIons
55+
# 2D and 3D tests for specified applications
56+
- app: structFactTest
57+
dim: 2
58+
artifact_name: structFactTest
59+
- app: structFactTest
60+
dim: 3
61+
artifact_name: structFactTest
62+
- app: reactDiff
63+
dim: 2
64+
artifact_name: reactDiff
65+
- app: reactDiff
66+
dim: 3
67+
artifact_name: reactDiff
68+
- app: multispec
69+
dim: 2
70+
artifact_name: multispec
71+
- app: multispec
72+
dim: 3
73+
artifact_name: multispec
74+
- app: dean_kow/singlelevel
75+
dim: 2
76+
artifact_name: dean_kow-singlelevel
77+
- app: dean_kow/singlelevel
78+
dim: 3
79+
artifact_name: dean_kow-singlelevel
80+
- app: dean_kow/multilevel
81+
dim: 2
82+
artifact_name: dean_kow-multilevel
83+
- app: dean_kow/multilevel
84+
dim: 3
85+
artifact_name: dean_kow-multilevel
86+
- app: hydro
87+
dim: 2
88+
artifact_name: hydro
89+
- app: hydro
90+
dim: 3
91+
artifact_name: hydro
4092
env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code"}
4193
steps:
42-
- uses: actions/checkout@v2
43-
- name: Dependencies
44-
run: .github/workflows/dependencies/dependencies_nvcc11.sh
45-
- name: Clone AMReX
46-
uses: actions/checkout@v2
94+
- uses: actions/checkout@v4
95+
- name: Clone AMReX for dependency scripts
96+
run: |
97+
git clone https://github.com/AMReX-Codes/amrex.git ${{runner.workspace}}/amrex
98+
- name: Prepare CUDA environment
99+
run: ${{runner.workspace}}/amrex/.github/workflows/dependencies/dependencies_nvcc.sh 12.0
100+
- name: Install Additional Dependencies
101+
run: |
102+
sudo apt-get update
103+
sudo apt-get install -y libfftw3-dev libfftw3-mpi-dev
104+
- name: Install CCache
105+
run: ${{runner.workspace}}/amrex/.github/workflows/dependencies/dependencies_ccache.sh
106+
- name: Set Up Cache
107+
uses: actions/cache@v4
47108
with:
48-
repository: AMReX-Codes/amrex
49-
ref: development
50-
path: amrex
51-
- name: Build & Install
109+
path: ~/.cache/ccache
110+
key: ccache-${{ github.workflow }}-${{ github.job }}-${{ matrix.app }}-git-${{ github.sha }}
111+
restore-keys: |
112+
ccache-${{ github.workflow }}-${{ github.job }}-${{ matrix.app }}-git-
113+
- name: Enable Problem Matchers
114+
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
115+
- name: Build & Install ${{ matrix.app }}
116+
continue-on-error: true
52117
run: |
118+
export CCACHE_COMPRESS=1
119+
export CCACHE_COMPRESSLEVEL=10
120+
export CCACHE_MAXSIZE=600M
121+
ccache -z
122+
53123
export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
54-
export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
124+
export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
55125
export LD_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
56-
cd exec/multispec
126+
cd exec/${{ matrix.app }}
57127
which nvcc || echo "nvcc not in PATH!"
58-
make -j2 VERBOSE=1 USE_MPI=TRUE USE_CUDA=TRUE USE_OMP=FALSE DIM=3 TINY_PROFILE=FALSE
128+
make -j2 VERBOSE=1 USE_MPI=TRUE USE_CUDA=TRUE USE_OMP=FALSE DIM=${{ matrix.dim }} TINY_PROFILE=FALSE 2>&1 | tee ${{runner.workspace}}/build-output.txt
129+
130+
ccache -s
131+
du -hs ~/.cache/ccache
59132
env:
60-
AMREX_HOME: $(GITHUB_WORKSPACE)/amrex
133+
AMREX_HOME: ${{runner.workspace}}/amrex
134+
135+
# Step 10 - hardcoded in summary link below
136+
- name: Report
137+
id: report
138+
if: always()
139+
continue-on-error: true
140+
run: |
141+
# Create empty files first
142+
touch ${{runner.workspace}}/build-warnings.txt
143+
touch ${{runner.workspace}}/build-errors.txt
144+
145+
if [ -f "${{runner.workspace}}/build-output.txt" ]; then
146+
# Extract warnings - broader patterns (include ALL warnings)
147+
egrep -i "(warning|Warning)" ${{runner.workspace}}/build-output.txt | \
148+
egrep -v "lto-wrapper:" | \
149+
sort | uniq | \
150+
awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-warnings.txt
61151
152+
# Extract errors - match specific compiler error patterns and make errors (include ALL errors)
153+
egrep -i "(\): error:|error:|Error:|errors detected|make: \*\*\*)" ${{runner.workspace}}/build-output.txt | \
154+
egrep -v "lto-wrapper:" | \
155+
sort | uniq | \
156+
awk 'BEGIN{i=0}{print $0}{i++}END{print "Errors: "i}' > ${{runner.workspace}}/build-errors.txt
157+
else
158+
echo "No build output captured" > ${{runner.workspace}}/build-errors.txt
159+
echo "Errors: 1" >> ${{runner.workspace}}/build-errors.txt
160+
fi
62161
162+
echo "=== WARNINGS ==="
163+
cat ${{runner.workspace}}/build-warnings.txt
164+
echo "=== ERRORS ==="
165+
cat ${{runner.workspace}}/build-errors.txt
63166
64-
#cmake -S . -B build \
65-
# -DCMAKE_VERBOSE_MAKEFILE=ON \
66-
# -DAMReX_ENABLE_TESTS=ON \
67-
# -DAMReX_FORTRAN=OFF \
68-
# -DAMReX_PARTICLES=ON \
69-
# -DAMReX_GPU_BACKEND=CUDA \
70-
# -DCMAKE_C_COMPILER=$(which gcc) \
71-
# -DCMAKE_CXX_COMPILER=$(which g++) \
72-
# -DCMAKE_CUDA_HOST_COMPILER=$(which g++) \
73-
# -DCMAKE_Fortran_COMPILER=$(which gfortran) \
74-
# -DCMAKE_CUDA_STANDARD=17 \
75-
# -DCMAKE_CXX_STANDARD=17 \
76-
# -DAMReX_CUDA_ARCH=8.0 \
77-
# -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
78-
# -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
167+
# Store error and warning counts for later use (but count total, not just displayed)
168+
total_errors=$(egrep -i "(\): error:|error:|Error:|errors detected|make: \*\*\*)" ${{runner.workspace}}/build-output.txt 2>/dev/null | egrep -v "lto-wrapper:" | wc -l)
169+
total_warnings=$(egrep -i "(warning|Warning)" ${{runner.workspace}}/build-output.txt 2>/dev/null | egrep -v "lto-wrapper:" | wc -l)
170+
echo "error_count=${total_errors}" >> $GITHUB_OUTPUT
171+
echo "warning_count=${total_warnings}" >> $GITHUB_OUTPUT
172+
echo "has_issues=$((total_errors > 0 || total_warnings > 0))" >> $GITHUB_OUTPUT
173+
- name: Upload Build Reports
174+
if: always()
175+
uses: actions/upload-artifact@v4
176+
with:
177+
name: build-reports-${{ matrix.artifact_name }}-${{ matrix.dim }}d
178+
path: |
179+
${{runner.workspace}}/build-warnings.txt
180+
${{runner.workspace}}/build-errors.txt
181+
retention-days: 30
182+
183+
- name: Generate Error Summary
184+
if: always() && steps.report.outputs.has_issues == '1'
185+
run: |
186+
echo "## Build Report 📊" >> $GITHUB_STEP_SUMMARY
187+
echo "**Job Details:**" >> $GITHUB_STEP_SUMMARY
188+
echo "- App: ${{ matrix.app }}" >> $GITHUB_STEP_SUMMARY
189+
echo "- Dimensions: ${{ matrix.dim }}" >> $GITHUB_STEP_SUMMARY
190+
echo "- Commit: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
191+
echo "- Branch: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY
192+
echo "- [📁 Download Build Reports](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts)" >> $GITHUB_STEP_SUMMARY
193+
echo "" >> $GITHUB_STEP_SUMMARY
194+
195+
# Add warnings if they exist (show only last 10 lines in summary)
196+
if [ "${{ steps.report.outputs.warning_count }}" != "0" ] && [ -f "${{runner.workspace}}/build-warnings.txt" ]; then
197+
echo "## ⚠️ Warnings (${{ steps.report.outputs.warning_count }} total, showing last 10)" >> $GITHUB_STEP_SUMMARY
198+
echo '```' >> $GITHUB_STEP_SUMMARY
199+
tail -10 ${{runner.workspace}}/build-warnings.txt >> $GITHUB_STEP_SUMMARY
200+
echo '```' >> $GITHUB_STEP_SUMMARY
201+
echo "" >> $GITHUB_STEP_SUMMARY
202+
fi
79203
80-
#cmake --build build -j 2
204+
# Add errors if they exist (show only last 10 lines in summary)
205+
if [ "${{ steps.report.outputs.error_count }}" != "0" ] && [ -f "${{runner.workspace}}/build-errors.txt" ]; then
206+
echo "## 🚨 Errors (${{ steps.report.outputs.error_count }} total, showing last 10)" >> $GITHUB_STEP_SUMMARY
207+
echo '```' >> $GITHUB_STEP_SUMMARY
208+
tail -10 ${{runner.workspace}}/build-errors.txt >> $GITHUB_STEP_SUMMARY
209+
echo '```' >> $GITHUB_STEP_SUMMARY
210+
fi
211+
212+
- name: Fail if errors detected
213+
if: steps.report.outputs.error_count != '0'
214+
run: |
215+
echo "Build failed with ${{ steps.report.outputs.error_count }} errors"
216+
exit 1
217+
218+
merge-reports:
219+
runs-on: ubuntu-latest
220+
needs: tests-cuda11
221+
if: always()
222+
steps:
223+
- name: Merge Build Reports
224+
uses: actions/upload-artifact/merge@v4
225+
with:
226+
name: build-reports-all
227+
pattern: build-reports-*
228+
delete-merged: true # Optional: removes individual artifacts
229+
separate-directories: true
81230

82-
# # Build 3D libamrex cuda build with configure
231+
# Build 3D libamrex cuda build with configure
83232
# configure-3d-cuda:
84233
# name: CUDA@11.2 GNU@9.3.0 [configure 3D]
85234
# runs-on: ubuntu-20.04

‎exec/immersedIons/Checkpoint.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ void WriteCheckPoint(int step,
2525
const MultiFab& pres,
2626
const FhdParticleContainer& particles,
2727
const MultiFab& particleMeans,
28-
const MultiFab& particleVars,
28+
const MultiFab& /*particleVars*/,
2929
const MultiFab& chargeM,
3030
const MultiFab& potential,
3131
const MultiFab& potentialM)
@@ -398,7 +398,7 @@ void ReadCheckPoint(int& step,
398398
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "potentialM"));
399399
}
400400

401-
void ReadCheckPointParticles(FhdParticleContainer& particles, species* particleInfo, const Real* dxp) {
401+
void ReadCheckPointParticles(FhdParticleContainer& particles, species* /*particleInfo*/, const Real* /*dxp*/) {
402402

403403
// timer for profiling
404404
BL_PROFILE_VAR("ReadCheckPointParticles()",ReadCheckPointParticles);
@@ -457,7 +457,7 @@ void ReadCheckPointParticles(FhdParticleContainer& particles, species* particleI
457457
DistributionMapping dm { bc, ParallelDescriptor::NProcs() };
458458

459459
//set number of ghost cells to fit whole peskin kernel
460-
int ang = 1;
460+
[[maybe_unused]] int ang = 1;
461461
if(*(std::max_element(pkernel_fluid.begin(),pkernel_fluid.begin()+nspecies)) == 3) {
462462
ang = 2;
463463
}

‎exec/immersedIons/WritePlotFile.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
void WritePlotFile(int step,
88
const amrex::Real time,
9-
const amrex::Geometry geom,
9+
const amrex::Geometry /*geom*/,
1010
const amrex::Geometry cgeom,
1111
const amrex::Geometry egeom,
1212
const MultiFab& particleInstant,

‎exec/immersedIons/main_driver.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ void main_driver(const char* argv)
8686
tempang = 4;
8787
}
8888
else {
89-
tempang = floor(eskernel_fluid[i]/2)+1;
89+
tempang = static_cast<int>(floor(eskernel_fluid[i]/2))+1;
9090
}
9191

9292
if(tempang > ang)
@@ -743,7 +743,7 @@ void main_driver(const char* argv)
743743

744744
//int num_neighbor_cells = 4; replaced by input var
745745
//Particles! Build on geom & box array for collision cells/ poisson grid?
746-
double relRefine = particle_grid_refine/es_grid_refine;
746+
[[maybe_unused]] double relRefine = particle_grid_refine/es_grid_refine;
747747
Real max_es_range = 0;
748748
Real max_sr_range = 0;
749749
Real max_range = 0;
@@ -972,7 +972,7 @@ void main_driver(const char* argv)
972972

973973

974974

975-
particles.initRankLists(simParticles);
975+
particles.initRankLists(static_cast<long>(simParticles));
976976

977977
Real init_time = ParallelDescriptor::second() - strt_time;
978978
ParallelDescriptor::ReduceRealMax(init_time);
@@ -1365,7 +1365,7 @@ void main_driver(const char* argv)
13651365

13661366
if (es_tog==2) {
13671367
// compute pairwise Coulomb force (currently hard-coded to work with y-wall).
1368-
particles.computeForcesCoulombGPU(simParticles);
1368+
particles.computeForcesCoulombGPU(static_cast<long>(simParticles));
13691369
}
13701370

13711371
// compute other forces and spread to grid
@@ -1530,7 +1530,7 @@ void main_driver(const char* argv)
15301530
Real time_PC1 = ParallelDescriptor::second();
15311531

15321532
// compute g(r)
1533-
particles.RadialDistribution(simParticles, istep, ionParticle);
1533+
particles.RadialDistribution(static_cast<long>(simParticles), istep, ionParticle);
15341534
//particles.potentialDistribution(simParticles, istep, ionParticle);
15351535

15361536
// timer
@@ -1546,7 +1546,7 @@ void main_driver(const char* argv)
15461546
Real time_PC1 = ParallelDescriptor::second();
15471547

15481548
// compute g(x), g(y), g(z)
1549-
particles.CartesianDistribution(simParticles, istep, ionParticle);
1549+
particles.CartesianDistribution(static_cast<long>(simParticles), istep, ionParticle);
15501550

15511551
// timer
15521552
Real time_PC2 = ParallelDescriptor::second() - time_PC1;

‎exec/immersedIons/particleGen.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ void FhdParticleContainer::InitParticles(species* particleInfo, const Real* dxp)
2424
auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)];
2525

2626
//Assuming tile=box for now, i.e. no tiling.
27-
IntVect smallEnd = tile_box.smallEnd();
28-
IntVect bigEnd = tile_box.bigEnd();
27+
[[maybe_unused]] IntVect smallEnd = tile_box.smallEnd();
28+
[[maybe_unused]] IntVect bigEnd = tile_box.bigEnd();
2929

3030
if(ParallelDescriptor::MyProc() == 0 && mfi.LocalTileIndex() == 0 && proc0_enter) {
3131

@@ -179,7 +179,7 @@ void FhdParticleContainer::ReInitParticles()
179179
const int lev = 0;
180180
const Geometry& geom = Geom(lev);
181181

182-
int pcount = 0;
182+
[[maybe_unused]] int pcount = 0;
183183

184184
bool proc0_enter = true;
185185
int pinnedParticles = 0;
@@ -199,8 +199,8 @@ void FhdParticleContainer::ReInitParticles()
199199
auto* pstruct = aos().dataPtr();
200200

201201
//Assuming tile=box for now, i.e. no tiling.
202-
IntVect smallEnd = tile_box.smallEnd();
203-
IntVect bigEnd = tile_box.bigEnd();
202+
[[maybe_unused]] IntVect smallEnd = tile_box.smallEnd();
203+
[[maybe_unused]] IntVect bigEnd = tile_box.bigEnd();
204204

205205
if (ParallelDescriptor::MyProc() == 0 && mfi.LocalTileIndex() == 0 && proc0_enter) {
206206

0 commit comments

Comments
 (0)