kitral regression tests

.github/workflows/build-debian-stable.yml

@@ -127,7 +127,8 @@ jobs:
           cd Cell2Fire
           make tests
 
-      - name: Bash Test
+      - name: Bash Tests
         run: |
           cd test
-          bash test.sh ${{ env.SUFFIX }}
+          bash test.sh ${{ env.SUFFIX }}
+          bash kitral_regression.sh ${{ env.SUFFIX }}

.github/workflows/build-macos.yml

@@ -140,4 +140,5 @@ jobs:
 #      - name: Bash Test
 #        run: |
 #          cd test
-#          bash test.sh _${{ env.suffix }}
+#          bash test.sh _${{ env.suffix }}
+#          bash kitral_regression.sh ${{ env.SUFFIX }}

.github/workflows/build-manylinux.yml

@@ -137,6 +137,7 @@ jobs:
         run: |
           cd test
           bash test.sh
+          bash kitral_regression.sh
 
       - name: Get Environment
         if: ${{ env.do_dynamic == 'true' }}

.github/workflows/build-windows.yml

@@ -148,6 +148,7 @@ jobs:
       run: |
         cd test
         ./test.ps1
+        ./kitral_regression.ps1
       shell: pwsh
 
     - name: Gather output

data/Kitral/biobio/Weather.csv

@@ -0,0 +1,13 @@
+Scenario,datetime,WS,WD,TM,RH
+DMC 1,2022-12-21 16:00:00,16.6,360,24.7,20
+DMC 1,2022-12-21 17:00:00,18.8,360,24.6,20
+DMC 1,2022-12-21 18:00:00,12.1,360,23.6,20
+DMC 1,2022-12-21 19:00:00,10.4,360,22.1,20
+DMC 1,2022-12-21 20:00:00,5.2,360,19.4,20
+DMC 1,2022-12-21 21:00:00,5.5,360,17.9,20
+DMC 1,2022-12-21 22:00:00,8.2,360,16.4,20
+DMC 1,2022-12-21 23:00:00,12.7,360,15.6,20
+DMC 1,2022-12-22 00:00:00,15.7,360,15.3,20
+DMC 1,2022-12-22 01:00:00,15.7,360,14.8,20
+DMC 1,2022-12-22 02:00:00,18.9,360,14.7,20
+DMC 1,2022-12-22 03:00:00,16.9,360,14.6,20

data/Kitral/biobio/kitral_lookup_table.csv

@@ -0,0 +1,38 @@
+grid_value,export_value,descriptive_name,fuel_type,r,g,b,h,s,l
+1,1,Pastizales Mesomorficos Densos,PCH1,209,255,115,57,255,185
+2,2,Pastizales Mesoomorficos Ralos,PCH2,34,102,51,95,128,68
+3,3,Pastizales Higromorficos Densos,PCH3,131,199,149,96,96,165
+4,4,Pastizales Higromorficos Ralos,PCH4,112,168,0,57,255,84
+5,5,Chacareria. Vinedos y Frutales,PCH5,223,184,230,206,122,207
+6,6,Matorrales y Arbustos Mesomorficos Densos,MT01,172,102,237,192,201,170
+7,7,Matorrales y Arbustos Mesomorficos Medios y Ralos,MT02,112,12,242,188,231,127
+8,8,Matorrales y Arbustos Higromorficos Densos,MT03,196,189,151,35,70,174
+9,9,Matorrales y Arbustos Higromorificos Medios y Ralos,MT04,137,112,68,27,86,103
+10,10,Formaciones con predominancia de Chuesquea spp,MT05,196,189,151,35,70,174
+11,11,Formaciones con predominancia de Ulex spp,MT06,251,190,185,3,227,218
+12,12,Renovales Nativos diferentes al Tipo Siempreverde,MT07,247,104,161,-272,229,176
+13,13,Renovales Nativos del Tipo Siempreverde,MT08,174,1,126,-285,252,88
+14,14,Formaciones con predominancia de Alerzales,BN01,255,255,190,42,255,223
+15,15,Formaciones con predominancia de Araucaria,BN02,230,230,0,42,255,115
+16,16,Arbolado Nativo Denso,BN03,255,211,127,28,255,191
+17,17,Arbolado Nativo de Densidad Media,BN04,255,170,0,28,255,128
+18,18,Arbolado Nativo de Densidad Baja,BN05,255,211,127,28,255,191
+19,19,Plantaciones Coniferas Nuevas (0-3) sin Manejo,PL01,99,0,0,0,255,50
+20,20,Plantaciones Coniferas Jovenes (4-11) sin Manejo,PL02,99,0,0,0,255,50
+21,21,Plantaciones Coniferas Adultas (12-17) sin Manejo,PL03,99,0,0,0,255,50
+22,22,Plantaciones Coniferas Mayores (>17) sin Manejo,PL04,99,0,0,0,255,50
+23,23,Plantaciones Coniferas Jovenes (4-11) con Manejo,PL05,99,0,0,0,255,50
+24,24,Plantaciones Coniferas Adultas (12-17) con Manejo,PL06,99,0,0,0,255,50
+25,25,Plantaciones Coniferas Mayores (>17) sin Manejo,PL07,99,0,0,0,255,50
+26,26,Plantaciones Eucaliptos Nuevas (0-3),PL08,99,0,0,0,255,50
+27,27,Plantaciones Eucaliptos Jovenes (4-10),PL09,99,0,0,0,255,50
+28,28,Plantaciones Eucalipto Adultas (>10),PL10,99,0,0,0,255,50
+29,29,Plantaciones Latifoliadas y Mixtas,PL11,99,0,0,0,255,50
+30,30,Desechos Explotacion a Tala Rasa de Plantaciones,DX01,255,211,127,28,255,191
+31,31,Desechos Explotacion a Tala Rasa de Bosque Nativo,DX02,255,211,127,28,255,191
+100,100,Not Available,Non-fuel,255,255,255,170,0,255
+101,101,Non-fuel,Non-fuel,130,130,130,170,0,130
+102,102,Water,Non-fuel,115,223,255,138,255,185
+103,103,Unknown,Non-fuel,0,0,0,170,0,0
+104,104,Unclassified,Non-fuel,166,166,166,170,0,166
+105,105,Vegetated Non-Fuel,Non-fuel,204,204,204,170,0,204

test/kitral_regression.ps1

@@ -0,0 +1,77 @@
+# unzip target results
+Expand-Archive -Path kitral_target_results.zip -DestinationPath "."
+
+# add Cell2Fire to PATH
+$env:PATH = "..\Cell2Fire;$env:PATH"
+
+# define runs: name|input_folder|extra_args
+$runs = @(
+    @{ name = "portillo"; input = "..\data\Kitral\Portillo-asc\"; extra = @("--nsims", "1", "--seed", "24", "--fmc", "50", "--ignitions", "--cros", "--out-crown", "--out-cfb") }
+    @{ name = "villarrica"; input = "..\data\Kitral\Villarrica-tif\"; extra = @("--nsims", "10", "--seed", "42", "--fmc", "66", "--FirebreakCells", "..\data\Kitral\Villarrica-tif\harvested_Random.csv") }
+    #@{ name = "biobio"; input = "..\data\Kitral\biobio\"; extra = @("--nsims", "3", "--seed", "5", "--fmc", "50", "--cros", "--out-crown", "--out-cfb") }
+    @{ name = "portezuelo"; input = "..\data\Kitral\Portillo-asc\"; extra = @("--nsims", "1", "--seed", "5", "--fmc", "50", "--ignitions") }
+)
+
+foreach ($run in $runs) {
+    $name = $run.name
+    $inputFolder = $run.input
+    $extraArgs = $run.extra
+    Write-Host "Running simulation: $name"
+
+    $outputFolder = "kitral_test_results\$name"
+    New-Item -ItemType Directory -Force -Path $outputFolder | Out-Null
+
+    $logFile = "$outputFolder/log.txt"
+    # enable debug tracing
+    Write-Host "Executing Cell2Fire..."
+    & .\..\Cell2Fire\x64\Release\Cell2Fire.exe --input-instance-folder $inputFolder --output-folder $outputFolder --output-messages --out-fl --out-intensity --ignitionsLog --sim K --weather rows --grids $extraArgs *> $logFile 2>$null
+    (Get-Content $logFile) -replace '\\', '/' | Set-Content $logFile
+    (Get-Content $logFile | Select-String -pattern 'version:' -notmatch) | Set-Content $logFile
+
+    # compare number of files
+    $target = "kitral_target_results\$name"
+    # Recursively get all files in each directory
+$dir1_files = Get-ChildItem -Path $target -Recurse -File
+$dir2_files = Get-ChildItem -Path $outputFolder -Recurse -File
+
+# Count and compare number of files
+$dir1_num_files = $dir1_files.Count
+$dir2_num_files = $dir2_files.Count
+
+if ($dir1_num_files -ne $dir2_num_files) {
+    Write-Output " Directories have different file counts:"
+    Write-Output "  ${target}: ${dir1_num_files} files"
+    Write-Output "  ${outputFolder}: ${dir2_num_files} files"
+    exit 1
+}
+
+# Compare file contents
+foreach ($file1 in $dir1_files) {
+    $file2_path = $file1.FullName.Replace($target, $outputFolder)
+
+    if (-not (Test-Path $file2_path)) {
+        Write-Output "Missing file in output: $file2_path"
+        exit 1
+    }
+
+    $diff = Compare-Object -ReferenceObject @((Get-Content $file1.FullName)|Select-Object) -DifferenceObject @((Get-Content $file2_path)|Select-Object)
+    if ($diff) {
+        Write-Output " Files differ: $($file1.FullName)"
+        Write-Output $diff
+        Write-Output "`nIf this is expected, you can update targets with:"
+        Write-Output "Remove-Item -Recurse -Force kitral_target_results"
+        Write-Output "Rename-Item test_results kitral_target_results"
+        Write-Output "Compress-Archive kitral_target_results kitral_target_results.zip -Force"
+        exit 1
+    }
+}
+
+
+    # optionally clean up after each run
+    Remove-Item -Recurse -Force $outputFolder
+}
+
+# clean up target results
+Remove-Item -Recurse -Force kitral_target_results
+
+exit 0

test/kitral_regression.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# generalized regression test script for multiple Cell2Fire runs
+
+# unzip target results
+unzip -q kitral_target_results.zip
+
+# add Cell2Fire to PATH
+PATH=../Cell2Fire:$PATH
+
+# define runs: name|input_folder|extra_args
+runs=(
+    "portillo|../data/Kitral/Portillo-asc/|--nsims 1 --seed 24 --fmc 50 --ignitions --cros --out-crown --out-cfb"
+    "villarrica|../data/Kitral/Villarrica-tif/|--nsims 10 --seed 42 --fmc 66 --FirebreakCells ../data/Kitral/Villarrica-tif/harvested_Random.csv"
+    #"biobio|../data/Kitral/biobio/|--nsims 3 --seed 5 --fmc 50 --cros --out-crown --out-cfb"
+    "portezuelo|../data/Kitral/Portezuelo-asc/|--nsims 1 --seed 5 --fmc 50 --ignitions"
+)
+
+# loop over each run
+for run in "${runs[@]}"; do
+    IFS='|' read -r name input_folder extra_args <<< "$run"
+    echo "Running simulation: $name"
+
+    output_folder=kitral_test_results/$name
+    mkdir -p "$output_folder"
+
+    set -x # enable debug tracing
+    Cell2Fire$1 --input-instance-folder "$input_folder" --output-folder "$output_folder" --grids --output-messages --out-fl --out-intensity --ignitionsLog --sim K --weather rows $extra_args > "$output_folder/log.txt"
+    set +x # disable debug tracing
+
+    # compare number of files
+    target="kitral_target_results/$name"
+    target_files=$(find "$target" -type f | sort)
+    target_num_files=$(echo "$target_files" | wc -l)
+    output_files=$(find "$output_folder" -type f | sort)
+    output_num_files=$(echo "$output_files" | wc -l)
+
+    if [ "$target_num_files" -ne "$output_num_files" ]; then
+        echo "Mismatch in file count for $name"
+        echo "Target: $target_num_files files"
+        echo "Output: $output_num_files files"
+        exit 1
+    fi
+
+    # clean version lines from logs
+    find "$output_folder" -name log.txt -type f | xargs sed -i -e '/version:/d'
+
+    # diff directories
+    diff_output=$(diff -rq "$target" "$output_folder")
+    if [ -z "$diff_output" ]; then
+        echo "$name: Directories are equal"
+    else
+        echo "$name: Differences found"
+        # detailed per-file check
+        for file1 in $(find "$target" -type f); do
+            file2=$(echo "$file1" | sed "s|$target|$output_folder|")
+            diff_result=$(diff "$file1" "$file2")
+            if [ -n "$diff_result" ]; then
+                echo "Difference in file: $file1"
+                echo "$diff_result"
+                exit 1
+            fi
+        done
+    fi
+
+    # optionally clean up after each run
+    rm -r "$output_folder"
+done
+
+# clean up target results
+rm -r kitral_target_results
+
+exit 0