Merge pull request #1307 from EnergySystemsModellingLab/debug-test-output

Generate debug output for all regression tests

Author: tsmbland

tests/cli.rs

@@ -109,4 +109,14 @@ fn check_example_extract_command(#[case] patch: bool) {
     );
 }
 
-// NB: `example run` is covered by regression tests
+/// Test the `example run` command with no additional flags
+#[test]
+fn check_example_run_command() {
+    let tmp = tempdir().unwrap();
+    let output_dir = tmp.path().join("out");
+    let output_dir_str = output_dir.to_string_lossy();
+
+    assert_muse2_runs(&["example", "run", "simple", "--output-dir", &output_dir_str]);
+}
+
+// NB: `example run` extra flags are covered by regression tests

tests/common.rs

@@ -29,7 +29,13 @@ macro_rules! define_regression_test_with_extra_args {
     ($example:ident, $extra_args:expr) => {
         #[test]
         fn $example() {
-            run_regression_test(stringify!($example), $extra_args);
+            run_regression_test(stringify!($example), $extra_args, false);
+        }
+    };
+    ($example:ident, $extra_args:expr, $debug_model:expr) => {
+        #[test]
+        fn $example() {
+            run_regression_test(stringify!($example), $extra_args, $debug_model);
         }
     };
 }
@@ -50,7 +56,7 @@ pub(crate) use define_regression_test;
 #[allow(unused_macros)]
 macro_rules! define_regression_test_with_debug_files {
     ($example:ident) => {
-        define_regression_test_with_extra_args!($example, &["--debug-model"]);
+        define_regression_test_with_extra_args!($example, &[], true);
     };
 }
 #[allow(unused_imports)]

tests/regression.rs

@@ -37,8 +37,11 @@ define_regression_test_with_patches!(simple_ironing_out);
 /// The tolerance when comparing floating-point values in CSV files
 const FLOAT_CMP_TOLERANCE: f64 = 1e-10;
 
-/// Run a regression test for the given example with optional extra arguments to `muse2 run`.
-fn run_regression_test(example: &str, extra_args: &[&str]) {
+/// Run a regression test for the given example with optional extra arguments to `muse2 example run`.
+///
+/// The `--debug-model` flag is always used so the debug files are available to examine. The debug
+/// files are only tested when the `debug_model` parameter is true.
+fn run_regression_test(example: &str, extra_args: &[&str], debug_model: bool) {
     // Allow user to set output dir for regression tests so they can examine results. This is
     // principally intended for use by CI.
     let tmp: TempDir;
@@ -51,17 +54,20 @@ fn run_regression_test(example: &str, extra_args: &[&str]) {
 
     // Invoke muse2
     let output_dir_str = output_dir.to_string_lossy();
-    let mut args = vec!["example", "run", example, "--output-dir", &output_dir_str];
+    let mut args = vec![
+        "example",
+        "run",
+        example,
+        "--debug-model",
+        "--output-dir",
+        &output_dir_str,
+    ];
     args.extend(extra_args);
     assert_muse2_runs(&args);
 
     // Check that the output files match (approximately)
     let test_data_dir = PathBuf::from(format!("tests/data/{example}"));
-    compare_output_dirs(
-        &output_dir,
-        &test_data_dir,
-        extra_args.contains(&"--debug-model"),
-    );
+    compare_output_dirs(&output_dir, &test_data_dir, debug_model);
 }
 
 fn compare_output_dirs(cur_output_dir1: &Path, test_data_dir: &Path, debug_model: bool) {