Allow the parser to read a directory of files

.clippy.toml

@@ -1,2 +1,5 @@
 allow-unwrap-in-tests = true
 allow-expect-in-tests = true
+allow-indexing-slicing-in-tests = true
+allow-panic-in-tests = true
+allow-dbg-in-tests = true

Cargo.toml

@@ -68,10 +68,29 @@ expect-used = "deny"
 expect-fun-call = "deny"
 unwrap-in-result = "forbid"
 unwrap-or-default = "forbid"
+panic-in-result-fn = "deny"
 get-unwrap = "forbid"
 missing-errors-doc = "forbid"
 missing-panics-doc = "forbid"
 missing-safety-doc = "forbid"
+undocumented-unsafe-blocks = "forbid"
+unchecked-time-subtraction = "forbid"
+string-slice = "forbid"
+let-underscore-future = "forbid"
+let-underscore-must-use = "forbid"
+unused-result-ok = "forbid"
+map-err-ignore = "forbid"
+assertions-on-result-states = "forbid"
+await-holding-lock = "forbid"
+await-holding-refcell-ref = "forbid"
+if-let-mutex = "forbid"
+float-cmp = "forbid"
+float-cmp-const = "forbid"
+lossy-float-literal = "forbid"
+cast-sign-loss = "forbid"
+invalid-upcast-comparisons = "forbid"
+allow-attributes = "warn"
+allow-attributes-without-reason = "forbid"
 explicit-iter-loop = "deny"
 items-after-statements = "deny"
 ref-option-ref = "deny"
@@ -82,17 +101,22 @@ if-not-else = "deny"
 redundant-closure-for-method-calls = "deny"
 single-char-pattern = "deny"
 use-self = "deny"
+clone-on-ref-ptr = "deny"
+cast-possible-wrap = "warn"
+cast-precision-loss = "warn"
+cast-possible-truncation = "warn"
+large-futures = "warn"
+future-not-send = "warn"
+# cognitive-complexity = "warn"
 redundant-pub-crate = "deny"
 option-if-let-else = "allow"
 unnecessary-wraps = "allow"
-future-not-send = "warn"
-# cognitive-complexity = "warn"
 uninlined-format-args = "allow"
 wildcard-imports = "allow"
 too-many-lines = "allow"
 needless-raw-string-hashes = "allow"
 pedantic = { level = "warn", priority = -1 }
-nursery = { level = "warn", priority = -1 }
+nursery = { level = "warn", priority = -3 }
 
 # Needed for -Zminimal-versions to pass.
 # The dependency is not used in the project itself.

benches/parser.rs

@@ -1,20 +1,23 @@
-#![allow(clippy::expect_used, reason = "Sir, this is a benchmark")]
-#![allow(clippy::unwrap_used, reason = "Sir, this is a benchmark")]
+#![allow(
+    clippy::unwrap_used,
+    clippy::panic_in_result_fn,
+    clippy::expect_used,
+    reason = "Sir, this is a benchmark"
+)]
 
 use criterion::{Criterion, criterion_group, criterion_main};
 use packet_generator::{
     intermediate::DefinitionRegistry,
-    kdl_parser::{ParserOpts, ParsingError, ParsingWarnings},
+    kdl_parser::{ParserOpts, ParsingError, ParsingWarnings, UnparsedKdl},
     vfs::{InMemoryFS, VfsPath},
 };
 use std::path::PathBuf;
 
 fn parse_files(
-    main_content: &'static str,
-    path: &PathBuf,
+    unparsed_kdls: &[UnparsedKdl<'_>],
     parser_opts: &ParserOpts<InMemoryFS>,
 ) -> Result<(DefinitionRegistry, ParsingWarnings), ParsingError> {
-    let res = packet_generator::parse_kdl(main_content, path, parser_opts);
+    let res = packet_generator::parse_kdl(unparsed_kdls, parser_opts);
 
     assert!(
         res.is_ok(),
@@ -35,7 +38,7 @@ fn add_all_paths(prefix: &str, directory: &str, fs: &mut InMemoryFS) {
 
         let vfs_path = path.strip_prefix(prefix).expect("can remove assets prefix");
 
-        let _ = fs.add_file(VfsPath::new(vfs_path), &content);
+        let _file = fs.add_file(VfsPath::new(vfs_path), &content);
     }
 }
 
@@ -59,7 +62,7 @@ fn build_gamefrontier_input() -> (&'static str, PathBuf, ParserOpts<InMemoryFS>)
     add_all_paths("assets", "mst", &mut fs);
     add_all_paths("assets", "net", &mut fs);
 
-    let _ = fs.add_file(VfsPath::new("all.kdl"), main_content);
+    let _file = fs.add_file(VfsPath::new("all.kdl"), main_content);
 
     let opts = ParserOpts::new(fs);
 
@@ -74,7 +77,11 @@ fn criterion_benchmark(c: &mut Criterion) {
         group.bench_with_input(
             criterion::BenchmarkId::from_parameter("stress-test files"),
             &(main_content, path, &opts),
-            |b, input| b.iter_with_large_drop(|| parse_files(input.0, &input.1, input.2)),
+            |b, input| {
+                let unparsed_kdl = UnparsedKdl::new(input.0, &input.1);
+                let files = &[unparsed_kdl];
+                b.iter_with_large_drop(|| parse_files(files, input.2));
+            },
         );
     }
 
@@ -83,7 +90,11 @@ fn criterion_benchmark(c: &mut Criterion) {
         group.bench_with_input(
             criterion::BenchmarkId::from_parameter("Brave Frontier files"),
             &(main_content, path, &opts),
-            |b, input| b.iter_with_large_drop(|| parse_files(input.0, &input.1, input.2)),
+            |b, input| {
+                let unparsed_kdl = UnparsedKdl::new(input.0, &input.1);
+                let files = &[unparsed_kdl];
+                b.iter_with_large_drop(|| parse_files(files, input.2));
+            },
         );
     }
 

packet-generator-cli/Cargo.toml

@@ -15,6 +15,7 @@ atomicow = { workspace = true }
 miette = { workspace = true }
 itertools = { workspace = true }
 thiserror = { workspace = true }
+glob = { workspace = true }
 
 mimalloc = { version = "0.1.37", optional = true }
 

packet-generator-cli/src/lib.rs

@@ -0,0 +1,97 @@
+use std::path::Path;
+
+use itertools::Itertools;
+use miette::{Context, IntoDiagnostic};
+use packet_generator::kdl_parser::{ParserOpts, ParsingWarnings, UnparsedKdl, schema::RawDocument};
+
+/// Reads KDL in a directory for the CLI.
+///
+/// # Errors
+///
+/// Errors if we cannot read the files from the filesystem.
+pub fn read_all_kdls_from_directory(
+    input: &Path,
+) -> Result<(RawDocument, ParsingWarnings), miette::Report> {
+    let files_to_read = glob::glob(&format!("{}/**/*.kdl", input.display()))
+        .into_diagnostic()
+        .wrap_err_with(|| format!("error creating glob pattern for '{}'", input.display()))?;
+
+    let paths = files_to_read
+        .process_results(|maybe_paths| {
+            maybe_paths
+                .map(|p| -> Result<_, miette::Report> {
+                    let kdl_document_content = std::fs::read_to_string(&p)
+                        .into_diagnostic()
+                        .wrap_err("cannot read file")?;
+
+                    Ok(UnparsedKdl::new_owned(kdl_document_content, p))
+                })
+                .collect::<Result<Vec<_>, _>>()
+        })
+        .into_diagnostic()
+        .wrap_err("cannot read globs")??;
+
+    Ok(packet_generator::kdl_parser::raw_parse_kdl(
+        &paths,
+        &ParserOpts::default(),
+    )?)
+}
+
+#[cfg(test)]
+mod tests {
+    #![allow(clippy::panic_in_result_fn, reason = "Sir, this is a test")]
+    use packet_generator::intermediate::schema::Definition;
+
+    use super::*;
+    use std::path::PathBuf;
+
+    #[test]
+    fn read_directory_works() -> Result<(), miette::Report> {
+        let assets_dir = PathBuf::from_iter(&[env!("CARGO_MANIFEST_DIR"), "tests", "assets"]);
+
+        dbg!(&assets_dir);
+
+        let (document, warnings) = read_all_kdls_from_directory(&assets_dir)?;
+
+        if warnings.are_there_any() {
+            warnings.print_warnings_if_any();
+            miette::bail!("There were warnings");
+        }
+
+        let document = document.finalize()?;
+
+        let definitions = packet_generator::kdl_parser::document_to_definitions(document)?;
+
+        dbg!(&definitions);
+
+        let (foo, _) = definitions.find("Foo").wrap_err("Cannot find Foo")?;
+        let (bar, _) = definitions.find("Bar").wrap_err("Cannot find Bar")?;
+        let (baz, _) = definitions.find("Baz").wrap_err("Cannot find Baz")?;
+
+        assert_eq!(foo.name(), "Foo");
+        if let Definition::Json(j) = foo {
+            assert_eq!(j.fields.len(), 1);
+            assert!(j.fields.contains("bar"));
+        } else {
+            miette::bail!("Foo is not a JSON!")
+        }
+
+        assert_eq!(bar.name(), "Bar");
+        if let Definition::Json(j) = bar {
+            assert_eq!(j.fields.len(), 1);
+            assert!(j.fields.contains("baz"));
+        } else {
+            miette::bail!("Bar is not a JSON!")
+        }
+
+        assert_eq!(baz.name(), "Baz");
+        if let Definition::StringEnum(j) = baz {
+            assert_eq!(j.variants.len(), 1);
+            assert!(j.variants.contains("test"));
+        } else {
+            miette::bail!("Baz is not a string enum!")
+        }
+
+        Ok(())
+    }
+}

packet-generator-cli/src/main.rs

@@ -6,44 +6,35 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 use std::{env::current_dir, path::PathBuf};
 
-use miette::{Context, miette};
+use miette::{Context, IntoDiagnostic, miette};
 use packet_generator::generators::write_sources;
+use packet_generator::kdl_parser::UnparsedKdl;
 use packet_generator::{
-    generators::{CxxGenerator, GenerationError, Generator, GlazeGenerator, WithAddons},
-    kdl_parser::{Diagnostic, ParserOpts, ParsingError},
+    generators::{CxxGenerator, Generator, GlazeGenerator, WithAddons},
+    kdl_parser::ParserOpts,
 };
 
+use packet_generator_cli::read_all_kdls_from_directory;
+
 use crate::cli::CxxSerializer;
 
 mod cli;
 
-#[derive(Debug, thiserror::Error)]
-#[expect(dead_code)]
-enum ApplicationError {
-    #[error(transparent)]
-    MietteReport(#[from] ParsingError),
-
-    #[error(transparent)]
-    Diagnostic(#[from] Diagnostic),
-
-    #[error(transparent)]
-    Generation(#[from] GenerationError),
-}
-
 fn main() -> Result<(), miette::Report> {
     let args = cli::parse_args();
 
     match args {
         cli::CliArgs::DumpRepresentation { input } => {
             let path = PathBuf::from(input);
 
-            let doc_str = std::fs::read_to_string(&path)
+            let kdl_document_content = std::fs::read_to_string(&path)
                 .map_err(|e| miette::miette!(e))
                 .wrap_err_with(|| format!("cannot open file: {}", path.display()))?;
 
+            let unparsed_document = UnparsedKdl::new(&kdl_document_content, &path);
+
             let (doc, warnings) = packet_generator::kdl_parser::raw_parse_kdl(
-                doc_str,
-                &path,
+                &[unparsed_document],
                 &ParserOpts::default(),
             )?;
 
@@ -68,6 +59,28 @@ fn main() -> Result<(), miette::Report> {
                 None => current_dir().map_err(|e| miette::miette!(e))?,
             };
 
+            let input_path = PathBuf::from(&input);
+
+            let (doc, warnings) = if input_path.is_dir() {
+                read_all_kdls_from_directory(&input_path)?
+            } else {
+                let kdl_document_content = std::fs::read_to_string(&input_path)
+                    .map_err(|e| miette::miette!(e))
+                    .wrap_err_with(|| format!("cannot open file: {}", input_path.display()))?;
+
+                let unparsed_document = UnparsedKdl::new(&kdl_document_content, &input_path);
+                packet_generator::kdl_parser::raw_parse_kdl(
+                    &[unparsed_document],
+                    &ParserOpts::default(),
+                )?
+            };
+
+            warnings.print_warnings_if_any();
+
+            let doc = doc.finalize()?;
+
+            let definitions = packet_generator::kdl_parser::document_to_definitions(doc)?;
+
             let generator = match language {
                 cli::ProgrammingLanguage::Cxx(options) => {
                     let mut cxx_generator = CxxGenerator::new();
@@ -79,7 +92,7 @@ fn main() -> Result<(), miette::Report> {
 
                         CxxSerializer::Simdjson => {
                             return Err(miette::miette!(
-                                "Simdjson secondary generator for Cxx is not implemented!"
+                                "Simdjson generator for C++ is not implemented!"
                             ));
                         }
                     }
@@ -94,24 +107,6 @@ fn main() -> Result<(), miette::Report> {
                 }
             };
 
-            let input_path = PathBuf::from(input);
-
-            let doc_str = std::fs::read_to_string(&input_path)
-                .map_err(|e| miette::miette!(e))
-                .wrap_err_with(|| format!("cannot open file: {}", input_path.display()))?;
-
-            let (doc, warnings) = packet_generator::kdl_parser::raw_parse_kdl(
-                doc_str,
-                &input_path,
-                &ParserOpts::default(),
-            )?;
-
-            warnings.print_warnings_if_any();
-
-            let doc = doc.finalize()?;
-
-            let definitions = packet_generator::kdl_parser::document_to_definitions(doc)?;
-
             let sources = generator
                 .generate(
                     &definitions,
@@ -131,7 +126,8 @@ fn main() -> Result<(), miette::Report> {
                             )
                         })?,
                 )
-                .map_err(|e| miette::miette!("could not generate sources: {e}"))?;
+                .into_diagnostic()
+                .wrap_err("could not generate sources")?;
 
             write_sources(&output_directory, &sources)?;
         }

packet-generator-cli/tests/assets/foo.kdl

@@ -0,0 +1,10 @@
+// No explicit include is present.
+
+json Foo {
+    doc "Contains a Bar"
+
+    field bar type="Bar" {
+        key "bar"
+        doc "The Bar"
+    }
+}

packet-generator-cli/tests/assets/nested/bar.kdl

@@ -0,0 +1,8 @@
+json Bar {
+    doc "Contains a Baz"
+
+    field baz type="Baz" {
+        key "baz"
+        doc "The Baz"
+    }
+}