enhancement: Add support for arrays to templated_json payload generator (#1799)

Author: bruceg

examples/lading-templated.yaml

@@ -0,0 +1,18 @@
+generator:
+  - file_gen:
+      traditional:
+        seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
+               59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
+        path_template: "/path/to/logs/part.%NNN%.json"
+        rotate: true
+        duplicates: 3
+        bytes_per_second: 100MiB
+        variant:
+          templated_json:
+            template_path: "examples/templates/templated_json_features.yaml"
+        maximum_bytes_per_file: 100MiB
+        maximum_prebuild_cache_size_bytes: 1GiB
+
+blackhole:
+  - tcp:
+      binding_addr: "0.0.0.0:8080"

examples/templates/templated_json_builtin.yaml

@@ -0,0 +1,18 @@
+# Approximate equivalent of the built-in Json generator.
+#
+# The template system cannot produce byte-identical output because:
+#
+#   - id and name are u64, but !range uses i64, restricting the
+#     upper bound to i64::MAX (roughly half the u64 range).
+#
+#   - !object emits fields alphabetically (BTreeMap); Json serializes in
+#     declaration order (id, name, seed, byte_parade).
+
+generator:
+  !object
+    id: !range { min: 0, max: 9223372036854775807 }
+    name: !range { min: 0, max: 9223372036854775807 }
+    seed: !range { min: 0, max: 65535 }
+    byte_parade: !array
+      length: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]
+      element: !range { min: 0, max: 255 }

…ples/templates/bench_templated_json.yaml …s/templates/templated_json_features.yamlexamples/templates/bench_templated_json.yaml renamed to examples/templates/templated_json_features.yaml examples/templates/bench_templated_json.yaml renamed to examples/templates/templated_json_features.yaml

@@ -12,6 +12,7 @@
 #   !weighted   - probability-weighted random selection
 #   !range      - random integer sampled from the closed interval [min, max]
 #   !format     - format string; {} placeholders filled by sub-generators
+#   !array      - generate a JSON array with generator-valued elements
 #   !object     - generate a JSON object with generator-valued fields
 #   !with       - bind one or more generated values into a local scope
 #   !var        - read back a value that was bound by an enclosing !with
@@ -71,6 +72,9 @@ definitions:
   request_id_suffix:
     !range { min: 100000, max: 999999 }
 
+  flag_words:
+    !choose ["one", "two", "three", "four", "five", "six", "seven", "eight"]
+
 generator:
   !with
     bind:
@@ -121,6 +125,16 @@ generator:
             - !var svc
             - !reference request_id_suffix
 
+      flags:
+        !array
+          length: { min: 0, max: 3 }
+          element: !reference flag_words
+
+      bytes:
+        !array
+          length: { min: 1, max: 20 }
+          element: !range { min: 0, max: 255 }
+
       # Nested !with: introduces a new binding (env) visible only inside its body
       metadata:
         !with
@@ -136,3 +150,4 @@ generator:
                 args:
                   - !var pod
                   - !var env
+

lading_payload/README.templated_json.md

@@ -70,6 +70,7 @@ The config DSL in `opw_tools::json_generator` supports these YAML tags:
 - `!with`: bind generated values to variables, then evaluate `in`
 - `!var`: read a bound variable
 - `!timestamp`: deterministic monotonically-advancing UTC timestamp (whole-second RFC-3339)
+- `!array`: generate a JSON array; element count is fixed, a uniform random range, or chosen from a set
 
 ## Example template
 
@@ -115,6 +116,11 @@ generator:
           args:
             - !var svc
             - !range { min: 1, max: 2000 }
+      data: !array
+        length: { min: 4, max: 32 }
+        # or: length: 3
+        # or: length: [1, 2, 4, 8, 16]
+        element: !range { min: 0, max: 255 }
 ```
 
 This produces newline-delimited JSON like:
@@ -137,3 +143,4 @@ For future work:
 
 - Strings of bounded random length range pulled from configurable text generators (hexadecimal,
   chosen words, etc).
+- Timestamps with sub-second fractions, different increment ranges, other configurations.

lading_payload/benches/templated_json.rs

@@ -18,23 +18,26 @@ use rand::{SeedableRng, rngs::SmallRng};
 
 const MIB: usize = 1_048_576;
 
-fn template_path() -> &'static Path {
-    Path::new(concat!(
-        env!("CARGO_MANIFEST_DIR"),
-        "/../examples/templates/bench_templated_json.yaml"
-    ))
-}
+const TEMPLATE_BUILTIN_PATH: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/../examples/templates/templated_json_builtin.yaml"
+);
+
+const TEMPLATE_FEATURES_PATH: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/../examples/templates/templated_json_features.yaml"
+);
 
 fn templated_json_setup(c: &mut Criterion) {
-    let path = template_path();
+    let path = Path::new(TEMPLATE_FEATURES_PATH);
     c.bench_function("templated_json_setup", |b| {
         b.iter(|| TemplatedJson::from_path(path).expect("failed to load template"));
     });
 }
 
-fn templated_json_throughput(c: &mut Criterion) {
-    let path = template_path();
-    let mut group = c.benchmark_group("templated_json_throughput");
+fn templated_json_throughput(c: &mut Criterion, path: &str, group_name: &str) {
+    let path = Path::new(path);
+    let mut group = c.benchmark_group(group_name);
     for size in &[MIB, 10 * MIB, 100 * MIB] {
         group.throughput(Throughput::Bytes(*size as u64));
         group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| {
@@ -55,6 +58,22 @@ fn templated_json_throughput(c: &mut Criterion) {
     group.finish();
 }
 
+fn templated_json_builtin_throughput(c: &mut Criterion) {
+    templated_json_throughput(
+        c,
+        TEMPLATE_BUILTIN_PATH,
+        "templated_json_builtin_throughput",
+    );
+}
+
+fn templated_json_features_throughput(c: &mut Criterion) {
+    templated_json_throughput(
+        c,
+        TEMPLATE_FEATURES_PATH,
+        "templated_json_features_throughput",
+    );
+}
+
 criterion_group!(
     name = setup_benches;
     config = Criterion::default()
@@ -68,7 +87,7 @@ criterion_group!(
     config = Criterion::default()
         .measurement_time(Duration::from_secs(30))
         .warm_up_time(Duration::from_secs(1));
-    targets = templated_json_throughput,
+    targets = templated_json_builtin_throughput, templated_json_features_throughput,
 );
 
 criterion_main!(setup_benches, throughput_benches);

lading_payload/src/templated_json/config.rs

@@ -136,6 +136,95 @@ pub(super) enum Generator {
     /// !timestamp
     /// ```
     Timestamp,
+
+    /// Generate a JSON array whose elements are drawn from a sub-generator.
+    ///
+    /// The number of elements is controlled by the `length` field, which
+    /// accepts three forms:
+    ///
+    /// - A plain integer: every array has exactly that many elements.
+    /// - A range mapping `{min: N, max: M}`: the count is drawn uniformly
+    ///   from `[N, M]` (inclusive) on each call.
+    /// - A sequence of integers `[N, M, ...]`: one length is chosen uniformly
+    ///   at random from the list on each call.
+    ///
+    /// ```yaml
+    /// # Fixed length
+    /// !array
+    ///   length: 3
+    ///   element: !range { min: 1, max: 100 }
+    ///
+    /// # Uniform random length
+    /// !array
+    ///   length: { min: 1, max: 5 }
+    ///   element: !const "hello"
+    ///
+    /// # Choose length from a set
+    /// !array
+    ///   length: [2, 3, 5]
+    ///   element: !reference my_def
+    /// ```
+    Array(ArraySpec),
+}
+
+// ── Array types ───────────────────────────────────────────────────────────────
+
+/// Determines how many elements a `!array` generator produces per call.
+///
+/// Deserializes from three YAML forms:
+/// - scalar integer   → `Fixed`
+/// - sequence         → `Choose`
+/// - mapping          → `Range`
+#[derive(Clone)]
+pub(super) enum ArrayLength {
+    /// Always produce exactly `n` elements.
+    Fixed(usize),
+    /// Draw the count uniformly from `[min, max]` inclusive.
+    Range { min: usize, max: usize },
+    /// Choose the count uniformly from the given list of values.
+    Choose(Vec<usize>),
+}
+
+impl<'de> Deserialize<'de> for ArrayLength {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        // Untagged: integer -> Fixed, sequence -> Choose, mapping -> Range.
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum RawArrayLength {
+            Fixed(usize),
+            Choose(Vec<usize>),
+            Range { min: usize, max: usize },
+        }
+        match RawArrayLength::deserialize(deserializer)? {
+            RawArrayLength::Fixed(n) => Ok(Self::Fixed(n)),
+            RawArrayLength::Choose(choices) => {
+                if choices.is_empty() {
+                    return Err(<D::Error as serde::de::Error>::custom(
+                        "!array length list must not be empty",
+                    ));
+                }
+                Ok(Self::Choose(choices))
+            }
+            RawArrayLength::Range { min, max } => {
+                if min > max {
+                    return Err(<D::Error as serde::de::Error>::custom(format!(
+                        "!array length range min ({min}) must be <= max ({max})"
+                    )));
+                }
+                Ok(Self::Range { min, max })
+            }
+        }
+    }
+}
+
+/// Configuration for a `!array` generator node.
+#[derive(Clone, Deserialize)]
+pub(super) struct ArraySpec {
+    pub length: ArrayLength,
+    pub element: Box<Generator>,
 }
 
 /// Return the JSON string body encoding of `s`: the bytes that would appear
@@ -231,11 +320,11 @@ impl<'de> Deserialize<'de> for RangeSpec {
     {
         #[derive(Deserialize)]
         #[serde(deny_unknown_fields)]
-        struct Raw {
+        struct RawRangeSpec {
             min: i64,
             max: i64,
         }
-        let raw = Raw::deserialize(deserializer)?;
+        let raw = RawRangeSpec::deserialize(deserializer)?;
         if raw.min > raw.max {
             return Err(<D::Error as serde::de::Error>::custom(format!(
                 "!range min ({}) must be <= max ({})",
@@ -279,11 +368,11 @@ impl<'de> Deserialize<'de> for FormatSpec {
     {
         #[derive(Deserialize)]
         #[serde(deny_unknown_fields)]
-        struct Raw {
+        struct RawFormatSpec {
             template: String,
             args: Vec<Generator>,
         }
-        let raw = Raw::deserialize(deserializer)?;
+        let raw = RawFormatSpec::deserialize(deserializer)?;
 
         // Split at each `{}` placeholder to get N+1 literal parts.
         let parts: Vec<&str> = raw.template.split("{}").collect();

lading_payload/src/templated_json/generator.rs

@@ -48,6 +48,7 @@ pub(super) enum Generator {
     /// Index into the context vec.
     Var(usize),
     Timestamp(Timestamp),
+    Array(ArraySpec),
 }
 
 pub(super) struct RangeSpec {
@@ -103,6 +104,19 @@ impl WithSpec {
     }
 }
 
+/// How many elements a `!array` generator produces per call.
+pub(super) enum ArrayLength {
+    Fixed(usize),
+    Range { min: usize, max: usize },
+    Choose(Vec<usize>),
+}
+
+/// Resolved `!array` generator node.
+pub(super) struct ArraySpec {
+    pub(super) length: ArrayLength,
+    pub(super) element: Box<Generator>,
+}
+
 /// State for a single `!timestamp` generator node.
 ///
 /// Wraps a `Cell<Option<NonZeroI64>>` holding the current value in
@@ -200,6 +214,8 @@ impl Generator {
             }
 
             Self::Timestamp(ts) => ts.generate(rng, out)?,
+
+            Self::Array(spec) => spec.generate(rng, ctx, defs, out)?,
         }
         Ok(())
     }
@@ -291,6 +307,48 @@ impl FormatSpec {
     }
 }
 
+impl ArraySpec {
+    /// Generate a JSON array into `out`.
+    ///
+    /// The element count is determined by `self.length`; then `self.element`
+    /// is called once per slot.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the length list is empty or if any element
+    /// generator fails.
+    pub(super) fn generate(
+        &self,
+        rng: &mut impl Rng,
+        ctx: &mut Context,
+        defs: &[Generator],
+        out: &mut JsonString,
+    ) -> Result<(), Error> {
+        let count = self.length.choose(rng)?;
+        out.push_char('[');
+        for i in 0..count {
+            if i > 0 {
+                out.push_char(',');
+            }
+            self.element.generate(rng, ctx, defs, out)?;
+        }
+        out.push_char(']');
+        Ok(())
+    }
+}
+
+impl ArrayLength {
+    fn choose(&self, rng: &mut impl Rng) -> Result<usize, Error> {
+        Ok(match self {
+            Self::Fixed(n) => *n,
+            Self::Range { min, max } => rng.random_range(*min..=*max),
+            Self::Choose(choices) => *choices
+                .choose(rng)
+                .ok_or_else(|| Error::TemplateError("!array length list is empty".to_string()))?,
+        })
+    }
+}
+
 impl WithSpec {
     /// Evaluate this `!with` block into `out`.
     ///