subsys: framed byte streams (SPEC-015 §7, OBS-019)

.github/workflows/ci.yml

@@ -358,7 +358,7 @@ jobs:
           packages: clang-format-18 python3-yaml
       - name: clang-format
         run: |
-          find include kernel arch boards soc drivers samples tests \( -name '*.c' -o -name '*.h' \) -print0 \
+          find include kernel arch boards soc drivers subsys samples tests \( -name '*.c' -o -name '*.h' \) -print0 \
             | xargs -0 clang-format-18 --dry-run -Werror
       - name: Banned constructs
         run: python3 -I scripts/check-banned.py
@@ -384,6 +384,14 @@ jobs:
         run: python3 -I -m unittest discover -s tools/esp -p 'test_*.py' -v
       - name: Debug descriptor reader (OBS-008, A107)
         run: python3 -I -m unittest discover -s tools/descriptor -p 'test_*.py' -v
+      - name: Framed byte streams, the host's side (SPEC-015 §7)
+        run: python3 -I -m unittest discover -s tools/transport -p 'test_*.py' -v
+      - name: Log decoder's tests (A127)
+        run: python3 -I -m unittest discover -s tools/logdec -p 'test_*.py' -v
+      - name: Trace converter's tests (A128)
+        run: python3 -I -m unittest discover -s tools/trace -p 'test_*.py' -v
+      - name: The emb command-line tool's tests (BLD-005)
+        run: python3 -I -m unittest discover -s tools/emb -p 'test_*.py' -v
       - name: EmbSIM runner, its project mode (SIM-002)
         run: python3 -I -m unittest discover -s tools/embsim -p 'test_*.py' -v
       - name: UF2 writer (A79)
@@ -807,7 +815,7 @@ jobs:
         timeout-minutes: 15
         uses: ./.github/actions/apt-install # archives cached, then scripts/ci-apt-install.sh
         with:
-          packages: ninja-build gcc-arm-none-eabi libnewlib-arm-none-eabi qemu-system-arm gcc-riscv64-unknown-elf picolibc-riscv64-unknown-elf qemu-system-misc gcc-avr avr-libc binutils-avr python3-yaml
+          packages: ninja-build gcc-arm-none-eabi libnewlib-arm-none-eabi qemu-system-arm gcc-riscv64-unknown-elf picolibc-riscv64-unknown-elf qemu-system-misc gcc-avr avr-libc binutils-avr python3-yaml babeltrace2
       - name: Copy the examples out of the tree
         run: |
           cp -r examples/hello_app "$RUNNER_TEMP/hello_app"
@@ -837,6 +845,46 @@ jobs:
             --qemu-args "-M mps2-an386 -semihosting-config enable=on,target=native -kernel" build/blinky-mps2_an386/blinky.elf
           python3 -I tools/qemu/run_qemu.py --qemu qemu-system-riscv32 --until "blinky: off" --timeout 60 \
             --qemu-args "-M virt -bios none -kernel" build/blinky-riscv32_virt/blinky.elf
+      - name: Logging with both backends print the same lines (A127)
+        run: |
+          cp -r examples/logging "$RUNNER_TEMP/logging"
+          for b in native mps2_an386 riscv32_virt; do
+            cmake -S "$RUNNER_TEMP/logging" -B "build/logging-$b" -G Ninja -DEmbLinkRTOS_DIR="$PWD/cmake" -DEMB_BOARD=$b
+            cmake --build "build/logging-$b"
+            cmake -S "$RUNNER_TEMP/logging" -B "build/logging-$b-framed" -G Ninja -DEmbLinkRTOS_DIR="$PWD/cmake" \
+                  -DEMB_BOARD=$b -DEMB_SET=EMB_CONSOLE_FRAMED=y
+            cmake --build "build/logging-$b-framed"
+          done
+          build/logging-native/logging > logging-native.log
+          build/logging-native-framed/logging > logging-native-framed.bin
+          for v in "" -framed; do
+            timeout 60 qemu-system-arm -M mps2-an386 -semihosting-config enable=on,target=native -nographic \
+              -monitor none -serial stdio -kernel "build/logging-mps2_an386$v/logging.elf" > "logging-mps2_an386$v.bin"
+            timeout 60 qemu-system-riscv32 -M virt -bios none -nographic -monitor none -serial stdio \
+              -kernel "build/logging-riscv32_virt$v/logging.elf" > "logging-riscv32_virt$v.bin"
+          done
+          mv logging-mps2_an386.bin logging-mps2_an386.log
+          mv logging-riscv32_virt.bin logging-riscv32_virt.log
+          python3 -I tools/logdec/backends_agree.py --plain logging-native.log --framed logging-native-framed.bin \
+                  --elf build/logging-native-framed/logging
+          for b in mps2_an386 riscv32_virt; do
+            python3 -I tools/logdec/backends_agree.py --plain "logging-$b.log" --framed "logging-$b-framed.bin" \
+                    --elf "build/logging-$b-framed/logging.elf"
+          done
+      - name: The trace ring dumped under QEMU, turned into CTF that Babeltrace 2 reads (A128)
+        run: |
+          cp -r examples/tracing "$RUNNER_TEMP/tracing"
+          cmake -S "$RUNNER_TEMP/tracing" -B build/tracing-mps2_an386 -G Ninja -DEmbLinkRTOS_DIR="$PWD/cmake" \
+                -DEMB_BOARD=mps2_an386
+          cmake --build build/tracing-mps2_an386
+          timeout 60 qemu-system-arm -M mps2-an386 -semihosting-config enable=on,target=native -nographic \
+            -monitor none -serial stdio -kernel build/tracing-mps2_an386/tracing.elf > tracing.bin
+          python3 -I tools/trace/trace.py tracing.bin --ctf build/trace-ctf --perfetto build/trace.json
+          babeltrace2 build/trace-ctf > trace-ctf.txt
+          n=$(wc -l < trace-ctf.txt)
+          echo "babeltrace2 read $n events"; head -5 trace-ctf.txt
+          test "$n" -ge 100
+          python3 -I -c "import json; d = json.load(open('build/trace.json')); assert len(d['traceEvents']) >= 100"
       - name: Build the reference controller for both NUCLEO-F446 boards (it runs on the board, not here)
         run: |
           cp -r examples/controller "$RUNNER_TEMP/controller"

Kconfig

@@ -12,3 +12,4 @@ mainmenu "EmbLinkRTOS configuration"
 
 source "arch/Kconfig"
 source "kernel/Kconfig"
+source "subsys/Kconfig"

boards/arm/mps2_an385/generated/.base/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an385/generated/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an386/generated/.base/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an386/generated/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an500/generated/.base/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an500/generated/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an505/generated/.base/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/arm/mps2_an505/generated/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/bbc/microbit/generated/.base/linker/regions.ld

@@ -88,4 +88,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/bbc/microbit/generated/linker/regions.ld

@@ -88,4 +88,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/espressif/esp32_devkitc/board.c

@@ -168,11 +168,13 @@ static void put_char(char c)
     UART_FIFO = (uint32_t)(unsigned char)c;
 }
 
-void emb_board_puts(const char *s)
+/* The console's bytes as they are (A127): emb_board_puts() and the framed console call it. */
+void emb_board_write(const void *buf, size_t len)
 {
-    while (*s != '\0') { /* bounded by the string */
-        put_char(*s);
-        s++;
+    const char *p = (const char *)buf;
+    size_t i;
+    for (i = 0u; i < len; i++) { /* bounded by len */
+        put_char(p[i]);
     }
 }
 

boards/espressif/esp32_devkitc/include/emb_board.h

@@ -18,6 +18,7 @@
 #include <emb/compiler.h>
 #include <emb/fault.h>
 
+#include <stddef.h>
 #include <stdint.h>
 
 #include <hw_config.h>
@@ -31,6 +32,7 @@ extern "C" {
 void emb_board_init(void);
 void emb_board_led_toggle(void);
 void emb_board_puts(const char *s);
+void emb_board_write(const void *buf, size_t len); /* raw bytes (A127) */
 void emb_board_put_u32(uint32_t v);
 EMB_NORETURN void emb_board_exit(int code);
 void emb_board_fault_report(const emb_fault_info_t *info);

boards/espressif/esp32c3_devkitm1/board.c

@@ -170,11 +170,13 @@ static void put_char(char c)
     UART_FIFO = (uint32_t)(unsigned char)c;
 }
 
-void emb_board_puts(const char *s)
+/* The console's bytes as they are (A127): emb_board_puts() and the framed console call it. */
+void emb_board_write(const void *buf, size_t len)
 {
-    while (*s != '\0') { /* bounded by the string */
-        put_char(*s);
-        s++;
+    const char *p = (const char *)buf;
+    size_t i;
+    for (i = 0u; i < len; i++) { /* bounded by len */
+        put_char(p[i]);
     }
 }
 

boards/espressif/esp32c3_devkitm1/include/emb_board.h

@@ -18,6 +18,7 @@
 #include <emb/compiler.h>
 #include <emb/fault.h>
 
+#include <stddef.h>
 #include <stdint.h>
 
 #include <hw_config.h>
@@ -31,6 +32,7 @@ extern "C" {
 void emb_board_init(void);
 void emb_board_led_toggle(void);
 void emb_board_puts(const char *s);
+void emb_board_write(const void *buf, size_t len); /* raw bytes (A127) */
 void emb_board_put_u32(uint32_t v);
 EMB_NORETURN void emb_board_exit(int code);
 void emb_board_fault_report(const emb_fault_info_t *info);

boards/native/native/board.c

@@ -36,10 +36,11 @@ void emb_board_led_toggle(void)
     /* the host has no LED */
 }
 
-void emb_board_puts(const char *s)
+/* The console's bytes as they are (A127): emb_board_puts() and the framed console call it. */
+void emb_board_write(const void *buf, size_t len)
 {
     emb_native_host_begin();
-    (void)fputs(s, stdout);
+    (void)fwrite(buf, 1u, len, stdout);
     emb_native_host_end();
 }
 

boards/native/native/include/emb_board.h

@@ -17,6 +17,9 @@
 
 #include <emb/compiler.h>
 
+#include <stddef.h>
+#include <stdint.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -26,6 +29,7 @@ extern "C" {
 void emb_board_init(void);
 void emb_board_led_toggle(void);
 void emb_board_puts(const char *s);
+void emb_board_write(const void *buf, size_t len); /* raw bytes (A127) */
 void emb_board_put_u32(uint32_t v);
 
 #ifdef __cplusplus

boards/qemu/riscv32_virt/generated/.base/linker/regions.ld

@@ -81,4 +81,15 @@ SECTIONS
   __emb_test_retained_end = __emb_retained_start;
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_test_retained_start;
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/qemu/riscv32_virt/generated/linker/regions.ld

@@ -81,4 +81,15 @@ SECTIONS
   __emb_test_retained_end = __emb_retained_start;
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_test_retained_start;
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }

boards/rpi/pico2_w/generated/.base/linker/regions.ld

@@ -90,4 +90,15 @@ SECTIONS
   __emb_test_retained_start = __emb_test_retained_end - 32;
   __stack_top = __emb_isr_stack_end;
   ASSERT(end <= __emb_test_retained_start, "RAM overflow: .data and .bss reach the retained memory")
+
+  /* the log's call sites (A127): kept in the ELF for tools/logdec and out of the image. At
+     address 0, from which a record's identifier is its metadata's offset plus 8; the first 8
+     bytes stay empty, so no metadata is at address 0, the null pointer. Last, since setting
+     its address moves the location counter. */
+  emb_log 0 (INFO) : {
+    __start_emb_log = .;
+    . += 8;
+    KEEP(*(emb_log))
+    __stop_emb_log = .;
+  }
 }