feat(java-vertx3): add JAX-RS/RESTEasy agent test app and exception capture docs

docker-compose/otel-compose.yaml

@@ -1,6 +1,6 @@
 services:
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.118.0
+    image: otel/opentelemetry-collector-contrib:0.144.0
     container_name: otel-collector
     command:
       [
@@ -14,10 +14,7 @@ services:
       # Host filesystem mounts so the hostmetrics receiver can read the VM's
       # CPU / memory / disk / network / process stats rather than the
       # collector container's own (containerized) view.
-      - /proc:/hostfs/proc:ro
-      - /sys:/hostfs/sys:ro
-      - /etc/os-release:/hostfs/etc/os-release:ro
-      - /:/hostfs/root:ro,rslave
+      - /:/hostfs:ro,rslave
     ports:
       - "4317:4317" # OTLP gRPC receiver
       - "4318:4318" # OTLP HTTP receiver
@@ -40,6 +37,11 @@ services:
       # this because they read via the Docker socket.
       - nginx_network
       - apache_network
+      # Note: bridge networking is required here for active prometheus scraping
+      # of named docker networks. For scraping host-networked containers
+      # (network_mode: host), switch to network_mode: host on this service,
+      # remove the networks: block, use localhost:<port> as scrape target,
+      # and change logspout command to syslog+tcp://localhost:2255.
 
   logspout:
     image: "gliderlabs/logspout:v3.2.14"

docker-compose/otel-config.yaml

@@ -89,19 +89,40 @@ receivers:
         metrics:
           system.filesystem.utilization:
             enabled: true
+        exclude_fs_types:
+          fs_types:
+            - squashfs
+            - tmpfs
+            - devtmpfs
+            - overlay
+            - nsfs
+            - autofs
+            - cgroup
+            - cgroup2
+            - proc
+            - sysfs
+          match_type: strict
         exclude_mount_points:
           mount_points:
-            - /hostfs/sys/*
-            - /hostfs/proc/*
-            - /var/lib/docker/*
-            - /hostfs/root/var/lib/docker/*
+            - /hostfs/sys/.*
+            - /hostfs/proc/.*
+            - /hostfs/dev/.*
+            - /hostfs/run/.*
+            - /hostfs/var/lib/docker/.*
+            - /hostfs/snap/.*
+            - /hostfs/boot.*
+            - /var/lib/docker/.*
           match_type: regexp
       network:
       paging:
       processes:
-      # mute_process_*_error: true silences the routine "permission denied"
-      # noise for kernel threads and short-lived processes whose
-      # /proc/<pid>/{exe,io,cmdline} entries can't be read.
+      # mute_process_*_error: true silences "permission denied" noise for
+      # kernel threads and short-lived processes. Note: these flags do NOT
+      # cover TOCTOU races — if a process exits between PID discovery and
+      # stat read, /proc/<pid>/stat returns ENOENT and is still logged as
+      # an error. On busy hosts with many short-lived processes (kworkers,
+      # sshd child processes), remove the process: scraper entirely and rely
+      # on docker_stats for per-container granularity instead.
       process:
         mute_process_user_error: true
         mute_process_io_error: true

gemini-cli/.env.example

@@ -0,0 +1,5 @@
+# Last9 OTLP credentials — get these from:
+# https://app.last9.io/integrations?integration=OpenTelemetry
+
+LAST9_OTLP_ENDPOINT=https://otlp.last9.io
+LAST9_OTLP_AUTH=Basic <your_credentials>

gemini-cli/.gitignore

@@ -0,0 +1,11 @@
+# Environment/secrets
+.env
+.env.local
+.env.*.local
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log

gemini-cli/README.md

@@ -0,0 +1,137 @@
+# Gemini CLI — OpenTelemetry to Last9
+
+Send Google's [Gemini CLI](https://github.com/google-gemini/gemini-cli) telemetry — traces, metrics, and logs — to Last9 via OpenTelemetry.
+
+Gemini CLI emits all three OpenTelemetry signal types. Keep all three on — combined volume is low and traces add per-call model latency / token attribution that metrics alone cannot.
+
+- **Traces** — spans for tool calls, API requests, agent runs (set `GEMINI_TELEMETRY_TRACES_ENABLED=true`)
+- **Metrics** — session counts, token usage, latency, file ops, agent durations
+- **Logs** — structured events for prompts, API requests/responses, slash commands, file operations
+
+## Prerequisites
+
+- A Last9 account ([app.last9.io](https://app.last9.io))
+- OTLP credentials from **Integrations → OpenTelemetry** in the Last9 dashboard
+- Gemini CLI installed (`npm install -g @google/gemini-cli`)
+
+## Option 1 — Direct Export to Last9 (no Collector)
+
+Gemini CLI's OTLP exporters take `url` only, but the underlying OpenTelemetry JS SDK reads standard `OTEL_EXPORTER_OTLP_HEADERS` from env for authentication — so direct export works.
+
+```bash
+# Gemini CLI telemetry switches
+export GEMINI_TELEMETRY_ENABLED=true
+export GEMINI_TELEMETRY_TARGET=local
+export GEMINI_TELEMETRY_OTLP_ENDPOINT="https://<your-last9-otlp-endpoint>"
+export GEMINI_TELEMETRY_OTLP_PROTOCOL=http
+export GEMINI_TELEMETRY_TRACES_ENABLED=true
+
+# Standard OTel env — picked up by SDK for auth headers
+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic <your-last9-auth-token>"
+```
+
+Reload and run:
+
+```bash
+source ~/.zshrc
+gemini -p "explain this repo"
+```
+
+## Option 2 — Local OpenTelemetry Collector
+
+Useful for batching, retries, or fan-out to multiple backends.
+
+1. Copy `.env.example` to `.env` and fill in your Last9 credentials:
+
+   ```bash
+   cp .env.example .env
+   ```
+
+2. Start the collector:
+
+   ```bash
+   docker compose up -d
+   ```
+
+3. Point Gemini CLI at the local collector:
+
+   ```bash
+   export GEMINI_TELEMETRY_ENABLED=true
+   export GEMINI_TELEMETRY_TARGET=local
+   export GEMINI_TELEMETRY_OTLP_ENDPOINT=http://localhost:4317
+   export GEMINI_TELEMETRY_OTLP_PROTOCOL=grpc
+   export GEMINI_TELEMETRY_TRACES_ENABLED=true
+   unset OTEL_EXPORTER_OTLP_HEADERS   # collector handles auth
+   ```
+
+4. Run Gemini CLI:
+
+   ```bash
+   gemini
+   ```
+
+## Configuration Reference
+
+### Gemini CLI-specific
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `GEMINI_TELEMETRY_ENABLED` | `false` | Master toggle |
+| `GEMINI_TELEMETRY_TARGET` | `local` | `local` (OTLP) or `gcp` (Google Cloud) |
+| `GEMINI_TELEMETRY_OTLP_ENDPOINT` | `http://localhost:4317` | OTLP endpoint (base URL — `/v1/traces` etc. appended) |
+| `GEMINI_TELEMETRY_OTLP_PROTOCOL` | `grpc` | `grpc` or `http` |
+| `GEMINI_TELEMETRY_TRACES_ENABLED` | `false` | Set `true` to export spans (recommended) |
+| `GEMINI_TELEMETRY_LOG_PROMPTS` | `true` | Include prompt text in logs |
+| `GEMINI_TELEMETRY_OUTFILE` | — | Write to local file instead of OTLP |
+
+### Standard OTel env (used for auth headers)
+
+| Variable | Purpose |
+|---|---|
+| `OTEL_EXPORTER_OTLP_HEADERS` | `Authorization=Basic <token>` |
+| `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_HEADERS` | Per-signal override |
+
+### `.env` variables (Collector path)
+
+| Variable | Purpose |
+|---|---|
+| `LAST9_OTLP_ENDPOINT` | Last9 OTLP endpoint (e.g. `https://otlp.last9.io`) |
+| `LAST9_OTLP_AUTH` | Last9 Basic auth header (`Basic <base64>`) |
+
+## Verification
+
+After running a Gemini CLI session for a minute:
+
+1. **Traces** — filter by `service.name = gemini-cli` in Last9 Traces Explorer. Span name: `llm_call`.
+2. **Metrics** — search for `gemini_cli_session_count_total`, `gemini_cli_api_request_count_total`, `gemini_cli_token_usage_total`
+3. **Logs** — filter by `service.name = gemini-cli` in Last9 Logs Explorer
+
+<details>
+<summary>Notable metrics</summary>
+
+| Metric | Type |
+|---|---|
+| `gemini_cli.session.count` | counter |
+| `gemini_cli.tool.call.count` | counter |
+| `gemini_cli.tool.call.latency` | histogram |
+| `gemini_cli.api.request.count` | counter |
+| `gemini_cli.api.request.latency` | histogram |
+| `gemini_cli.token.usage` | counter |
+| `gemini_cli.file.operation.count` | counter |
+| `gemini_cli.lines.changed` | counter |
+| `gemini_cli.agent.run.count` | counter |
+| `gemini_cli.agent.duration` | histogram |
+| `gemini_cli.startup.duration` | histogram |
+| `gemini_cli.memory.usage` | gauge |
+| `gemini_cli.cpu.usage` | gauge |
+| `gen_ai.client.token.usage` | counter (GenAI semconv) |
+| `gen_ai.client.operation.duration` | histogram (GenAI semconv) |
+
+</details>
+
+## Troubleshooting
+
+- **No data in Last9** — confirm `echo $GEMINI_TELEMETRY_ENABLED` returns `true` in the shell that ran `gemini`. Restart shell after editing `~/.zshrc`.
+- **Authentication errors** — verify `OTEL_EXPORTER_OTLP_HEADERS` is `Authorization=Basic <token>` (key=value format, not HTTP colon syntax). Trailing whitespace breaks it.
+- **Traces missing** — make sure `GEMINI_TELEMETRY_TRACES_ENABLED=true` is set. Gemini CLI defaults to `false`, so spans only flow when you explicitly enable them.
+- **gRPC connection refused** — make sure `-p 4317:4317` is exposed (collector path) and you're using `grpc` protocol with `http://localhost:4317` (not `https`).

gemini-cli/docker-compose.yaml

@@ -0,0 +1,19 @@
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.144.0
+    container_name: gemini-cli-otel-collector
+    command: ["--config=/etc/otel/config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel/config.yaml:ro
+    env_file: .env
+    environment:
+      - DEPLOYMENT_ENV=${DEPLOYMENT_ENV:-development}
+    ports:
+      - "4317:4317"   # gRPC receiver
+      - "4318:4318"   # HTTP receiver
+      - "13133:13133" # Health check
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133"]
+      interval: 10s
+      timeout: 5s
+      retries: 3

gemini-cli/otel-collector-config.yaml

@@ -0,0 +1,44 @@
+receivers:
+  # Receive logs, traces, and metrics from Gemini CLI via gRPC and HTTP
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 1000
+
+  # Add organization-level resource attributes
+  resource:
+    attributes:
+      - key: deployment.environment
+        value: "${env:DEPLOYMENT_ENV}"
+        action: upsert
+
+exporters:
+  otlp/last9:
+    endpoint: "${env:LAST9_OTLP_ENDPOINT}"
+    headers:
+      "Authorization": "${env:LAST9_OTLP_AUTH}"
+
+  debug:
+    verbosity: basic
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]
+    metrics:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]
+    logs:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]

go/custom-metrics/.gitignore

@@ -0,0 +1,2 @@
+custom-metrics-example
+example

go/custom-metrics/go.mod

@@ -0,0 +1,31 @@
+module custom-metrics-example
+
+go 1.22.7
+
+toolchain go1.22.12
+
+require (
+	go.opentelemetry.io/otel v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0
+	go.opentelemetry.io/otel/metric v1.33.0
+	go.opentelemetry.io/otel/sdk v1.33.0
+	go.opentelemetry.io/otel/sdk/metric v1.33.0
+)
+
+require (
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel/trace v1.33.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
+	google.golang.org/grpc v1.68.1 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
+)

go/custom-metrics/go.sum

@@ -0,0 +1,53 @@
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
+go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0 h1:bSjzTvsXZbLSWU8hnZXcKmEVaJjjnandxD0PxThhVU8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0/go.mod h1:aj2rilHL8WjXY1I5V+ra+z8FELtk681deydgYT8ikxU=
+go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
+go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
+go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM=
+go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
+go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU=
+go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q=
+go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
+go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
+go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
+go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
+golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
+golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
+google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=
+google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0=
+google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=