fix(docker-compose): fix hostmetrics mount layout, filesystem exclusions, and image version

docker-compose/otel-compose.yaml

@@ -1,6 +1,6 @@
 services:
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.118.0
+    image: otel/opentelemetry-collector-contrib:0.144.0
     container_name: otel-collector
     command:
       [
@@ -14,10 +14,7 @@ services:
       # Host filesystem mounts so the hostmetrics receiver can read the VM's
       # CPU / memory / disk / network / process stats rather than the
       # collector container's own (containerized) view.
-      - /proc:/hostfs/proc:ro
-      - /sys:/hostfs/sys:ro
-      - /etc/os-release:/hostfs/etc/os-release:ro
-      - /:/hostfs/root:ro,rslave
+      - /:/hostfs:ro,rslave
     ports:
       - "4317:4317" # OTLP gRPC receiver
       - "4318:4318" # OTLP HTTP receiver
@@ -40,6 +37,11 @@ services:
       # this because they read via the Docker socket.
       - nginx_network
       - apache_network
+      # Note: bridge networking is required here for active prometheus scraping
+      # of named docker networks. For scraping host-networked containers
+      # (network_mode: host), switch to network_mode: host on this service,
+      # remove the networks: block, use localhost:<port> as scrape target,
+      # and change logspout command to syslog+tcp://localhost:2255.
 
   logspout:
     image: "gliderlabs/logspout:v3.2.14"

docker-compose/otel-config.yaml

@@ -89,19 +89,40 @@ receivers:
         metrics:
           system.filesystem.utilization:
             enabled: true
+        exclude_fs_types:
+          fs_types:
+            - squashfs
+            - tmpfs
+            - devtmpfs
+            - overlay
+            - nsfs
+            - autofs
+            - cgroup
+            - cgroup2
+            - proc
+            - sysfs
+          match_type: strict
         exclude_mount_points:
           mount_points:
-            - /hostfs/sys/*
-            - /hostfs/proc/*
-            - /var/lib/docker/*
-            - /hostfs/root/var/lib/docker/*
+            - /hostfs/sys/.*
+            - /hostfs/proc/.*
+            - /hostfs/dev/.*
+            - /hostfs/run/.*
+            - /hostfs/var/lib/docker/.*
+            - /hostfs/snap/.*
+            - /hostfs/boot.*
+            - /var/lib/docker/.*
           match_type: regexp
       network:
       paging:
       processes:
-      # mute_process_*_error: true silences the routine "permission denied"
-      # noise for kernel threads and short-lived processes whose
-      # /proc/<pid>/{exe,io,cmdline} entries can't be read.
+      # mute_process_*_error: true silences "permission denied" noise for
+      # kernel threads and short-lived processes. Note: these flags do NOT
+      # cover TOCTOU races — if a process exits between PID discovery and
+      # stat read, /proc/<pid>/stat returns ENOENT and is still logged as
+      # an error. On busy hosts with many short-lived processes (kworkers,
+      # sshd child processes), remove the process: scraper entirely and rely
+      # on docker_stats for per-container granularity instead.
       process:
         mute_process_user_error: true
         mute_process_io_error: true

gemini-cli/.env.example

@@ -0,0 +1,5 @@
+# Last9 OTLP credentials — get these from:
+# https://app.last9.io/integrations?integration=OpenTelemetry
+
+LAST9_OTLP_ENDPOINT=https://otlp.last9.io
+LAST9_OTLP_AUTH=Basic <your_credentials>

gemini-cli/.gitignore

@@ -0,0 +1,11 @@
+# Environment/secrets
+.env
+.env.local
+.env.*.local
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log

gemini-cli/README.md

@@ -0,0 +1,137 @@
+# Gemini CLI — OpenTelemetry to Last9
+
+Send Google's [Gemini CLI](https://github.com/google-gemini/gemini-cli) telemetry — traces, metrics, and logs — to Last9 via OpenTelemetry.
+
+Gemini CLI emits all three OpenTelemetry signal types. Keep all three on — combined volume is low and traces add per-call model latency / token attribution that metrics alone cannot.
+
+- **Traces** — spans for tool calls, API requests, agent runs (set `GEMINI_TELEMETRY_TRACES_ENABLED=true`)
+- **Metrics** — session counts, token usage, latency, file ops, agent durations
+- **Logs** — structured events for prompts, API requests/responses, slash commands, file operations
+
+## Prerequisites
+
+- A Last9 account ([app.last9.io](https://app.last9.io))
+- OTLP credentials from **Integrations → OpenTelemetry** in the Last9 dashboard
+- Gemini CLI installed (`npm install -g @google/gemini-cli`)
+
+## Option 1 — Direct Export to Last9 (no Collector)
+
+Gemini CLI's OTLP exporters take `url` only, but the underlying OpenTelemetry JS SDK reads standard `OTEL_EXPORTER_OTLP_HEADERS` from env for authentication — so direct export works.
+
+```bash
+# Gemini CLI telemetry switches
+export GEMINI_TELEMETRY_ENABLED=true
+export GEMINI_TELEMETRY_TARGET=local
+export GEMINI_TELEMETRY_OTLP_ENDPOINT="https://<your-last9-otlp-endpoint>"
+export GEMINI_TELEMETRY_OTLP_PROTOCOL=http
+export GEMINI_TELEMETRY_TRACES_ENABLED=true
+
+# Standard OTel env — picked up by SDK for auth headers
+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic <your-last9-auth-token>"
+```
+
+Reload and run:
+
+```bash
+source ~/.zshrc
+gemini -p "explain this repo"
+```
+
+## Option 2 — Local OpenTelemetry Collector
+
+Useful for batching, retries, or fan-out to multiple backends.
+
+1. Copy `.env.example` to `.env` and fill in your Last9 credentials:
+
+   ```bash
+   cp .env.example .env
+   ```
+
+2. Start the collector:
+
+   ```bash
+   docker compose up -d
+   ```
+
+3. Point Gemini CLI at the local collector:
+
+   ```bash
+   export GEMINI_TELEMETRY_ENABLED=true
+   export GEMINI_TELEMETRY_TARGET=local
+   export GEMINI_TELEMETRY_OTLP_ENDPOINT=http://localhost:4317
+   export GEMINI_TELEMETRY_OTLP_PROTOCOL=grpc
+   export GEMINI_TELEMETRY_TRACES_ENABLED=true
+   unset OTEL_EXPORTER_OTLP_HEADERS   # collector handles auth
+   ```
+
+4. Run Gemini CLI:
+
+   ```bash
+   gemini
+   ```
+
+## Configuration Reference
+
+### Gemini CLI-specific
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `GEMINI_TELEMETRY_ENABLED` | `false` | Master toggle |
+| `GEMINI_TELEMETRY_TARGET` | `local` | `local` (OTLP) or `gcp` (Google Cloud) |
+| `GEMINI_TELEMETRY_OTLP_ENDPOINT` | `http://localhost:4317` | OTLP endpoint (base URL — `/v1/traces` etc. appended) |
+| `GEMINI_TELEMETRY_OTLP_PROTOCOL` | `grpc` | `grpc` or `http` |
+| `GEMINI_TELEMETRY_TRACES_ENABLED` | `false` | Set `true` to export spans (recommended) |
+| `GEMINI_TELEMETRY_LOG_PROMPTS` | `true` | Include prompt text in logs |
+| `GEMINI_TELEMETRY_OUTFILE` | — | Write to local file instead of OTLP |
+
+### Standard OTel env (used for auth headers)
+
+| Variable | Purpose |
+|---|---|
+| `OTEL_EXPORTER_OTLP_HEADERS` | `Authorization=Basic <token>` |
+| `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_HEADERS` | Per-signal override |
+
+### `.env` variables (Collector path)
+
+| Variable | Purpose |
+|---|---|
+| `LAST9_OTLP_ENDPOINT` | Last9 OTLP endpoint (e.g. `https://otlp.last9.io`) |
+| `LAST9_OTLP_AUTH` | Last9 Basic auth header (`Basic <base64>`) |
+
+## Verification
+
+After running a Gemini CLI session for a minute:
+
+1. **Traces** — filter by `service.name = gemini-cli` in Last9 Traces Explorer. Span name: `llm_call`.
+2. **Metrics** — search for `gemini_cli_session_count_total`, `gemini_cli_api_request_count_total`, `gemini_cli_token_usage_total`
+3. **Logs** — filter by `service.name = gemini-cli` in Last9 Logs Explorer
+
+<details>
+<summary>Notable metrics</summary>
+
+| Metric | Type |
+|---|---|
+| `gemini_cli.session.count` | counter |
+| `gemini_cli.tool.call.count` | counter |
+| `gemini_cli.tool.call.latency` | histogram |
+| `gemini_cli.api.request.count` | counter |
+| `gemini_cli.api.request.latency` | histogram |
+| `gemini_cli.token.usage` | counter |
+| `gemini_cli.file.operation.count` | counter |
+| `gemini_cli.lines.changed` | counter |
+| `gemini_cli.agent.run.count` | counter |
+| `gemini_cli.agent.duration` | histogram |
+| `gemini_cli.startup.duration` | histogram |
+| `gemini_cli.memory.usage` | gauge |
+| `gemini_cli.cpu.usage` | gauge |
+| `gen_ai.client.token.usage` | counter (GenAI semconv) |
+| `gen_ai.client.operation.duration` | histogram (GenAI semconv) |
+
+</details>
+
+## Troubleshooting
+
+- **No data in Last9** — confirm `echo $GEMINI_TELEMETRY_ENABLED` returns `true` in the shell that ran `gemini`. Restart shell after editing `~/.zshrc`.
+- **Authentication errors** — verify `OTEL_EXPORTER_OTLP_HEADERS` is `Authorization=Basic <token>` (key=value format, not HTTP colon syntax). Trailing whitespace breaks it.
+- **Traces missing** — make sure `GEMINI_TELEMETRY_TRACES_ENABLED=true` is set. Gemini CLI defaults to `false`, so spans only flow when you explicitly enable them.
+- **gRPC connection refused** — make sure `-p 4317:4317` is exposed (collector path) and you're using `grpc` protocol with `http://localhost:4317` (not `https`).

gemini-cli/docker-compose.yaml

@@ -0,0 +1,19 @@
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.144.0
+    container_name: gemini-cli-otel-collector
+    command: ["--config=/etc/otel/config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel/config.yaml:ro
+    env_file: .env
+    environment:
+      - DEPLOYMENT_ENV=${DEPLOYMENT_ENV:-development}
+    ports:
+      - "4317:4317"   # gRPC receiver
+      - "4318:4318"   # HTTP receiver
+      - "13133:13133" # Health check
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133"]
+      interval: 10s
+      timeout: 5s
+      retries: 3

gemini-cli/otel-collector-config.yaml

@@ -0,0 +1,44 @@
+receivers:
+  # Receive logs, traces, and metrics from Gemini CLI via gRPC and HTTP
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 1000
+
+  # Add organization-level resource attributes
+  resource:
+    attributes:
+      - key: deployment.environment
+        value: "${env:DEPLOYMENT_ENV}"
+        action: upsert
+
+exporters:
+  otlp/last9:
+    endpoint: "${env:LAST9_OTLP_ENDPOINT}"
+    headers:
+      "Authorization": "${env:LAST9_OTLP_AUTH}"
+
+  debug:
+    verbosity: basic
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]
+    metrics:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]
+    logs:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [otlp/last9, debug]

otel-collector/delta-cumulative-routing/.gitignore

@@ -0,0 +1,2 @@
+.env
+delta_cumulative_demo

otel-collector/delta-cumulative-routing/docker-compose.yaml

@@ -0,0 +1,73 @@
+services:
+
+  # Simulates K8s Service load-balancing across N collector pods
+  nginx:
+    image: nginx:alpine
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "4318:4318"
+    depends_on:
+      - collector-1
+      - collector-2
+    networks:
+      - demo
+
+  # Collector pod 1
+  collector-1:
+    image: otel/opentelemetry-collector-contrib:0.137.0
+    hostname: collector-1
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-config.yaml:/etc/otel-collector-config.yaml:ro
+    environment:
+      - HOSTNAME=collector-1
+      - LAST9_OTLP_ENDPOINT=${LAST9_OTLP_ENDPOINT}
+      - LAST9_OTLP_AUTH_HEADER=${LAST9_OTLP_AUTH_HEADER}
+    networks:
+      - demo
+
+  # Collector pod 2 — identical config, different hostname → different collector_id on delta series
+  collector-2:
+    image: otel/opentelemetry-collector-contrib:0.137.0
+    hostname: collector-2
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-config.yaml:/etc/otel-collector-config.yaml:ro
+    environment:
+      - HOSTNAME=collector-2
+      - LAST9_OTLP_ENDPOINT=${LAST9_OTLP_ENDPOINT}
+      - LAST9_OTLP_AUTH_HEADER=${LAST9_OTLP_AUTH_HEADER}
+    networks:
+      - demo
+
+  # Metrics generator — sends both delta and cumulative counters
+  generator:
+    build: ./generator
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=nginx:4318
+    depends_on:
+      - nginx
+    restart: on-failure
+    networks:
+      - demo
+
+  # Local VictoriaMetrics — receives all metrics via remote write
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:v1.115.0
+    command:
+      - --storageDataPath=/storage
+      - --retentionPeriod=1d
+    ports:
+      - "8428:8428"
+    volumes:
+      - vm-data:/storage
+    networks:
+      - demo
+
+networks:
+  demo:
+    name: delta_cumulative_demo
+
+volumes:
+  vm-data:

otel-collector/delta-cumulative-routing/generator/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.24-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY main.go .
+RUN CGO_ENABLED=0 go build -o generator .
+
+FROM alpine:3.20
+COPY --from=builder /app/generator /generator
+ENTRYPOINT ["/generator"]