DLPX-96312 Filter garbage stat names from estat metaslab-alloc output

Wraps estat metaslab-alloc in a shell script that drops JSON lines whose
"name" tag contains non-standard characters (backslashes, hashes, etc.).
Addresses DLPX-88427 where a kernel bug causes random memory bytes or C
macro strings to appear as stat names, producing unreadable metrics.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: dbshah12

telegraf/connstat-stats.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Collect per-connection TCP stats from connstat and aggregate by remote
-# endpoint (laddr:raddr:rport) to bound cardinality on engines with many
+# endpoint (laddr:raddr:service) to bound cardinality on engines with many
 # connections — e.g. Oracle dNFS (hundreds of connections per VDB host) or
 # Elastic Data (many connections per object storage endpoint IP).
 # Mirrors the aggregation done by LocalTCPStatsCollector in the mgmt stack.
@@ -11,7 +11,7 @@
 # engine is the server) are identified correctly. Falls back to "unknown".
 #
 # Output fields per aggregated endpoint:
-#   laddr, raddr, rport, service
+#   laddr, raddr, service
 #   inbytes, outbytes, retranssegs, suna, unsent  (summed across connections)
 #   swnd, cwnd, rwnd, rtt                          (averaged across connections)
 #   connections                                    (count of aggregated conns)
@@ -41,7 +41,7 @@ BEGIN {
     for (key in cnt) {
         n = cnt[key]
         split(key, k, SUBSEP)
-        print k[1] "," k[2] "," k[3] "," k[4] "," \
+        print k[1] "," k[2] "," k[3] "," \
               inb[key] "," outb[key] "," ret[key] "," sun[key] "," uns[key] "," \
               int(sw[key]/n) "," int(cw[key]/n) "," int(rw[key]/n) "," int(rt[key]/n) "," n
     }
@@ -50,16 +50,14 @@ BEGIN {
     next
 }
 NF == 13 {
-    lport = $2 + 0
-    rport = $4 + 0
-    if (lport in svc) {
-        service = svc[lport]
-    } else if (rport in svc) {
-        service = svc[rport]
+    if (($2 + 0) in svc) {
+        service = svc[$2 + 0]
+    } else if (($4 + 0) in svc) {
+        service = svc[$4 + 0]
     } else {
         service = "unknown"
     }
-    key = $1 SUBSEP $3 SUBSEP rport SUBSEP service
+    key = $1 SUBSEP $3 SUBSEP service
     inb[key] += $5;  outb[key] += $6;  ret[key] += $7
     sun[key] += $8;  uns[key]  += $9
     sw[key]  += $10; cw[key]   += $11; rw[key]  += $12; rt[key] += $13

telegraf/metaslab-alloc-stats.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+#
+# Wrapper around "estat metaslab-alloc -jm 10" that filters out metrics whose
+# "name" tag contains garbage characters (DLPX-88427). A kernel bug causes
+# estat to occasionally emit stat names containing raw memory bytes or C macro
+# strings. Only names consisting of printable ASCII letters, digits, spaces,
+# and common punctuation are passed through.
+#
+estat metaslab-alloc -jm 10 | grep -E '"name":"[A-Za-z0-9 ,_()/.-]+"'

telegraf/telegraf.base

@@ -68,9 +68,9 @@
   data_format = "csv"
   csv_delimiter = ","
   csv_trim_space = true
-  csv_column_names = ["laddr", "raddr", "rport", "service", "inbytes", "outbytes", "retranssegs", "suna", "unsent", "swnd", "cwnd", "rwnd", "rtt", "connections"]
-  csv_column_types = ["string", "string", "int", "string", "int", "int", "int", "int", "int", "int", "int", "int", "int", "int"]
-  csv_tag_columns = ["laddr", "raddr", "rport", "service"]
+  csv_column_names = ["laddr", "raddr", "service", "inbytes", "outbytes", "retranssegs", "suna", "unsent", "swnd", "cwnd", "rwnd", "rtt", "connections"]
+  csv_column_types = ["string", "string", "string", "int", "int", "int", "int", "int", "int", "int", "int", "int", "int"]
+  csv_tag_columns = ["laddr", "raddr", "service"]
 
 # Track CPU and Memory for the "delphix-mgmt" service (and children).
 [[inputs.procstat]]

telegraf/telegraf.inputs.playbook

@@ -55,9 +55,10 @@
   ]
   json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)", "microseconds"]
 
-# Collect output from "estat metaslab-alloc -jm 10"
+# Collect output from "estat metaslab-alloc -jm 10" via wrapper script.
+# The wrapper filters out metrics with garbage "name" tags (DLPX-88427).
 [[inputs.execd]]
-  command = ["estat", "metaslab-alloc", "-jm", "10"]
+  command = ["/etc/telegraf/metaslab-alloc-stats.sh"]
   name_override = "estat_metaslab-alloc"
   signal = "none"
   restart_delay = "30s"