DLPX-96312 Filter garbage stat names from estat metaslab-alloc output

Wraps estat metaslab-alloc in a shell script that drops JSON lines whose
"name" tag contains non-standard characters (backslashes, hashes, etc.).
Addresses DLPX-88427 where a kernel bug causes random memory bytes or C
macro strings to appear as stat names, producing unreadable metrics.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: dbshah12

telegraf/connstat-stats.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Collect per-connection TCP stats from connstat and aggregate by remote
-# endpoint (laddr:raddr:rport) to bound cardinality on engines with many
+# endpoint (laddr:raddr:service) to bound cardinality on engines with many
 # connections — e.g. Oracle dNFS (hundreds of connections per VDB host) or
 # Elastic Data (many connections per object storage endpoint IP).
 # Mirrors the aggregation done by LocalTCPStatsCollector in the mgmt stack.
@@ -11,7 +11,7 @@
 # engine is the server) are identified correctly. Falls back to "unknown".
 #
 # Output fields per aggregated endpoint:
-#   laddr, raddr, rport, service
+#   laddr, raddr, service
 #   inbytes, outbytes, retranssegs, suna, unsent  (summed across connections)
 #   swnd, cwnd, rwnd, rtt                          (averaged across connections)
 #   connections                                    (count of aggregated conns)
@@ -33,33 +33,35 @@ BEGIN {
         }
     }
     close("/etc/services")
-    # Delphix DSP (Session Protocol) port — not present in /etc/services.
-    # Matches the ServiceProtocol special-case in LocalTCPStatsCollector.
-    svc[50001] = "dlpx-sp"
+    # Delphix-specific ports not present in /etc/services.
+    # Matches LocalTCPStatsCollector.getService() special-cases exactly.
+    svc[8415]  = "dlpx-sp"              # DSP (ServiceProtocol.PORT)
+    svc[50001] = "network-throughput-test" # TtcpPerfSession.DEFAULT_PORT
+    svc[8341]  = "oracle-logsync"       # HTTP server (TunableRegistry.HTTP_SERVER_PORT default)
+    svc[9100]  = "dlpx-connector"       # Host Connector (Connector.DEFAULT_PORT)
 }
 /^=/ {
     for (key in cnt) {
         n = cnt[key]
         split(key, k, SUBSEP)
-        print k[1] "," k[2] "," k[3] "," k[4] "," \
+        print k[1] "," k[2] "," k[3] "," \
               inb[key] "," outb[key] "," ret[key] "," sun[key] "," uns[key] "," \
               int(sw[key]/n) "," int(cw[key]/n) "," int(rw[key]/n) "," int(rt[key]/n) "," n
     }
+    fflush()
     delete cnt; delete inb; delete outb; delete ret
     delete sun; delete uns; delete sw; delete cw; delete rw; delete rt
     next
 }
 NF == 13 {
-    lport = $2 + 0
-    rport = $4 + 0
-    if (lport in svc) {
-        service = svc[lport]
-    } else if (rport in svc) {
-        service = svc[rport]
+    if (($2 + 0) in svc) {
+        service = svc[$2 + 0]
+    } else if (($4 + 0) in svc) {
+        service = svc[$4 + 0]
     } else {
         service = "unknown"
     }
-    key = $1 SUBSEP $3 SUBSEP rport SUBSEP service
+    key = $1 SUBSEP $3 SUBSEP service
     inb[key] += $5;  outb[key] += $6;  ret[key] += $7
     sun[key] += $8;  uns[key]  += $9
     sw[key]  += $10; cw[key]   += $11; rw[key]  += $12; rt[key] += $13

telegraf/metaslab-alloc-stats.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+#
+# Wrapper around "estat metaslab-alloc -jm 10" that filters out metrics whose
+# "name" tag contains garbage characters (DLPX-88427). A kernel bug causes
+# estat to occasionally emit stat names containing raw memory bytes or C macro
+# strings. Only names consisting of printable ASCII letters, digits, spaces,
+# and common punctuation are passed through.
+#
+estat metaslab-alloc -jm 10 | grep -E '"name":"[A-Za-z0-9 ,_()/.-]+"'

telegraf/telegraf.base

@@ -68,9 +68,9 @@
   data_format = "csv"
   csv_delimiter = ","
   csv_trim_space = true
-  csv_column_names = ["laddr", "raddr", "rport", "service", "inbytes", "outbytes", "retranssegs", "suna", "unsent", "swnd", "cwnd", "rwnd", "rtt", "connections"]
-  csv_column_types = ["string", "string", "int", "string", "int", "int", "int", "int", "int", "int", "int", "int", "int", "int"]
-  csv_tag_columns = ["laddr", "raddr", "rport", "service"]
+  csv_column_names = ["laddr", "raddr", "service", "inbytes", "outbytes", "retranssegs", "suna", "unsent", "swnd", "cwnd", "rwnd", "rtt", "connections"]
+  csv_column_types = ["string", "string", "string", "int", "int", "int", "int", "int", "int", "int", "int", "int", "int"]
+  csv_tag_columns = ["laddr", "raddr", "service"]
 
 # Track CPU and Memory for the "delphix-mgmt" service (and children).
 [[inputs.procstat]]

telegraf/telegraf.inputs.playbook

@@ -55,9 +55,10 @@
   ]
   json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)", "microseconds"]
 
-# Collect output from "estat metaslab-alloc -jm 10"
+# Collect output from "estat metaslab-alloc -jm 10" via wrapper script.
+# The wrapper filters out metrics with garbage "name" tags (DLPX-88427).
 [[inputs.execd]]
-  command = ["estat", "metaslab-alloc", "-jm", "10"]
+  command = ["/etc/telegraf/metaslab-alloc-stats.sh"]
   name_override = "estat_metaslab-alloc"
   signal = "none"
   restart_delay = "30s"

telegraf/telegraf.inputs.storage_io

@@ -67,5 +67,32 @@
   namepass = ["estat_*"]
   fieldexclude = ["microseconds"]
 
+# Expand hist_estat_* microseconds histogram strings into per-bucket rows for
+# Grafana heatmap support. Each "{upper_bound_us,count}" pair becomes a
+# separate metric with le=<upper_bound_us> tag and count field, replacing the
+# opaque string. Runs after clone (order=1) and strings (order=2) so the
+# microseconds field is still present in hist_estat_* at this point.
+[[processors.starlark]]
+  order = 3
+  namepass = ["hist_estat_*"]
+  source = '''
+def apply(metric):
+    ms = metric.fields.get("microseconds")
+    if ms == None:
+        return [metric]
+
+    result = []
+    for pair in ms[1:-1].split("},{"):
+        parts = pair.split(",")
+        if len(parts) == 2:
+            m = deepcopy(metric)
+            m.tags["le"] = parts[0]
+            m.fields["count"] = int(parts[1])
+            m.fields.pop("microseconds")
+            result.append(m)
+
+    return result if result else [metric]
+'''
+
 # End of Storage I/O section
 ##############################################################################