Back merged

Author: dbshah12

debian/control

@@ -13,6 +13,6 @@ Standards-Version: 4.1.2
 
 Package: performance-diagnostics
 Architecture: any
-Depends: python3-bpfcc, python3-minimal, python3-psutil, telegraf, docker.io, influxdb2, curl
+Depends: python3-bpfcc, python3-minimal, python3-psutil, telegraf, docker-ce, influxdb2, curl
 Description: eBPF-based Performance Diagnostic Tools
   A collection of eBPF-based tools for diagnosing performance issues.

influxdb/delphix-influxdb-init

@@ -212,3 +212,7 @@ umask "$old_umask"
 
 rm -f "$INFLUXDB_SETUP_STATE_FILE"
 log "InfluxDB initialized successfully."
+
+# Restart Telegraf so it picks up the InfluxDB output stanza written above.
+# This is only reached on first boot (subsequent boots exit early at the top).
+systemctl restart delphix-telegraf 2>/dev/null || true

influxdb/perf_influxdb

@@ -32,7 +32,7 @@ function enable_influxdb() {
 function disable_influxdb() {
 	date
 	echo "Disabling InfluxDB Metric Output"
-	rm -rf $INFLUXDB_FLAG
+	rm -f $INFLUXDB_FLAG
 	systemctl restart delphix-telegraf
 }
 

telegraf/delphix-telegraf-service

@@ -51,7 +51,14 @@ else
 fi
 
 if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then
-	cat $STORAGE_IO_INPUTS $INFLUXDB_OUTPUT >> $TELEGRAF_CONFIG
+	if [[ -f $STORAGE_IO_INPUTS ]]; then
+		cat $STORAGE_IO_INPUTS >> $TELEGRAF_CONFIG
+	fi
+	cat $INFLUXDB_OUTPUT >> $TELEGRAF_CONFIG
+else
+	# No InfluxDB output configured. Add discard so Telegraf can start —
+	# it requires at least one output plugin.
+	echo "[[outputs.discard]]" >> $TELEGRAF_CONFIG
 fi
 
 # Restrict permissions so the InfluxDB write token is not world-readable.

telegraf/telegraf.inputs.playbook

@@ -1,7 +1,7 @@
 ##############################################################################
 # Performance Playbook (estat, nfs_threads) collection
 # Note: estat_nfs, estat_iscsi, and estat_backend-io live in
-# telegraf.inputs.nfs_iscsi and are always collected when InfluxDB is
+# telegraf.inputs.storage_io and are always collected when InfluxDB is
 # enabled, independent of playbook state.
 
 # Collect output from "estat zpl -jm 10"
@@ -88,31 +88,25 @@
 #                         PROCESSOR PLUGINS                                   #
 ###############################################################################
 # Convert strings from estat into integer values so they don't get dropped.
-# Scoped to playbook-only metrics; estat_nfs/iscsi have their own converter
-# in telegraf.inputs.nfs_iscsi.
+# Scoped to playbook-only metrics; estat_nfs/iscsi/backend-io have their own
+# converter in telegraf.inputs.storage_io.
 [[processors.converter]]
   namepass = ["estat_zpl", "estat_zio", "estat_zvol", "estat_zio-queue", "estat_metaslab-alloc"]
   [processors.converter.fields]
     integer = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)"]
 
-# The estat output contains a nested latency histogram, so we need to 
-# parse that out as a new array metric rather than a non-JSON string.
-#
-# From this:
-#	"microseconds":"{20000,5},{30000,15},{40000,3},{50000,24}"
-# to this:
-#	"microseconds":"{20000:5,30000:15,40000:3,50000:24}"
-#
-# Clone the original so we have a "new" metric with a "hist_" name prefix
+# Parse microseconds latency histograms for playbook estat measurements.
+# Transforms "{20000,5},{30000,15}" → {"20000":5,"30000":15} as a JSON object.
+# Scoped to playbook-only measurements; storage_io (estat_nfs/iscsi/backend-io)
+# does not collect microseconds to keep always-on data volume low.
 [[processors.clone]]
   order = 1
   name_prefix = "hist_"
-  namepass = ["estat_*"]
+  namepass = ["estat_zpl", "estat_zvol", "estat_zio", "estat_zio-queue", "estat_metaslab-alloc"]
 
-# Rewrite the histograms for the "hist_estat_*" metrics as JSON objects
 [[processors.regex]]
   order = 2
-  namepass = ["hist_estat_*"]
+  namepass = ["hist_estat_zpl", "hist_estat_zvol", "hist_estat_zio", "hist_estat_zio-queue", "hist_estat_metaslab-alloc"]
   [[processors.regex.fields]]
     key = "microseconds"
     pattern = "{(\\d+),(\\d+)}"
@@ -122,14 +116,13 @@
     pattern = ".*"
     replacement = "{$0}"
 
-# Now parse out the arrays for "hist_estat_*" metrics
 [[processors.parser]]
   order = 3
   merge = "override"
   parse_fields = ["microseconds"]
   drop_original = false
   data_format = "json"
-  namepass = ["hist_estat_*"]
+  namepass = ["hist_estat_zpl", "hist_estat_zvol", "hist_estat_zio", "hist_estat_zio-queue", "hist_estat_metaslab-alloc"]
   fieldpass = ["microseconds"]
 
 # End of Processor section

telegraf/telegraf.inputs.storage_io

@@ -1,6 +1,9 @@
 ##############################################################################
 # Storage I/O collection: NFS server, iSCSI target, and backend disk I/O.
 # Always included when InfluxDB is enabled, independent of playbook state.
+# Note: microseconds histogram is NOT collected here to keep always-on data
+# volume low. Full histogram expansion lives in telegraf.inputs.playbook and
+# is only active when the playbook is enabled.
 
 # Collect output from "estat nfs -jm 10"
 [[inputs.execd]]
@@ -13,7 +16,7 @@
     "name",
     "axis"
   ]
-  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)", "microseconds"]
+  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)"]
 
 # Collect output from "estat iscsi -jm 10"
 [[inputs.execd]]
@@ -26,7 +29,7 @@
     "name",
     "axis"
   ]
-  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)", "microseconds"]
+  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)"]
 
 # Collect output from "estat backend-io -jm 10" (stbtrace io equivalent)
 [[inputs.execd]]
@@ -39,7 +42,7 @@
     "name",
     "axis"
   ]
-  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)", "microseconds"]
+  json_string_fields = ["iops(/s)", "avg latency(us)", "stddev(us)", "throughput(k/s)"]
 
 # Convert estat string fields to integers so they are not dropped by Telegraf.
 [[processors.converter]]