hd-debug-k8s.sh

#!/bin/bash
#
# This is a helper script to collet and log diagnostics for the Hybrid Deployment Agent in a Kubernetes environment.
#
# Requirements:
#   - kubectl must be installed and configured
#   - The HD agent should be installed and running
#
# Usage:
#   ./hd-debug-k8s.sh [-n namespace] [-h]
#
# set -x
# set -e

# Image for the Hybrid Deployment Agent container
HD_AGENT_IMAGE="us-docker.pkg.dev/prod-eng-fivetran-ldp/public-docker-us/ldp-agent:production"
# defined in templates/deployment.yaml
HD_AGENT_DEPLOYMENT_CONTAINER_NAME="hd-agent"
# defined in templates/configmap.yaml
HD_AGENT_CONFIG_NAME="hd-agent-config"
NAMESPACE="default"
# Set to true to collect full node provisioner (Karpenter or Cluster Autoscaler) configuration (YAML).
# Set to false to collect basic listing only (names and status), or use the -s flag at runtime.
GET_PROVISIONER_DETAILS=true
SCRIPT_PATH="$(realpath "$0")"
BASE_DIR="$(dirname "$SCRIPT_PATH")"
DIAG_DIR="$BASE_DIR/k8s_stats"
TIMESTAMP=$(date +%Y-%m-%d_%H-%M)
AGENT_DEPLOYMENT=""
AGENT_POD=""
CONTROLLER_ID=""

rm -r "$DIAG_DIR" 2>/dev/null
mkdir -p "$DIAG_DIR" 2>/dev/null

if [ "$UID" -eq 0 ]; then
    echo -e "This script should not be run as root user.\n"
    exit 1
fi

function usage() {
    echo -e "Usage: $0 [-n <namespace>] [-s] [-h]"
    echo -e "  -n  Kubernetes namespace (default: default)"
    echo -e "  -s  Skip full node provisioner (Karpenter or Cluster Autoscaler) YAML dump (only collect basic names and info)"
    echo -e "  -h  Show this help message\n"
    exit 1
}

function check_kubectl() {
 if ! command -v kubectl &> /dev/null; then
    echo "kubectl command line utility not found. Please install the latest version."
    echo -e "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/\n"
    exit 1
 fi
}

function check_helm() {
 if ! command -v helm &> /dev/null; then
    echo "helm command line utility not found. Please install the latest version."
    echo -e "https://helm.sh/docs/intro/install/\n"
    exit 1
 fi
}

function get_controller_id () {
    # Token secret is named <deployment>-token-secret; controller ID is the prefix before ':' in the decoded token
    local TOKEN_SECRET="${AGENT_DEPLOYMENT}-token-secret"
    local TOKEN
    # stringData in the secret template causes k8s to re-encode the already-base64 token, so decode twice
    TOKEN=$(kubectl get secret "$TOKEN_SECRET" -n "$NAMESPACE" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null | base64 -d 2>/dev/null)
    CONTROLLER_ID=$(echo "$TOKEN" | cut -d: -f1)

    if [ -n "$CONTROLLER_ID" ]; then
        echo "Controller ID: $CONTROLLER_ID"
    else
        echo "Warning: Could not extract controller_id from secret $TOKEN_SECRET"
        CONTROLLER_ID="unknown"
    fi
}

function list_kubectl_current_context(){
    local CURRENT_CONTEXT=$(kubectl config current-context)
    if [ -z "$CURRENT_CONTEXT" ]; then
        echo -e "No current kubectl context set.\nPlease set a valid context pointing to your cluster.\n"
        exit 1
    else
        echo -e "Current kubectl context: $CURRENT_CONTEXT\n"
    fi
}

function get_agent_deployment_name() {
    AGENT_DEPLOYMENT=$(kubectl get deployments -n "$NAMESPACE" -l app.kubernetes.io/name=hd-agent --no-headers -o custom-columns=":metadata.name")
    if [ -z "$AGENT_DEPLOYMENT" ]; then
        echo "No Agent deployment found in '$NAMESPACE'"
    fi
}

function get_agent_pod_name() {
    AGENT_POD=$(kubectl get pod -n "$NAMESPACE" -l app.kubernetes.io/name=hd-agent --no-headers -o custom-columns=":metadata.name")
    if [ -z "$AGENT_POD" ]; then
        echo "No Agent pod found in '$NAMESPACE'"
    fi
}

function get_helm_manifest_for_deployment() {
    helm get manifest "$AGENT_DEPLOYMENT" -n "$NAMESPACE" | sed -E 's/^(\s*token: ).*/\1*****/' > "$DIAG_DIR/helm_manifest.log" 2>&1
}

function log_agent_info() {
    echo -e "Collecting HD Agent environment diagnostics...\n"

    get_agent_deployment_name
    get_agent_pod_name
    get_controller_id

    if [ -z "$AGENT_DEPLOYMENT" ] || [ -z "$AGENT_POD" ]; then
        echo "No HD Agent deployment or pod found in namespace '$NAMESPACE'."
        exit 1
    else
        echo "Found HD Agent deployment: $AGENT_DEPLOYMENT"
        echo "Found HD Agent pod: $AGENT_POD"

        echo "This may take a few seconds..."
    
        kubectl describe pod "$AGENT_POD" -n "$NAMESPACE" > "$DIAG_DIR/pod_description.log" 2>&1
        # Get events for all objects related to the agent deployment (including terminated pods) with timestamp and node info
        kubectl get events -n "$NAMESPACE" -o custom-columns=Timestamp:.lastTimestamp,Node:.source.host,Name:.involvedObject.name,Message:.message --no-headers | egrep "donkey|worker|hd-agent|setup|standard" > "$DIAG_DIR/pod_events.log" 2>&1
        kubectl top pod "$AGENT_POD" -n "$NAMESPACE" > "$DIAG_DIR/pod_resource_usage.log" 2>&1
        kubectl get pod "$AGENT_POD" -n "$NAMESPACE" -o yaml > "$DIAG_DIR/pod_definition.log" 2>&1
        kubectl get deployment $AGENT_DEPLOYMENT -n "$NAMESPACE" -o yaml > "$DIAG_DIR/agent_deployment.log" 2>&1
        kubectl get pods -n "$NAMESPACE" -o wide > "$DIAG_DIR/pods.log" 2>&1
        kubectl get jobs -n "$NAMESPACE" -o wide > "$DIAG_DIR/jobs.log" 2>&1
	    kubectl logs "$AGENT_POD" -n "$NAMESPACE" > "$DIAG_DIR/agent.log" 2>&1

        # attempt to get the helm manifest for the deployment
        get_helm_manifest_for_deployment

        kubectl version > "$DIAG_DIR/kubectl_version.log" 2>&1
        helm version > "$DIAG_DIR/helm_version.log" 2>&1
    fi

    kubectl get configmaps -n "$NAMESPACE" > "$DIAG_DIR/configmap_listing.log" 2>&1
    kubectl get configmap "$HD_AGENT_CONFIG_NAME" -n "$NAMESPACE" -o yaml | grep -v 'token:' > "$DIAG_DIR/agent_config_map.log" 2>&1
    kubectl get secrets -n "$NAMESPACE" > "$DIAG_DIR/secrets_listing.log" 2>&1

    # Collect full namespace events in yaml (captures pod sandbox/CNI failures across all nodes, not just the agent node)
    kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' -o yaml > "$DIAG_DIR/namespace_events.log" 2>&1

    local HD_AGENT_NODE_NAME=$(kubectl get pod "$AGENT_POD" -n "$NAMESPACE" -o jsonpath="{.spec.nodeName}")
    kubectl describe node "$HD_AGENT_NODE_NAME" > "$DIAG_DIR/node_description.log" 2>&1
    kubectl get nodes -o custom-columns=Name:.metadata.name,Created:.metadata.creationTimestamp,nCPU:.status.capacity.cpu,Memory:.status.capacity.memory  > "$DIAG_DIR/node_listing.log" 2>&1
    kubectl get nodes -o wide > "$DIAG_DIR/nodes_wide.log" 2>&1
    kubectl get nodes --show-labels > "$DIAG_DIR/nodes_labels.log" 2>&1

    # Collect node provisioner details (Karpenter or Cluster Autoscaler)
    local HAS_KARPENTER=false
    if kubectl api-resources --api-group=karpenter.sh --no-headers 2>/dev/null | grep -q nodepools; then
        HAS_KARPENTER=true
    fi

    if [ "$HAS_KARPENTER" = true ]; then
        if [ "$GET_PROVISIONER_DETAILS" = true ]; then
            if ! kubectl get nodepools.karpenter.sh -o yaml > "$DIAG_DIR/karpenter_nodepools.log" 2>&1; then
                kubectl get nodepools.karpenter.sh/v1beta1 -o yaml > "$DIAG_DIR/karpenter_nodepools.log" 2>/dev/null
            fi
            kubectl get ec2nodeclasses.karpenter.k8s.aws -o yaml > "$DIAG_DIR/karpenter_ec2nodeclass.log" 2>/dev/null
        else
            if ! kubectl get nodepools.karpenter.sh -o wide > "$DIAG_DIR/karpenter_nodepools.log" 2>&1; then
                kubectl get nodepools.karpenter.sh/v1beta1 -o wide > "$DIAG_DIR/karpenter_nodepools.log" 2>/dev/null
            fi
            kubectl get ec2nodeclasses.karpenter.k8s.aws -o wide > "$DIAG_DIR/karpenter_ec2nodeclass.log" 2>/dev/null
        fi
    else
        # Check for Cluster Autoscaler
        if [ "$GET_PROVISIONER_DETAILS" = true ]; then
            kubectl get deployment -n kube-system -l app=cluster-autoscaler -o yaml > "$DIAG_DIR/cluster_autoscaler.log" 2>/dev/null
            kubectl get configmap -n kube-system cluster-autoscaler-status -o yaml > "$DIAG_DIR/cluster_autoscaler_status.log" 2>/dev/null
        else
            kubectl get deployment -n kube-system -l app=cluster-autoscaler -o wide > "$DIAG_DIR/cluster_autoscaler.log" 2>/dev/null
            kubectl get configmap -n kube-system cluster-autoscaler-status -o wide > "$DIAG_DIR/cluster_autoscaler_status.log" 2>/dev/null
        fi
    fi

    kubectl get serviceaccounts -n "$NAMESPACE" > "$DIAG_DIR/service_accounts.log" 2>&1
    kubectl get roles -n "$NAMESPACE" > "$DIAG_DIR/roles.log" 2>&1
    kubectl get rolebindings -n "$NAMESPACE" > "$DIAG_DIR/role_bindings.log" 2>&1

    kubectl get pv -n "$NAMESPACE" -o wide > "$DIAG_DIR/pv.log" 2>&1
    kubectl get pvc -n "$NAMESPACE" -o wide > "$DIAG_DIR/pvc.log" 2>&1
    kubectl get pvc -n "$NAMESPACE" -o yaml > "$DIAG_DIR/pvc-detail.log" 2>&1

    # Collect full pod spec (including affinity) for all hybrid-deployment pods, redacting cert/key env vars
    local HD_PODS
    HD_PODS=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/part-of=hybrid-deployment" --no-headers -o custom-columns=":metadata.name")
    if [ -n "$HD_PODS" ]; then
        for POD in $HD_PODS; do
            kubectl get pod "$POD" -n "$NAMESPACE" -o yaml \
                | awk '/- name: orchestrator\.client_cert|- name: orchestrator\.client_private_key/{skip=1;next} skip && /- name:/{skip=0} !skip{print}' \
                >> "$DIAG_DIR/pod_specs.log" 2>&1
            echo -e "\n" >> "$DIAG_DIR/pod_specs.log"
        done
    fi
}

while getopts "n:sh" opt; do
    case $opt in
        n) NAMESPACE="$OPTARG" ;;
        s) GET_PROVISIONER_DETAILS=false ;;
        h) usage ;;
        *) usage ;;
    esac
done
shift $((OPTIND-1))

if [ $# -gt 0 ]; then
    echo "Error: Extra arguments provided: $*"
    usage
    exit 1
fi

check_kubectl
check_helm
list_kubectl_current_context
log_agent_info

echo -e "done.\n"
echo -e "Packing logs into hd-$CONTROLLER_ID-$TIMESTAMP.tar.gz\n"

cd $DIAG_DIR 
tar czf hd-$CONTROLLER_ID-$TIMESTAMP.tar.gz ./*.log
ls -altr hd-$CONTROLLER_ID-$TIMESTAMP.tar.gz
cd - 

echo -e "done.\n"
echo -e "Logs are available in $DIAG_DIR/hd-$CONTROLLER_ID-$TIMESTAMP.tar.gz\n"