esunar / health-check.sh

#!/usr/bin/env bash set -euo pipefail # k8s-health.sh — single-context Kubernetes health report (Markdown) # Focus: core cluster + Calico + Istio + common apps (Redis, RabbitMQ, MinIO) # Deps: kubectl, jq, awk, sed, grep, base64, openssl; (optional) gdate # ----------------------------- CLI & Globals ----------------------------- OUT_DIR="" REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-8s}" DNS_TEST="${DNS_TEST:-true}" # set false to skip ephemeral DNS test TLS_WARN_DAYS="${TLS_WARN_DAYS:-14}" TLS_ERR_DAYS="${TLS_ERR_DAYS:-7}" usage() { cat <<EOF Usage: $0 [--context NAME] [--namespace NS] [--out DIR] [--no-dns] Options: --context NAME Use a specific kubeconfig context (default: current) --namespace NS Limit app checks to a namespace (default: all) --out DIR Output directory (default: ./k8s-health-<timestamp>) --no-dns Skip ephemeral DNS resolution test Env overrides: REQUEST_TIMEOUT (default: 8s) | TLS_WARN_DAYS (14) | TLS_ERR_DAYS (7) EOF } CTX_ARGS=() APP_NS="" while [[ $# -gt 0 ]]; do case "$1" in --context) shift; CTX_ARGS+=(--context "$1");; --namespace) shift; APP_NS="$1";; --out) shift; OUT_DIR="$1";; --no-dns) DNS_TEST=false;; -h|--help) usage; exit 0;; *) echo "Unknown arg: $1" >&2; usage; exit 3;; esac shift done ts_now() { date -Is; } to_ts() { # portable timestamp from RFC date string (uses gdate if available) local d="$1" if command -v gdate >/dev/null 2>&1; then gdate -d "$d" +%s; else date -d "$d" +%s; fi 2>/dev/null || echo 0 } days_until() { local end="$1"; local end_ts; end_ts=$(to_ts "$end") local now_ts; now_ts=$(date +%s) echo $(( (end_ts - now_ts) / 86400 )) } # Normalize possibly multi-line/non-numeric values to a single non-negative integer (default 0) to_int() { local v="$1" # replace newlines/tabs with spaces, take first token v="${v//$'\n'/ }"; v="${v//$'\t'/ }"; v="${v%% *}" # strip non-digits v="$(printf '%s' "$v" | sed -E 's/[^0-9-]//g')" [[ "$v" =~ ^-?[0-9]+$ ]] || v=0 printf '%s' "$v" } if ! command -v kubectl >/dev/null; then echo "kubectl not found" >&2; exit 3; fi if ! command -v jq >/dev/null; then echo "jq not found" >&2; exit 3; fi if [[ -z "$OUT_DIR" ]]; then OUT_DIR="./k8s-health-$(date +%Y%m%d-%H%M%S)" fi ART="$OUT_DIR/artifacts" mkdir -p "$ART" REPORT="$OUT_DIR/report.md" JSONL="$OUT_DIR/summary.jsonl" touch "$REPORT" "$JSONL" # ----------------------------- Emit Helpers ------------------------------ emit_json() { # emit_json LEVEL AREA CHECK MESSAGE HINT printf '{"ts":"%s","level":"%s","area":"%s","check":"%s","message":%s,"hint":%s}\n' \ "$(ts_now)" "$1" "$2" "$3" "$(jq -Rs . <<<"$4")" "$(jq -Rs . <<<"${5:-}")" >> "$JSONL" } emit_md_h1() { echo -e "# $1\n" >> "$REPORT"; } emit_md_h2() { echo -e "## $1\n" >> "$REPORT"; } emit_md_h3() { echo -e "### $1\n" >> "$REPORT"; } emit_md_kv() { echo "- **$1:** $2" >> "$REPORT"; } emit_md_code() { echo -e "\n\`\`\`\n$1\n\`\`\`\n" >> "$REPORT"; } # ----------------------------- Prefetch Cache ---------------------------- echo "Collecting cluster state..." set +e KUBECTL=(kubectl "${CTX_ARGS[@]}") # Use a lightweight API call instead of 'kubectl version' which can fail for reasons unrelated to reachability. if ! "${KUBECTL[@]}" --request-timeout="$REQUEST_TIMEOUT" get --raw='/version' >/dev/null 2>&1; then # Fallback to a simple resource list in case /version endpoint is blocked by a proxy. if ! "${KUBECTL[@]}" --request-timeout="$REQUEST_TIMEOUT" get nodes -o name >/dev/null 2>&1; then echo "Cannot reach cluster with kubectl" >&2; exit 3 fi fi set -e "${KUBECTL[@]}" version -o json > "$ART/version.json" 2>/dev/null "${KUBECTL[@]}" api-resources > "$ART/apiresources.txt" || true "${KUBECTL[@]}" get nodes -o json > "$ART/nodes.json" if [[ -n "$APP_NS" ]]; then ns_arg=(-n "$APP_NS") else ns_arg=(--all-namespaces) fi "${KUBECTL[@]}" get pods "${ns_arg[@]}" -o json > "$ART/pods.json" "${KUBECTL[@]}" get ns -o json > "$ART/namespaces.json" "${KUBECTL[@]}" get events --all-namespaces --sort-by=.lastTimestamp -o json --request-timeout="$REQUEST_TIMEOUT" > "$ART/events.json" || true "${KUBECTL[@]}" get svc --all-namespaces -o json > "$ART/svc.json" "${KUBECTL[@]}" get endpoints --all-namespaces -o json > "$ART/endpoints.json" "${KUBECTL[@]}" get endpointslices.discovery.k8s.io --all-namespaces -o json > "$ART/epslices.json" 2>/dev/null || true "${KUBECTL[@]}" get deploy,ds,sts,job,cronjob,hpa,pdb --all-namespaces -o json > "$ART/workloads.json" 2>/dev/null || true "${KUBECTL[@]}" get pvc --all-namespaces -o json > "$ART/pvc.json" 2>/dev/null || true "${KUBECTL[@]}" get pv -o json > "$ART/pv.json" 2>/dev/null || true "${KUBECTL[@]}" get storageclasses.storage.k8s.io -o json > "$ART/sc.json" 2>/dev/null || true "${KUBECTL[@]}" get secrets --all-namespaces -o json > "$ART/secrets.json" 2>/dev/null || true "${KUBECTL[@]}" get csidrivers.storage.k8s.io,csinodes.storage.k8s.io -o json > "$ART/csi.json" 2>/dev/null || true # Istio + Calico artifacts (best effort) "${KUBECTL[@]}" -n istio-system get deploy,ds,pods,svc -o wide > "$ART/istio_ls.txt" 2>/dev/null || true "${KUBECTL[@]}" -n calico-system get deploy,ds,pods -o wide > "$ART/calico_ls.txt" 2>/dev/null || true "${KUBECTL[@]}" get crd tenants.minio.min.io rabbitmqclusters.rabbitmq.com 2>/dev/null | sed '1d' > "$ART/app_crds.txt" || true # Cache Istio Gateways (best effort) "${KUBECTL[@]}" get gateway.networking.istio.io --all-namespaces -o json > "$ART/istio_gateways.json" 2>/dev/null || true # ----------------------------- Report Header ----------------------------- cluster_server=$( jq -r '.serverVersion.gitVersion + " (" + .serverVersion.platform + ")"' "$ART/version.json" 2>/dev/null \ || echo "unknown") client_ver=$( jq -r '.clientVersion.gitVersion' "$ART/version.json" 2>/dev/null \ || echo "unknown") ctx_name=$("${KUBECTL[@]}" config current-context 2>/dev/null || echo "current") emit_md_h1 "Kubernetes Health Report — ${ctx_name}" emit_md_kv "Generated" "$(ts_now)" emit_md_kv "kubectl client" "$client_ver" emit_md_kv "APIServer" "$cluster_server" emit_md_kv "Namespace scope (apps)" "${APP_NS:-all}" echo "" >> "$REPORT" # ----------------------------- Versions & Skew ---------------------------- emit_md_h2 "Cluster Versions & Skew" node_versions=$(jq -r '.items[]?.status.nodeInfo.kubeletVersion' "$ART/nodes.json" | sort | uniq -c | sed 's/^/ /') emit_md_code "Kubelet versions:\n${node_versions}" server_minor=$(jq -r 'try (.serverVersion.minor|tonumber) catch 0' "$ART/version.json" 2>/dev/null || echo 0) first_kubelet_minor=$(jq -r '[.items[]?.status.nodeInfo.kubeletVersion | capture("v(?<maj>\\d+)\\.(?<min>\\d+)") | .min | tonumber] | (min // 0)' "$ART/nodes.json" 2>/dev/null || echo 0) last_kubelet_minor=$(jq -r '[.items[]?.status.nodeInfo.kubeletVersion | capture("v(?<maj>\\d+)\\.(?<min>\\d+)") | .min | tonumber] | (max // 0)' "$ART/nodes.json" 2>/dev/null || echo 0) : "${server_minor:=0}" : "${first_kubelet_minor:=0}" : "${last_kubelet_minor:=0}" # Normalize minor versions server_minor="$(to_int "$server_minor")" first_kubelet_minor="$(to_int "$first_kubelet_minor")" last_kubelet_minor="$(to_int "$last_kubelet_minor")" # Ensure single-token integers before computing skew (avoid bash arithmetic on malformed values) lm="$(to_int "$last_kubelet_minor")" sm="$(to_int "$server_minor")" fm="$(to_int "$first_kubelet_minor")" # Compute absolute skew with awk only (robust even if inputs are "0") abs_skew="$(awk -v a="$lm" -v b="$sm" 'BEGIN{d=a-b; if (d<0) d=-d; print d+0}' 2>/dev/null)" abs_skew="$(to_int "$abs_skew")" if [ "$abs_skew" -gt 1 ]; then emit_json "ERROR" "version" "skew" "Kubelet/APIServer minor skew > 1 (server minor ${sm}, kubelet min/max ${fm}/${lm})" "Align versions per K8s skew policy." echo "- **Version Skew:** ❌ kubelet/APIServer minor skew > 1 (server=${sm}, kubelet min/max=${fm}/${lm})" >> "$REPORT" else echo "- **Version Skew:** ✅ within supported range (server=${sm}, kubelet min/max=${fm}/${lm})" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- API Server Health -------------------------- emit_md_h2 "API Server Ready/Liveness" readyz="$ART/readyz.txt" livez="$ART/livez.txt" set +e "${KUBECTL[@]}" get --raw='/readyz?verbose' >"$readyz" 2>&1 r_rc=$? "${KUBECTL[@]}" get --raw='/livez?verbose' >"$livez" 2>&1 l_rc=$? set -e emit_md_code "readyz:\n$(cat "$readyz" 2>/dev/null || true)" emit_md_code "livez:\n$(cat "$livez" 2>/dev/null || true)" fail_cnt=$(grep -c 'fail' "$readyz" 2>/dev/null || true) fail_cnt="$(to_int "${fail_cnt:-0}")" if [ "$r_rc" -ne 0 ] || [ "$fail_cnt" -gt 0 ]; then emit_json "ERROR" "control-plane" "readyz" "APIServer readyz reports failures" "Check control-plane component health." echo "- **APIServer readyz:** ❌ failures detected" >> "$REPORT" else echo "- **APIServer readyz:** ✅ ok" >> "$REPORT" fi if [[ $l_rc -ne 0 ]]; then emit_json "ERROR" "control-plane" "livez" "APIServer livez not reachable" "" echo "- **APIServer livez:** ❌ unreachable" >> "$REPORT" else echo "- **APIServer livez:** ✅ ok" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Nodes ------------------------------------- emit_md_h2 "Node Health" nodes_json="$ART/nodes.json" not_ready=$(jq -r '.items[] | select([.status.conditions[]?|select(.type=="Ready")][0].status!="True") | .metadata.name' "$nodes_json") if [[ -n "$not_ready" ]]; then emit_json "ERROR" "nodes" "ready" "Some nodes NotReady" "$not_ready" echo "- **Ready:** ❌ NotReady nodes present:" >> "$REPORT"; echo "$not_ready" | sed 's/^/ - /' >> "$REPORT" else echo "- **Ready:** ✅ all nodes Ready" >> "$REPORT" fi pressures=$(jq -r ' .items[] as $n | ($n.status.conditions[] | select((.type=="DiskPressure" or .type=="MemoryPressure" or .type=="PIDPressure") and .status=="True")) as $p | "\($n.metadata.name)\t\($p.type)\t\($p.message)"' "$nodes_json") if [[ -n "$pressures" ]]; then emit_json "WARN" "nodes" "pressure" "Node pressure conditions detected" "$pressures" echo "- **Pressure:** ⚠️" >> "$REPORT"; echo "$pressures" | sed 's/^/ - /' >> "$REPORT" else echo "- **Pressure:** ✅ none" >> "$REPORT" fi unsched=$(jq -r '.items[] | select(.spec.unschedulable==true) | .metadata.name' "$nodes_json") if [[ -n "$unsched" ]]; then emit_json "WARN" "nodes" "unschedulable" "Unschedulable nodes present" "$unsched" echo "- **Unschedulable:** ⚠️ $(echo "$unsched" | tr '\n' ' ')" >> "$REPORT" else echo "- **Unschedulable:** ✅ none" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Networking (DNS + Calico) ------------------ emit_md_h2 "Networking & DNS" # CoreDNS pods status core_dns=$("${KUBECTL[@]}" -n kube-system get deploy -l k8s-app=kube-dns,app.kubernetes.io/name=coredns -o json 2>/dev/null || true) dn_unavail=$(jq -r '([.items[]?|.status.unavailableReplicas // 0] | add) // 0' <<<"$core_dns" 2>/dev/null || echo 0) dn_unavail="$(to_int "$dn_unavail")" if [ "$dn_unavail" -gt 0 ]; then emit_json "ERROR" "networking" "coredns" "CoreDNS has unavailable replicas" "" echo "- **CoreDNS:** ❌ unavailable replicas: $dn_unavail" >> "$REPORT" else echo "- **CoreDNS:** ✅ deployment healthy or not found" >> "$REPORT" fi # Optional ephemeral DNS nslookup test if [[ "$DNS_TEST" == "true" ]]; then echo "- **DNS test:** running ephemeral busybox nslookup ..." >> "$REPORT" set +e "${KUBECTL[@]}" run dnscheck-$$ --image=busybox:1.36 --restart=Never --command -- /bin/sh -c 'nslookup kubernetes.default.svc.cluster.local >/dev/null' \ --image-pull-policy=IfNotPresent --quiet --timeout=30s 1>/dev/null 2>&1 run_rc=$? "${KUBECTL[@]}" delete pod dnscheck-$$ --now --wait=false 1>/dev/null 2>&1 set -e if [[ $run_rc -ne 0 ]]; then emit_json "ERROR" "networking" "dns" "In-pod DNS resolution failed" "Check CoreDNS, network policies, kube-dns Service." echo " ❌ DNS resolution failed" >> "$REPORT" else echo " ✅ DNS resolution ok" >> "$REPORT" fi else echo "- **DNS test:** (skipped)" >> "$REPORT" fi echo "" >> "$REPORT" # Calico basic health emit_md_h3 "Calico" calico_ds=$("${KUBECTL[@]}" -n calico-system get ds calico-node -o json 2>/dev/null || true) if [[ -n "$calico_ds" ]]; then desire=$(jq -r '.status.desiredNumberScheduled // 0' <<<"$calico_ds") ready=$(jq -r '.status.numberReady // 0' <<<"$calico_ds") desire="$(to_int "$desire")"; ready="$(to_int "$ready")" if [ "$ready" -lt "$desire" ]; then emit_json "ERROR" "calico" "daemonset" "calico-node not fully Ready ($ready/$desire)" "Check calico-node pods and CNI errors." echo "- **calico-node:** ❌ $ready/$desire Ready" >> "$REPORT" else echo "- **calico-node:** ✅ $ready/$desire Ready" >> "$REPORT" fi else echo "- **calico-node:** (DaemonSet not found)" >> "$REPORT" fi typha=$("${KUBECTL[@]}" -n calico-system get deploy -l k8s-app=calico-typha -o json 2>/dev/null || true) if [[ -n "$typha" ]]; then unavail=$(jq -r '[.items[]?|.status.unavailableReplicas // 0] | add' <<<"$typha") unavail="$(to_int "$unavail")" if [ "$unavail" -gt 0 ]; then emit_json "WARN" "calico" "typha" "Calico Typha unavailable replicas: $unavail" "" echo "- **calico-typha:** ⚠️ unavailable: $unavail" >> "$REPORT" else echo "- **calico-typha:** ✅ healthy" >> "$REPORT" fi fi echo "" >> "$REPORT" # ----------------------------- Storage & CSI ------------------------------ emit_md_h2 "Storage" sc_json="$ART/sc.json" if [[ -s "$sc_json" ]]; then defaults=$(jq -r '.items[]|select(.metadata.annotations["storageclass.kubernetes.io/is-default-class"]=="true")|.metadata.name' "$sc_json") if [[ -z "$defaults" ]]; then emit_json "WARN" "storage" "default-sc" "No default StorageClass set" "Annotate one SC as default." echo "- **Default StorageClass:** ⚠️ none set" >> "$REPORT" else echo "- **Default StorageClass:** ✅ $defaults" >> "$REPORT" fi fi pvc_pending=$(jq -r '.items[]|select(.status.phase=="Pending")|.metadata.namespace + "/" + .metadata.name' "$ART/pvc.json" 2>/dev/null || true) if [[ -n "$pvc_pending" ]]; then emit_json "ERROR" "storage" "pvc" "Pending PVCs detected" "$pvc_pending" echo "- **PVCs:** ❌ Pending:\n$(echo "$pvc_pending" | sed 's/^/ - /')" >> "$REPORT" else echo "- **PVCs:** ✅ none Pending" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Workloads --------------------------------- emit_md_h2 "Workloads" # Pending pods >5m pending=$(jq -r ' .items[] | select(.status.phase=="Pending") | select((now - (.metadata.creationTimestamp|fromdate)) > 300) | .metadata.namespace + "/" + .metadata.name + " — " + ((.status.conditions // [] | map(select(.type=="PodScheduled"))[0].reason) // "Pending") ' "$ART/pods.json") if [[ -n "$pending" ]]; then emit_json "ERROR" "workloads" "pending" "Pending pods >5m" "$pending" echo "- **Pending Pods (>5m):** ❌" >> "$REPORT"; echo "$pending" | sed 's/^/ - /' >> "$REPORT" else echo "- **Pending Pods (>5m):** ✅ none" >> "$REPORT" fi # CrashLoop / high restarts crash=$(jq -r ' .items[] as $p | ($p.status.containerStatuses // [])[] | select((.restartCount // 0) >= 3) | "\($p.metadata.namespace)/\($p.metadata.name) — \(.name) restarts=\(.restartCount) lastState=\(.lastState|tojson)" ' "$ART/pods.json") if [[ -n "$crash" ]]; then emit_json "WARN" "workloads" "restarts" "Containers with >=3 restarts" "$crash" echo "- **High Restarts (>=3):** ⚠️" >> "$REPORT"; echo "$crash" | sed 's/^/ - /' >> "$REPORT" else echo "- **High Restarts (>=3):** ✅ none" >> "$REPORT" fi # Deployments with unavailable replicas unavail=$(jq -r ' .items[]?|select(.kind=="Deployment")|select((.status.unavailableReplicas // 0) > 0) | .metadata.namespace + "/" + .metadata.name + " — unavailable=" + ((.status.unavailableReplicas|tostring)) ' "$ART/workloads.json" 2>/dev/null || true) if [[ -n "$unavail" ]]; then emit_json "ERROR" "workloads" "deploy-unavailable" "Deployments with unavailable replicas" "$unavail" echo "- **Deployments:** ❌ unavailable replicas:\n$(echo "$unavail" | sed 's/^/ - /')" >> "$REPORT" else echo "- **Deployments:** ✅ all available" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Services & Endpoints ----------------------- emit_md_h2 "Services & Endpoints" svc_0ep=$(jq -r ' ( input | .items[] | {ns:.metadata.namespace, name:.metadata.name} ) as $svc | . as $eps | $svc.ns + "/" + $svc.name as $k ' "$ART/svc.json" "$ART/endpoints.json" 2>/dev/null | sort | uniq -u || true) # Alternative: compute zero endpoints properly svc_zero=$( jq -r ' .items[] | [.metadata.namespace,.metadata.name, (.spec.selector|type)] | @tsv' "$ART/svc.json" \ | while IFS=$'\t' read -r ns name seltype; do # Skip headless/ExternalName? Keep simple: check subsets len subsets=$(jq -r --arg ns "$ns" --arg name "$name" \ '.items[]|select(.metadata.namespace==$ns and .metadata.name==$name)|(.subsets|length)' "$ART/endpoints.json" 2>/dev/null | head -n1) subsets=${subsets:-0} subsets="$(to_int "$subsets")" if [[ "$seltype" != "null" && "$subsets" -eq 0 ]]; then echo "$ns/$name" fi done ) if [[ -n "$svc_zero" ]]; then emit_json "ERROR" "networking" "svc-no-endpoints" "Services with zero Endpoints" "$svc_zero" echo "- **Services with 0 endpoints:** ❌" >> "$REPORT"; echo "$svc_zero" | sed 's/^/ - /' >> "$REPORT" else echo "- **Services with 0 endpoints:** ✅ none" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- TLS Secret Expiry ------------------------- emit_md_h2 "TLS Certificates (Secrets)" # Build a set of TLS secrets actually referenced by Istio Gateways (credentialName) ISTIO_GW_SECRETS_FILE="$ART/istio_gateway_tls_secrets.tsv" : > "$ISTIO_GW_SECRETS_FILE" if [[ -s "$ART/istio_gateways.json" ]]; then jq -r ' .items[] | .metadata.namespace as $ns | (.spec.servers // []) | map(select(.tls.credentialName != null) | [$ns, .tls.credentialName]) | .[] | @tsv ' "$ART/istio_gateways.json" 2>/dev/null | sort -u > "$ISTIO_GW_SECRETS_FILE" || true fi tls_list=$(jq -r '.items[]|select(.type=="kubernetes.io/tls")|.metadata.namespace + "\t" + .metadata.name + "\t" + (.data["tls.crt"]//"")' "$ART/secrets.json" 2>/dev/null || true) if [[ -n "$tls_list" ]]; then exp_rows_inuse="" exp_rows_unused="" while IFS=$'\t' read -r ns name b64; do [[ -z "$b64" ]] && continue crt="$ART/${ns}_${name}.crt" echo "$b64" | base64 -d > "$crt" 2>/dev/null || continue end=$(openssl x509 -enddate -noout -in "$crt" 2>/dev/null | cut -d= -f2) [[ -z "$end" ]] && continue days=$(days_until "$end"); days="$(to_int "$days")" # Is this secret referenced by any Istio Gateway in the same namespace? in_use="no" if grep -q -P "^${ns}\t${name}$" "$ISTIO_GW_SECRETS_FILE" 2>/dev/null; then in_use="yes" fi if [ "$in_use" = "yes" ]; then # Severity: only for IN-USE secrets level="INFO" if [ "$days" -le "$TLS_WARN_DAYS" ]; then level="WARN"; fi if [ "$days" -le "$TLS_ERR_DAYS" ]; then level="ERROR"; fi exp_rows_inuse+="$ns/$name — expires in ${days}d (${level}) [IN-USE]"$'\n' if [ "$level" = "ERROR" ]; then emit_json "ERROR" "security" "tls-expiry" "$ns/$name expiring in ${days}d [IN-USE]" "Referenced by an Istio Gateway; renew certificate." elif [ "$level" = "WARN" ]; then emit_json "WARN" "security" "tls-expiry" "$ns/$name expiring in ${days}d [IN-USE]" "Plan renewal." fi else # UNUSED secrets: do NOT alert; just list under an informational subheader exp_rows_unused+="$ns/$name — expires in ${days}d [unused]"$'\n' fi done <<< "$tls_list" # Print IN-USE expiries (with levels) if [[ -n "$exp_rows_inuse" ]]; then echo "- **TLS expiries (in-use secrets):**" >> "$REPORT" echo "$exp_rows_inuse" | sed 's/^/ - /' >> "$REPORT" else echo "- **TLS expiries (in-use secrets):** none" >> "$REPORT" fi # Print UNUSED secrets as information only if [[ -n "$exp_rows_unused" ]]; then emit_md_h3 "Unused Secrets" echo "$exp_rows_unused" | sed 's/^/ - /' >> "$REPORT" else emit_md_h3 "Unused Secrets" echo " - none" >> "$REPORT" fi else echo "- **TLS expiries (in-use secrets):** (no kubernetes.io/tls secrets found)" >> "$REPORT" emit_md_h3 "Unused Secrets" echo " - none" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Istio Checks ------------------------------ emit_md_h2 "Istio" # istiod deployment istiod=$("${KUBECTL[@]}" -n istio-system get deploy istiod -o json 2>/dev/null || true) if [[ -n "$istiod" ]]; then un=$(jq -r '.status.unavailableReplicas // 0' <<<"$istiod") un="$(to_int "$un")" if [ "$un" -gt 0 ]; then emit_json "ERROR" "istio" "istiod" "istiod has unavailable replicas: $un" "" echo "- **istiod:** ❌ unavailable=$un" >> "$REPORT" else echo "- **istiod:** ✅ healthy" >> "$REPORT" fi else echo "- **istiod:** (not found)" >> "$REPORT" fi # ingress gateway (classic) igw=$("${KUBECTL[@]}" -n istio-system get deploy -l app=istio-ingressgateway -o json 2>/dev/null || true) if [[ -n "$igw" ]]; then un=$(jq -r '[.items[]?|.status.unavailableReplicas // 0] | add' <<<"$igw") un="$(to_int "$un")" if [ "$un" -gt 0 ]; then emit_json "WARN" "istio" "ingressgateway" "IngressGateway unavailable: $un" "" echo "- **IngressGateway:** ⚠️ unavailable=$un" >> "$REPORT" else echo "- **IngressGateway:** ✅ healthy" >> "$REPORT" fi fi # namespaces with auto-injection enabled but pods missing sidecar emit_md_h3 "Sidecar Injection Coverage" # Detect namespaces with auto-injection enabled either by legacy label or revision label inj_ns=$(jq -r '.items[] | select(.metadata.labels["istio-injection"]=="enabled" or (.metadata.labels["istio.io/rev"] != null)) | .metadata.name' "$ART/namespaces.json") missing_list="" if [[ -n "$inj_ns" ]]; then while IFS= read -r ns; do pods=$(jq -r --arg ns "$ns" ' .items[] | select(.metadata.namespace==$ns and (.status.phase=="Running" or .status.phase=="Pending")) | .metadata.name as $n | ((.spec.containers // []) | any(.name=="istio-proxy")) as $has | (.metadata.annotations["sidecar.istio.io/inject"] // "") as $inject | [$n, ($has|tostring), $inject] | @tsv ' "$ART/pods.json") while IFS=$'\t' read -r pn has inject; do [[ -z "$pn" ]] && continue # If a pod explicitly disables injection, don't flag it as missing. if [[ "$has" != "true" && "$inject" != "false" ]]; then missing_list+="$ns/$pn"$'\n' fi done <<< "$pods" done <<< "$inj_ns" fi if [[ -n "$missing_list" ]]; then emit_json "WARN" "istio" "sidecar-missing" "Pods missing istio-proxy in injection-enabled namespaces" "$missing_list" echo "- **Missing sidecars (in injection-enabled ns):** ⚠️" >> "$REPORT"; echo "$missing_list" | sed 's/^/ - /' >> "$REPORT" else echo "- **Missing sidecars:** ✅ none (or no injection-enabled namespaces)" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- App Discovery: Redis ----------------------- emit_md_h2 "App Health — Redis / RabbitMQ / MinIO" emit_md_h3 "Redis" # detect by common labels & names redis_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app=redis,app.kubernetes.io/name=redis -o json 2>/dev/null || true) if [[ -z "$redis_objs" || "$(jq '.items|length' <<<"$redis_objs")" -eq 0 ]]; then # fallback: name contains redis redis_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("redis"))]}' || true) fi if [[ "$(jq '.items|length' <<<"$redis_objs" 2>/dev/null)" -gt 0 ]]; then while read -r line; do ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line") obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json) desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj") desired="$(to_int "$desired")"; ready="$(to_int "$ready")" status="ok"; marker="✅" if [ "$ready" -lt "$desired" ]; then status="unavailable"; marker="❌"; emit_json "ERROR" "apps.redis" "$kind" "$ns/$name unavailable ($ready/$desired)" "Check pod logs and PVCs."; fi echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT" # Endpoints svc=$("${KUBECTL[@]}" -n "$ns" get svc -l "app=redis,app.kubernetes.io/name=redis" -o json 2>/dev/null || true) if [[ -n "$svc" && "$(jq '.items|length' <<<"$svc")" -gt 0 ]]; then while read -r sname; do eps=$(jq -r --arg ns "$ns" --arg s "$sname" '.items[]|select(.metadata.namespace==$ns and .metadata.name==$s)|(.subsets|length)' "$ART/endpoints.json") eps=${eps:-0} eps="$(to_int "$eps")" echo " - svc/$sname endpoints: $eps" >> "$REPORT" if [ "$eps" -eq 0 ]; then emit_json "ERROR" "apps.redis" "endpoints" "$ns/svc/$sname has 0 endpoints" ""; fi done < <(jq -r '.items[].metadata.name' <<<"$svc") fi done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$redis_objs") else echo "- (no Redis discovered)" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- App Discovery: RabbitMQ -------------------- emit_md_h3 "RabbitMQ" rabbit_crd=$(grep -c rabbitmqclusters.rabbitmq.com "$ART/app_crds.txt" 2>/dev/null || echo 0) if (( rabbit_crd > 0 )); then # Operator CRD health (best effort) "${KUBECTL[@]}" get rabbitmqclusters.rabbitmq.com --all-namespaces -o json > "$ART/rabbit_cr.json" 2>/dev/null || true if [[ -s "$ART/rabbit_cr.json" ]]; then while read -r ns name phase; do marker="✅"; lvl="INFO" if [[ "$phase" != "Running" && "$phase" != "Ready" ]]; then marker="❌"; lvl="ERROR"; fi echo "- **$ns/$name (RabbitmqCluster):** $marker phase=$phase" >> "$REPORT" [[ "$lvl" == "ERROR" ]] && emit_json "ERROR" "apps.rabbitmq" "cluster" "$ns/$name phase=$phase" "Check operator and pods." done < <(jq -r '.items[]|.metadata.namespace+" "+.metadata.name+" "+(.status.conditions[]?|select(.type=="Ready")|.status // "Unknown")' "$ART/rabbit_cr.json" 2>/dev/null || true) fi fi # Fallback to Deploy/STS named rabbit rabbit_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app.kubernetes.io/name=rabbitmq,app=rabbitmq -o json 2>/dev/null || true) if [[ -z "$rabbit_objs" || "$(jq '.items|length' <<<"$rabbit_objs")" -eq 0 ]]; then rabbit_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("rabbit"))]}' || true) fi if [[ "$(jq '.items|length' <<<"$rabbit_objs" 2>/dev/null)" -gt 0 ]]; then while read -r line; do ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line") obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json) desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj") desired="$(to_int "$desired")"; ready="$(to_int "$ready")" marker="✅"; if [ "$ready" -lt "$desired" ]; then marker="❌"; emit_json "ERROR" "apps.rabbitmq" "$kind" "$ns/$name unavailable ($ready/$desired)" ""; fi echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT" done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$rabbit_objs") else echo "- (no RabbitMQ discovered)" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- App Discovery: MinIO ----------------------- emit_md_h3 "MinIO" minio_tenants_crd=$(grep -c tenants.minio.min.io "$ART/app_crds.txt" 2>/dev/null || echo 0) if (( minio_tenants_crd > 0 )); then "${KUBECTL[@]}" get tenants.minio.min.io --all-namespaces -o json > "$ART/minio_tenants.json" 2>/dev/null || true if [[ -s "$ART/minio_tenants.json" ]]; then while read -r ns name ready; do marker="✅" [[ "$ready" != "True" ]] && marker="❌" && emit_json "ERROR" "apps.minio" "tenant" "$ns/$name not Ready" "" echo "- **$ns/$name (Tenant):** $marker Ready=$ready" >> "$REPORT" done < <(jq -r '.items[]|.metadata.namespace+" "+.metadata.name+" "+((.status.conditions[]?|select(.type=="Available")|.status)//"Unknown")' "$ART/minio_tenants.json") fi fi # Fallback: Deploy/STS named/labeled minio minio_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app=minio,app.kubernetes.io/name=minio -o json 2>/dev/null || true) if [[ -z "$minio_objs" || "$(jq '.items|length' <<<"$minio_objs")" -eq 0 ]]; then minio_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("minio"))]}' || true) fi if [[ "$(jq '.items|length' <<<"$minio_objs" 2>/dev/null)" -gt 0 ]]; then while read -r line; do ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line") obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json) desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj") desired="$(to_int "$desired")"; ready="$(to_int "$ready")" marker="✅"; if [ "$ready" -lt "$desired" ]; then marker="❌"; emit_json "ERROR" "apps.minio" "$kind" "$ns/$name unavailable ($ready/$desired)" ""; fi echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT" # PVCs bound? claim_names=$(jq -r '.spec.volumeClaimTemplates[]?.metadata.name' <<<"$obj" 2>/dev/null || true) if [[ -n "$claim_names" ]]; then for cn in $claim_names; do # StatefulSets name-ordinal claim pattern echo " - PVC template: $cn" >> "$REPORT" done fi done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$minio_objs") else echo "- (no MinIO discovered)" >> "$REPORT" fi echo "" >> "$REPORT" # ----------------------------- Events Snapshot --------------------------- emit_md_h2 "Recent Warning/Error Events (top 30)" events_tsv=$(jq -r ' .items[] | select(.type=="Warning" or (.reason|test("BackOff|Failed|Error"))) | [.lastTimestamp, .involvedObject.namespace, .involvedObject.kind, .involvedObject.name, .reason, (.message|gsub("\n"; " "))] | @tsv' "$ART/events.json" 2>/dev/null | tail -n 30 || true) if [[ -n "$events_tsv" ]]; then echo -e "\n| Time | NS | Kind | Name | Reason | Message |" >> "$REPORT" echo "|---|---|---|---|---|---|" >> "$REPORT" while IFS=$'\t' read -r t ns k n r m; do echo "| $t | ${ns:-} | ${k:-} | ${n:-} | ${r:-} | ${m:-} |" >> "$REPORT" done <<< "$events_tsv" else echo "- No recent warnings/errors." >> "$REPORT" fi # ----------------------------- Rollup & Exit ----------------------------- emit_md_h2 "Summary & Exit Code" # produce a compact rollup LEVEL="OK" if grep -q '"level":"ERROR"' "$JSONL" 2>/dev/null; then LEVEL="ERROR" elif grep -q '"level":"WARN"' "$JSONL" 2>/dev/null; then LEVEL="WARN" fi echo "- **Overall:** ${LEVEL}" >> "$REPORT" # finalize JSON summary array jq -s '.' "$JSONL" > "$OUT_DIR/summary.json" 2>/dev/null || echo "[]">"$OUT_DIR/summary.json" echo echo "Report written to: $REPORT" echo "Artifacts in: $ART" case "$LEVEL" in ERROR) exit 2;; WARN) exit 1;; *) exit 0;; esac