#!/usr/bin/env bash
set -euo pipefail
# k8s-health.sh — single-context Kubernetes health report (Markdown)
# Focus: core cluster + Calico + Istio + common apps (Redis, RabbitMQ, MinIO)
# Deps: kubectl, jq, awk, sed, grep, base64, openssl; (optional) gdate
# ----------------------------- CLI & Globals -----------------------------
OUT_DIR=""
REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-8s}"
DNS_TEST="${DNS_TEST:-true}"         # set false to skip ephemeral DNS test
TLS_WARN_DAYS="${TLS_WARN_DAYS:-14}"
TLS_ERR_DAYS="${TLS_ERR_DAYS:-7}"
usage() {
  cat <<EOF
Usage: $0 [--context NAME] [--namespace NS] [--out DIR] [--no-dns]
Options:
  --context NAME    Use a specific kubeconfig context (default: current)
  --namespace NS    Limit app checks to a namespace (default: all)
  --out DIR         Output directory (default: ./k8s-health-<timestamp>)
  --no-dns          Skip ephemeral DNS resolution test
Env overrides:
  REQUEST_TIMEOUT (default: 8s) | TLS_WARN_DAYS (14) | TLS_ERR_DAYS (7)
EOF
}
CTX_ARGS=()
APP_NS=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --context) shift; CTX_ARGS+=(--context "$1");;
    --namespace) shift; APP_NS="$1";;
    --out) shift; OUT_DIR="$1";;
    --no-dns) DNS_TEST=false;;
    -h|--help) usage; exit 0;;
    *) echo "Unknown arg: $1" >&2; usage; exit 3;;
  esac
  shift
done
ts_now() { date -Is; }
to_ts() {
  # portable timestamp from RFC date string (uses gdate if available)
  local d="$1"
  if command -v gdate >/dev/null 2>&1; then gdate -d "$d" +%s; else date -d "$d" +%s; fi 2>/dev/null || echo 0
}
days_until() {
  local end="$1"; local end_ts; end_ts=$(to_ts "$end")
  local now_ts; now_ts=$(date +%s)
  echo $(( (end_ts - now_ts) / 86400 ))
}
# Normalize possibly multi-line/non-numeric values to a single non-negative integer (default 0)
to_int() {
  local v="$1"
  # replace newlines/tabs with spaces, take first token
  v="${v//$'\n'/ }"; v="${v//$'\t'/ }"; v="${v%% *}"
  # strip non-digits
  v="$(printf '%s' "$v" | sed -E 's/[^0-9-]//g')"
  [[ "$v" =~ ^-?[0-9]+$ ]] || v=0
  printf '%s' "$v"
}
if ! command -v kubectl >/dev/null; then echo "kubectl not found" >&2; exit 3; fi
if ! command -v jq >/dev/null; then echo "jq not found" >&2; exit 3; fi
if [[ -z "$OUT_DIR" ]]; then
  OUT_DIR="./k8s-health-$(date +%Y%m%d-%H%M%S)"
fi
ART="$OUT_DIR/artifacts"
mkdir -p "$ART"
REPORT="$OUT_DIR/report.md"
JSONL="$OUT_DIR/summary.jsonl"
touch "$REPORT" "$JSONL"
# ----------------------------- Emit Helpers ------------------------------
emit_json() {
  # emit_json LEVEL AREA CHECK MESSAGE HINT
  printf '{"ts":"%s","level":"%s","area":"%s","check":"%s","message":%s,"hint":%s}\n' \
    "$(ts_now)" "$1" "$2" "$3" "$(jq -Rs . <<<"$4")" "$(jq -Rs . <<<"${5:-}")" >> "$JSONL"
}
emit_md_h1() { echo -e "# $1\n" >> "$REPORT"; }
emit_md_h2() { echo -e "## $1\n" >> "$REPORT"; }
emit_md_h3() { echo -e "### $1\n" >> "$REPORT"; }
emit_md_kv() { echo "- **$1:** $2" >> "$REPORT"; }
emit_md_code() { echo -e "\n\`\`\`\n$1\n\`\`\`\n" >> "$REPORT"; }
# ----------------------------- Prefetch Cache ----------------------------
echo "Collecting cluster state..."
set +e
KUBECTL=(kubectl "${CTX_ARGS[@]}")
# Use a lightweight API call instead of 'kubectl version' which can fail for reasons unrelated to reachability.
if ! "${KUBECTL[@]}" --request-timeout="$REQUEST_TIMEOUT" get --raw='/version' >/dev/null 2>&1; then
  # Fallback to a simple resource list in case /version endpoint is blocked by a proxy.
  if ! "${KUBECTL[@]}" --request-timeout="$REQUEST_TIMEOUT" get nodes -o name >/dev/null 2>&1; then
    echo "Cannot reach cluster with kubectl" >&2; exit 3
  fi
fi
set -e
"${KUBECTL[@]}" version -o json > "$ART/version.json" 2>/dev/null
"${KUBECTL[@]}" api-resources > "$ART/apiresources.txt" || true
"${KUBECTL[@]}" get nodes -o json > "$ART/nodes.json"
if [[ -n "$APP_NS" ]]; then
  ns_arg=(-n "$APP_NS")
else
  ns_arg=(--all-namespaces)
fi
"${KUBECTL[@]}" get pods "${ns_arg[@]}" -o json > "$ART/pods.json"
"${KUBECTL[@]}" get ns -o json > "$ART/namespaces.json"
"${KUBECTL[@]}" get events --all-namespaces --sort-by=.lastTimestamp -o json --request-timeout="$REQUEST_TIMEOUT" > "$ART/events.json" || true
"${KUBECTL[@]}" get svc --all-namespaces -o json > "$ART/svc.json"
"${KUBECTL[@]}" get endpoints --all-namespaces -o json > "$ART/endpoints.json"
"${KUBECTL[@]}" get endpointslices.discovery.k8s.io --all-namespaces -o json > "$ART/epslices.json" 2>/dev/null || true
"${KUBECTL[@]}" get deploy,ds,sts,job,cronjob,hpa,pdb --all-namespaces -o json > "$ART/workloads.json" 2>/dev/null || true
"${KUBECTL[@]}" get pvc --all-namespaces -o json > "$ART/pvc.json" 2>/dev/null || true
"${KUBECTL[@]}" get pv -o json > "$ART/pv.json" 2>/dev/null || true
"${KUBECTL[@]}" get storageclasses.storage.k8s.io -o json > "$ART/sc.json" 2>/dev/null || true
"${KUBECTL[@]}" get secrets --all-namespaces -o json > "$ART/secrets.json" 2>/dev/null || true
"${KUBECTL[@]}" get csidrivers.storage.k8s.io,csinodes.storage.k8s.io -o json > "$ART/csi.json" 2>/dev/null || true
# Istio + Calico artifacts (best effort)
"${KUBECTL[@]}" -n istio-system get deploy,ds,pods,svc -o wide > "$ART/istio_ls.txt" 2>/dev/null || true
"${KUBECTL[@]}" -n calico-system get deploy,ds,pods -o wide > "$ART/calico_ls.txt" 2>/dev/null || true
"${KUBECTL[@]}" get crd tenants.minio.min.io rabbitmqclusters.rabbitmq.com 2>/dev/null | sed '1d' > "$ART/app_crds.txt" || true
# Cache Istio Gateways (best effort)
"${KUBECTL[@]}" get gateway.networking.istio.io --all-namespaces -o json > "$ART/istio_gateways.json" 2>/dev/null || true
# ----------------------------- Report Header -----------------------------
cluster_server=$(
  jq -r '.serverVersion.gitVersion + " (" + .serverVersion.platform + ")"' "$ART/version.json" 2>/dev/null \
  || echo "unknown")
client_ver=$(
  jq -r '.clientVersion.gitVersion' "$ART/version.json" 2>/dev/null \
  || echo "unknown")
ctx_name=$("${KUBECTL[@]}" config current-context 2>/dev/null || echo "current")
emit_md_h1 "Kubernetes Health Report — ${ctx_name}"
emit_md_kv "Generated" "$(ts_now)"
emit_md_kv "kubectl client" "$client_ver"
emit_md_kv "APIServer" "$cluster_server"
emit_md_kv "Namespace scope (apps)" "${APP_NS:-all}"
echo "" >> "$REPORT"
# ----------------------------- Versions & Skew ----------------------------
emit_md_h2 "Cluster Versions & Skew"
node_versions=$(jq -r '.items[]?.status.nodeInfo.kubeletVersion' "$ART/nodes.json" | sort | uniq -c | sed 's/^/  /')
emit_md_code "Kubelet versions:\n${node_versions}"
server_minor=$(jq -r 'try (.serverVersion.minor|tonumber) catch 0' "$ART/version.json" 2>/dev/null || echo 0)
first_kubelet_minor=$(jq -r '[.items[]?.status.nodeInfo.kubeletVersion
                              | capture("v(?<maj>\\d+)\\.(?<min>\\d+)")
                              | .min
                              | tonumber] 
                              | (min // 0)' "$ART/nodes.json" 2>/dev/null || echo 0)
last_kubelet_minor=$(jq -r  '[.items[]?.status.nodeInfo.kubeletVersion
                              | capture("v(?<maj>\\d+)\\.(?<min>\\d+)")
                              | .min
                              | tonumber] 
                              | (max // 0)' "$ART/nodes.json" 2>/dev/null || echo 0)
: "${server_minor:=0}"
: "${first_kubelet_minor:=0}"
: "${last_kubelet_minor:=0}"
# Normalize minor versions
server_minor="$(to_int "$server_minor")"
first_kubelet_minor="$(to_int "$first_kubelet_minor")"
last_kubelet_minor="$(to_int "$last_kubelet_minor")"
# Ensure single-token integers before computing skew (avoid bash arithmetic on malformed values)
lm="$(to_int "$last_kubelet_minor")"
sm="$(to_int "$server_minor")"
fm="$(to_int "$first_kubelet_minor")"
# Compute absolute skew with awk only (robust even if inputs are "0")
abs_skew="$(awk -v a="$lm" -v b="$sm" 'BEGIN{d=a-b; if (d<0) d=-d; print d+0}' 2>/dev/null)"
abs_skew="$(to_int "$abs_skew")"
if [ "$abs_skew" -gt 1 ]; then
  emit_json "ERROR" "version" "skew" "Kubelet/APIServer minor skew > 1 (server minor ${sm}, kubelet min/max ${fm}/${lm})" "Align versions per K8s skew policy."
  echo "- **Version Skew:** ❌ kubelet/APIServer minor skew > 1 (server=${sm}, kubelet min/max=${fm}/${lm})" >> "$REPORT"
else
  echo "- **Version Skew:** ✅ within supported range (server=${sm}, kubelet min/max=${fm}/${lm})" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- API Server Health --------------------------
emit_md_h2 "API Server Ready/Liveness"
readyz="$ART/readyz.txt"
livez="$ART/livez.txt"
set +e
"${KUBECTL[@]}" get --raw='/readyz?verbose' >"$readyz" 2>&1
r_rc=$?
"${KUBECTL[@]}" get --raw='/livez?verbose' >"$livez" 2>&1
l_rc=$?
set -e
emit_md_code "readyz:\n$(cat "$readyz" 2>/dev/null || true)"
emit_md_code "livez:\n$(cat "$livez" 2>/dev/null || true)"
fail_cnt=$(grep -c 'fail' "$readyz" 2>/dev/null || true)
fail_cnt="$(to_int "${fail_cnt:-0}")"
if [ "$r_rc" -ne 0 ] || [ "$fail_cnt" -gt 0 ]; then
  emit_json "ERROR" "control-plane" "readyz" "APIServer readyz reports failures" "Check control-plane component health."
  echo "- **APIServer readyz:** ❌ failures detected" >> "$REPORT"
else
  echo "- **APIServer readyz:** ✅ ok" >> "$REPORT"
fi
if [[ $l_rc -ne 0 ]]; then
  emit_json "ERROR" "control-plane" "livez" "APIServer livez not reachable" ""
  echo "- **APIServer livez:** ❌ unreachable" >> "$REPORT"
else
  echo "- **APIServer livez:** ✅ ok" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Nodes -------------------------------------
emit_md_h2 "Node Health"
nodes_json="$ART/nodes.json"
not_ready=$(jq -r '.items[] | select([.status.conditions[]?|select(.type=="Ready")][0].status!="True") | .metadata.name' "$nodes_json")
if [[ -n "$not_ready" ]]; then
  emit_json "ERROR" "nodes" "ready" "Some nodes NotReady" "$not_ready"
  echo "- **Ready:** ❌ NotReady nodes present:" >> "$REPORT"; echo "$not_ready" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **Ready:** ✅ all nodes Ready" >> "$REPORT"
fi
pressures=$(jq -r '
  .items[] as $n
  | ($n.status.conditions[] | select((.type=="DiskPressure" or .type=="MemoryPressure" or .type=="PIDPressure") and .status=="True")) as $p
  | "\($n.metadata.name)\t\($p.type)\t\($p.message)"' "$nodes_json")
if [[ -n "$pressures" ]]; then
  emit_json "WARN" "nodes" "pressure" "Node pressure conditions detected" "$pressures"
  echo "- **Pressure:** ⚠️" >> "$REPORT"; echo "$pressures" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **Pressure:** ✅ none" >> "$REPORT"
fi
unsched=$(jq -r '.items[] | select(.spec.unschedulable==true) | .metadata.name' "$nodes_json")
if [[ -n "$unsched" ]]; then
  emit_json "WARN" "nodes" "unschedulable" "Unschedulable nodes present" "$unsched"
  echo "- **Unschedulable:** ⚠️ $(echo "$unsched" | tr '\n' ' ')" >> "$REPORT"
else
  echo "- **Unschedulable:** ✅ none" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Networking (DNS + Calico) ------------------
emit_md_h2 "Networking & DNS"
# CoreDNS pods status
core_dns=$("${KUBECTL[@]}" -n kube-system get deploy -l k8s-app=kube-dns,app.kubernetes.io/name=coredns -o json 2>/dev/null || true)
dn_unavail=$(jq -r '([.items[]?|.status.unavailableReplicas // 0] | add) // 0' <<<"$core_dns" 2>/dev/null || echo 0)
dn_unavail="$(to_int "$dn_unavail")"
if [ "$dn_unavail" -gt 0 ]; then
  emit_json "ERROR" "networking" "coredns" "CoreDNS has unavailable replicas" ""
  echo "- **CoreDNS:** ❌ unavailable replicas: $dn_unavail" >> "$REPORT"
else
  echo "- **CoreDNS:** ✅ deployment healthy or not found" >> "$REPORT"
fi
# Optional ephemeral DNS nslookup test
if [[ "$DNS_TEST" == "true" ]]; then
  echo "- **DNS test:** running ephemeral busybox nslookup ..." >> "$REPORT"
  set +e
  "${KUBECTL[@]}" run dnscheck-$$ --image=busybox:1.36 --restart=Never --command -- /bin/sh -c 'nslookup kubernetes.default.svc.cluster.local >/dev/null' \
    --image-pull-policy=IfNotPresent --quiet --timeout=30s 1>/dev/null 2>&1
  run_rc=$?
  "${KUBECTL[@]}" delete pod dnscheck-$$ --now --wait=false 1>/dev/null 2>&1
  set -e
  if [[ $run_rc -ne 0 ]]; then
    emit_json "ERROR" "networking" "dns" "In-pod DNS resolution failed" "Check CoreDNS, network policies, kube-dns Service."
    echo "  ❌ DNS resolution failed" >> "$REPORT"
  else
    echo "  ✅ DNS resolution ok" >> "$REPORT"
  fi
else
  echo "- **DNS test:** (skipped)" >> "$REPORT"
fi
echo "" >> "$REPORT"
# Calico basic health
emit_md_h3 "Calico"
calico_ds=$("${KUBECTL[@]}" -n calico-system get ds calico-node -o json 2>/dev/null || true)
if [[ -n "$calico_ds" ]]; then
  desire=$(jq -r '.status.desiredNumberScheduled // 0' <<<"$calico_ds")
  ready=$(jq -r '.status.numberReady // 0' <<<"$calico_ds")
  desire="$(to_int "$desire")"; ready="$(to_int "$ready")"
  if [ "$ready" -lt "$desire" ]; then
    emit_json "ERROR" "calico" "daemonset" "calico-node not fully Ready ($ready/$desire)" "Check calico-node pods and CNI errors."
    echo "- **calico-node:** ❌ $ready/$desire Ready" >> "$REPORT"
  else
    echo "- **calico-node:** ✅ $ready/$desire Ready" >> "$REPORT"
  fi
else
  echo "- **calico-node:** (DaemonSet not found)" >> "$REPORT"
fi
typha=$("${KUBECTL[@]}" -n calico-system get deploy -l k8s-app=calico-typha -o json 2>/dev/null || true)
if [[ -n "$typha" ]]; then
  unavail=$(jq -r '[.items[]?|.status.unavailableReplicas // 0] | add' <<<"$typha")
  unavail="$(to_int "$unavail")"
  if [ "$unavail" -gt 0 ]; then
    emit_json "WARN" "calico" "typha" "Calico Typha unavailable replicas: $unavail" ""
    echo "- **calico-typha:** ⚠️ unavailable: $unavail" >> "$REPORT"
  else
    echo "- **calico-typha:** ✅ healthy" >> "$REPORT"
  fi
fi
echo "" >> "$REPORT"
# ----------------------------- Storage & CSI ------------------------------
emit_md_h2 "Storage"
sc_json="$ART/sc.json"
if [[ -s "$sc_json" ]]; then
  defaults=$(jq -r '.items[]|select(.metadata.annotations["storageclass.kubernetes.io/is-default-class"]=="true")|.metadata.name' "$sc_json")
  if [[ -z "$defaults" ]]; then
    emit_json "WARN" "storage" "default-sc" "No default StorageClass set" "Annotate one SC as default."
    echo "- **Default StorageClass:** ⚠️ none set" >> "$REPORT"
  else
    echo "- **Default StorageClass:** ✅ $defaults" >> "$REPORT"
  fi
fi
pvc_pending=$(jq -r '.items[]|select(.status.phase=="Pending")|.metadata.namespace + "/" + .metadata.name' "$ART/pvc.json" 2>/dev/null || true)
if [[ -n "$pvc_pending" ]]; then
  emit_json "ERROR" "storage" "pvc" "Pending PVCs detected" "$pvc_pending"
  echo "- **PVCs:** ❌ Pending:\n$(echo "$pvc_pending" | sed 's/^/  - /')" >> "$REPORT"
else
  echo "- **PVCs:** ✅ none Pending" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Workloads ---------------------------------
emit_md_h2 "Workloads"
# Pending pods >5m
pending=$(jq -r '
  .items[]
  | select(.status.phase=="Pending")
  | select((now - (.metadata.creationTimestamp|fromdate)) > 300)
  | .metadata.namespace + "/" + .metadata.name + " — " + ((.status.conditions // [] | map(select(.type=="PodScheduled"))[0].reason) // "Pending")
' "$ART/pods.json")
if [[ -n "$pending" ]]; then
  emit_json "ERROR" "workloads" "pending" "Pending pods >5m" "$pending"
  echo "- **Pending Pods (>5m):** ❌" >> "$REPORT"; echo "$pending" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **Pending Pods (>5m):** ✅ none" >> "$REPORT"
fi
# CrashLoop / high restarts
crash=$(jq -r '
  .items[] as $p
  | ($p.status.containerStatuses // [])[]
  | select((.restartCount // 0) >= 3)
  | "\($p.metadata.namespace)/\($p.metadata.name) — \(.name) restarts=\(.restartCount) lastState=\(.lastState|tojson)"
' "$ART/pods.json")
if [[ -n "$crash" ]]; then
  emit_json "WARN" "workloads" "restarts" "Containers with >=3 restarts" "$crash"
  echo "- **High Restarts (>=3):** ⚠️" >> "$REPORT"; echo "$crash" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **High Restarts (>=3):** ✅ none" >> "$REPORT"
fi
# Deployments with unavailable replicas
unavail=$(jq -r '
  .items[]?|select(.kind=="Deployment")|select((.status.unavailableReplicas // 0) > 0)
  | .metadata.namespace + "/" + .metadata.name + " — unavailable=" + ((.status.unavailableReplicas|tostring))
' "$ART/workloads.json" 2>/dev/null || true)
if [[ -n "$unavail" ]]; then
  emit_json "ERROR" "workloads" "deploy-unavailable" "Deployments with unavailable replicas" "$unavail"
  echo "- **Deployments:** ❌ unavailable replicas:\n$(echo "$unavail" | sed 's/^/  - /')" >> "$REPORT"
else
  echo "- **Deployments:** ✅ all available" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Services & Endpoints -----------------------
emit_md_h2 "Services & Endpoints"
svc_0ep=$(jq -r '
  ( input | .items[] | {ns:.metadata.namespace, name:.metadata.name} ) as $svc
  | . as $eps
  | $svc.ns + "/" + $svc.name as $k
' "$ART/svc.json" "$ART/endpoints.json" 2>/dev/null | sort | uniq -u || true)
# Alternative: compute zero endpoints properly
svc_zero=$(
  jq -r '
    .items[] | [.metadata.namespace,.metadata.name, (.spec.selector|type)] | @tsv' "$ART/svc.json" \
  | while IFS=$'\t' read -r ns name seltype; do
      # Skip headless/ExternalName? Keep simple: check subsets len
      subsets=$(jq -r --arg ns "$ns" --arg name "$name" \
        '.items[]|select(.metadata.namespace==$ns and .metadata.name==$name)|(.subsets|length)' "$ART/endpoints.json" 2>/dev/null | head -n1)
      subsets=${subsets:-0}
      subsets="$(to_int "$subsets")"
      if [[ "$seltype" != "null" && "$subsets" -eq 0 ]]; then
        echo "$ns/$name"
      fi
    done
)
if [[ -n "$svc_zero" ]]; then
  emit_json "ERROR" "networking" "svc-no-endpoints" "Services with zero Endpoints" "$svc_zero"
  echo "- **Services with 0 endpoints:** ❌" >> "$REPORT"; echo "$svc_zero" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **Services with 0 endpoints:** ✅ none" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- TLS Secret Expiry -------------------------
emit_md_h2 "TLS Certificates (Secrets)"
# Build a set of TLS secrets actually referenced by Istio Gateways (credentialName)
ISTIO_GW_SECRETS_FILE="$ART/istio_gateway_tls_secrets.tsv"
: > "$ISTIO_GW_SECRETS_FILE"
if [[ -s "$ART/istio_gateways.json" ]]; then
  jq -r '
    .items[]
    | .metadata.namespace as $ns
    | (.spec.servers // [])
    | map(select(.tls.credentialName != null) | [$ns, .tls.credentialName])
    | .[]
    | @tsv
  ' "$ART/istio_gateways.json" 2>/dev/null | sort -u > "$ISTIO_GW_SECRETS_FILE" || true
fi
tls_list=$(jq -r '.items[]|select(.type=="kubernetes.io/tls")|.metadata.namespace + "\t" + .metadata.name + "\t" + (.data["tls.crt"]//"")' "$ART/secrets.json" 2>/dev/null || true)
if [[ -n "$tls_list" ]]; then
  exp_rows_inuse=""
  exp_rows_unused=""
  while IFS=$'\t' read -r ns name b64; do
    [[ -z "$b64" ]] && continue
    crt="$ART/${ns}_${name}.crt"
    echo "$b64" | base64 -d > "$crt" 2>/dev/null || continue
    end=$(openssl x509 -enddate -noout -in "$crt" 2>/dev/null | cut -d= -f2)
    [[ -z "$end" ]] && continue
    days=$(days_until "$end"); days="$(to_int "$days")"
    # Is this secret referenced by any Istio Gateway in the same namespace?
    in_use="no"
    if grep -q -P "^${ns}\t${name}$" "$ISTIO_GW_SECRETS_FILE" 2>/dev/null; then
      in_use="yes"
    fi
    if [ "$in_use" = "yes" ]; then
      # Severity: only for IN-USE secrets
      level="INFO"
      if [ "$days" -le "$TLS_WARN_DAYS" ]; then level="WARN"; fi
      if [ "$days" -le "$TLS_ERR_DAYS" ]; then level="ERROR"; fi
      exp_rows_inuse+="$ns/$name — expires in ${days}d (${level}) [IN-USE]"$'\n'
      if [ "$level" = "ERROR" ]; then
        emit_json "ERROR" "security" "tls-expiry" "$ns/$name expiring in ${days}d [IN-USE]" "Referenced by an Istio Gateway; renew certificate."
      elif [ "$level" = "WARN" ]; then
        emit_json "WARN" "security" "tls-expiry" "$ns/$name expiring in ${days}d [IN-USE]" "Plan renewal."
      fi
    else
      # UNUSED secrets: do NOT alert; just list under an informational subheader
      exp_rows_unused+="$ns/$name — expires in ${days}d [unused]"$'\n'
    fi
  done <<< "$tls_list"
  # Print IN-USE expiries (with levels)
  if [[ -n "$exp_rows_inuse" ]]; then
    echo "- **TLS expiries (in-use secrets):**" >> "$REPORT"
    echo "$exp_rows_inuse" | sed 's/^/  - /' >> "$REPORT"
  else
    echo "- **TLS expiries (in-use secrets):** none" >> "$REPORT"
  fi
  # Print UNUSED secrets as information only
  if [[ -n "$exp_rows_unused" ]]; then
    emit_md_h3 "Unused Secrets"
    echo "$exp_rows_unused" | sed 's/^/  - /' >> "$REPORT"
  else
    emit_md_h3 "Unused Secrets"
    echo "  - none" >> "$REPORT"
  fi
else
  echo "- **TLS expiries (in-use secrets):** (no kubernetes.io/tls secrets found)" >> "$REPORT"
  emit_md_h3 "Unused Secrets"
  echo "  - none" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Istio Checks ------------------------------
emit_md_h2 "Istio"
# istiod deployment
istiod=$("${KUBECTL[@]}" -n istio-system get deploy istiod -o json 2>/dev/null || true)
if [[ -n "$istiod" ]]; then
  un=$(jq -r '.status.unavailableReplicas // 0' <<<"$istiod")
  un="$(to_int "$un")"
  if [ "$un" -gt 0 ]; then
    emit_json "ERROR" "istio" "istiod" "istiod has unavailable replicas: $un" ""
    echo "- **istiod:** ❌ unavailable=$un" >> "$REPORT"
  else
    echo "- **istiod:** ✅ healthy" >> "$REPORT"
  fi
else
  echo "- **istiod:** (not found)" >> "$REPORT"
fi
# ingress gateway (classic)
igw=$("${KUBECTL[@]}" -n istio-system get deploy -l app=istio-ingressgateway -o json 2>/dev/null || true)
if [[ -n "$igw" ]]; then
  un=$(jq -r '[.items[]?|.status.unavailableReplicas // 0] | add' <<<"$igw")
  un="$(to_int "$un")"
  if [ "$un" -gt 0 ]; then
    emit_json "WARN" "istio" "ingressgateway" "IngressGateway unavailable: $un" ""
    echo "- **IngressGateway:** ⚠️ unavailable=$un" >> "$REPORT"
  else
    echo "- **IngressGateway:** ✅ healthy" >> "$REPORT"
  fi
fi
# namespaces with auto-injection enabled but pods missing sidecar
emit_md_h3 "Sidecar Injection Coverage"
# Detect namespaces with auto-injection enabled either by legacy label or revision label
inj_ns=$(jq -r '.items[]
                  | select(.metadata.labels["istio-injection"]=="enabled" or (.metadata.labels["istio.io/rev"] != null))
                  | .metadata.name' "$ART/namespaces.json")
missing_list=""
if [[ -n "$inj_ns" ]]; then
  while IFS= read -r ns; do
    pods=$(jq -r --arg ns "$ns" '
      .items[]
      | select(.metadata.namespace==$ns and (.status.phase=="Running" or .status.phase=="Pending"))
      | .metadata.name as $n
      | ((.spec.containers // []) | any(.name=="istio-proxy")) as $has
      | (.metadata.annotations["sidecar.istio.io/inject"] // "") as $inject
      | [$n, ($has|tostring), $inject] | @tsv
    ' "$ART/pods.json")
    while IFS=$'\t' read -r pn has inject; do
      [[ -z "$pn" ]] && continue
      # If a pod explicitly disables injection, don't flag it as missing.
      if [[ "$has" != "true" && "$inject" != "false" ]]; then
        missing_list+="$ns/$pn"$'\n'
      fi
    done <<< "$pods"
  done <<< "$inj_ns"
fi
if [[ -n "$missing_list" ]]; then
  emit_json "WARN" "istio" "sidecar-missing" "Pods missing istio-proxy in injection-enabled namespaces" "$missing_list"
  echo "- **Missing sidecars (in injection-enabled ns):** ⚠️" >> "$REPORT"; echo "$missing_list" | sed 's/^/  - /' >> "$REPORT"
else
  echo "- **Missing sidecars:** ✅ none (or no injection-enabled namespaces)" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- App Discovery: Redis -----------------------
emit_md_h2 "App Health — Redis / RabbitMQ / MinIO"
emit_md_h3 "Redis"
# detect by common labels & names
redis_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app=redis,app.kubernetes.io/name=redis -o json 2>/dev/null || true)
if [[ -z "$redis_objs" || "$(jq '.items|length' <<<"$redis_objs")" -eq 0 ]]; then
  # fallback: name contains redis
  redis_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("redis"))]}' || true)
fi
if [[ "$(jq '.items|length' <<<"$redis_objs" 2>/dev/null)" -gt 0 ]]; then
  while read -r line; do
    ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line")
    obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json)
    desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj")
    desired="$(to_int "$desired")"; ready="$(to_int "$ready")"
    status="ok"; marker="✅"
    if [ "$ready" -lt "$desired" ]; then status="unavailable"; marker="❌"; emit_json "ERROR" "apps.redis" "$kind" "$ns/$name unavailable ($ready/$desired)" "Check pod logs and PVCs."; fi
    echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT"
    # Endpoints
    svc=$("${KUBECTL[@]}" -n "$ns" get svc -l "app=redis,app.kubernetes.io/name=redis" -o json 2>/dev/null || true)
    if [[ -n "$svc" && "$(jq '.items|length' <<<"$svc")" -gt 0 ]]; then
      while read -r sname; do
        eps=$(jq -r --arg ns "$ns" --arg s "$sname" '.items[]|select(.metadata.namespace==$ns and .metadata.name==$s)|(.subsets|length)' "$ART/endpoints.json")
        eps=${eps:-0}
        eps="$(to_int "$eps")"
        echo "  - svc/$sname endpoints: $eps" >> "$REPORT"
        if [ "$eps" -eq 0 ]; then emit_json "ERROR" "apps.redis" "endpoints" "$ns/svc/$sname has 0 endpoints" ""; fi
      done < <(jq -r '.items[].metadata.name' <<<"$svc")
    fi
  done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$redis_objs")
else
  echo "- (no Redis discovered)" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- App Discovery: RabbitMQ --------------------
emit_md_h3 "RabbitMQ"
rabbit_crd=$(grep -c rabbitmqclusters.rabbitmq.com "$ART/app_crds.txt" 2>/dev/null || echo 0)
if (( rabbit_crd > 0 )); then
  # Operator CRD health (best effort)
  "${KUBECTL[@]}" get rabbitmqclusters.rabbitmq.com --all-namespaces -o json > "$ART/rabbit_cr.json" 2>/dev/null || true
  if [[ -s "$ART/rabbit_cr.json" ]]; then
    while read -r ns name phase; do
      marker="✅"; lvl="INFO"
      if [[ "$phase" != "Running" && "$phase" != "Ready" ]]; then marker="❌"; lvl="ERROR"; fi
      echo "- **$ns/$name (RabbitmqCluster):** $marker phase=$phase" >> "$REPORT"
      [[ "$lvl" == "ERROR" ]] && emit_json "ERROR" "apps.rabbitmq" "cluster" "$ns/$name phase=$phase" "Check operator and pods."
    done < <(jq -r '.items[]|.metadata.namespace+" "+.metadata.name+" "+(.status.conditions[]?|select(.type=="Ready")|.status // "Unknown")' "$ART/rabbit_cr.json" 2>/dev/null || true)
  fi
fi
# Fallback to Deploy/STS named rabbit
rabbit_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app.kubernetes.io/name=rabbitmq,app=rabbitmq -o json 2>/dev/null || true)
if [[ -z "$rabbit_objs" || "$(jq '.items|length' <<<"$rabbit_objs")" -eq 0 ]]; then
  rabbit_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("rabbit"))]}' || true)
fi
if [[ "$(jq '.items|length' <<<"$rabbit_objs" 2>/dev/null)" -gt 0 ]]; then
  while read -r line; do
    ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line")
    obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json)
    desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj")
    desired="$(to_int "$desired")"; ready="$(to_int "$ready")"
    marker="✅"; if [ "$ready" -lt "$desired" ]; then marker="❌"; emit_json "ERROR" "apps.rabbitmq" "$kind" "$ns/$name unavailable ($ready/$desired)" ""; fi
    echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT"
  done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$rabbit_objs")
else
  echo "- (no RabbitMQ discovered)" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- App Discovery: MinIO -----------------------
emit_md_h3 "MinIO"
minio_tenants_crd=$(grep -c tenants.minio.min.io "$ART/app_crds.txt" 2>/dev/null || echo 0)
if (( minio_tenants_crd > 0 )); then
  "${KUBECTL[@]}" get tenants.minio.min.io --all-namespaces -o json > "$ART/minio_tenants.json" 2>/dev/null || true
  if [[ -s "$ART/minio_tenants.json" ]]; then
    while read -r ns name ready; do
      marker="✅"
      [[ "$ready" != "True" ]] && marker="❌" && emit_json "ERROR" "apps.minio" "tenant" "$ns/$name not Ready" ""
      echo "- **$ns/$name (Tenant):** $marker Ready=$ready" >> "$REPORT"
    done < <(jq -r '.items[]|.metadata.namespace+" "+.metadata.name+" "+((.status.conditions[]?|select(.type=="Available")|.status)//"Unknown")' "$ART/minio_tenants.json")
  fi
fi
# Fallback: Deploy/STS named/labeled minio
minio_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -l app=minio,app.kubernetes.io/name=minio -o json 2>/dev/null || true)
if [[ -z "$minio_objs" || "$(jq '.items|length' <<<"$minio_objs")" -eq 0 ]]; then
  minio_objs=$("${KUBECTL[@]}" get deploy,sts --all-namespaces -o json 2>/dev/null | jq '{items:[.items[]|select(.metadata.name|test("minio"))]}' || true)
fi
if [[ "$(jq '.items|length' <<<"$minio_objs" 2>/dev/null)" -gt 0 ]]; then
  while read -r line; do
    ns=$(cut -d' ' -f1 <<<"$line"); kind=$(cut -d' ' -f2 <<<"$line"); name=$(cut -d' ' -f3- <<<"$line")
    obj=$("${KUBECTL[@]}" -n "$ns" get "$kind" "$name" -o json)
    desired=$(jq -r '.spec.replicas // 1' <<<"$obj"); ready=$(jq -r '.status.readyReplicas // 0' <<<"$obj")
    desired="$(to_int "$desired")"; ready="$(to_int "$ready")"
    marker="✅"; if [ "$ready" -lt "$desired" ]; then marker="❌"; emit_json "ERROR" "apps.minio" "$kind" "$ns/$name unavailable ($ready/$desired)" ""; fi
    echo "- **$ns/$name ($kind):** $marker $ready/$desired ready" >> "$REPORT"
    # PVCs bound?
    claim_names=$(jq -r '.spec.volumeClaimTemplates[]?.metadata.name' <<<"$obj" 2>/dev/null || true)
    if [[ -n "$claim_names" ]]; then
      for cn in $claim_names; do
        # StatefulSets name-ordinal claim pattern
        echo "  - PVC template: $cn" >> "$REPORT"
      done
    fi
  done < <(jq -r '.items[]|.metadata.namespace+" "+.kind+" "+.metadata.name' <<<"$minio_objs")
else
  echo "- (no MinIO discovered)" >> "$REPORT"
fi
echo "" >> "$REPORT"
# ----------------------------- Events Snapshot ---------------------------
emit_md_h2 "Recent Warning/Error Events (top 30)"
events_tsv=$(jq -r '
  .items[]
  | select(.type=="Warning" or (.reason|test("BackOff|Failed|Error")))
  | [.lastTimestamp, .involvedObject.namespace, .involvedObject.kind, .involvedObject.name, .reason, (.message|gsub("\n"; " "))]
  | @tsv' "$ART/events.json" 2>/dev/null | tail -n 30 || true)
if [[ -n "$events_tsv" ]]; then
  echo -e "\n| Time | NS | Kind | Name | Reason | Message |" >> "$REPORT"
  echo "|---|---|---|---|---|---|" >> "$REPORT"
  while IFS=$'\t' read -r t ns k n r m; do
    echo "| $t | ${ns:-} | ${k:-} | ${n:-} | ${r:-} | ${m:-} |" >> "$REPORT"
  done <<< "$events_tsv"
else
  echo "- No recent warnings/errors." >> "$REPORT"
fi
# ----------------------------- Rollup & Exit -----------------------------
emit_md_h2 "Summary & Exit Code"
# produce a compact rollup
LEVEL="OK"
if grep -q '"level":"ERROR"' "$JSONL" 2>/dev/null; then LEVEL="ERROR"
elif grep -q '"level":"WARN"' "$JSONL" 2>/dev/null; then LEVEL="WARN"
fi
echo "- **Overall:** ${LEVEL}" >> "$REPORT"
# finalize JSON summary array
jq -s '.' "$JSONL" > "$OUT_DIR/summary.json" 2>/dev/null || echo "[]">"$OUT_DIR/summary.json"
echo
echo "Report written to: $REPORT"
echo "Artifacts in:      $ART"
case "$LEVEL" in
  ERROR) exit 2;;
  WARN)  exit 1;;
  *)     exit 0;;
esac