#!/bin/bash # Cross-component diagnostic helpers for catalog install/update/delete scripts. # Sourced by per-component install scripts (cluster + taskserv modes). # # Usage in install-.sh: # # source "${SCRIPT_DIR}/diagnostics.sh" # _diag_install_err_trap # auto-dump on ERR # _diag_register_cluster_workload deployment fleet-daemon fleet-system \ # "app.kubernetes.io/name=fleet-daemon" # _diag_wait_for_deployment "$NAME" "$NAMESPACE" 180 # use this, not bare rollout # # Or for taskserv: # _diag_register_systemd_unit fleet-agent.service # _diag_install_err_trap # ...do work... # # Both helpers assume _kubectl exists in the parent script (cluster mode) or # systemctl/journalctl are on PATH (taskserv mode). set -o pipefail # ── env flag resolution (canonical + PRVNG_ alias) ────────────────────── # Use these helpers in component install.sh: # if _diag_is_force; then ... ; fi # if _diag_is_debug; then ... ; fi # Both accept "1", "true", "yes" (case-insensitive) as truthy. _diag_env_truthy() { local raw="${1:-}" case "$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" in 1|true|yes) return 0 ;; *) return 1 ;; esac } _diag_is_debug() { _diag_env_truthy "${PROVISIONING_DEBUG:-${PRVNG_DEBUG:-}}" } _diag_is_force() { _diag_env_truthy "${PROVISIONING_FORCE:-${PRVNG_FORCE:-}}" } _diag_is_verbose() { _diag_env_truthy "${PROVISIONING_VERBOSE:-${PRVNG_VERBOSE:-}}" } # Step echo gated by verbose. In quiet mode, only major milestones print. _diag_log_step() { if _diag_is_verbose; then echo " → $*" fi } # Run a kubectl-or-similar command, gating output by verbose: # - verbose: stream output as usual # - quiet : capture output, drop on success, print on failure (so errors # are never hidden, while routine "unchanged" lines disappear) _diag_run_quiet() { if _diag_is_verbose; then "$@" return $? fi local out rc out=$("$@" 2>&1) rc=$? if [ $rc -ne 0 ]; then printf '%s\n' "$out" >&2 fi return $rc } # Wrap a kubectl-apply-via-stdin invocation, gating output by verbose. # Usage: # echo "$manifest" | _diag_apply_quiet -f - # Equivalent to: _kubectl apply -f - with verbose-aware output. _diag_apply_quiet() { if _diag_is_verbose; then _kubectl apply "$@" return $? fi local out rc out=$(_kubectl apply "$@" 2>&1) rc=$? if [ $rc -ne 0 ]; then printf '%s\n' "$out" >&2 fi return $rc } # Force-recreate a kube workload: delete (wait for termination, with timeout) # then return success so the caller can re-apply. Idempotent — no-op when the # workload doesn't exist. Use BEFORE `kubectl apply` of the same resource: # # if _diag_is_force; then # _diag_force_delete_workload deployment fleet-daemon fleet-system # fi # _kubectl apply -f - </dev/null 2>&1; then return 0 fi echo " [force] deleting ${kind}/${name} before re-apply (timeout ${timeout}s)" >&2 _kubectl delete "$kind" "$name" \ --namespace "$namespace" \ --wait=true --timeout="${timeout}s" --ignore-not-found 2>&1 || { echo " [force] WARN: delete did not complete cleanly within ${timeout}s — proceeding with apply anyway" >&2 } } # ── workload registration (used by ERR trap) ──────────────────────────── _DIAG_KIND="" # 'cluster' or 'taskserv' _DIAG_K8S_KIND="" # deployment|statefulset|daemonset|cronjob|job _DIAG_K8S_NAME="" _DIAG_K8S_NAMESPACE="" _DIAG_K8S_SELECTOR="" _DIAG_SYSTEMD_UNIT="" _DIAG_TAIL="${_DIAG_TAIL:-60}" _diag_register_cluster_workload() { _DIAG_KIND="cluster" _DIAG_K8S_KIND="$1" _DIAG_K8S_NAME="$2" _DIAG_K8S_NAMESPACE="$3" _DIAG_K8S_SELECTOR="${4:-app.kubernetes.io/name=$2}" } _diag_register_systemd_unit() { _DIAG_KIND="taskserv" _DIAG_SYSTEMD_UNIT="$1" } # ── ERR trap installation ─────────────────────────────────────────────── _diag_install_err_trap() { trap '_diag_on_error $? "$BASH_COMMAND" $LINENO' ERR } _diag_on_error() { local rc="$1" cmd="$2" lineno="$3" { echo "" echo "═══ FAILURE ═══════════════════════════════════════════════════════" echo "exit code : $rc" echo "line : $lineno" echo "command : $cmd" echo "" case "$_DIAG_KIND" in cluster) _diag_dump_cluster ;; taskserv) _diag_dump_taskserv ;; "") echo "(no diagnostic context registered)" ;; *) echo "(unknown _DIAG_KIND=$_DIAG_KIND)" ;; esac echo "═══════════════════════════════════════════════════════════════════" } >&2 exit "$rc" } # ── cluster mode diagnostics ──────────────────────────────────────────── _diag_dump_cluster() { if ! command -v kubectl >/dev/null 2>&1 && ! command -v k0s >/dev/null 2>&1; then echo " (no kubectl/k0s on this host — cannot collect k8s diagnostics)" return fi if [ -z "$_DIAG_K8S_NAME" ]; then echo " (no cluster workload registered — call _diag_register_cluster_workload first)" return fi echo "─── ${_DIAG_K8S_KIND}/${_DIAG_K8S_NAME} (namespace=${_DIAG_K8S_NAMESPACE}) ───" _kubectl get "${_DIAG_K8S_KIND}" "${_DIAG_K8S_NAME}" \ --namespace "${_DIAG_K8S_NAMESPACE}" -o wide 2>&1 || true case "$_DIAG_K8S_KIND" in deployment|statefulset|daemonset) _diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE" ;; cronjob) echo "─── recent jobs from cronjob/${_DIAG_K8S_NAME} ───" _kubectl get jobs --namespace "${_DIAG_K8S_NAMESPACE}" \ -l "app.kubernetes.io/managed-by=cronjob,batch.kubernetes.io/cronjob-name=${_DIAG_K8S_NAME}" \ 2>/dev/null || _kubectl get jobs --namespace "${_DIAG_K8S_NAMESPACE}" 2>/dev/null | grep "${_DIAG_K8S_NAME}" || true _diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE" ;; *) _diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE" ;; esac } _diag_dump_pods() { local selector="$1" namespace="$2" tail="${3:-$_DIAG_TAIL}" echo "" echo "─── pods (selector=${selector}) ───" _kubectl get pods --namespace "$namespace" --selector "$selector" -o wide 2>&1 || true local pods pods=$(_kubectl get pods --namespace "$namespace" --selector "$selector" \ -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true) if [ -z "$pods" ]; then echo " (no pods match selector)" return fi for pod in $pods; do # Smart dump: if logs are accessible the daemon HAS started, so the # error is in the daemon's startup — logs are sufficient. Skip # describe in that case (it doesn't add info). Only when logs are # unavailable (pod never started: image pull failure, volume mount # error, scheduling issue) do we fall back to describe. local logs_out prev_out rc_logs rc_prev logs_out=$(_kubectl logs "$pod" --namespace "$namespace" \ --tail="$tail" --all-containers=true 2>&1) rc_logs=$? if [ $rc_logs -eq 0 ] && [ -n "$logs_out" ]; then echo "" echo "─── logs pod/${pod} --tail=${tail} ───" printf '%s\n' "$logs_out" # If the pod has restarted (CrashLoop), --previous shows the # last instance — usually the most useful for diagnosis. prev_out=$(_kubectl logs "$pod" --namespace "$namespace" \ --tail="$tail" --all-containers=true --previous 2>/dev/null) rc_prev=$? if [ $rc_prev -eq 0 ] && [ -n "$prev_out" ] \ && [ "$prev_out" != "$logs_out" ]; then echo "" echo "─── logs pod/${pod} --previous --tail=${tail} (CrashLoop) ───" printf '%s\n' "$prev_out" fi else # No logs accessible → pod never reached running. Describe + events # are the right diagnostic here. echo "" echo "─── describe pod/${pod} (last 40 lines) ───" _kubectl describe pod "$pod" --namespace "$namespace" 2>&1 | tail -40 || true fi done } # Wrap kubectl rollout with timeout + auto-dump-on-failure. _diag_wait_for_deployment() { local name="$1" namespace="$2" timeout="${3:-180}" local selector="${4:-app.kubernetes.io/name=$name}" _diag_register_cluster_workload deployment "$name" "$namespace" "$selector" if _kubectl rollout status deployment/"$name" \ --namespace "$namespace" --timeout="${timeout}s"; then return 0 fi local rc=$? echo "" >&2 echo "ERROR: deployment/${name} did not become ready within ${timeout}s" >&2 _diag_dump_cluster >&2 return "$rc" } _diag_wait_for_statefulset() { local name="$1" namespace="$2" timeout="${3:-300}" local selector="${4:-app.kubernetes.io/name=$name}" _diag_register_cluster_workload statefulset "$name" "$namespace" "$selector" if _kubectl rollout status statefulset/"$name" \ --namespace "$namespace" --timeout="${timeout}s"; then return 0 fi local rc=$? echo "" >&2 echo "ERROR: statefulset/${name} did not become ready within ${timeout}s" >&2 _diag_dump_cluster >&2 return "$rc" } # ── taskserv mode diagnostics ─────────────────────────────────────────── _diag_dump_taskserv() { if [ -z "$_DIAG_SYSTEMD_UNIT" ]; then echo " (no systemd unit registered — call _diag_register_systemd_unit first)" return fi if ! command -v systemctl >/dev/null 2>&1; then echo " (no systemctl on this host — cannot collect systemd diagnostics)" return fi echo "─── systemctl status ${_DIAG_SYSTEMD_UNIT} ───" systemctl status "$_DIAG_SYSTEMD_UNIT" --no-pager --lines=20 2>&1 || true echo "" echo "─── journalctl -u ${_DIAG_SYSTEMD_UNIT} -n ${_DIAG_TAIL} ───" journalctl --no-pager -u "$_DIAG_SYSTEMD_UNIT" -n "$_DIAG_TAIL" 2>&1 || true } # Idempotent "wait for unit Active" with timeout + auto-dump-on-failure. _diag_wait_for_systemd() { local unit="$1" timeout="${2:-60}" _diag_register_systemd_unit "$unit" local elapsed=0 while [ "$elapsed" -lt "$timeout" ]; do if systemctl is-active --quiet "$unit"; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "" >&2 echo "ERROR: ${unit} did not become Active within ${timeout}s" >&2 _diag_dump_taskserv >&2 return 1 }