307 lines
11 KiB
Bash
307 lines
11 KiB
Bash
|
#!/bin/bash
|
||
|
|
# Cross-component diagnostic helpers for catalog install/update/delete scripts.
|
||
|
|
# Sourced by per-component install scripts (cluster + taskserv modes).
|
||
|
|
#
|
||
|
|
# Usage in install-<name>.sh:
|
||
|
|
#
|
||
|
|
# source "${SCRIPT_DIR}/diagnostics.sh"
|
||
|
|
# _diag_install_err_trap # auto-dump on ERR
|
||
|
|
# _diag_register_cluster_workload deployment fleet-daemon fleet-system \
|
||
|
|
# "app.kubernetes.io/name=fleet-daemon"
|
||
|
|
# _diag_wait_for_deployment "$NAME" "$NAMESPACE" 180 # use this, not bare rollout
|
||
|
|
#
|
||
|
|
# Or for taskserv:
|
||
|
|
# _diag_register_systemd_unit fleet-agent.service
|
||
|
|
# _diag_install_err_trap
|
||
|
|
# ...do work...
|
||
|
|
#
|
||
|
|
# Both helpers assume _kubectl exists in the parent script (cluster mode) or
|
||
|
|
# systemctl/journalctl are on PATH (taskserv mode).
|
||
|
|
|
||
|
|
set -o pipefail
|
||
|
|
|
||
|
|
# ── env flag resolution (canonical + PRVNG_ alias) ──────────────────────
|
||
|
|
# Use these helpers in component install.sh:
|
||
|
|
# if _diag_is_force; then ... ; fi
|
||
|
|
# if _diag_is_debug; then ... ; fi
|
||
|
|
# Both accept "1", "true", "yes" (case-insensitive) as truthy.
|
||
|
|
|
||
|
|
_diag_env_truthy() {
|
||
|
|
local raw="${1:-}"
|
||
|
|
case "$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" in
|
||
|
|
1|true|yes) return 0 ;;
|
||
|
|
*) return 1 ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_is_debug() {
|
||
|
|
_diag_env_truthy "${PROVISIONING_DEBUG:-${PRVNG_DEBUG:-}}"
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_is_force() {
|
||
|
|
_diag_env_truthy "${PROVISIONING_FORCE:-${PRVNG_FORCE:-}}"
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_is_verbose() {
|
||
|
|
_diag_env_truthy "${PROVISIONING_VERBOSE:-${PRVNG_VERBOSE:-}}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Step echo gated by verbose. In quiet mode, only major milestones print.
|
||
|
|
_diag_log_step() {
|
||
|
|
if _diag_is_verbose; then
|
||
|
|
echo " → $*"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# Run a kubectl-or-similar command, gating output by verbose:
|
||
|
|
# - verbose: stream output as usual
|
||
|
|
# - quiet : capture output, drop on success, print on failure (so errors
|
||
|
|
# are never hidden, while routine "unchanged" lines disappear)
|
||
|
|
_diag_run_quiet() {
|
||
|
|
if _diag_is_verbose; then
|
||
|
|
"$@"
|
||
|
|
return $?
|
||
|
|
fi
|
||
|
|
local out rc
|
||
|
|
out=$("$@" 2>&1)
|
||
|
|
rc=$?
|
||
|
|
if [ $rc -ne 0 ]; then
|
||
|
|
printf '%s\n' "$out" >&2
|
||
|
|
fi
|
||
|
|
return $rc
|
||
|
|
}
|
||
|
|
|
||
|
|
# Wrap a kubectl-apply-via-stdin invocation, gating output by verbose.
|
||
|
|
# Usage:
|
||
|
|
# echo "$manifest" | _diag_apply_quiet -f -
|
||
|
|
# Equivalent to: _kubectl apply -f - with verbose-aware output.
|
||
|
|
_diag_apply_quiet() {
|
||
|
|
if _diag_is_verbose; then
|
||
|
|
_kubectl apply "$@"
|
||
|
|
return $?
|
||
|
|
fi
|
||
|
|
local out rc
|
||
|
|
out=$(_kubectl apply "$@" 2>&1)
|
||
|
|
rc=$?
|
||
|
|
if [ $rc -ne 0 ]; then
|
||
|
|
printf '%s\n' "$out" >&2
|
||
|
|
fi
|
||
|
|
return $rc
|
||
|
|
}
|
||
|
|
|
||
|
|
# Force-recreate a kube workload: delete (wait for termination, with timeout)
|
||
|
|
# then return success so the caller can re-apply. Idempotent — no-op when the
|
||
|
|
# workload doesn't exist. Use BEFORE `kubectl apply` of the same resource:
|
||
|
|
#
|
||
|
|
# if _diag_is_force; then
|
||
|
|
# _diag_force_delete_workload deployment fleet-daemon fleet-system
|
||
|
|
# fi
|
||
|
|
# _kubectl apply -f - <<MANIFEST
|
||
|
|
# ...
|
||
|
|
#
|
||
|
|
# Requires _kubectl to be defined in the parent script (cluster mode).
|
||
|
|
_diag_force_delete_workload() {
|
||
|
|
local kind="$1" name="$2" namespace="$3" timeout="${4:-30}"
|
||
|
|
if ! _kubectl get "$kind" "$name" --namespace "$namespace" >/dev/null 2>&1; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
echo " [force] deleting ${kind}/${name} before re-apply (timeout ${timeout}s)" >&2
|
||
|
|
_kubectl delete "$kind" "$name" \
|
||
|
|
--namespace "$namespace" \
|
||
|
|
--wait=true --timeout="${timeout}s" --ignore-not-found 2>&1 || {
|
||
|
|
echo " [force] WARN: delete did not complete cleanly within ${timeout}s — proceeding with apply anyway" >&2
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# ── workload registration (used by ERR trap) ────────────────────────────
|
||
|
|
_DIAG_KIND="" # 'cluster' or 'taskserv'
|
||
|
|
_DIAG_K8S_KIND="" # deployment|statefulset|daemonset|cronjob|job
|
||
|
|
_DIAG_K8S_NAME=""
|
||
|
|
_DIAG_K8S_NAMESPACE=""
|
||
|
|
_DIAG_K8S_SELECTOR=""
|
||
|
|
_DIAG_SYSTEMD_UNIT=""
|
||
|
|
_DIAG_TAIL="${_DIAG_TAIL:-60}"
|
||
|
|
|
||
|
|
_diag_register_cluster_workload() {
|
||
|
|
_DIAG_KIND="cluster"
|
||
|
|
_DIAG_K8S_KIND="$1"
|
||
|
|
_DIAG_K8S_NAME="$2"
|
||
|
|
_DIAG_K8S_NAMESPACE="$3"
|
||
|
|
_DIAG_K8S_SELECTOR="${4:-app.kubernetes.io/name=$2}"
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_register_systemd_unit() {
|
||
|
|
_DIAG_KIND="taskserv"
|
||
|
|
_DIAG_SYSTEMD_UNIT="$1"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ── ERR trap installation ───────────────────────────────────────────────
|
||
|
|
_diag_install_err_trap() {
|
||
|
|
trap '_diag_on_error $? "$BASH_COMMAND" $LINENO' ERR
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_on_error() {
|
||
|
|
local rc="$1" cmd="$2" lineno="$3"
|
||
|
|
{
|
||
|
|
echo ""
|
||
|
|
echo "═══ FAILURE ═══════════════════════════════════════════════════════"
|
||
|
|
echo "exit code : $rc"
|
||
|
|
echo "line : $lineno"
|
||
|
|
echo "command : $cmd"
|
||
|
|
echo ""
|
||
|
|
case "$_DIAG_KIND" in
|
||
|
|
cluster) _diag_dump_cluster ;;
|
||
|
|
taskserv) _diag_dump_taskserv ;;
|
||
|
|
"") echo "(no diagnostic context registered)" ;;
|
||
|
|
*) echo "(unknown _DIAG_KIND=$_DIAG_KIND)" ;;
|
||
|
|
esac
|
||
|
|
echo "═══════════════════════════════════════════════════════════════════"
|
||
|
|
} >&2
|
||
|
|
exit "$rc"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ── cluster mode diagnostics ────────────────────────────────────────────
|
||
|
|
_diag_dump_cluster() {
|
||
|
|
if ! command -v kubectl >/dev/null 2>&1 && ! command -v k0s >/dev/null 2>&1; then
|
||
|
|
echo " (no kubectl/k0s on this host — cannot collect k8s diagnostics)"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if [ -z "$_DIAG_K8S_NAME" ]; then
|
||
|
|
echo " (no cluster workload registered — call _diag_register_cluster_workload first)"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "─── ${_DIAG_K8S_KIND}/${_DIAG_K8S_NAME} (namespace=${_DIAG_K8S_NAMESPACE}) ───"
|
||
|
|
_kubectl get "${_DIAG_K8S_KIND}" "${_DIAG_K8S_NAME}" \
|
||
|
|
--namespace "${_DIAG_K8S_NAMESPACE}" -o wide 2>&1 || true
|
||
|
|
|
||
|
|
case "$_DIAG_K8S_KIND" in
|
||
|
|
deployment|statefulset|daemonset)
|
||
|
|
_diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE"
|
||
|
|
;;
|
||
|
|
cronjob)
|
||
|
|
echo "─── recent jobs from cronjob/${_DIAG_K8S_NAME} ───"
|
||
|
|
_kubectl get jobs --namespace "${_DIAG_K8S_NAMESPACE}" \
|
||
|
|
-l "app.kubernetes.io/managed-by=cronjob,batch.kubernetes.io/cronjob-name=${_DIAG_K8S_NAME}" \
|
||
|
|
2>/dev/null || _kubectl get jobs --namespace "${_DIAG_K8S_NAMESPACE}" 2>/dev/null | grep "${_DIAG_K8S_NAME}" || true
|
||
|
|
_diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE"
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
_diag_dump_pods "$_DIAG_K8S_SELECTOR" "$_DIAG_K8S_NAMESPACE"
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_dump_pods() {
|
||
|
|
local selector="$1" namespace="$2" tail="${3:-$_DIAG_TAIL}"
|
||
|
|
echo ""
|
||
|
|
echo "─── pods (selector=${selector}) ───"
|
||
|
|
_kubectl get pods --namespace "$namespace" --selector "$selector" -o wide 2>&1 || true
|
||
|
|
local pods
|
||
|
|
pods=$(_kubectl get pods --namespace "$namespace" --selector "$selector" \
|
||
|
|
-o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
|
||
|
|
if [ -z "$pods" ]; then
|
||
|
|
echo " (no pods match selector)"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
for pod in $pods; do
|
||
|
|
# Smart dump: if logs are accessible the daemon HAS started, so the
|
||
|
|
# error is in the daemon's startup — logs are sufficient. Skip
|
||
|
|
# describe in that case (it doesn't add info). Only when logs are
|
||
|
|
# unavailable (pod never started: image pull failure, volume mount
|
||
|
|
# error, scheduling issue) do we fall back to describe.
|
||
|
|
local logs_out prev_out rc_logs rc_prev
|
||
|
|
logs_out=$(_kubectl logs "$pod" --namespace "$namespace" \
|
||
|
|
--tail="$tail" --all-containers=true 2>&1)
|
||
|
|
rc_logs=$?
|
||
|
|
if [ $rc_logs -eq 0 ] && [ -n "$logs_out" ]; then
|
||
|
|
echo ""
|
||
|
|
echo "─── logs pod/${pod} --tail=${tail} ───"
|
||
|
|
printf '%s\n' "$logs_out"
|
||
|
|
# If the pod has restarted (CrashLoop), --previous shows the
|
||
|
|
# last instance — usually the most useful for diagnosis.
|
||
|
|
prev_out=$(_kubectl logs "$pod" --namespace "$namespace" \
|
||
|
|
--tail="$tail" --all-containers=true --previous 2>/dev/null)
|
||
|
|
rc_prev=$?
|
||
|
|
if [ $rc_prev -eq 0 ] && [ -n "$prev_out" ] \
|
||
|
|
&& [ "$prev_out" != "$logs_out" ]; then
|
||
|
|
echo ""
|
||
|
|
echo "─── logs pod/${pod} --previous --tail=${tail} (CrashLoop) ───"
|
||
|
|
printf '%s\n' "$prev_out"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
# No logs accessible → pod never reached running. Describe + events
|
||
|
|
# are the right diagnostic here.
|
||
|
|
echo ""
|
||
|
|
echo "─── describe pod/${pod} (last 40 lines) ───"
|
||
|
|
_kubectl describe pod "$pod" --namespace "$namespace" 2>&1 | tail -40 || true
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
}
|
||
|
|
|
||
|
|
# Wrap kubectl rollout with timeout + auto-dump-on-failure.
|
||
|
|
_diag_wait_for_deployment() {
|
||
|
|
local name="$1" namespace="$2" timeout="${3:-180}"
|
||
|
|
local selector="${4:-app.kubernetes.io/name=$name}"
|
||
|
|
_diag_register_cluster_workload deployment "$name" "$namespace" "$selector"
|
||
|
|
if _kubectl rollout status deployment/"$name" \
|
||
|
|
--namespace "$namespace" --timeout="${timeout}s"; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
local rc=$?
|
||
|
|
echo "" >&2
|
||
|
|
echo "ERROR: deployment/${name} did not become ready within ${timeout}s" >&2
|
||
|
|
_diag_dump_cluster >&2
|
||
|
|
return "$rc"
|
||
|
|
}
|
||
|
|
|
||
|
|
_diag_wait_for_statefulset() {
|
||
|
|
local name="$1" namespace="$2" timeout="${3:-300}"
|
||
|
|
local selector="${4:-app.kubernetes.io/name=$name}"
|
||
|
|
_diag_register_cluster_workload statefulset "$name" "$namespace" "$selector"
|
||
|
|
if _kubectl rollout status statefulset/"$name" \
|
||
|
|
--namespace "$namespace" --timeout="${timeout}s"; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
local rc=$?
|
||
|
|
echo "" >&2
|
||
|
|
echo "ERROR: statefulset/${name} did not become ready within ${timeout}s" >&2
|
||
|
|
_diag_dump_cluster >&2
|
||
|
|
return "$rc"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ── taskserv mode diagnostics ───────────────────────────────────────────
|
||
|
|
_diag_dump_taskserv() {
|
||
|
|
if [ -z "$_DIAG_SYSTEMD_UNIT" ]; then
|
||
|
|
echo " (no systemd unit registered — call _diag_register_systemd_unit first)"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
||
|
|
echo " (no systemctl on this host — cannot collect systemd diagnostics)"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
echo "─── systemctl status ${_DIAG_SYSTEMD_UNIT} ───"
|
||
|
|
systemctl status "$_DIAG_SYSTEMD_UNIT" --no-pager --lines=20 2>&1 || true
|
||
|
|
echo ""
|
||
|
|
echo "─── journalctl -u ${_DIAG_SYSTEMD_UNIT} -n ${_DIAG_TAIL} ───"
|
||
|
|
journalctl --no-pager -u "$_DIAG_SYSTEMD_UNIT" -n "$_DIAG_TAIL" 2>&1 || true
|
||
|
|
}
|
||
|
|
|
||
|
|
# Idempotent "wait for unit Active" with timeout + auto-dump-on-failure.
|
||
|
|
_diag_wait_for_systemd() {
|
||
|
|
local unit="$1" timeout="${2:-60}"
|
||
|
|
_diag_register_systemd_unit "$unit"
|
||
|
|
local elapsed=0
|
||
|
|
while [ "$elapsed" -lt "$timeout" ]; do
|
||
|
|
if systemctl is-active --quiet "$unit"; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 2
|
||
|
|
elapsed=$((elapsed + 2))
|
||
|
|
done
|
||
|
|
echo "" >&2
|
||
|
|
echo "ERROR: ${unit} did not become Active within ${timeout}s" >&2
|
||
|
|
_diag_dump_taskserv >&2
|
||
|
|
return 1
|
||
|
|
}
|