From 6f0de7ae89bd731eb076ad99a75aec3dc9f0552d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesu=CC=81s=20Pe=CC=81rez?= Date: Fri, 10 Jul 2026 07:24:20 +0100 Subject: [PATCH] core: move cluster introspection helpers to domain/cluster (ADR-026 slice 2b) Complete the cluster decomposition: the 23 introspection helpers (capacity, kubectl/ssh summaries, live-check, badges) move from cli/components_cluster.nu to domain/cluster/introspect.nu. cli/components_cluster.nu now holds ONLY the 5 `main cluster *` verbs (963 -> 452 lines). introspect.nu imports downward only (tools/nickel, domain/cluster/access, platform/clients/ssh, primitives/logging). Exported 4 previously-private helpers (fmt-gib, k8s-quantity-bytes, node-operator-labels, cluster-top-legend) so the cli verbs can call them across the module boundary; dropped a dead comp-dag-summary import. component.nu repoints its 11 helper imports to introspect.nu. Verified: module-load of all touched modules (no cycle, no upward import); ADR-026 topology clean across the whole cluster subsystem; cluster --help + component list run. Concludes slices 2a+2b: cli/components.nu 3611 -> 2410 (-33%). --- nulib/cli/component.nu | 2 +- nulib/cli/components_cluster.nu | 518 +------------------------------- 2 files changed, 3 insertions(+), 517 deletions(-) diff --git a/nulib/cli/component.nu b/nulib/cli/component.nu index 61ae663..b031ca3 100644 --- a/nulib/cli/component.nu +++ b/nulib/cli/component.nu @@ -44,7 +44,7 @@ use cli/components.nu [ use tools/nickel/process.nu [comp-ncl-export] use domain/cluster/access.nu [comp-kubectl-access] use domain/workspace/resolve.nu [comp-state-server comp-ws-path comp-ws-infra] -use cli/components_cluster.nu [ +use domain/cluster/introspect.nu [ comp-live-check comp-pods-for comp-ssh-pods diff --git a/nulib/cli/components_cluster.nu b/nulib/cli/components_cluster.nu index 2017415..7c79051 100644 --- a/nulib/cli/components_cluster.nu +++ b/nulib/cli/components_cluster.nu @@ -12,250 +12,12 @@ use platform/integrations/context_assembler.nu [build-cabling-credentials-env] # Cluster-observability slice extracted from components.nu (ADR-026 layer hygiene). -use cli/components.nu [cli-err comp-dag-summary] +use cli/components.nu [cli-err] +use domain/cluster/introspect.nu [cluster-top-legend comp-load-server-info comp-server-capacity comp-ssh-k8s-data fmt-gib k8s-quantity-bytes node-operator-labels] use domain/cluster/access.nu [comp-kubectl-access comp-kubectl-ssh] use platform/clients/ssh.nu [comp-server-ssh-info comp-ssh-run] use domain/workspace/resolve.nu [comp-ws-path comp-ws-infra comp-state-server comp-resolve-ws-path] -# Structured compute capacity for a Hetzner server type code: parsed CPU and RAM -# for capacity math (cluster top, the placement simulator) plus a human label. -# Unknown codes yield zero capacity and the code itself as label, so callers can -# degrade gracefully instead of mis-reporting headroom. -export def comp-server-capacity [type_code: string]: nothing -> record { - let tbl = { - cax11: { cpu: 2, mem_gb: 4, arch: "ARM" }, cax21: { cpu: 4, mem_gb: 8, arch: "ARM" }, - cax31: { cpu: 8, mem_gb: 16, arch: "ARM" }, cax41: { cpu: 16, mem_gb: 32, arch: "ARM" }, - cpx11: { cpu: 2, mem_gb: 2, arch: "AMD" }, cpx21: { cpu: 3, mem_gb: 4, arch: "AMD" }, - cpx31: { cpu: 4, mem_gb: 8, arch: "AMD" }, cpx41: { cpu: 8, mem_gb: 16, arch: "AMD" }, - cpx51: { cpu: 16, mem_gb: 32, arch: "AMD" }, - cx22: { cpu: 2, mem_gb: 4, arch: "Intel" }, cx32: { cpu: 4, mem_gb: 8, arch: "Intel" }, - cx42: { cpu: 8, mem_gb: 16, arch: "Intel" }, cx52: { cpu: 16, mem_gb: 32, arch: "Intel" }, - } - let spec = ($tbl | get -o $type_code) - if ($spec == null) { - { type: $type_code, cpu: 0, cpu_millis: 0, mem_gb: 0, mem_bytes: 0, arch: "", label: $type_code } - } else { - { - type: $type_code, - cpu: $spec.cpu, - cpu_millis: ($spec.cpu * 1000), - mem_gb: $spec.mem_gb, - mem_bytes: ($spec.mem_gb * 1024 * 1024 * 1024), - arch: $spec.arch, - label: $"($spec.cpu)CPU ($spec.mem_gb)GB ($spec.arch)", - } - } -} - -# Translate Hetzner server type codes to a human-readable CPU/RAM/arch label. -def hetzner-type-label [type_code: string]: nothing -> string { - (comp-server-capacity $type_code).label -} - -def status-dot [status: string]: nothing -> string { - match $status { - "running" => "●" - "off" => "○" - "starting" => "◐" - _ => "○" - } -} - -# Load server runtime + declared metadata for all servers in a workspace infra. -# Cache mode: .servers-state.json (status/public_ip/location) + settings.ncl (type/private_ip). -# Live mode: hcloud API directly. -# Returns record keyed by hostname. -export def comp-load-server-info [ - ws_root: string - infra: string - --live -]: nothing -> record { - let state_path = ($ws_root | path join "infra" $infra ".servers-state.json") - let cache = if ($state_path | path exists) { open $state_path } else { {} } - - let declared = do -i { - let raw = (comp-ncl-export $ws_root $"infra/($infra)/settings.ncl") - $raw | get -o servers | default [] - | reduce --fold {} {|srv, acc| - $acc | insert $srv.hostname { - type: ($srv.server_type? | default ""), - private_ip: ($srv | get -o networking.private_ip | default ""), - provider: ($srv.provider? | default "hetzner"), - } - } - } | default {} - - if $live { - let res = (do { ^hcloud server list -o json } | complete) - if $res.exit_code != 0 { return $declared } - ($res.stdout | from json) - | reduce --fold {} {|h, acc| - let d = ($declared | get -o $h.name | default {}) - $acc | insert $h.name { - status: $h.status, - type: ($h | get -o server_type.name | default ($d.type? | default "")), - location: ($h | get -o datacenter.location.name | default ""), - public_ip: ($h | get -o public_net.ipv4.ip | default ""), - private_ip: ($d.private_ip? | default ""), - provider: ($d.provider? | default "hetzner"), - } - } - } else { - $cache | items {|hostname entry| - let d = ($declared | get -o $hostname | default {}) - { - key: $hostname, - value: { - status: ($entry.status? | default ""), - type: ($d.type? | default ""), - location: ($entry.location? | default ""), - public_ip: ($entry.public_ip? | default ""), - private_ip: ($d.private_ip? | default ""), - provider: ($d.provider? | default "hetzner"), - }, - } - } | reduce --fold {} {|it, acc| $acc | insert $it.key $it.value} - } -} - -# One-line server info badge for the TASKSERV section header. -export def comp-server-badge [info: record]: nothing -> string { - if ($info | is-empty) { return "" } - let dot = (status-dot ($info.status? | default "")) - let status = ($info.status? | default "") - let type_l = if ($info.type? | default "" | is-not-empty) { - hetzner-type-label $info.type - } else { "" } - let loc = ($info.location? | default "") - let pub = ($info.public_ip? | default "") - let priv = ($info.private_ip? | default "") - let ip_str = if ($pub | is-not-empty) and ($priv | is-not-empty) { - $"($pub) / ($priv)" - } else if ($pub | is-not-empty) { $pub } else { $priv } - let parts = [$"($dot) ($status)" $type_l $loc $ip_str] | where { $in | is-not-empty } - if ($parts | is-empty) { "" } else { $parts | str join " " } -} - -def cluster-cache-path [ws_root: string, infra: string]: nothing -> string { - $ws_root | path join "infra" $infra ".cluster-state.json" -} - -# Read the declared k0s version from settings.ncl (settings.k0s.version). -export def comp-settings-k0s-version [ws_root: string, infra: string]: nothing -> string { - let raw = (comp-ncl-export $ws_root $"infra/($infra)/settings.ncl") - $raw | get -o settings | default {} | get -o k0s | default {} | get -o version | default "" -} - -# Fetch cluster summary counts via the declared remote kubectl (single SSH call). -# Returns the same shape as comp-cluster-summary's result record. node/pod totals -# come from the caller's already-fetched k8s_data; this adds ns/dply/pvc counts. -def comp-kubectl-summary-counts [ws_root: string, infra: string, version: string, k8s_data: record]: nothing -> record { - let nodes = ($k8s_data.nodes? | default []) - let pods = ($k8s_data.pods? | default []) - let nodes_total = ($nodes | length) - let nodes_ready = ($nodes | where {|n| - ($n | get -o status | default {} | get -o conditions | default []) | any { ($in.type? == "Ready") and ($in.status? == "True") } - } | length) - let pods_total = ($pods | length) - let pods_running = ($pods | where { ($in | get -o status | default {} | get -o phase | default "") == "Running" } | length) - let cp_ip = ($nodes | get 0? | default {} | get -o status | default {} | get -o addresses | default [] | where { $in.type? == "InternalIP" } | get 0? | get -o address | default "") - let r = (comp-kubectl-ssh $ws_root $infra "get namespaces,deployments,pvc -A --no-headers -o json") - let counts = if $r.exit_code == 0 { - let items = (try { $r.stdout | from json | get -o items | default [] } catch { [] }) - { - ns: ($items | where { ($in.kind? | default "") == "Namespace" } | length), - dply: ($items | where { ($in.kind? | default "") == "Deployment" } | length), - pvc: ($items | where { (($in.kind? | default "") == "PersistentVolumeClaim") and (($in.status?.phase? | default "") == "Bound") } | length), - } - } else { { ns: 0, dply: 0, pvc: 0 } } - { - version: $version, - namespaces: $counts.ns, - deployments: $counts.dply, - pvcs_bound: $counts.pvc, - nodes_ready: $nodes_ready, - nodes_total: $nodes_total, - pods_running: $pods_running, - pods_total: $pods_total, - cp_ip: $cp_ip, - last_sync: (date now | format date "%Y-%m-%dT%H:%M:%SZ"), - } -} - -# Fetch (--live) or read cached cluster summary: version, ns/dply/pvc counts, node+pod status. -# Live: second lightweight SSH (counts only — separate from comp-ssh-k8s-data heavy JSON call). -# Cache: writes .cluster-state.json on live fetch; reads it on non-live call. -export def comp-cluster-summary [ - ws_root: string - infra: string - k8s_data: record # node/pod data already fetched by comp-ssh-k8s-data - version: string # declared k0s/k8s version from settings — caller resolves via comp-settings-k0s-version - --live -]: nothing -> record { - let empty = { - version: "", namespaces: 0, deployments: 0, pvcs_bound: 0, last_sync: "", - nodes_ready: 0, nodes_total: 0, pods_running: 0, pods_total: 0, cp_ip: "", - } - if not $live { - let cache = (cluster-cache-path $ws_root $infra) - let base = if ($cache | path exists) { open $cache } else { $empty } - return ($base | merge { version: (if ($version | is-not-empty) { $version } else { $base.version? | default "" }) }) - } - # Live: counts come from the declared remote kubectl (SSH to control-plane). - # provisioning never runs kubectl locally — see comp-kubectl-access. - let result = (comp-kubectl-summary-counts $ws_root $infra $version $k8s_data) - $result | to json | save -f (cluster-cache-path $ws_root $infra) - $result -} - -# One-line cluster summary badge for the CLUSTER COMPONENTS header. -export def comp-cluster-badge [summary: record]: nothing -> string { - if ($summary | is-empty) { return "" } - let ver = ($summary.version? | default "") - let ns = ($summary.namespaces? | default 0) - let pods = ($summary.pods_total? | default 0) - if ($ver | is-empty) and $ns == 0 and $pods == 0 { return "" } - let cp_str = if ($summary.cp_ip? | default "" | is-not-empty) { $"CP: ($summary.cp_ip)" } else { "" } - let ver_str = if ($ver | is-not-empty) { $ver } else { "" } - let nodes_str = if ($summary.nodes_total? | default 0) > 0 { $"($summary.nodes_ready)/($summary.nodes_total) nodes" } else { "" } - let pods_str = if ($summary.pods_total? | default 0) > 0 { $"($summary.pods_running)/($summary.pods_total) pods" } else { "" } - let ns_str = if $ns > 0 { $"($ns) ns" } else { "" } - let dply_str = if ($summary.deployments? | default 0) > 0 { $"($summary.deployments) dply" } else { "" } - let pvcs_str = if ($summary.pvcs_bound? | default 0) > 0 { $"($summary.pvcs_bound) pvcs" } else { "" } - [$cp_str $ver_str $nodes_str $pods_str $ns_str $dply_str $pvcs_str] | where { $in | is-not-empty } | str join " " -} - -# Parse a Kubernetes resource quantity (e.g. "529Mi", "1Gi", "1024Ki", "2G") to -# bytes for numeric sorting and capacity math. Unparseable input yields 0. -def k8s-quantity-bytes [q: string]: nothing -> int { - let s = ($q | str trim) - if ($s | is-empty) { return 0 } - let m = ($s | parse --regex '(?[0-9.]+)(?[A-Za-z]*)' | get 0?) - if ($m == null) { return 0 } - let num = ($m.num | into float) - let mult = (match $m.unit { - "Ki" => 1024.0, "Mi" => 1048576.0, "Gi" => 1073741824.0, "Ti" => 1099511627776.0, - "k" | "K" => 1000.0, "M" => 1000000.0, "G" => 1000000000.0, "T" => 1000000000000.0, - _ => 1.0, - }) - (($num * $mult) | math round | into int) -} - -# Operator-relevant node labels: drop the well-known system/cloud domains so only -# placement-meaningful labels (node_class, can_use_fip.*, custom) remain. Returns -# a "k=v k=v" string (valueless labels shown as bare keys). -def node-operator-labels [labels: record]: nothing -> string { - let system = ["kubernetes.io", "k8s.io", "hetzner.cloud", "topology.", "beta.", "csi."] - $labels | transpose key value - | where {|r| not ($system | any {|p| $r.key | str contains $p }) } - | each {|r| - # Boolean labels (value "true") and valueless labels show as the bare key — - # if it's present it's set. Non-boolean labels keep key=value. - if (($r.value | default "" | is-empty) or ($r.value == "true")) { $r.key } else { $"($r.key)=($r.value)" } - } - | str join " " -} - # Live per-node resource usage via `kubectl top`, enriched with the declared # Hetzner server type and its nominal capacity. provisioning never runs kubectl # locally — metrics arrive over the bounded remote-kubectl SSH path @@ -376,24 +138,6 @@ export def "main cluster top" [ } } -# Interpretive legend for `cluster top` columns — printed under the node table so -# the numbers can be read without external context. Suppressed in json/yaml output. -def cluster-top-legend []: nothing -> nothing { - print "" - print "How to read it:" - print " nominal physical CPU/RAM of the Hetzner type (cax11=2c/4Gi, cax21=4c/8Gi)." - print " cpu /mem LIVE usage now (kubectl top). mem% is vs ALLOCATABLE (RAM minus" - print " kernel+kube reservations), so it reads higher than vs nominal." - print " req% sum of pod memory REQUESTS vs allocatable = what the scheduler" - print " reserved. Healthy < 80%; it is the real 'fullness' for placement." - print " lim% sum of pod memory LIMITS vs allocatable = overcommit. > 100% means" - print " pods may collectively exceed RAM → OOM-kill risk under pressure." - print " no-req containers with NO memory request — invisible to the scheduler," - print " cause blind bin-packing and are first to be OOM-killed." - print " labels operator node labels (node_class, can_use_fip.*, custom) that drive" - print " placement; system/cloud domains (kubernetes.io, hetzner.cloud) hidden." -} - # Reconcile declared operator node labels onto the live Kubernetes nodes. Source # of truth: servers.ncl node_labels (record) + node_class (list, projected as # node-class.=true). Closes the drift where declared labels never reach the @@ -548,11 +292,6 @@ export def "main cluster plan" [ } } -# Format a byte count as a compact Gi/Mi string. -def fmt-gib [b: int]: nothing -> string { - if $b >= 1073741824 { $"(($b / 1073741824 * 10 | math round) / 10)Gi" } else { $"(($b / 1048576) | math round)Mi" } -} - # Cluster capacity analysis: per-node working-set broken down by category # (movable = general app classes, pinned = datastore/storage_engine, infra = # kube-system/longhorn/cilium/daemonsets), plus an aggregate capacity check by @@ -711,256 +450,3 @@ export def "main cluster services" [ } } -# Oldest last_sync across .servers-state.json and .cluster-state.json (= most stale data shown). -export def comp-cache-last-sync [ws_root: string, infra: string]: nothing -> string { - let srv_ts = do { - let p = ($ws_root | path join "infra" $infra ".servers-state.json") - if ($p | path exists) { - open $p | values | each { $in.last_sync? | default "" } | where { $in | is-not-empty } - } else { [] } - } - let cl_ts = do { - let p = (cluster-cache-path $ws_root $infra) - if ($p | path exists) { - let v = ((open $p).last_sync? | default "") - if ($v | is-not-empty) { [$v] } else { [] } - } else { [] } - } - let all = ($srv_ts | append $cl_ts) | where { $in | is-not-empty } - if ($all | is-empty) { "" } else { $all | sort | first } -} - -# Single SSH session to CP: fetch pods + nodes as JSON in one connection. -# Returns {pods: list, nodes: list} — parsed from kubectl -o json output. -# Pods have metadata.namespace/name and status.phase/containerStatuses fields. -# Nodes have metadata.name and status.conditions fields. -export def comp-ssh-k8s-data [ - ws_root: string, - infra: string, - server: string, - --systemd-services: list = [], # service names to check via systemctl is-active -]: nothing -> record { - let empty = { pods: [], nodes: [], systemd: {} } - if ($server | is-empty) { - print --stderr "[live] no server resolved — cannot fetch k8s data" - return $empty - } - let info = (comp-server-ssh-info $ws_root $infra $server) - if (is-debug-level 2) { - let key_part = if ($info.key | is-not-empty) { $" -i ($info.key)" } else { "" } - print --stderr $"[live] ssh($key_part) ($info.user)@($info.host) kubectl get pods+nodes -o json" - } - # PATH: SSH non-interactive sessions may not have /usr/local/bin (where k0s and kubeadm kubectl live). - # KUBECONFIG: kubeadm clusters set this in ~/.bashrc (not sourced in BatchMode SSH). - # Probe /root/.kube/config then /etc/kubernetes/admin.conf; k0s kubectl ignores KUBECONFIG entirely. - let systemd_snippet = if ($systemd_services | is-not-empty) { - let checks = ($systemd_services | each {|svc| $"printf '($svc):'; systemctl is-active ($svc) 2>/dev/null || true"} | str join "; ") - $"echo '===SYSTEMD==='; ($checks)" - } else { "" } - let cmd_base = "export PATH=/usr/local/bin:/usr/bin:/bin:$PATH; for _kc in /root/.kube/config /etc/kubernetes/admin.conf; do [ -f \"$_kc\" ] && { export KUBECONFIG=\"$_kc\"; break; }; done; kc() { if command -v kubectl >/dev/null 2>&1; then kubectl \"$@\"; else k0s kubectl \"$@\"; fi; }; echo '===PODS==='; kc get pods --all-namespaces -o json 2>/dev/null || echo '{\"items\":[]}'; echo '===NODES==='; kc get nodes -o json 2>/dev/null || echo '{\"items\":[]}'" - let cmd = if ($systemd_snippet | is-not-empty) { $"($cmd_base); ($systemd_snippet)" } else { $cmd_base } - let result = (comp-ssh-run $info.user $info.key $info.host $cmd) - if (is-debug-level 2) { - print --stderr $"[live] SSH exit=($result.exit_code) stdout_len=($result.stdout | str length)" - } - if $result.exit_code != 0 { return $empty } - let all_lines = ($result.stdout | str replace --all "\r" "" | lines) - let pods_idx = ($all_lines | enumerate | where { ($in.item | str trim) == "===PODS===" } | get 0? | get -o index | default (-1)) - let nodes_idx = ($all_lines | enumerate | where { ($in.item | str trim) == "===NODES===" } | get 0? | get -o index | default (-1)) - let systemd_idx = ($all_lines | enumerate | where { ($in.item | str trim) == "===SYSTEMD===" } | get 0? | get -o index | default (-1)) - if (is-debug-level 3) { - print --stderr $"[live] pods_idx=($pods_idx) nodes_idx=($nodes_idx) systemd_idx=($systemd_idx) total_lines=($all_lines | length)" - } - let pods = if $pods_idx >= 0 { - let from = ($pods_idx + 1) - let raw = if $nodes_idx > $pods_idx { $all_lines | skip $from | first ($nodes_idx - $from) } else { $all_lines | skip $from } - try { $raw | str join "\n" | from json | get -o items | default [] } catch {|e| - if (is-debug-level 1) { - print --stderr $"[live] pods JSON parse error: ($e.msg | str substring 0..120)" - print --stderr $"[live] pods first line: ($raw | get 0? | default '')" - } - [] - } - } else { [] } - let nodes = if $nodes_idx >= 0 { - let from = ($nodes_idx + 1) - let raw = if $systemd_idx > $nodes_idx { $all_lines | skip $from | first ($systemd_idx - $from) } else { $all_lines | skip $from } - try { $raw | str join "\n" | from json | get -o items | default [] } catch {|e| - if (is-debug-level 1) { - print --stderr $"[live] nodes JSON parse error: ($e.msg | str substring 0..120)" - } - [] - } - } else { [] } - let systemd = if $systemd_idx >= 0 { - $all_lines | skip ($systemd_idx + 1) - | each {|ln| - let parts = ($ln | str trim | split column ":" name status) - if ($parts | length) >= 1 and ($parts | get 0 | get name | is-not-empty) { - { ($parts | get 0 | get name): ($parts | get 0 | get status | str trim) } - } else { {} } - } - | reduce -f {} {|it, acc| $acc | merge $it } - } else { {} } - if (is-debug-level 2) { - print --stderr $"[live] parsed pods=($pods | length) nodes=($nodes | length) systemd_keys=($systemd | columns | length)" - } - { pods: $pods, nodes: $nodes, systemd: $systemd } -} - -# Public API: return pod records for a server (used by describe-pod and external callers). -export def comp-ssh-pods [ws_root: string, infra: string, server: string]: nothing -> list { - (comp-ssh-k8s-data $ws_root $infra $server).pods -} - -# Run kubectl describe pod -n via SSH. Returns describe output as string. -export def comp-ssh-describe-pod [ws_root: string, infra: string, server: string, namespace: string, pod: string]: nothing -> string { - let info = (comp-server-ssh-info $ws_root $infra $server) - let cmd = $"if command -v kubectl >/dev/null 2>&1; then kubectl describe pod -n ($namespace) ($pod); else k0s kubectl describe pod -n ($namespace) ($pod); fi" - let result = (comp-ssh-run $info.user $info.key $info.host $cmd) - if $result.exit_code != 0 { $result.stderr | str trim } else { $result.stdout | str trim } -} - -# Summarise pod records (from kubectl -o json) into {live, detail}. -def comp-kubectl-summarise [pods: list]: nothing -> record { - if ($pods | is-empty) { return { live: "deployed", detail: "no pods" } } - # Exclude Succeeded pods (completed jobs/migrations) — they never return to Running - # and would distort the ratio. Failed phase and error-waiting pods are kept and - # classified as degraded. - let active = ($pods | where { ($in | get -o status | default {} | get -o phase | default "") != "Succeeded" }) - let total = ($active | length) - let running = ($active | where { ($in | get -o status | default {} | get -o phase | default "") == "Running" } | length) - let pending = ($active | where { ($in | get -o status | default {} | get -o phase | default "") == "Pending" } | length) - - # Any pod in Failed phase OR any container waiting with an error reason → degraded. - let error_reasons = ["CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", - "Error", "OOMKilled", "CreateContainerError", - "CreateContainerConfigError", "RunContainerError", - "InvalidImageName", "PostStartHookError"] - let failing = ($active | where { |p| - let phase = ($p | get -o status | default {} | get -o phase | default "") - if $phase == "Failed" { true } else { - let cs = ($p | get -o status | default {} | get -o containerStatuses | default []) - let ics = ($p | get -o status | default {} | get -o initContainerStatuses | default []) - ($cs | append $ics) | any { |c| - ($c | get -o state | default {} | get -o waiting | default {} | get -o reason | default "") in $error_reasons - } - } - } | length) - - let live = if $failing > 0 { "degraded" - } else if $pending > 0 { "starting" - } else if $total == 0 { "deployed" - } else if $running == $total { "healthy" - } else { "partial" } - - let detail = if $failing > 0 { - $"($running)/($total) pods running ($failing) failing" - } else { - $"($running)/($total) pods running" - } - { live: $live, detail: $detail } -} - -# Filter pod records by namespace (exact match) and name substring. -def comp-filter-pods [namespace: string, search: string, pods: list]: nothing -> list { - let ns_filtered = if ($namespace | is-not-empty) { - $pods | where { ($in | get -o metadata | default {} | get -o namespace | default "") == $namespace } - } else { $pods } - $ns_filtered | where { ($in | get -o metadata | default {} | get -o name | default "") | str contains $search } -} - -# Summarise node records (from kubectl -o json) into {live, detail}. -def comp-kubectl-nodes-summarise [nodes: list]: nothing -> record { - if ($nodes | is-empty) { return { live: "—", detail: "no node data" } } - let total = ($nodes | length) - let ready = ($nodes | where { |n| - $n | get -o status | default {} | get -o conditions | default [] - | any { ($in | get -o type | default "") == "Ready" and ($in | get -o status | default "") == "True" } - } | length) - let live = if $ready == $total { "healthy" - } else if $ready == 0 { "degraded" - } else { "partial" } - { live: $live, detail: $"($ready)/($total) nodes Ready" } -} - -# Live check for a component using pre-fetched k8s_data {pods, nodes}. -# Resolves strategy from comp.live_check; falls back to namespace/pod_selector on the component. -# 'systemd strategy always returns "—" in live (fast) mode — requires prvng c health. -# Returns {live: string, detail: string}. -export def comp-live-check [name: string, comp: record, k8s_data: record]: nothing -> record { - let lc = ($comp | get -o live_check | default {}) - let strategy = ($lc | get -o strategy | default "none" | into string) - match $strategy { - "k8s_pods" => { - if ($k8s_data.pods | is-empty) { return { live: "—", detail: "no pod data" } } - let ns = do { - let explicit = ($lc | get -o namespace | default "") - if ($explicit | is-not-empty) { $explicit } else { $comp | get -o namespace | default "" } - } - let sel = do { - let explicit = ($lc | get -o selector | default "") - if ($explicit | is-not-empty) { $explicit } else { - let ps = ($comp | get -o pod_selector | default "") - if ($ps | is-not-empty) { $ps } else { $name | str replace --all "_" "-" } - } - } - let matching = (comp-filter-pods $ns $sel $k8s_data.pods) - if ($matching | is-empty) { return { live: "—", detail: $"no pods: ($sel)" } } - comp-kubectl-summarise $matching - } - "k8s_nodes" => { - if ($k8s_data.nodes | is-empty) { return { live: "—", detail: "no node data" } } - let sel = ($lc | get -o selector | default "") - let filtered = if ($sel | is-not-empty) { - $k8s_data.nodes | where { ($in | get -o metadata | default {} | get -o name | default "") | str contains $sel } - } else { - $k8s_data.nodes - } - comp-kubectl-nodes-summarise $filtered - } - "k8s_api" => { - # apiserver healthy if kubectl returned any node data - if ($k8s_data.nodes | is-not-empty) or ($k8s_data.pods | is-not-empty) { - { live: "healthy", detail: "apiserver ok" } - } else { - { live: "—", detail: "apiserver unreachable" } - } - } - "systemd" => { - let svc = ($lc | get -o service | default "") - if ($svc | is-empty) { return { live: "—", detail: "health: systemd" } } - let units = ($k8s_data | get -o systemd | default {}) - if ($units | is-empty) { return { live: "—", detail: "health: systemd" } } - let active = ($units | get -o $svc | default "") - match $active { - "active" => { live: "healthy", detail: $"systemd: ($svc) active" }, - "inactive" => { live: "degraded", detail: $"systemd: ($svc) inactive" }, - "failed" => { live: "degraded", detail: $"systemd: ($svc) failed" }, - _ => { live: "—", detail: $"systemd: ($svc) ($active)" }, - } - } - "on_demand" => { live: "—", detail: "on-demand" } - _ => { live: "—", detail: "no live check" } - } -} - -# Extract pod summary records for a component from pre-fetched pod JSON records. -# Returns list of {namespace, pod, ready, status}. -export def comp-pods-for [namespace: string, name: string, pods: list, pod_selector: string = ""]: nothing -> list { - let search = if ($pod_selector | is-not-empty) { $pod_selector } else { $name | str replace --all "_" "-" } - (comp-filter-pods $namespace $search $pods) - | each {|p| - let meta = ($p | get -o metadata | default {}) - let status = ($p | get -o status | default {}) - let cs = ($status | get -o containerStatuses | default []) - let ready = ($cs | where { $in.ready? | default false } | length) - { - namespace: ($meta | get -o namespace | default ""), - pod: ($meta | get -o name | default ""), - ready: $"($ready)/($cs | length)", - status: ($status | get -o phase | default ""), - } - } -} -