# Workspace k8s monitoring model generator for mirador. # # Generates a MonitoringModel data record for a Kubernetes workspace. # Two base collectors are always produced: # kube-nodes — cluster-wide, no namespace/label filter (node health only) # kube-ns — namespace-scoped (pods, PVCs, deployment ratio for the workspace) # # Each enabled service produces a dedicated label-selector-scoped collector so # pods.not_ready is attributed per service instead of namespace-wide. # # Usage (workspace instantiation file or provisioning CLI): # let mk = import "templates/workspace/nickel/monitoring.ncl" in # mk.for_workspace { # workspace_id = "libre-wuji", # namespace = "libre-wuji", # defaults to workspace_id when absent # has_nats = true, # has_postgres = true, # has_forgejo = true, # has_woodpecker = true, # has_zot = true, # notification_channels = [ # { id = "ops-local", kind = 'Local, config = {} }, # ], # alert_channel_ids = ["ops-local"], # } # # The returned record conforms to MonitoringModel but carries no imports — it # is pure data that the mirador agent validates through the Rust loader when # it runs `nickel export` on the instantiation file. { for_workspace = fun cfg => ( let ws_id = cfg.workspace_id in let ns = if std.record.has_field "namespace" cfg then cfg.namespace else cfg.workspace_id in let has_nats = if std.record.has_field "has_nats" cfg then cfg.has_nats else false in let has_pg = if std.record.has_field "has_postgres" cfg then cfg.has_postgres else false in let has_fg = if std.record.has_field "has_forgejo" cfg then cfg.has_forgejo else false in let has_wp = if std.record.has_field "has_woodpecker" cfg then cfg.has_woodpecker else false in let has_zot = if std.record.has_field "has_zot" cfg then cfg.has_zot else false in let notif_chs = if std.record.has_field "notification_channels" cfg then cfg.notification_channels else [] in let alert_chs = if std.record.has_field "alert_channel_ids" cfg then cfg.alert_channel_ids else [] in # kubeconfig context name — used for out-of-cluster access (dev/testing). # In-cluster: omit; kube::Config::infer() picks up the ServiceAccount token automatically. let kube_ctx = if std.record.has_field "kube_context" cfg then cfg.kube_context else null in # Base config fragment applied to every Kube collector. # context is injected only when explicitly provided (non-null). let kube_base_cfg = if kube_ctx != null then { context = kube_ctx } else {} in # Produces a Notify action iff channels are configured. # When alert_chs is empty monitors still fire and are logged — just not routed. let notify = fun msg => if (std.array.length alert_chs) > 0 then [{ kind = 'Notify, channels = alert_chs, message = msg }] else [] in # Collector scoped to a namespace + label selector for per-service attribution. let svc_collector = fun cid label_sel => { id = cid, kind = 'Kube, config = kube_base_cfg & { namespace = ns, label_selector = label_sel }, probe_timeout_secs = 10, } in let base_collectors = [ # No namespace or label selector — nodes are cluster-scoped; filtering them # by a workload label selector would return an empty list. { id = "kube-nodes", kind = 'Kube, config = kube_base_cfg, probe_timeout_secs = 10, }, # Namespace-scoped: whole-workspace pod, PVC, and deployment health. { id = "kube-ns", kind = 'Kube, config = kube_base_cfg & { namespace = ns }, probe_timeout_secs = 10, }, ] in let svc_collectors = (if has_nats then [svc_collector "kube-nats" "app.kubernetes.io/name=nats"] else []) @ (if has_pg then [svc_collector "kube-postgres" "app.kubernetes.io/name=postgresql"] else []) @ (if has_fg then [svc_collector "kube-forgejo" "app.kubernetes.io/name=forgejo"] else []) @ (if has_wp then [svc_collector "kube-woodpecker" "app=woodpecker"] else []) @ (if has_zot then [svc_collector "kube-zot" "app.kubernetes.io/name=zot"] else []) in let base_monitors = [ # --- Node health ------------------------------------------------------- # Root monitors — no depends_on so the graph always has a root. { id = "%{ws_id}.nodes.ready", name = "Nodes ready", collector = "kube-nodes", threshold = { metric = "nodes.ready", op = 'Lt, value = 1, unit = "" }, for_secs = 60, actions = notify "[%{ws_id}] No ready node — cluster may be unreachable", }, { id = "%{ws_id}.nodes.not_ready", name = "Nodes not ready", collector = "kube-nodes", threshold = { metric = "nodes.not_ready", op = 'Gt, value = 0, unit = "" }, for_secs = 120, actions = notify "[%{ws_id}] Node(s) degraded", }, # --- Namespace health -------------------------------------------------- { id = "%{ws_id}.pvcs.lost", name = "PVCs lost", collector = "kube-ns", threshold = { metric = "pvcs.lost", op = 'Gt, value = 0, unit = "" }, for_secs = 0, actions = notify "[%{ws_id}] Lost PVC — data at risk", }, { id = "%{ws_id}.pods.failed", name = "Pods failed", collector = "kube-ns", threshold = { metric = "pods.failed", op = 'Gt, value = 0, unit = "" }, for_secs = 60, actions = notify "[%{ws_id}] Pod(s) in Failed phase", }, { id = "%{ws_id}.deployments.ratio", name = "Deployment ready ratio", collector = "kube-ns", threshold = { metric = "deployments.ready_ratio", op = 'Lt, value = 90.0, unit = "%" }, for_secs = 120, actions = notify "[%{ws_id}] Deployment(s) degraded — ready ratio below 90%", }, ] in # Per-service monitor: tracks pods.not_ready scoped to the service's label selector. # This fires when pods belonging to that service are running but not passing readiness. let svc_monitor = fun svc_name collector_id => { id = "%{ws_id}.%{svc_name}.not_ready", name = "%{svc_name} pods not ready", collector = collector_id, depends_on = [], threshold = { metric = "pods.not_ready", op = 'Gt, value = 0, unit = "" }, for_secs = 60, actions = notify "[%{ws_id}] %{svc_name} pod(s) failing readiness probe", } in let svc_monitors = (if has_nats then [svc_monitor "nats" "kube-nats"] else []) @ (if has_pg then [svc_monitor "postgres" "kube-postgres"] else []) @ (if has_fg then [svc_monitor "forgejo" "kube-forgejo"] else []) @ (if has_wp then [svc_monitor "woodpecker" "kube-woodpecker"] else []) @ (if has_zot then [svc_monitor "zot" "kube-zot"] else []) in { collectors = base_collectors @ svc_collectors, channels = notif_chs, monitors = base_monitors @ svc_monitors, } ), }