169 lines
7.2 KiB
Text
169 lines
7.2 KiB
Text
# Workspace k8s monitoring model generator for mirador.
|
|
#
|
|
# Generates a MonitoringModel data record for a Kubernetes workspace.
|
|
# Two base collectors are always produced:
|
|
# kube-nodes — cluster-wide, no namespace/label filter (node health only)
|
|
# kube-ns — namespace-scoped (pods, PVCs, deployment ratio for the workspace)
|
|
#
|
|
# Each enabled service produces a dedicated label-selector-scoped collector so
|
|
# pods.not_ready is attributed per service instead of namespace-wide.
|
|
#
|
|
# Usage (workspace instantiation file or provisioning CLI):
|
|
# let mk = import "templates/workspace/nickel/monitoring.ncl" in
|
|
# mk.for_workspace {
|
|
# workspace_id = "libre-wuji",
|
|
# namespace = "libre-wuji", # defaults to workspace_id when absent
|
|
# has_nats = true,
|
|
# has_postgres = true,
|
|
# has_forgejo = true,
|
|
# has_woodpecker = true,
|
|
# has_zot = true,
|
|
# notification_channels = [
|
|
# { id = "ops-local", kind = 'Local, config = {} },
|
|
# ],
|
|
# alert_channel_ids = ["ops-local"],
|
|
# }
|
|
#
|
|
# The returned record conforms to MonitoringModel but carries no imports — it
|
|
# is pure data that the mirador agent validates through the Rust loader when
|
|
# it runs `nickel export` on the instantiation file.
|
|
|
|
{
|
|
for_workspace = fun cfg => (
|
|
let ws_id = cfg.workspace_id in
|
|
let ns = if std.record.has_field "namespace" cfg then cfg.namespace else cfg.workspace_id in
|
|
|
|
let has_nats = if std.record.has_field "has_nats" cfg then cfg.has_nats else false in
|
|
let has_pg = if std.record.has_field "has_postgres" cfg then cfg.has_postgres else false in
|
|
let has_fg = if std.record.has_field "has_forgejo" cfg then cfg.has_forgejo else false in
|
|
let has_wp = if std.record.has_field "has_woodpecker" cfg then cfg.has_woodpecker else false in
|
|
let has_zot = if std.record.has_field "has_zot" cfg then cfg.has_zot else false in
|
|
|
|
let notif_chs = if std.record.has_field "notification_channels" cfg then cfg.notification_channels else [] in
|
|
let alert_chs = if std.record.has_field "alert_channel_ids" cfg then cfg.alert_channel_ids else [] in
|
|
# kubeconfig context name — used for out-of-cluster access (dev/testing).
|
|
# In-cluster: omit; kube::Config::infer() picks up the ServiceAccount token automatically.
|
|
let kube_ctx = if std.record.has_field "kube_context" cfg then cfg.kube_context else null in
|
|
|
|
# Base config fragment applied to every Kube collector.
|
|
# context is injected only when explicitly provided (non-null).
|
|
let kube_base_cfg = if kube_ctx != null then { context = kube_ctx } else {} in
|
|
|
|
# Produces a Notify action iff channels are configured.
|
|
# When alert_chs is empty monitors still fire and are logged — just not routed.
|
|
let notify = fun msg =>
|
|
if (std.array.length alert_chs) > 0 then
|
|
[{ kind = 'Notify, channels = alert_chs, message = msg }]
|
|
else
|
|
[]
|
|
in
|
|
|
|
# Collector scoped to a namespace + label selector for per-service attribution.
|
|
let svc_collector = fun cid label_sel => {
|
|
id = cid,
|
|
kind = 'Kube,
|
|
config = kube_base_cfg & { namespace = ns, label_selector = label_sel },
|
|
probe_timeout_secs = 10,
|
|
} in
|
|
|
|
let base_collectors = [
|
|
# No namespace or label selector — nodes are cluster-scoped; filtering them
|
|
# by a workload label selector would return an empty list.
|
|
{
|
|
id = "kube-nodes",
|
|
kind = 'Kube,
|
|
config = kube_base_cfg,
|
|
probe_timeout_secs = 10,
|
|
},
|
|
# Namespace-scoped: whole-workspace pod, PVC, and deployment health.
|
|
{
|
|
id = "kube-ns",
|
|
kind = 'Kube,
|
|
config = kube_base_cfg & { namespace = ns },
|
|
probe_timeout_secs = 10,
|
|
},
|
|
] in
|
|
|
|
let svc_collectors =
|
|
(if has_nats then [svc_collector "kube-nats" "app.kubernetes.io/name=nats"] else [])
|
|
@ (if has_pg then [svc_collector "kube-postgres" "app.kubernetes.io/name=postgresql"] else [])
|
|
@ (if has_fg then [svc_collector "kube-forgejo" "app.kubernetes.io/name=forgejo"] else [])
|
|
@ (if has_wp then [svc_collector "kube-woodpecker" "app=woodpecker"] else [])
|
|
@ (if has_zot then [svc_collector "kube-zot" "app.kubernetes.io/name=zot"] else [])
|
|
in
|
|
|
|
let base_monitors = [
|
|
# --- Node health -------------------------------------------------------
|
|
# Root monitors — no depends_on so the graph always has a root.
|
|
{
|
|
id = "%{ws_id}.nodes.ready",
|
|
name = "Nodes ready",
|
|
collector = "kube-nodes",
|
|
threshold = { metric = "nodes.ready", op = 'Lt, value = 1, unit = "" },
|
|
for_secs = 60,
|
|
actions = notify "[%{ws_id}] No ready node — cluster may be unreachable",
|
|
},
|
|
{
|
|
id = "%{ws_id}.nodes.not_ready",
|
|
name = "Nodes not ready",
|
|
collector = "kube-nodes",
|
|
threshold = { metric = "nodes.not_ready", op = 'Gt, value = 0, unit = "" },
|
|
for_secs = 120,
|
|
actions = notify "[%{ws_id}] Node(s) degraded",
|
|
},
|
|
# --- Namespace health --------------------------------------------------
|
|
{
|
|
id = "%{ws_id}.pvcs.lost",
|
|
name = "PVCs lost",
|
|
collector = "kube-ns",
|
|
threshold = { metric = "pvcs.lost", op = 'Gt, value = 0, unit = "" },
|
|
for_secs = 0,
|
|
actions = notify "[%{ws_id}] Lost PVC — data at risk",
|
|
},
|
|
{
|
|
id = "%{ws_id}.pods.failed",
|
|
name = "Pods failed",
|
|
collector = "kube-ns",
|
|
threshold = { metric = "pods.failed", op = 'Gt, value = 0, unit = "" },
|
|
for_secs = 60,
|
|
actions = notify "[%{ws_id}] Pod(s) in Failed phase",
|
|
},
|
|
{
|
|
id = "%{ws_id}.deployments.ratio",
|
|
name = "Deployment ready ratio",
|
|
collector = "kube-ns",
|
|
threshold = { metric = "deployments.ready_ratio", op = 'Lt, value = 90.0, unit = "%" },
|
|
for_secs = 120,
|
|
actions = notify "[%{ws_id}] Deployment(s) degraded — ready ratio below 90%",
|
|
},
|
|
] in
|
|
|
|
# Per-service monitor: tracks pods.not_ready scoped to the service's label selector.
|
|
# This fires when pods belonging to that service are running but not passing readiness.
|
|
let svc_monitor = fun svc_name collector_id =>
|
|
{
|
|
id = "%{ws_id}.%{svc_name}.not_ready",
|
|
name = "%{svc_name} pods not ready",
|
|
collector = collector_id,
|
|
depends_on = [],
|
|
threshold = { metric = "pods.not_ready", op = 'Gt, value = 0, unit = "" },
|
|
for_secs = 60,
|
|
actions = notify "[%{ws_id}] %{svc_name} pod(s) failing readiness probe",
|
|
}
|
|
in
|
|
|
|
let svc_monitors =
|
|
(if has_nats then [svc_monitor "nats" "kube-nats"] else [])
|
|
@ (if has_pg then [svc_monitor "postgres" "kube-postgres"] else [])
|
|
@ (if has_fg then [svc_monitor "forgejo" "kube-forgejo"] else [])
|
|
@ (if has_wp then [svc_monitor "woodpecker" "kube-woodpecker"] else [])
|
|
@ (if has_zot then [svc_monitor "zot" "kube-zot"] else [])
|
|
in
|
|
|
|
{
|
|
collectors = base_collectors @ svc_collectors,
|
|
channels = notif_chs,
|
|
monitors = base_monitors @ svc_monitors,
|
|
}
|
|
),
|
|
}
|