#!/usr/bin/env nu # Derived variables for prometheus bundle rendering. # Reads the component JSON, outputs extra Tera context vars as JSON. # Derive job_name from a scrape target string: take everything before the first dot. def job_name_from_target [target: string]: nothing -> string { $target | split row "." | first } # Build a single scrape_config block string for a job. def scrape_config_block [job: string, target: string]: nothing -> string { $" - job_name: ($job)\n static_configs:\n - targets: ['($target)']" } # Render a single alert rule block. def alert_rule_block [rule: record]: nothing -> string { let lines = [ $" - alert: ($rule.name)", $" expr: ($rule.expr)", $" for: ($rule.for_)", " labels:", $" severity: ($rule.severity)", " annotations:", $" summary: \"($rule.summary)\"", $" description: \"($rule.description)\"", ] $lines | str join "\n" } def main [component_json: string]: nothing -> nothing { let d = ($component_json | from json) let scrape_interval = ($d | get -o params.scrape_interval | default "30s") let retention = ($d | get -o params.retention | default "30d") let scrape_targets = ($d | get -o params.scrape_targets | default []) let alert_rules = ($d | get -o params.alert_rules | default []) # Build scrape_configs: prometheus self + longhorn hardcoded + params targets let self_block = " - job_name: prometheus\n static_configs:\n - targets: ['localhost:9090']" # Use short name — search domain (svc.libre-wuji.local) handles resolution. # svc.cluster.local is the kubeadm default but this cluster uses libre-wuji.local. let dns_domain = ($d | get -o params.cluster_dns_domain | default "svc.libre-wuji.local") let longhorn_block = $" - job_name: longhorn\n static_configs:\n - targets: ['longhorn-backend.longhorn-system.($dns_domain):9500']" let param_blocks = ($scrape_targets | each {|t| scrape_config_block (job_name_from_target $t) $t }) let all_scrape_blocks = ([$self_block $longhorn_block] | append $param_blocks) let grafana_am = ($d | get -o params.grafana_alertmanager | default "") let alerting_block = if ($grafana_am | is-not-empty) { $"alerting: alertmanagers: - static_configs: - targets: ['($grafana_am)'] path_prefix: /api/alertmanager/grafana timeout: 10s\n\n" } else { "" } # Regex literals that contain parens — must be plain variables to avoid # Nushell treating (...) as subexpressions inside $"..." strings. let re_any = "(.+)" let re_node_label = "__meta_kubernetes_node_label_(.+)" let backref_one = "${1}" # Build kubernetes_sd_configs blocks for components that need pod-level discovery. let sd_scrapes = ($d | get -o params.kubernetes_sd_scrapes | default []) let sd_blocks = ($sd_scrapes | each {|s| let role = ($s | get -o role | default "endpoints") if $role == "node" { # kubelet/cadvisor: role=node, proxy via kube API, HTTPS with SA token $" - job_name: ($s.job_name) scheme: https tls_config: insecure_skip_verify: true authorization: credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node metrics_path: ($s.path) relabel_configs: - action: labelmap regex: ($re_node_label) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: ($re_any) target_label: __metrics_path__ replacement: /api/v1/nodes/($backref_one)/proxy($s.path)" } else { $" - job_name: ($s.job_name) kubernetes_sd_configs: - role: endpoints namespaces: names: ['($s.namespace)'] relabel_configs: - source_labels: [__meta_kubernetes_service_label_app] action: keep regex: ($s.app_label) - source_labels: [__meta_kubernetes_endpoint_port_name] action: keep regex: metrics - source_labels: [__meta_kubernetes_pod_node_name] target_label: node - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod" } }) # Build cross-cluster static scrapes with explicit labels. let cc_scrapes = ($d | get -o params.cross_cluster_scrapes | default []) let cc_blocks = ($cc_scrapes | each {|s| let label_lines = ($s.labels | transpose key val | each {|kv| $" ($kv.key): \"($kv.val)\"" } | str join "\n") $" - job_name: ($s.job_name) static_configs: - targets: ['($s.target)'] labels: ($label_lines)" }) let prometheus_yml = $"global: scrape_interval: ($scrape_interval) evaluation_interval: ($scrape_interval) ($alerting_block)rule_files: - /etc/prometheus/alert-rules.yml scrape_configs: ($all_scrape_blocks | str join "\n") ($cc_blocks | str join "\n") ($sd_blocks | str join "\n")" # Build alert rules: hardcoded Longhorn rules first, then params rules let longhorn_rules = [ # longhorn_volume_read_only does not exist — Longhorn has no visibility into kernel-level RO. # RO volumes appear as degraded (robustness=2) once Longhorn detects I/O errors. # Catch them via robustness transition to degraded with short window. " - alert: LonghornVolumeReadOnly", " expr: longhorn_volume_robustness == 2", " for: 2m", " labels:", " severity: critical", " annotations:", " summary: \"Longhorn volume possibly read-only: {{ $labels.volume }}\"", " description: \"Volume {{ $labels.volume }} is degraded — may be read-only (aborted journal). Run: nu scripts/repair-longhorn-fs.nu -n \"", " - alert: LonghornVolumeRobustnessDegraded", " expr: longhorn_volume_robustness == 2", " for: 5m", " labels:", " severity: warning", " annotations:", " summary: \"Longhorn volume degraded: {{ $labels.volume }}\"", " - alert: LonghornVolumeRobustnessFaulted", " expr: longhorn_volume_robustness == 3", " for: 2m", " labels:", " severity: critical", " annotations:", " summary: \"Longhorn volume faulted: {{ $labels.volume }}\"", " - alert: PodCrashLoopStateful", " expr: kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"} == 1", " for: 10m", " labels:", " severity: warning", " annotations:", " summary: \"Pod {{ $labels.pod }} in CrashLoopBackOff\"", " description: \"Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been crash-looping for 10m.\"", ] let param_rule_blocks = ($alert_rules | each {|r| alert_rule_block $r }) let all_rule_lines = ($longhorn_rules | append ($param_rule_blocks | each {|b| $b | lines } | flatten)) let alert_rules_yml = $"groups: - name: platform rules: ($all_rule_lines | str join "\n")" { prometheus_yml: $prometheus_yml, alert_rules_yml: $alert_rules_yml, } | to json --raw | print }