191 lines
7.5 KiB
Text
191 lines
7.5 KiB
Text
|
#!/usr/bin/env nu
|
||
|
|
# Derived variables for prometheus bundle rendering.
|
||
|
|
# Reads the component JSON, outputs extra Tera context vars as JSON.
|
||
|
|
|
||
|
|
# Derive job_name from a scrape target string: take everything before the first dot.
|
||
|
|
def job_name_from_target [target: string]: nothing -> string {
|
||
|
|
$target | split row "." | first
|
||
|
|
}
|
||
|
|
|
||
|
|
# Build a single scrape_config block string for a job.
|
||
|
|
def scrape_config_block [job: string, target: string]: nothing -> string {
|
||
|
|
$" - job_name: ($job)\n static_configs:\n - targets: ['($target)']"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Render a single alert rule block.
|
||
|
|
def alert_rule_block [rule: record]: nothing -> string {
|
||
|
|
let lines = [
|
||
|
|
$" - alert: ($rule.name)",
|
||
|
|
$" expr: ($rule.expr)",
|
||
|
|
$" for: ($rule.for_)",
|
||
|
|
" labels:",
|
||
|
|
$" severity: ($rule.severity)",
|
||
|
|
" annotations:",
|
||
|
|
$" summary: \"($rule.summary)\"",
|
||
|
|
$" description: \"($rule.description)\"",
|
||
|
|
]
|
||
|
|
$lines | str join "\n"
|
||
|
|
}
|
||
|
|
|
||
|
|
def main [component_json: string]: nothing -> nothing {
|
||
|
|
let d = ($component_json | from json)
|
||
|
|
|
||
|
|
let scrape_interval = ($d | get -o params.scrape_interval | default "30s")
|
||
|
|
let retention = ($d | get -o params.retention | default "30d")
|
||
|
|
let scrape_targets = ($d | get -o params.scrape_targets | default [])
|
||
|
|
let alert_rules = ($d | get -o params.alert_rules | default [])
|
||
|
|
|
||
|
|
# Build scrape_configs: prometheus self + longhorn hardcoded + params targets
|
||
|
|
let self_block = " - job_name: prometheus\n static_configs:\n - targets: ['localhost:9090']"
|
||
|
|
# Use short name — search domain (svc.libre-wuji.local) handles resolution.
|
||
|
|
# svc.cluster.local is the kubeadm default but this cluster uses libre-wuji.local.
|
||
|
|
let dns_domain = ($d | get -o params.cluster_dns_domain | default "svc.libre-wuji.local")
|
||
|
|
let longhorn_block = $" - job_name: longhorn\n static_configs:\n - targets: ['longhorn-backend.longhorn-system.($dns_domain):9500']"
|
||
|
|
|
||
|
|
let param_blocks = ($scrape_targets | each {|t|
|
||
|
|
scrape_config_block (job_name_from_target $t) $t
|
||
|
|
})
|
||
|
|
|
||
|
|
let all_scrape_blocks = ([$self_block $longhorn_block] | append $param_blocks)
|
||
|
|
|
||
|
|
let grafana_am = ($d | get -o params.grafana_alertmanager | default "")
|
||
|
|
let alerting_block = if ($grafana_am | is-not-empty) {
|
||
|
|
$"alerting:
|
||
|
|
alertmanagers:
|
||
|
|
- static_configs:
|
||
|
|
- targets: ['($grafana_am)']
|
||
|
|
path_prefix: /api/alertmanager/grafana
|
||
|
|
timeout: 10s\n\n"
|
||
|
|
} else { "" }
|
||
|
|
|
||
|
|
# Regex literals that contain parens — must be plain variables to avoid
|
||
|
|
# Nushell treating (...) as subexpressions inside $"..." strings.
|
||
|
|
let re_any = "(.+)"
|
||
|
|
let re_node_label = "__meta_kubernetes_node_label_(.+)"
|
||
|
|
let backref_one = "${1}"
|
||
|
|
|
||
|
|
# Build kubernetes_sd_configs blocks for components that need pod-level discovery.
|
||
|
|
let sd_scrapes = ($d | get -o params.kubernetes_sd_scrapes | default [])
|
||
|
|
let sd_blocks = ($sd_scrapes | each {|s|
|
||
|
|
let role = ($s | get -o role | default "endpoints")
|
||
|
|
if $role == "node" {
|
||
|
|
# kubelet/cadvisor: role=node, proxy via kube API, HTTPS with SA token
|
||
|
|
$" - job_name: ($s.job_name)
|
||
|
|
scheme: https
|
||
|
|
tls_config:
|
||
|
|
insecure_skip_verify: true
|
||
|
|
authorization:
|
||
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||
|
|
kubernetes_sd_configs:
|
||
|
|
- role: node
|
||
|
|
metrics_path: ($s.path)
|
||
|
|
relabel_configs:
|
||
|
|
- action: labelmap
|
||
|
|
regex: ($re_node_label)
|
||
|
|
- target_label: __address__
|
||
|
|
replacement: kubernetes.default.svc:443
|
||
|
|
- source_labels: [__meta_kubernetes_node_name]
|
||
|
|
regex: ($re_any)
|
||
|
|
target_label: __metrics_path__
|
||
|
|
replacement: /api/v1/nodes/($backref_one)/proxy($s.path)"
|
||
|
|
} else {
|
||
|
|
$" - job_name: ($s.job_name)
|
||
|
|
kubernetes_sd_configs:
|
||
|
|
- role: endpoints
|
||
|
|
namespaces:
|
||
|
|
names: ['($s.namespace)']
|
||
|
|
relabel_configs:
|
||
|
|
- source_labels: [__meta_kubernetes_service_label_app]
|
||
|
|
action: keep
|
||
|
|
regex: ($s.app_label)
|
||
|
|
- source_labels: [__meta_kubernetes_endpoint_port_name]
|
||
|
|
action: keep
|
||
|
|
regex: metrics
|
||
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
||
|
|
target_label: node
|
||
|
|
- source_labels: [__meta_kubernetes_namespace]
|
||
|
|
target_label: namespace
|
||
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
||
|
|
target_label: pod"
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
# Build cross-cluster static scrapes with explicit labels.
|
||
|
|
let cc_scrapes = ($d | get -o params.cross_cluster_scrapes | default [])
|
||
|
|
let cc_blocks = ($cc_scrapes | each {|s|
|
||
|
|
let label_lines = ($s.labels | transpose key val | each {|kv|
|
||
|
|
$" ($kv.key): \"($kv.val)\""
|
||
|
|
} | str join "\n")
|
||
|
|
$" - job_name: ($s.job_name)
|
||
|
|
static_configs:
|
||
|
|
- targets: ['($s.target)']
|
||
|
|
labels:
|
||
|
|
($label_lines)"
|
||
|
|
})
|
||
|
|
|
||
|
|
let prometheus_yml = $"global:
|
||
|
|
scrape_interval: ($scrape_interval)
|
||
|
|
evaluation_interval: ($scrape_interval)
|
||
|
|
|
||
|
|
($alerting_block)rule_files:
|
||
|
|
- /etc/prometheus/alert-rules.yml
|
||
|
|
|
||
|
|
scrape_configs:
|
||
|
|
($all_scrape_blocks | str join "\n")
|
||
|
|
($cc_blocks | str join "\n")
|
||
|
|
($sd_blocks | str join "\n")"
|
||
|
|
|
||
|
|
# Build alert rules: hardcoded Longhorn rules first, then params rules
|
||
|
|
let longhorn_rules = [
|
||
|
|
# longhorn_volume_read_only does not exist — Longhorn has no visibility into kernel-level RO.
|
||
|
|
# RO volumes appear as degraded (robustness=2) once Longhorn detects I/O errors.
|
||
|
|
# Catch them via robustness transition to degraded with short window.
|
||
|
|
" - alert: LonghornVolumeReadOnly",
|
||
|
|
" expr: longhorn_volume_robustness == 2",
|
||
|
|
" for: 2m",
|
||
|
|
" labels:",
|
||
|
|
" severity: critical",
|
||
|
|
" annotations:",
|
||
|
|
" summary: \"Longhorn volume possibly read-only: {{ $labels.volume }}\"",
|
||
|
|
" description: \"Volume {{ $labels.volume }} is degraded — may be read-only (aborted journal). Run: nu scripts/repair-longhorn-fs.nu <workload> -n <ns>\"",
|
||
|
|
" - alert: LonghornVolumeRobustnessDegraded",
|
||
|
|
" expr: longhorn_volume_robustness == 2",
|
||
|
|
" for: 5m",
|
||
|
|
" labels:",
|
||
|
|
" severity: warning",
|
||
|
|
" annotations:",
|
||
|
|
" summary: \"Longhorn volume degraded: {{ $labels.volume }}\"",
|
||
|
|
" - alert: LonghornVolumeRobustnessFaulted",
|
||
|
|
" expr: longhorn_volume_robustness == 3",
|
||
|
|
" for: 2m",
|
||
|
|
" labels:",
|
||
|
|
" severity: critical",
|
||
|
|
" annotations:",
|
||
|
|
" summary: \"Longhorn volume faulted: {{ $labels.volume }}\"",
|
||
|
|
" - alert: PodCrashLoopStateful",
|
||
|
|
" expr: kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"} == 1",
|
||
|
|
" for: 10m",
|
||
|
|
" labels:",
|
||
|
|
" severity: warning",
|
||
|
|
" annotations:",
|
||
|
|
" summary: \"Pod {{ $labels.pod }} in CrashLoopBackOff\"",
|
||
|
|
" description: \"Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been crash-looping for 10m.\"",
|
||
|
|
]
|
||
|
|
|
||
|
|
let param_rule_blocks = ($alert_rules | each {|r| alert_rule_block $r })
|
||
|
|
|
||
|
|
let all_rule_lines = ($longhorn_rules | append ($param_rule_blocks | each {|b| $b | lines } | flatten))
|
||
|
|
|
||
|
|
let alert_rules_yml = $"groups:
|
||
|
|
- name: platform
|
||
|
|
rules:
|
||
|
|
($all_rule_lines | str join "\n")"
|
||
|
|
|
||
|
|
{
|
||
|
|
prometheus_yml: $prometheus_yml,
|
||
|
|
alert_rules_yml: $alert_rules_yml,
|
||
|
|
}
|
||
|
|
| to json --raw
|
||
|
|
| print
|
||
|
|
}
|