130 lines
4.5 KiB
JSON
130 lines
4.5 KiB
JSON
{
|
|
"description": "Verify Vector → Loki → Grafana pipeline is healthy in libre-wuji; Prometheus is scraping ops-controller and keeper-daemon; required queue metrics are present (ADR-037 constraint pending-queue-ttl-monitored).",
|
|
"id": "validate-observability",
|
|
"params": {
|
|
"grafana_url": "http://grafana.observability.svc.cluster.local:3000",
|
|
"kubeconfig": "",
|
|
"loki_url": "http://loki.observability.svc.cluster.local:3100",
|
|
"prometheus_url": "http://prometheus.observability.svc.cluster.local:9090",
|
|
"workspace": "libre-wuji"
|
|
},
|
|
"steps": [
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "kubectl -n observability rollout status daemonset/vector --timeout=30s",
|
|
"depends_on": [],
|
|
"id": "vector_daemonset_ready",
|
|
"name": "Vector DaemonSet is fully rolled out",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf {loki_url}/ready",
|
|
"depends_on": [],
|
|
"id": "loki_ready",
|
|
"name": "Loki /ready endpoint responds 200",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf {prometheus_url}/-/ready",
|
|
"depends_on": [],
|
|
"id": "prometheus_ready",
|
|
"name": "Prometheus /-/ready endpoint responds 200",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf {grafana_url}/api/health | jq -e '.database == \"ok\"'",
|
|
"depends_on": [],
|
|
"id": "grafana_ready",
|
|
"name": "Grafana /api/health endpoint responds 200",
|
|
"on_error": "warn"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{prometheus_url}/api/v1/query?query=up{job=\"ops-controller\"}' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"prometheus_ready"
|
|
],
|
|
"id": "prometheus_scrapes_ops_controller",
|
|
"name": "Prometheus is scraping ops-controller metrics",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{prometheus_url}/api/v1/query?query=ops_pending_queue_depth' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"prometheus_scrapes_ops_controller"
|
|
],
|
|
"id": "metric_ops_pending_queue_depth",
|
|
"name": "ops_pending_queue_depth metric is present",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{prometheus_url}/api/v1/query?query=ops_pending_oldest_age_seconds' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"prometheus_scrapes_ops_controller"
|
|
],
|
|
"id": "metric_ops_pending_oldest_age",
|
|
"name": "ops_pending_oldest_age_seconds metric is present",
|
|
"on_error": "abort"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{prometheus_url}/api/v1/query?query=keeper_signs_total' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"prometheus_scrapes_ops_controller"
|
|
],
|
|
"id": "metric_keeper_signs_total",
|
|
"name": "keeper_signs_total metric is present",
|
|
"on_error": "warn"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{prometheus_url}/api/v1/query?query=audit_mirror_lag_seconds' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"prometheus_scrapes_ops_controller"
|
|
],
|
|
"id": "metric_audit_mirror_lag",
|
|
"name": "audit_mirror_lag_seconds metric is present",
|
|
"on_error": "warn"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{grafana_url}/api/dashboards/uid/ops-controller-queue' | jq -e '.dashboard.uid == \"ops-controller-queue\"'",
|
|
"depends_on": [
|
|
"grafana_ready"
|
|
],
|
|
"id": "grafana_dashboards_loaded",
|
|
"name": "All four ops dashboards provisioned in Grafana",
|
|
"on_error": "warn"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "curl -sf '{loki_url}/loki/api/v1/query_range?query=%7Bnamespace%3D%22ops-system%22%7D&limit=1&start=$(date -d '15 minutes ago' +%s)000000000' | jq -e '.data.result | length > 0'",
|
|
"depends_on": [
|
|
"loki_ready"
|
|
],
|
|
"id": "loki_receives_ops_logs",
|
|
"name": "Loki has received at least one log entry from ops-system namespace in last 15 minutes",
|
|
"on_error": "warn"
|
|
},
|
|
{
|
|
"actor": "Agent",
|
|
"cmd": "echo 'observability validation complete for workspace={workspace}'",
|
|
"depends_on": [
|
|
"metric_ops_pending_queue_depth",
|
|
"metric_ops_pending_oldest_age",
|
|
"grafana_dashboards_loaded",
|
|
"loki_receives_ops_logs"
|
|
],
|
|
"id": "summary_report",
|
|
"name": "Emit observability validation summary",
|
|
"on_error": "warn"
|
|
}
|
|
],
|
|
"strategy": "Override",
|
|
"version": "1.0"
|
|
}
|