{
  "description": "Verify Vector → Loki → Grafana pipeline is healthy in libre-wuji; Prometheus is scraping ops-controller and keeper-daemon; required queue metrics are present (ADR-037 constraint pending-queue-ttl-monitored).",
  "id": "validate-observability",
  "params": {
    "grafana_url": "http://grafana.observability.svc.cluster.local:3000",
    "kubeconfig": "",
    "loki_url": "http://loki.observability.svc.cluster.local:3100",
    "prometheus_url": "http://prometheus.observability.svc.cluster.local:9090",
    "workspace": "libre-wuji"
  },
  "steps": [
    {
      "actor": "Agent",
      "cmd": "kubectl -n observability rollout status daemonset/vector --timeout=30s",
      "depends_on": [],
      "id": "vector_daemonset_ready",
      "name": "Vector DaemonSet is fully rolled out",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf {loki_url}/ready",
      "depends_on": [],
      "id": "loki_ready",
      "name": "Loki /ready endpoint responds 200",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf {prometheus_url}/-/ready",
      "depends_on": [],
      "id": "prometheus_ready",
      "name": "Prometheus /-/ready endpoint responds 200",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf {grafana_url}/api/health | jq -e '.database == \"ok\"'",
      "depends_on": [],
      "id": "grafana_ready",
      "name": "Grafana /api/health endpoint responds 200",
      "on_error": "warn"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{prometheus_url}/api/v1/query?query=up{job=\"ops-controller\"}' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "prometheus_ready"
      ],
      "id": "prometheus_scrapes_ops_controller",
      "name": "Prometheus is scraping ops-controller metrics",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{prometheus_url}/api/v1/query?query=ops_pending_queue_depth' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "prometheus_scrapes_ops_controller"
      ],
      "id": "metric_ops_pending_queue_depth",
      "name": "ops_pending_queue_depth metric is present",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{prometheus_url}/api/v1/query?query=ops_pending_oldest_age_seconds' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "prometheus_scrapes_ops_controller"
      ],
      "id": "metric_ops_pending_oldest_age",
      "name": "ops_pending_oldest_age_seconds metric is present",
      "on_error": "abort"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{prometheus_url}/api/v1/query?query=keeper_signs_total' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "prometheus_scrapes_ops_controller"
      ],
      "id": "metric_keeper_signs_total",
      "name": "keeper_signs_total metric is present",
      "on_error": "warn"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{prometheus_url}/api/v1/query?query=audit_mirror_lag_seconds' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "prometheus_scrapes_ops_controller"
      ],
      "id": "metric_audit_mirror_lag",
      "name": "audit_mirror_lag_seconds metric is present",
      "on_error": "warn"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{grafana_url}/api/dashboards/uid/ops-controller-queue' | jq -e '.dashboard.uid == \"ops-controller-queue\"'",
      "depends_on": [
        "grafana_ready"
      ],
      "id": "grafana_dashboards_loaded",
      "name": "All four ops dashboards provisioned in Grafana",
      "on_error": "warn"
    },
    {
      "actor": "Agent",
      "cmd": "curl -sf '{loki_url}/loki/api/v1/query_range?query=%7Bnamespace%3D%22ops-system%22%7D&limit=1&start=$(date -d '15 minutes ago' +%s)000000000' | jq -e '.data.result | length > 0'",
      "depends_on": [
        "loki_ready"
      ],
      "id": "loki_receives_ops_logs",
      "name": "Loki has received at least one log entry from ops-system namespace in last 15 minutes",
      "on_error": "warn"
    },
    {
      "actor": "Agent",
      "cmd": "echo 'observability validation complete for workspace={workspace}'",
      "depends_on": [
        "metric_ops_pending_queue_depth",
        "metric_ops_pending_oldest_age",
        "grafana_dashboards_loaded",
        "loki_receives_ops_logs"
      ],
      "id": "summary_report",
      "name": "Emit observability validation summary",
      "on_error": "warn"
    }
  ],
  "strategy": "Override",
  "version": "1.0"
}