# Monitoring and Observability Examples

Production monitoring, alerting, and observability patterns.

## Prometheus Monitoring Stack

Complete Prometheus setup with Grafana dashboards:

```nickel
{
  monitoring = {
    prometheus = {
      enabled = true,
      version = "latest",
      retention = "30d",
      storage = {
        size = 500,
        type = "io2",
        iops = 5000
      },

      # Scrape configurations
      scrape_configs = [
        {
          job_name = "kubernetes-nodes",
          kubernetes_sd_configs = [{
            role = "node"
          }],
          relabel_configs = [
            {
              source_labels = ["__address__"],
              regex = "([^:]+)(?::\\d+)?",
              replacement = "${1}:9100",
              target_label = "__address__"
            }
          ]
        },
        {
          job_name = "kubernetes-pods",
          kubernetes_sd_configs = [{
            role = "pod"
          }],
          relabel_configs = [
            {
              source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"],
              action = "keep",
              regex = "true"
            }
          ]
        }
      ],

      # Alert rules
      alert_rules = [
        {
          alert = "HighCPUUsage",
          expr = "node_cpu_seconds_total > 0.8",
          for = "5m",
          annotations = {
            summary = "High CPU usage detected"
          }
        },
        {
          alert = "HighMemoryUsage",
          expr = "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1",
          for = "5m",
          annotations = {
            summary = "Low memory available"
          }
        }
      ]
    },

    # Grafana dashboards
    grafana = {
      enabled = true,
      version = "latest",
      replicas = 2,
      persistence = {
        enabled = true,
        size = 10
      },

      dashboards = [
        {
          name = "Kubernetes Cluster",
          uid = "kubernetes-cluster",
          panels = [
            { title = "CPU Usage", metric = "rate(container_cpu_usage_seconds_total[5m])" },
            { title = "Memory Usage", metric = "container_memory_usage_bytes" },
            { title = "Network I/O", metric = "rate(container_network_transmit_bytes_total[5m])" }
          ]
        },
        {
          name = "Application Performance",
          uid = "app-performance",
          panels = [
            { title = "Request Rate", metric = "http_requests_total" },
            { title = "Error Rate", metric = "http_requests_total{status=~\"5..\"}"},
            { title = "Latency (p95)", metric = "histogram_quantile(0.95, http_request_duration_seconds)" }
          ]
        }
      ]
    }
  }
}
```

## Alert Rule Configuration

Comprehensive alerting rules:

```yaml
# alerts/kubernetes.yml
groups:
  - name: kubernetes.rules
    rules:
      # Node alerts
      - alert: NodeNotReady
        expr: kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.node }} is not ready"
          runbook_url: " [https://wiki.example.com/runbooks/node_not_ready"](https://wiki.example.com/runbooks/node_not_ready")

      - alert: NodeMemoryPressure
        expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Node {{ $labels.node }} has memory pressure"

      # Pod alerts
      - alert: PodCrashLooping
        expr: rate(kube_pod_container_status_restarts_total[15m]) > 0.1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Pod {{ $labels.pod }} is crash looping"

      - alert: PodNotHealthy
        expr: |
          min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pendin| g Unknow| n Failed"})[15m:1m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is not healthy"

      # Container alerts
      - alert: ContainerOOMKiller
        expr: kube_pod_container_status_last_state_reason{reason="OOMKilled"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.container }} in {{ $labels.pod }} was OOM killed"
```

## ELK Stack Logging

Elasticsearch, Logstash, Kibana configuration:

```nickel
{
  logging = {
    elasticsearch = {
      enabled = true,
      version = "8.0",
      replicas = 3,
      node_type = "data_content",
      storage = {
        size = 2000,
        type = "io2",
        iops = 10000
      },

      # Index lifecycle management
      ilm_policy = {
        name = "logs-policy",
        phases = {
          hot = {
            min_age = "0d",
            rollover = {
              max_primary_shard_size = "50gb"
            }
          },
          warm = {
            min_age = "7d",
            set_priority = 50
          },
          cold = {
            min_age = "30d",
            searchable_snapshot = "snapshot-repo"
          },
          delete = {
            min_age = "90d"
          }
        }
      },

      # Indices
      indices = [
        {
          name = "logs-kubernetes",
          shards = 5,
          replicas = 2,
          retention = 30
        },
        {
          name = "logs-application",
          shards = 3,
          replicas = 2,
          retention = 30
        }
      ]
    },

    # Filebeat for log collection
    filebeat = {
      enabled = true,
      inputs = [
        {
          type = "container",
          enabled = true,
          paths = ["/var/lib/docker/containers/*/*.log"]
        },
        {
          type = "journald",
          enabled = true
        }
      ],
      processors = [
        { add_kubernetes_metadata = {} },
        { drop_fields = { fields = ["host.name"] } }
      ]
    },

    # Kibana for visualization
    kibana = {
      enabled = true,
      replicas = 2,
      elasticsearch_hosts = ["elasticsearch-0:9200"],
      plugins = ["security", "alerting", "cases"]
    }
  }
}
```

## Distributed Tracing

Jaeger distributed tracing setup:

```nickel
{
  tracing = {
    jaeger = {
      enabled = true,
      version = "latest",
      strategy = "probabilistic",
      sampling_rate = 0.1,  # 10% sampling

      # Jaeger collector
      collector = {
        replicas = 2,
        storage = {
          type = "elasticsearch",
          elasticsearch = {
            server_urls = [" [http://elasticsearch:9200"],](http://elasticsearch:9200"],)
            index_prefix = "jaeger"
          }
        }
      },

      # Jaeger query UI
      query = {
        replicas = 2,
        ui = {
          enabled = true,
          tracking = {
            google_analytics_id = "UA-XXXXXXXXXXXX-X"
          }
        }
      },

      # Jaeger agent (sidecar)
      agent = {
        enabled = true,
        port = 6831,
        protocol = "udp"
      }
    }
  }
}
```

## Distributed Tracing in Application Code

Example instrument application with Jaeger:

```nushell
#!/usr/bin/env nu

# Initialize tracer
let jaeger_config = {
  agent_host = "jaeger-agent",
  agent_port = 6831,
  service_name = "api-service",
  sampler = {
    type = "const",
    param = 1
  }
}

# Example function with tracing
def call-database-service [query] {
  # Start span
  let span = opentelemetry start-span "database_query" {
    attributes = {
      "db.system" = "postgres",
      "db.statement" = $query,
      "db.operation" = "select"
    }
  }

  try {
    # Execute query (simulate)
    let result = {
      rows = 100,
      duration = 45
    }

    # Add event
    $span | opentelemetry add-event "query_completed" {
      "db.rows_returned" = $result.rows,
      "db.duration_ms" = $result.duration
    }

    $result
  } catch { | err |
    # Record error
    $span | opentelemetry record-error $err

    error $err
  } finally {
    # End span
    $span | opentelemetry end-span
  }
}

# HTTP request tracing
def handle-http-request [path method] {
  let span = opentelemetry start-span "http_request" {
    attributes = {
      "http.method" = $method,
      "http.url" = $path
    }
  }

  try {
    # Handle request
    let response = call-database-service "SELECT * FROM users"

    $span | opentelemetry set-status "OK"
    $response
  } catch {
    $span | opentelemetry set-status "ERROR"
    error "Request failed"
  } finally {
    $span | opentelemetry end-span
  }
}
```

## APM with Application Performance Monitoring

Elastic APM setup:

```nickel
{
  apm = {
    enabled = true,
    type = "elastic",

    # APM Server
    apm_server = {
      replicas = 2,
      elasticsearch = {
        hosts = ["elasticsearch:9200"],
        index_prefix = "apm"
      },

      # Sampling rules
      sampling = {
        enabled = true,
        default_rate = 0.1,
        rules = [
          {
            service_name = "api-service",
            trace_name = "critical-transaction",
            sample_rate = 1.0  # Always sample
          },
          {
            service_name = "*",
            sample_rate = 0.1  # 10% for all others
          }
        ]
      }
    },

    # APM Agents
    agents = [
      {
        language = "go",
        service_name = "api-service",
        server_url = " [http://apm-server:8200",](http://apm-server:8200",)
        enabled = true
      },
      {
        language = "python",
        service_name = "worker-service",
        server_url = " [http://apm-server:8200",](http://apm-server:8200",)
        enabled = true
      }
    ]
  }
}
```

## SLO Monitoring

Service Level Objectives tracking:

```bash
#!/usr/bin/env nu

# Define SLOs
let slos = {
  "api-service" = {
    availability = {
      target = 0.999,      # 99.9% availability
      window = "30d"
    },
    latency = {
      p99 = 200,           # 200ms p99
      p95 = 100,           # 100ms p95
      window = "30d"
    },
    error_rate = {
      target = 0.001,      # 0.1% error rate
      window = "30d"
    }
  },
  "database-service" = {
    availability = {
      target = 0.9999,     # 99.99% availability
      window = "30d"
    },
    latency = {
      p99 = 50,
      p95 = 20,
      window = "30d"
    }
  }
}

# Calculate SLO compliance
def calculate-slo-compliance [service_name slo_type] {
  # Query Prometheus for metrics
  let metric = match $slo_type {
    "availability" => "up",
    "latency" => "http_request_duration_seconds",
    "error_rate" => "http_requests_total{status=~\"5..\"}"
  }

  # Fetch metrics
  let results = (http get $" [http://prometheus:9090/api/v1/query?query=($metri](http://prometheus:9090/api/v1/query?query=($metri)c)").data

  # Calculate compliance percentage
  let compliance = ($results.values | length) / ($slos | get $service_name | get $slo_type | get target)

  print $"($service_name) ($slo_type): ($compliance * 100)% compliance"

  { service: $service_name, slo_type: $slo_type, compliance: $compliance }
}

# Monitor all SLOs
print "SLO Compliance Report"
print "=================="

$slos | columns | each { | service |
  $slos | get $service | columns | each { | slo_type |
    calculate-slo-compliance $service $slo_type
  }
}
```

## Custom Metrics

Publish custom application metrics:

```nushell
#!/usr/bin/env nu

# Define custom metric collectors
let metric_collectors = {
  deployment_duration = {
    type = "histogram",
    help = "Time taken for deployment",
    buckets = [10, 30, 60, 120, 300, 600]
  },
  active_deployments = {
    type = "gauge",
    help = "Number of active deployments"
  },
  deployment_failures = {
    type = "counter",
    help = "Total number of failed deployments",
    labels = ["reason", "component"]
  }
}

# Record metrics
def record-deployment-metric [deployment_name duration_seconds] {
  # Record histogram
  provisioning metrics histogram \
    --name deployment_duration \
    --value $duration_seconds \
    --labels workspace=$env.PROVISIONING_WORKSPACE

  # Increment gauge
  provisioning metrics gauge \
    --name active_deployments \
    --increment

  # Record counter on failure
  if $env.LAST_EXIT_CODE != 0 {
    provisioning metrics counter \
      --name deployment_failures \
      --increment \
      --labels reason="execution_error",component="orchestrator"
  }
}
```

## Related Documentation

- [Operations Guide](../operations/README.md)
- [Platform Health](../operations/platform-health.md)
- [Troubleshooting](../troubleshooting/README.md)
- [Logging Configuration](../operations/monitoring.md)
