# High Availability Configuration Examples

Production-grade HA patterns across multiple cloud providers.

## Multi-AZ Kubernetes Cluster

Distribute Kubernetes across availability zones:

```nickel
{
  workspace = "production",

  kubernetes = {
    name = "prod-k8s-cluster",
    version = "1.28",
    provider = "aws",

    # Multi-AZ control plane
    control_plane = {
      count = 3,
      instance_type = "t3.large",
      availability_zones = [
        "us-east-1a",
        "us-east-1b",
        "us-east-1c"
      ],
      etcd = {
        storage_size = 200,
        storage_type = "io2",
        iops = 5000,
        multi_az = true
      }
    },

    # Worker nodes distributed across AZs
    worker_pools = [
      {
        name = "general-purpose",
        machine_type = "t3.xlarge",
        replicas = 3,
        availability_zones = ["us-east-1a", "us-east-1b", "us-east-1c"],
        auto_scaling = {
          min = 3,
          max = 10,
          target_cpu_utilization = 70
        }
      },
      {
        name = "compute-intensive",
        machine_type = "c5.2xlarge",
        replicas = 2,
        availability_zones = ["us-east-1a", "us-east-1c"],
        auto_scaling = {
          min = 2,
          max = 8
        }
      }
    ],

    # CNI with network policies
    networking = {
      cni = "cilium",
      pod_cidr = "10.0.0.0/8",
      service_cidr = "172.30.0.0/16",
      network_policies = "enforced"
    },

    # High availability configuration
    ha_settings = {
      pod_disruption_budgets = "enabled",
      preemption_policy = "preemptive",
      priority_classes = [
        { name = "critical", value = 1000000 },
        { name = "high", value = 100000 },
        { name = "medium", value = 10000 }
      ]
    }
  }
}
```

## Multi-Region Load Balancing

Geographic load balancing across regions:

```nickel
{
  load_balancing = {
    global_load_balancer = {
      name = "global-app-lb",
      type = "geographic",
      routing_policy = "geolocation",

      regions = [
        {
          name = "us-east",
          provider = "aws",
          region = "us-east-1",
          weight = 50,
          backends = [
            {
              name = "regional-lb",
              health_check = {
                protocol = "HTTPS",
                path = "/health",
                interval = 30,
                timeout = 5,
                threshold = 2
              }
            }
          ]
        },
        {
          name = "eu-west",
          provider = "upcloud",
          region = "nl-ams",
          weight = 30,
          backends = [
            {
              name = "regional-lb",
              health_check = {
                protocol = "HTTPS",
                path = "/health",
                interval = 30,
                timeout = 5,
                threshold = 2
              }
            }
          ]
        },
        {
          name = "ap-southeast",
          provider = "hetzner",
          location = "fsn1",
          weight = 20,
          backends = [
            {
              name = "regional-lb",
              health_check = {
                protocol = "HTTPS",
                path = "/health",
                interval = 30,
                timeout = 5,
                threshold = 2
              }
            }
          ]
        }
      ],

      # Failover configuration
      failover = {
        enabled = true,
        health_threshold = 2,
        failover_chain = ["us-east", "eu-west", "ap-southeast"]
      }
    }
  }
}
```

## Database Replication with Automatic Failover

PostgreSQL streaming replication with automatic failover:

```nickel
{
  database = {
    name = "primary-db",
    engine = "postgres",
    version = "15",

    ha = {
      enabled = true,
      mode = "streaming-replication",
      automatic_failover = true
    },

    # Primary node
    primary = {
      instance_class = "db.r6i.2xlarge",
      availability_zone = "us-east-1a",
      storage = {
        size = 1000,
        type = "io2",
        iops = 10000
      }
    },

    # Standby replicas
    standbys = [
      {
        name = "standby-01",
        instance_class = "db.r6i.xlarge",
        availability_zone = "us-east-1b",
        promotion_priority = 1,
        replication_lag_threshold = 100
      },
      {
        name = "standby-02",
        instance_class = "db.r6i.xlarge",
        availability_zone = "us-east-1c",
        promotion_priority = 2,
        replication_lag_threshold = 100
      }
    ],

    # HA cluster configuration
    cluster = {
      name = "prod-db-cluster",
      members = ["primary", "standby-01", "standby-02"],
      vip = "db.prod.internal",
      heartbeat_interval = 5,
      heartbeat_timeout = 15,
      election_timeout = 30
    },

    # Backup for disaster recovery
    backups = {
      enabled = true,
      retention_days = 35,
      backup_window = "03:00-04:00",
      wal_backup_enabled = true,
      wal_retention_days = 7
    }
  }
}
```

## Service Mesh with High Availability

Istio/Linkerd service mesh for resilience:

```nickel
{
  service_mesh = {
    name = "production-mesh",
    type = "istio",
    version = "1.18",

    # Control plane HA
    control_plane = {
      replicas = 3,
      anti_affinity = "required",
      resource_limits = {
        cpu = "2000m",
        memory = "2Gi"
      }
    },

    # Ingress gateway HA
    ingress_gateways = [
      {
        name = "main-gateway",
        replicas = 3,
        anti_affinity = "required",
        load_balancer_type = "nlb"
      }
    ],

    # Virtual services with circuit breaking
    virtual_services = [
      {
        name = "api-service",
        hosts = ["api.prod.internal"],
        http = [
          {
            match = [{ uri = { prefix = "/" } }],
            route = [
              {
                destination = {
                  host = "api-service.default.svc.cluster.local",
                  subset = "v1"
                },
                weight = 90
              },
              {
                destination = {
                  host = "api-service.default.svc.cluster.local",
                  subset = "v2"
                },
                weight = 10
              }
            ],
            timeout = "30s",
            retries = {
              attempts = 3,
              per_try_timeout = "10s"
            }
          }
        ]
      }
    ],

    # Destination rules with circuit breaker
    destination_rules = [
      {
        name = "api-service",
        host = "api-service.default.svc.cluster.local",
        traffic_policy = {
          connection_pool = {
            tcp = { max_connections = 1000 },
            http = {
              http1_max_pending_requests = 100,
              max_requests_per_connection = 2
            }
          },
          outlier_detection = {
            consecutive_errors = 5,
            interval = "30s",
            base_ejection_time = "30s"
          }
        }
      }
    ]
  }
}
```

## Cache Layer with Replication

Redis cluster with automatic failover and replication:

```nickel
{
  cache_layer = {
    primary = {
      engine = "redis",
      version = "7.0",
      cluster_mode = "enabled",
      num_cache_clusters = 3,
      auto_failover = true,

      nodes = [
        {
          id = "shard-001",
          primary_zone = "us-east-1a",
          instance_type = "cache.r6g.xlarge",
          replicas = 2,
          replica_zones = ["us-east-1b", "us-east-1c"]
        },
        {
          id = "shard-002",
          primary_zone = "us-east-1b",
          instance_type = "cache.r6g.xlarge",
          replicas = 2,
          replica_zones = ["us-east-1c", "us-east-1a"]
        },
        {
          id = "shard-003",
          primary_zone = "us-east-1c",
          instance_type = "cache.r6g.xlarge",
          replicas = 2,
          replica_zones = ["us-east-1a", "us-east-1b"]
        }
      ]
    },

    # Secondary cache in another region (read-only)
    secondary = {
      enabled = true,
      region = "eu-west-1",
      replication_mode = "read-only",
      replication_lag_threshold = 1000  # milliseconds
    }
  }
}
```

## Circuit Breaker Pattern

Resilient service calls with circuit breakers:

```nushell
#!/usr/bin/env nu

# Circuit breaker state machine
let circuit_breaker = {
  state: "closed",        # closed, open, half-open
  failure_count: 0,
  failure_threshold: 5,
  success_count: 0,
  success_threshold: 3,
  timeout_duration: 60
}

def call_service [service_name] {
  # Check circuit state
  match $circuit_breaker.state {
    "closed" => {
      # Normal operation
      try {
        http get $" [https://($service_nam](https://($service_nam)e).prod.internal/api"
      } catch {
        # Increment failure count
        $circuit_breaker.failure_count += 1

        # Open circuit if threshold reached
        if $circuit_breaker.failure_count >= $circuit_breaker.failure_threshold {
          $circuit_breaker.state = "open"
          error "Circuit breaker opened"
        }
      }
    },
    "open" => {
      # Reject all requests
      error "Circuit breaker is open - service unavailable"
    },
    "half-open" => {
      # Test if service is recovered
      try {
        http get $" [https://($service_nam](https://($service_nam)e).prod.internal/health"

        # Increment success count
        $circuit_breaker.success_count += 1

        # Close circuit if recovered
        if $circuit_breaker.success_count >= $circuit_breaker.success_threshold {
          $circuit_breaker.state = "closed"
          $circuit_breaker.failure_count = 0
          $circuit_breaker.success_count = 0
        }
      } catch {
        # Reopen circuit
        $circuit_breaker.state = "open"
      }
    }
  }
}
```

## Health Check Orchestration

Comprehensive health checking:

```bash
#!/usr/bin/env nu

# Define health check targets
let targets = [
  { name: "database", url: "postgres://db.prod.internal:5432", type: "postgres" },
  { name: "cache", url: "redis://cache.prod.internal:6379", type: "redis" },
  { name: "api", url: " [https://api.prod.internal/health",](https://api.prod.internal/health",) type: "http" },
  { name: "web", url: " [https://web.prod.internal/health",](https://web.prod.internal/health",) type: "http" }
]

# Run health checks
def perform_health_checks [] {
  $targets | each { | target |
    let result = match $target.type {
      "http" => {
        try {
          http get --headers [user-agent: "health-checker"] $target.url
          { status: "healthy", timestamp: (now) }
        } catch {
          { status: "unhealthy", error: $"HTTP error", timestamp: (now) }
        }
      },
      "postgres" => {
        try {
          # Check database connectivity
          { status: "healthy", timestamp: (now) }
        } catch {
          { status: "unhealthy", error: $"Database unreachable", timestamp: (now) }
        }
      },
      "redis" => {
        try {
          # Check Redis connectivity
          { status: "healthy", timestamp: (now) }
        } catch {
          { status: "unhealthy", error: $"Redis unreachable", timestamp: (now) }
        }
      }
    }

    { target: $target.name, ...$result }
  }
}

# Monitor continuously
print "Starting health checks..."
loop {
  let results = perform_health_checks
  let unhealthy = $results | where status == "unhealthy"

  if ($unhealthy | is-empty) {
    print "✓ All services healthy"
  } else {
    print "✗ Unhealthy services detected:"
    $unhealthy | each { | svc |
      print $"  - ($svc.target): ($svc.error)"
    }
  }

  sleep 30sec
}
```

## Related Documentation

- [Orchestrator Guide](../features/orchestrator.md)
- [Disaster Recovery](../guides/disaster-recovery.md)
- [Monitoring](../operations/monitoring.md)
- [Multi-Cloud Deployment](./multi-cloud.md)
