# Hybrid Cloud Deployment Guide

Deploy infrastructure across public clouds (AWS, UpCloud) and on-premise data centers simultaneously.

## Architecture Overview

```text
┌─────────────────────────────────────────────────────────────┐
│                    Global Load Balancer                     │
└────────┬──────────────────────────────────────────────┬─────┘
         │                                              │
         ▼                                              ▼
    ┌─────────────┐                         ┌──────────────────┐
    │  AWS Cloud  │                         │  On-Premise DC   │
    ├─────────────┤                         ├──────────────────┤
    │ Region: us-east-1                    │ Location: Chicago │
    │ AZs: 3                                │ HA: 2 datacenters│
    │ DB: RDS Multi-AZ                      │ DB: PostgreSQL HA│
    │ Load: 70% of traffic                  │ Load: 30% traffic│
    └─────────────┘                         └──────────────────┘
         │                                              │
         └──────────────────┬───────────────────────────┘
                            │
                    ┌───────▼────────┐
                    │ SurrealDB Sync │ (Cross-DC replication)
                    └────────────────┘
```

## Hybrid Deployment Configuration

### Multi-Cloud Nickel Schema

```nickel
{
  infrastructure = {
    deployment_type = "hybrid",

    regions = [
      {
        name = "aws-primary",
        provider = "aws",
        region = "us-east-1",
        availability_zones = ["us-east-1a", "us-east-1b", "us-east-1c"],
        instance_count = 3,
        workload_percentage = 70,

        compute = {
          instance_type = "t3.xlarge",
          count = 3
        },

        database = {
          engine = "postgres",
          instance_class = "db.r6i.2xlarge",
          multi_az = true,
          backup_retention_days = 30
        },

        networking = {
          vpc_cidr = "10.0.0.0/16",
          nat_gateway_count = 3,
          load_balancer = "alb"
        }
      },
      {
        name = "onprem-secondary",
        provider = "onprem",
        location = "chicago",
        datacenters = 2,
        instance_count = 2,
        workload_percentage = 30,

        compute = {
          instance_type = "physical-server",
          count = 2,
          cpu_cores = 64,
          memory_gb = 512
        },

        database = {
          engine = "postgres",
          replication_type = "streaming",
          standby_server = true
        },

        networking = {
          bandwidth_gbps = 10,
          vpn_to_aws = {
            type = "ipsec",
            encryption = "aes-256-gcm"
          }
        }
      }
    ],

    # Global configuration
    global = {
      load_balancer_type = "global",
      health_check_interval_seconds = 10,
      failover_threshold = 2,  # 2 failed health checks before failover
      max_failure_rate = 0.05  # 5% error rate triggers failover
    }
  }
}
```

## Deployment Process

### Step 1: Validate Connectivity

```nushell
def validate-hybrid-connectivity [] {
    print "Validating hybrid cloud connectivity..."

    # Check AWS connectivity
    let aws_health = (provisioning health check aws-primary)
    if $aws_health.status != "healthy" {
        print "✗ AWS connectivity issue"
        return 1
    }
    print "✓ AWS connectivity OK"

    # Check on-premise VPN
    let vpn_status = (provisioning vpn status --connection aws-onprem)
    if $vpn_status.status != "connected" {
        print "✗ VPN to on-premise disconnected"
        return 1
    }
    print "✓ VPN to on-premise OK"

    # Check inter-DC latency
    let latency = (provisioning network latency --from aws-primary --to onprem-secondary)
    if $latency > 100 {  # 100ms
        print $"⚠️  High latency between clouds: ($latency)ms"
    } else {
        print $"✓ Latency acceptable: ($latency)ms"
    }

    print "✓ All connectivity checks passed"
}
```

### Step 2: Deploy to AWS

```bash
provisioning infra apply \
  --config hybrid-deployment.ncl \
  --target aws-primary \
  --region us-east-1 \
  --auto-approve
```

### Step 3: Deploy to On-Premise

```bash
provisioning infra apply \
  --config hybrid-deployment.ncl \
  --target onprem-secondary \
  --auto-approve \
  --provisioning-daemon-host on-prem-node-1.chicago.internal
```

### Step 4: Setup Replication

```nushell
def setup-database-replication [] {
    print "Setting up hybrid database replication..."

    # Create AWS database
    provisioning database create \
        --provider aws \
        --region us-east-1 \
        --instance-class db.r6i.2xlarge \
        --db-name production

    # Create on-premise standby
    provisioning database create \
        --provider onprem \
        --location chicago \
        --replication-role standby \
        --replica-of "production-aws-us-east-1"

    # Enable streaming replication
    provisioning database replicate \
        --primary "production-aws-us-east-1" \
        --standby "production-onprem-chicago" \
        --replication-type streaming \
        --wal-level logical \
        --max_wal_senders 5

    print "✓ Database replication configured"
}
```

### Step 5: Configure Global Load Balancing

```nickel
{
  global_load_balancer = {
    type = "dns_with_health_checks",

    primary_endpoint = "aws-lb-us-east-1.example.com",
    secondary_endpoint = "onprem-lb-chicago.example.com",

    # Route 53 or equivalent DNS failover
    routing_policy = "failover",

    health_checks = [
      {
        endpoint = " [https://aws-lb-us-east-1.example.com/health",](https://aws-lb-us-east-1.example.com/health",)
        interval_seconds = 10,
        failure_threshold = 2,
        timeout_seconds = 5
      },
      {
        endpoint = " [https://onprem-lb-chicago.example.com/health",](https://onprem-lb-chicago.example.com/health",)
        interval_seconds = 10,
        failure_threshold = 2,
        timeout_seconds = 5
      }
    ],

    # Weighted routing (70% AWS, 30% on-prem)
    weights = {
      aws_primary = 70,
      onprem_secondary = 30
    },

    # Geo-proximity (AWS for US East, on-prem for Chicago users)
    geo_routing = {
      "us-east" = "aws-primary",
      "midwest" = "onprem-secondary",
      default = "aws-primary"
    }
  }
}
```

## Data Synchronization

### Real-Time Replication

```nushell
# PostgreSQL streaming replication (synchronous)
def enable-sync-replication [] {
    provisioning database config set \
        --database production-aws \
        --setting synchronous_commit \
        --value "on"

    provisioning database config set \
        --database production-aws \
        --setting synchronous_standby_names \
        --value "production-onprem-chicago"
}

# Every write waits for both AWS and on-prem confirmation
# Ensures zero data loss if AWS fails
```

### Conflict Resolution

```nickel
{
  data_sync = {
    replication_mode = "bi-directional",

    conflict_resolution = {
      strategy = "last-write-wins",
      timestamp_source = "database",

      # Special handling for certain tables
      table_policies = [
        {
          table = "configuration",
          conflict_resolution = "manual-review",  # Require operator review
          notifications = {
            slack = "#infrastructure-conflicts"
          }
        },
        {
          table = "metrics",
          conflict_resolution = "sum",  # Combine metrics from both
          aggregation = "total"
        }
      ]
    }
  }
}
```

## Failover Scenarios

### AWS Primary Failure

```nushell
# Automatic failover when AWS becomes unhealthy
def failover-to-onprem [] {
    print "AWS primary failed, initiating failover..."

    # 1. Stop sending traffic to AWS
    provisioning lb update-weights \
        --target aws-primary \
        --weight 0

    # 2. Promote on-premise standby to primary
    provisioning database promote-replica \
        --database production-onprem-chicago

    # 3. Repoint primary endpoint to on-prem
    provisioning dns update \
        --record api.example.com \
        --target "onprem-lb-chicago.example.com"

    # 4. Verify on-prem is accepting traffic
    let health = (provisioning health check onprem-secondary)
    if $health.status == "healthy" {
        print "✓ Failover complete, on-premise is now primary"
    } else {
        print "✗ Failover failed, on-premise is unhealthy"
        return 1
    }
}
```

### AWS Primary Recovery

```nushell
# When AWS recovers, gradually rebalance traffic
def failback-to-aws [] {
    print "AWS recovered, initiating gradual failback..."

    # 1. Verify AWS health
    let aws_health = (provisioning health check aws-primary)
    if $aws_health.status != "healthy" {
        print "✗ AWS still unhealthy, aborting failback"
        return 1
    }

    # 2. Sync AWS database from on-prem
    provisioning database sync \
        --source production-onprem-chicago \
        --target production-aws-us-east-1 \
        --mode full

    # 3. Gradually increase AWS traffic (canary-style)
    for percent in [10, 25, 50, 75, 100] {
        provisioning lb update-weights \
            --target aws-primary \
            --weight $percent

        print $"Sent ($percent)% traffic to AWS"
        sleep 5m  # Wait 5 minutes between increases

        # Check error rates
        let error_rate = (provisioning metrics error-rate --window 5m)
        if $error_rate > 0.01 {  # >1% error rate
            print $"High error rate detected: ($error_rate), reverting..."
            provisioning lb update-weights \
                --target aws-primary \
                --weight 0
            return 1
        }
    }

    print "✓ Successfully failed back to AWS"
}
```

## Monitoring Hybrid Infrastructure

```bash
# Global dashboard
provisioning dashboard open --type hybrid-global

# Metrics:
# AWS Latency: 45ms  | On-Prem Latency: 15ms
# AWS Errors: 0.02%  | On-Prem Errors: 0.01%
# Replication Lag: <1s
# Inter-DC Bandwidth: 450Mbps / 1000Mbps (45% utilized)
```

## Cost Optimization

```nickel
{
  cost_optimization = {
    # Shift non-critical workloads to on-premise during peak AWS prices
    intelligent_placement = {
      enabled = true,
      criteria = [
        {
          metric = "aws_on_demand_price",
          threshold = 0.50,  # If exceeds $0.50/hour
          action = "migrate_to_onprem"
        }
      ]
    },

    # Reserve AWS capacity for baseline, burst to on-prem
    capacity_planning = {
      aws_reserved = 30,    # Reserved for steady-state
      aws_on_demand = 40,   # For peak scaling
      onprem_baseline = 30  # Always available
    }
  }
}
```

## Best Practices

1. **Connectivity**: Maintain redundant network links (multiple VPNs, direct fiber)
2. **Replication**: Use synchronous replication for critical data
3. **Monitoring**: Monitor replication lag continuously
4. **Testing**: Regular failover drills to verify automation
5. **Documentation**: Document on-premise infrastructure comprehensively
6. **Compliance**: Ensure data residency requirements met in each region
7. **Backup**: Multiple backup copies in geographically diverse locations

## See Also

- [Multi-Cloud Deployment Guide](./managing-multiple-clouds.md)
- [Disaster Recovery Guide](./disaster-recovery.md)
- [Monitoring Guide](../operations/monitoring.md)
