# Secrets Rotation Strategy Guide

Implement automated secret rotation for passwords, API keys, certificates, and encryption keys.

## Rotation Policy Framework

```nickel
{
  secrets_rotation = {
    policies = [
      {
        name = "database_passwords",
        resource_type = "database",
        rotation_interval_days = 30,
        rotation_type = "automatic",
        notification_days_before = 7,
        retention_days = 90
      },
      {
        name = "api_keys",
        resource_type = "api_key",
        rotation_interval_days = 90,
        rotation_type = "automatic",
        retention_days = 30
      },
      {
        name = "tls_certificates",
        resource_type = "certificate",
        rotation_interval_days = 365,
        rotation_type = "automatic",
        notification_days_before = 30,
        renewal_days_before = 30
      },
      {
        name = "encryption_keys",
        resource_type = "encryption_key",
        rotation_interval_days = 180,
        rotation_type = "automatic",
        retention_days = 365
      }
    ]
  }
}
```

## Database Password Rotation

### Automatic PostgreSQL Password Rotation

```nushell
def rotate-db-password [--database: string, --user: string] {
    print $"Rotating password for ($user)@($database)..."

    let start_time = (date now)

    # Step 1: Generate new password (20 chars, mixed case, symbols)
    let new_password = (provisioning secret generate --length 20 --symbols true)

    # Step 2: Update database user with new password
    let update_result = (
        provisioning database user-password-update \
            --database $database \
            --user $user \
            --new-password $new_password
    )

    if $update_result.status != "success" {
        print $"✗ Failed to update password: ($update_result.error)"
        return 1
    }

    # Step 3: Update secret in vault
    let vault_result = (
        provisioning secret update \
            --secret-name $"($database)-($user)-password" \
            --value $new_password
    )

    if $vault_result.status != "success" {
        print $"✗ Failed to update vault: ($vault_result.error)"
        # Rollback the database change
        provisioning database user-password-update \
            --database $database \
            --user $user \
            --new-password $old_password
        return 1
    }

    # Step 4: Verify applications can connect with new password
    let verification_attempts = 0
    let max_attempts = 5

    while $verification_attempts < $max_attempts {
        let test = (
            provisioning database test-connection \
                --database $database \
                --user $user \
                --password $new_password
        )

        if $test.status == "success" {
            print "✓ Applications verified connection with new password"
            break
        }

        $verification_attempts = $verification_attempts + 1
        if $verification_attempts < $max_attempts {
            sleep 5s
        }
    }

    if $verification_attempts == $max_attempts {
        print "✗ Could not verify application connections"
        return 1
    }

    # Step 5: Remove old password from vault (after retention period)
    let elapsed = ((date now) - $start_time)
    print $"✓ Password rotated successfully in ($elapsed.ms)ms"
}
```

## API Key Rotation

### Multi-Stage API Key Rotation

```nushell
def rotate-api-keys [] {
    print "Rotating API keys with zero-downtime..."

    # Step 1: Generate new keys
    let new_keys = (
        provisioning secret generate-api-keys \
            --count 1 \
            --environment production
    )

    let new_key = $new_keys.0

    # Step 2: Add new key as "inactive" in vault
    provisioning secret update \
        --secret-name "api-key-prod-new" \
        --value $new_key \
        --status "inactive"

    print "New API key staged (inactive)"

    # Step 3: Update applications to accept both old and new keys
    provisioning config update \
        --setting "api_keys_accepted" \
        --value ([$old_key, $new_key])

    # Deploy updated configuration
    provisioning deployment apply --config api-keys-dual.ncl

    print "Applications now accepting both old and new keys"

    # Step 4: Monitor for all services using new key
    sleep 5m

    let still_using_old = (
        provisioning audit query \
            --filter "api_key = $old_key" \
            --last-hour
        | length
    )

    if $still_using_old > 0 {
        print $"⚠️  ($still_using_old) requests still using old key"
        sleep 5m
    }

    # Step 5: Mark new key as active, old as deprecated
    provisioning secret update \
        --secret-name "api-key-prod" \
        --value $new_key \
        --status "active"

    provisioning secret update \
        --secret-name "api-key-prod-old" \
        --value $old_key \
        --status "deprecated"

    print "✓ Old key marked deprecated (will be revoked in 30 days)"

    # Step 6: Schedule old key revocation
    provisioning secret revoke \
        --secret-name "api-key-prod-old" \
        --at "2025-02-15"
}
```

## Certificate Rotation

### Automated TLS Certificate Rotation

```nushell
def rotate-certificates [] {
    print "Rotating TLS certificates..."

    let certificates = (
        provisioning cert list --environment production
    )

    $certificates | each { | cert |
        let days_until_expiry = ((parse date $cert.expiry_date) - (date now)) / 86400

        if $days_until_expiry < 30 {
            print $"Rotating ($cert.domain) - expires in ($days_until_expiry) days"

            # Generate new certificate
            let new_cert = (
                provisioning cert generate \
                    --domain $cert.domain \
                    --issuer "letsencrypt" \
                    --key-length 4096
            )

            # Install new certificate with zero-downtime reload
            provisioning cert install \
                --certificate $new_cert \
                --service "load-balancer" \
                --reload-strategy "graceful"

            # Verify new certificate is active
            let verification = (
                provisioning cert verify \
                    --domain $cert.domain
            )

            if $verification.status == "valid" {
                print $"✓ Certificate rotated for ($cert.domain)"
            } else {
                print $"✗ Certificate verification failed for ($cert.domain)"
                return 1
            }
        }
    }
}
```

## Encryption Key Rotation

### HSM-Backed Key Rotation

```nushell
def rotate-encryption-keys [] {
    print "Rotating encryption keys..."

    # Step 1: Generate new key
    let new_key_response = (
        provisioning kms generate-key \
            --key-spec AES-256 \
            --usage ENCRYPT_DECRYPT \
            --origin HSM
    )

    let new_key_id = $new_key_response.key_id
    print $"Generated new key: ($new_key_id)"

    # Step 2: Mark old key as deprecated
    let old_key_id = (
        provisioning kms list-keys --current-key-only
    )

    provisioning kms update-key-status \
        --key-id $old_key_id \
        --status "deprecated"

    # Step 3: Re-encrypt data with new key
    let data_count = (provisioning database count --table secrets)
    print $"Re-encrypting ($data_count) secret values..."

    let batch_size = 1000
    let total_batches = (($data_count / $batch_size) | math ceil)

    for batch_num in (1..$total_batches) {
        let offset = ($batch_num - 1) * $batch_size

        let batch = (
            provisioning database select \
                --table secrets \
                --limit $batch_size \
                --offset $offset
        )

        # Decrypt with old key, re-encrypt with new key
        $batch | each { | record |
            let plaintext = (
                provisioning kms decrypt \
                    --key-id $old_key_id \
                    --ciphertext $record.encrypted_value
            )

            let new_ciphertext = (
                provisioning kms encrypt \
                    --key-id $new_key_id \
                    --plaintext $plaintext
            )

            provisioning database update \
                --table secrets \
                --where $"id = ($record.id)" \
                --set $"encrypted_value = '($new_ciphertext)', key_id = '($new_key_id)'"
        }

        print $"Re-encrypted batch ($batch_num) / ($total_batches)"
    }

    # Step 4: Verify new key is working
    let test_secret = (
        provisioning secret retrieve --secret-name test-secret
    )

    if $test_secret.status == "success" {
        print "✓ All data successfully re-encrypted with new key"
    } else {
        print "✗ Key rotation verification failed"
        return 1
    }

    # Step 5: Schedule old key destruction
    provisioning kms schedule-key-deletion \
        --key-id $old_key_id \
        --waiting-period-days 30
}
```

## Automated Rotation Scheduling

```nushell
def schedule-secret-rotations [] {
    # Database passwords - monthly
    provisioning schedule create \
        --name "Rotate database passwords" \
        --cron "0 2 1 * *" \
        --command "rotate-db-password --database production --user appuser" \
        --notifications {
            slack = "#infrastructure",
            email = "[ops@example.com](mailto:ops@example.com)"
        }

    # API keys - quarterly
    provisioning schedule create \
        --name "Rotate API keys" \
        --cron "0 3 1 1,4,7,10 *" \
        --command "rotate-api-keys"

    # Certificates - monthly
    provisioning schedule create \
        --name "Rotate TLS certificates" \
        --cron "0 4 1 * *" \
        --command "rotate-certificates"

    # Encryption keys - semi-annually
    provisioning schedule create \
        --name "Rotate encryption keys" \
        --cron "0 5 1 1,7 *" \
        --command "rotate-encryption-keys"
}
```

## Rotation Monitoring

```nushell
def monitor-secret-rotation [] {
    let secrets = (provisioning secret list --show-rotation-status)

    let overdue = ($secrets | where rotation_due == true)

    if ($overdue | length) > 0 {
        print "⚠️  Secrets overdue for rotation:"
        $overdue | each { | secret |
            print $"  - ($secret.name): last rotated ($secret.last_rotated)"
        }
    }

    let upcoming = ($secrets | where days_until_rotation < 7)
    if ($upcoming | length) > 0 {
        print "📋 Upcoming rotations (next 7 days):"
        $upcoming | table
    }
}
```

## Compliance Reporting

```bash
# Generate rotation compliance report
provisioning secret rotation-report --period quarterly

# Output:
# Secret Rotation Compliance Report - Q1 2025
#
# Database Passwords:
#   Total: 25
#   Rotated On-Schedule: 25
#   Overdue: 0
#   Compliance: 100%
#
# API Keys:
#   Total: 15
#   Rotated On-Schedule: 15
#   Overdue: 0
#   Compliance: 100%
#
# TLS Certificates:
#   Total: 12
#   Rotated On-Schedule: 12
#   Overdue: 0
#   Compliance: 100%
#
# Encryption Keys:
#   Total: 8
#   Rotated On-Schedule: 8
#   Overdue: 0
#   Compliance: 100%
#
# Overall Compliance: 100%
```

## See Also

- [Secrets Management Guide](../security/secrets-management.md)
- [Post-Quantum Cryptography](../architecture/adr-006-post-quantum-cryptography.md)
- [Data Encryption Strategy](../architecture/adr-007-data-encryption-strategy.md)
