74 lines
4.2 KiB
Text
74 lines
4.2 KiB
Text
|
#!/usr/bin/env nu
|
||
|
|
# dr_wuji_lost/run.nu — catastrophic wuji recovery: provision → zot restore → Radicle reconnect → audit replay
|
||
|
|
|
||
|
|
def main [
|
||
|
|
--workspace: string = ""
|
||
|
|
--hetzner-context: string = ""
|
||
|
|
--s3-bucket: string = ""
|
||
|
|
--ops-vm-host: string = ""
|
||
|
|
--nats-account-url: string = ""
|
||
|
|
--skip-provision: string = "false"
|
||
|
|
--dry-run
|
||
|
|
]: nothing -> nothing {
|
||
|
|
let ws = if ($workspace | is-not-empty) { $workspace } else { $env.PLAYBOOK_PARAM_WORKSPACE? | default "libre-wuji" }
|
||
|
|
let hetzner_ctx = if ($hetzner_context | is-not-empty) { $hetzner_context } else { $env.PLAYBOOK_PARAM_HETZNER_CONTEXT? | default "" }
|
||
|
|
let s3_bucket = if ($s3_bucket | is-not-empty) { $s3_bucket } else { $env.PLAYBOOK_PARAM_S3_BUCKET? | default "libre-wuji-zot" }
|
||
|
|
let vm_host = if ($ops_vm_host | is-not-empty) { $ops_vm_host } else { $env.PLAYBOOK_PARAM_OPS_VM_HOST? | default "" }
|
||
|
|
let nats_url = if ($nats_account_url | is-not-empty) { $nats_account_url } else { $env.PLAYBOOK_PARAM_NATS_ACCOUNT_URL? | default "" }
|
||
|
|
let skip_prov = if ($skip_provision | is-not-empty) { $skip_provision } else { $env.PLAYBOOK_PARAM_SKIP_PROVISION? | default "false" }
|
||
|
|
|
||
|
|
if ($ws | is-empty) { error make { msg: "workspace is required" } }
|
||
|
|
if ($vm_host | is-empty) { error make { msg: "ops-vm-host is required" } }
|
||
|
|
|
||
|
|
let playbook_dir = $env.PLAYBOOK_DIR? | default ($env.FILE_PWD | path expand)
|
||
|
|
let dry_flag = if $dry_run { "--dry-run" } else { "" }
|
||
|
|
|
||
|
|
print $"[dr_wuji_lost] workspace=($ws) dry-run=($dry_run)"
|
||
|
|
print "WARNING: This is a disaster recovery playbook. Verify preconditions before proceeding."
|
||
|
|
print ""
|
||
|
|
|
||
|
|
print "[step 1/7] verifying Radicle governance ledgers are accessible..."
|
||
|
|
let r1 = do { ^nu ($playbook_dir | path join "steps/verify_radicle_ledgers_accessible.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete
|
||
|
|
if $r1.exit_code != 0 { error make { msg: $"Radicle ledger check failed: ($r1.stderr | str trim)" } }
|
||
|
|
print $r1.stdout
|
||
|
|
|
||
|
|
print "[step 2/7] provisioning new cluster..."
|
||
|
|
let r2 = do { ^nu ($playbook_dir | path join "steps/provision_new_cluster.nu") --workspace $ws --hetzner-context $hetzner_ctx --skip-provision $skip_prov $dry_flag } | complete
|
||
|
|
if $r2.exit_code != 0 { error make { msg: $"Cluster provision failed: ($r2.stderr | str trim)" } }
|
||
|
|
print $r2.stdout
|
||
|
|
|
||
|
|
print "[step 3/7] restoring zot from S3..."
|
||
|
|
let r3 = do { ^nu ($playbook_dir | path join "steps/restore_zot_from_s3.nu") --workspace $ws --s3-bucket $s3_bucket $dry_flag } | complete
|
||
|
|
if $r3.exit_code != 0 { error make { msg: $"zot S3 restore failed: ($r3.stderr | str trim)" } }
|
||
|
|
print $r3.stdout
|
||
|
|
|
||
|
|
print "[step 4/7] reconnecting Radicle seed..."
|
||
|
|
let r4 = do { ^nu ($playbook_dir | path join "steps/reconnect_radicle_seed.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete
|
||
|
|
if $r4.exit_code != 0 { error make { msg: $"Radicle reconnect failed: ($r4.stderr | str trim)" } }
|
||
|
|
print $r4.stdout
|
||
|
|
|
||
|
|
print "[step 5/7] verifying ops-controller..."
|
||
|
|
let r5 = do { ^nu ($playbook_dir | path join "steps/verify_ops_controller.nu") --workspace $ws --nats-account-url $nats_url $dry_flag } | complete
|
||
|
|
if $r5.exit_code != 0 {
|
||
|
|
print $" WARNING: ops-controller verification failed: ($r5.stderr | str trim)"
|
||
|
|
} else {
|
||
|
|
print $r5.stdout
|
||
|
|
}
|
||
|
|
|
||
|
|
print "[step 6/7] replaying stranded audit events..."
|
||
|
|
let r6 = do { ^nu ($playbook_dir | path join "steps/replay_stranded_audit_events.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete
|
||
|
|
if $r6.exit_code != 0 {
|
||
|
|
print $" WARNING: audit replay reported errors: ($r6.stderr | str trim)"
|
||
|
|
} else {
|
||
|
|
print $r6.stdout
|
||
|
|
}
|
||
|
|
|
||
|
|
print "[step 7/7] emitting dr-wuji-recovered audit event..."
|
||
|
|
let r7 = do { ^nu ($playbook_dir | path join "steps/emit_audit.nu") --workspace $ws --event-type "dr_wuji_recovered" $dry_flag } | complete
|
||
|
|
if $r7.exit_code != 0 { print $" WARNING: audit emit failed: ($r7.stderr | str trim)" }
|
||
|
|
|
||
|
|
print ""
|
||
|
|
print "[dr_wuji_lost] recovery complete"
|
||
|
|
print "Verify: kubectl get pods -n ops-system; rad sync --repo policy-libre-wuji; nats stream info OPS_PENDING_LIBRE-WUJI"
|
||
|
|
}
|