#!/usr/bin/env nu # dr_wuji_lost/run.nu — catastrophic wuji recovery: provision → zot restore → Radicle reconnect → audit replay def main [ --workspace: string = "" --hetzner-context: string = "" --s3-bucket: string = "" --ops-vm-host: string = "" --nats-account-url: string = "" --skip-provision: string = "false" --dry-run ]: nothing -> nothing { let ws = if ($workspace | is-not-empty) { $workspace } else { $env.PLAYBOOK_PARAM_WORKSPACE? | default "libre-wuji" } let hetzner_ctx = if ($hetzner_context | is-not-empty) { $hetzner_context } else { $env.PLAYBOOK_PARAM_HETZNER_CONTEXT? | default "" } let s3_bucket = if ($s3_bucket | is-not-empty) { $s3_bucket } else { $env.PLAYBOOK_PARAM_S3_BUCKET? | default "libre-wuji-zot" } let vm_host = if ($ops_vm_host | is-not-empty) { $ops_vm_host } else { $env.PLAYBOOK_PARAM_OPS_VM_HOST? | default "" } let nats_url = if ($nats_account_url | is-not-empty) { $nats_account_url } else { $env.PLAYBOOK_PARAM_NATS_ACCOUNT_URL? | default "" } let skip_prov = if ($skip_provision | is-not-empty) { $skip_provision } else { $env.PLAYBOOK_PARAM_SKIP_PROVISION? | default "false" } if ($ws | is-empty) { error make { msg: "workspace is required" } } if ($vm_host | is-empty) { error make { msg: "ops-vm-host is required" } } let playbook_dir = $env.PLAYBOOK_DIR? | default ($env.FILE_PWD | path expand) let dry_flag = if $dry_run { "--dry-run" } else { "" } print $"[dr_wuji_lost] workspace=($ws) dry-run=($dry_run)" print "WARNING: This is a disaster recovery playbook. Verify preconditions before proceeding." print "" print "[step 1/7] verifying Radicle governance ledgers are accessible..." let r1 = do { ^nu ($playbook_dir | path join "steps/verify_radicle_ledgers_accessible.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete if $r1.exit_code != 0 { error make { msg: $"Radicle ledger check failed: ($r1.stderr | str trim)" } } print $r1.stdout print "[step 2/7] provisioning new cluster..." let r2 = do { ^nu ($playbook_dir | path join "steps/provision_new_cluster.nu") --workspace $ws --hetzner-context $hetzner_ctx --skip-provision $skip_prov $dry_flag } | complete if $r2.exit_code != 0 { error make { msg: $"Cluster provision failed: ($r2.stderr | str trim)" } } print $r2.stdout print "[step 3/7] restoring zot from S3..." let r3 = do { ^nu ($playbook_dir | path join "steps/restore_zot_from_s3.nu") --workspace $ws --s3-bucket $s3_bucket $dry_flag } | complete if $r3.exit_code != 0 { error make { msg: $"zot S3 restore failed: ($r3.stderr | str trim)" } } print $r3.stdout print "[step 4/7] reconnecting Radicle seed..." let r4 = do { ^nu ($playbook_dir | path join "steps/reconnect_radicle_seed.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete if $r4.exit_code != 0 { error make { msg: $"Radicle reconnect failed: ($r4.stderr | str trim)" } } print $r4.stdout print "[step 5/7] verifying ops-controller..." let r5 = do { ^nu ($playbook_dir | path join "steps/verify_ops_controller.nu") --workspace $ws --nats-account-url $nats_url $dry_flag } | complete if $r5.exit_code != 0 { print $" WARNING: ops-controller verification failed: ($r5.stderr | str trim)" } else { print $r5.stdout } print "[step 6/7] replaying stranded audit events..." let r6 = do { ^nu ($playbook_dir | path join "steps/replay_stranded_audit_events.nu") --workspace $ws --ops-vm-host $vm_host $dry_flag } | complete if $r6.exit_code != 0 { print $" WARNING: audit replay reported errors: ($r6.stderr | str trim)" } else { print $r6.stdout } print "[step 7/7] emitting dr-wuji-recovered audit event..." let r7 = do { ^nu ($playbook_dir | path join "steps/emit_audit.nu") --workspace $ws --event-type "dr_wuji_recovered" $dry_flag } | complete if $r7.exit_code != 0 { print $" WARNING: audit emit failed: ($r7.stderr | str trim)" } print "" print "[dr_wuji_lost] recovery complete" print "Verify: kubectl get pods -n ops-system; rad sync --repo policy-libre-wuji; nats stream info OPS_PENDING_LIBRE-WUJI" }