let pb = import "schemas/lib/playbook.ncl" in pb.make_playbook { id = "dr_wuji_lost", name = "DR — Libre-Wuji Cluster Lost", description = "Disaster recovery for catastrophic loss of libre-wuji. Provisions a new cluster, restores zot from S3, reconnects to existing Radicle governance ledgers, replays any stranded audit events. Governance ledgers survive in Radicle; only the runtime is lost.", version = 1, preconditions = [ "Radicle governance ledgers (policy-libre-wuji, desired-libre-wuji, state-libre-wuji) are accessible from ops-vm or Radicle peers", "S3 bucket for zot (libre-wuji-zot) is intact with versioning enabled", "ops-vm is operational and keeper-daemon is running", "Hetzner API credentials are available for cluster reprovisioning", "NATS cluster config (account server URL, credentials) is documented in ops runbook", ], params = [ { name = "workspace", description = "Target workspace (must be libre-wuji)", required = true, default_val = "libre-wuji" }, { name = "hetzner_context", description = "hcloud context name for cluster provisioning", required = true, default_val = "" }, { name = "s3_bucket", description = "S3 bucket name for zot restore", required = true, default_val = "libre-wuji-zot" }, { name = "ops_vm_host", description = "ops-vm SSH host (for Radicle sync)", required = true, default_val = "" }, { name = "nats_account_url", description = "NATS account server URL for NATS reconnection", required = false, default_val = "" }, { name = "skip_provision", description = "Skip cluster provision if new cluster already up", required = false, default_val = "false" }, ], steps = [ { id = "verify_radicle_ledgers_accessible", name = "Verify Radicle governance ledgers are accessible", script = "steps/verify_radicle_ledgers_accessible.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = [], }, { id = "provision_new_cluster", name = "Provision new libre-wuji cluster (skip if already up)", script = "steps/provision_new_cluster.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = ["verify_radicle_ledgers_accessible"], }, { id = "restore_zot_from_s3", name = "Restore zot registry pointing to existing S3 bucket", script = "steps/restore_zot_from_s3.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = ["provision_new_cluster"], }, { id = "reconnect_radicle_seed", name = "Reconnect Radicle seed node — sync governance ledgers", script = "steps/reconnect_radicle_seed.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = ["provision_new_cluster"], }, { id = "verify_ops_controller", name = "Verify ops-controller is running and NATS-connected", script = "steps/verify_ops_controller.nu", dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["restore_zot_from_s3", "reconnect_radicle_seed"], }, { id = "replay_stranded_audit_events", name = "Replay stranded audit events from Radicle state ledger", script = "steps/replay_stranded_audit_events.nu", dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["verify_ops_controller"], }, { id = "emit_audit", name = "Emit dr-wuji-recovered audit event", script = "steps/emit_audit.nu", params = { event_type = "dr_wuji_recovered" }, dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["replay_stranded_audit_events"], }, ], rollback_strategy = 'manual, success_criteria = [ "New libre-wuji cluster is up with all components running", "zot registry serving images from existing S3 bucket (no data loss)", "Radicle seed synced policy-libre-wuji, desired-libre-wuji, state-libre-wuji from peers", "ops-controller connected to NATS and processing ops from queue", "Any stranded audit events replayed into SurrealDB mirror", "ops history shows dr-wuji-recovered audit event", ], emit_audit = true, adr_refs = ["adr-037", "adr-038", "adr-039"], }