let pb = import "schemas/lib/playbook.ncl" in pb.make_playbook { id = "dr_daoshi_lost", name = "DR — Libre-Daoshi Cluster Lost", description = "Disaster recovery for catastrophic loss of libre-daoshi (CI/build cluster). Provisions a new daoshi cluster, restores forgejo repos from Radicle mirrors, reissues daoshi-CI NATS keys, reconnects Radicle seed, and verifies CI pipeline. Because daoshi is declared replaceable in ADR-038, data loss risk is limited to in-flight builds.", version = 1, preconditions = [ "libre-wuji (wuji NATS, zot, ops-controller) is operational", "Radicle governance ledgers for libre-daoshi are accessible from peers", "ops-vm keeper-daemon is running", "Hetzner API credentials available for cluster reprovisioning", "Forgejo repos are mirrored in Radicle (or last-known-good repo archives exist in zot)", ], params = [ { name = "workspace", description = "Target workspace (must be libre-daoshi)", required = true, default_val = "libre-daoshi" }, { name = "hetzner_context", description = "hcloud context name for cluster provisioning", required = true, default_val = "" }, { name = "wuji_nats_url", description = "NATS URL for libre-wuji (connectivity check)", required = true, default_val = "" }, { name = "ops_vm_host", description = "ops-vm SSH host (for CI key reissuance)", required = true, default_val = "" }, { name = "skip_provision", description = "Skip cluster provision if new cluster already up", required = false, default_val = "false" }, ], steps = [ { id = "verify_wuji_accessible", name = "Verify libre-wuji (NATS + zot) is reachable", script = "steps/verify_wuji_accessible.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = [], }, { id = "provision_new_cluster", name = "Provision new libre-daoshi cluster (skip if already up)", script = "steps/provision_new_cluster.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = ["verify_wuji_accessible"], }, { id = "restore_forgejo_from_radicle", name = "Restore forgejo repositories from Radicle mirrors", script = "steps/restore_forgejo_from_radicle.nu", dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["provision_new_cluster"], }, { id = "reissue_ci_keys", name = "Reissue daoshi-CI NATS JWT (deploy+rollback scope)", script = "steps/reissue_ci_keys.nu", dry_run_arg = "--dry-run", on_error = 'Stop, depends_on = ["provision_new_cluster"], }, { id = "reconnect_radicle_seed", name = "Reconnect Radicle seed node — sync daoshi governance ledgers", script = "steps/reconnect_radicle_seed.nu", dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["provision_new_cluster"], }, { id = "verify_ci_pipeline", name = "Verify CI pipeline connectivity (woodpecker → NATS → ops-controller)", script = "steps/verify_ci_pipeline.nu", dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["restore_forgejo_from_radicle", "reissue_ci_keys"], }, { id = "emit_audit", name = "Emit dr-daoshi-recovered audit event", script = "steps/emit_audit.nu", params = { event_type = "dr_daoshi_recovered" }, dry_run_arg = "--dry-run", on_error = 'Continue, depends_on = ["verify_ci_pipeline", "reconnect_radicle_seed"], }, ], rollback_strategy = 'manual, success_criteria = [ "New libre-daoshi cluster is up with all components running", "forgejo repositories restored from Radicle mirrors", "daoshi-CI NATS JWT reissued with deploy+rollback scope only", "Radicle seed synced daoshi governance ledgers from peers", "woodpecker can trigger a test build and publish OCI image to wuji zot", "ops history shows dr-daoshi-recovered audit event", ], emit_audit = true, adr_refs = ["adr-037", "adr-038", "adr-039"], }