{ "adr_refs": [ "adr-037", "adr-038", "adr-039" ], "description": "Disaster recovery for catastrophic loss of libre-daoshi (CI/build cluster). Provisions a new daoshi cluster, restores forgejo repos from Radicle mirrors, reissues daoshi-CI NATS keys, reconnects Radicle seed, and verifies CI pipeline. Because daoshi is declared replaceable in ADR-038, data loss risk is limited to in-flight builds.", "emit_audit": true, "id": "dr_daoshi_lost", "name": "DR — Libre-Daoshi Cluster Lost", "params": [ { "default_val": "libre-daoshi", "description": "Target workspace (must be libre-daoshi)", "name": "workspace", "required": true }, { "default_val": "", "description": "hcloud context name for cluster provisioning", "name": "hetzner_context", "required": true }, { "default_val": "", "description": "NATS URL for libre-wuji (connectivity check)", "name": "wuji_nats_url", "required": true }, { "default_val": "", "description": "ops-vm SSH host (for CI key reissuance)", "name": "ops_vm_host", "required": true }, { "default_val": "false", "description": "Skip cluster provision if new cluster already up", "name": "skip_provision", "required": false } ], "preconditions": [ "libre-wuji (wuji NATS, zot, ops-controller) is operational", "Radicle governance ledgers for libre-daoshi are accessible from peers", "ops-vm keeper-daemon is running", "Hetzner API credentials available for cluster reprovisioning", "Forgejo repos are mirrored in Radicle (or last-known-good repo archives exist in zot)" ], "rollback_strategy": "manual", "steps": [ { "depends_on": [], "dry_run_arg": "--dry-run", "id": "verify_wuji_accessible", "name": "Verify libre-wuji (NATS + zot) is reachable", "on_error": "Stop", "params": {}, "script": "steps/verify_wuji_accessible.nu" }, { "depends_on": [ "verify_wuji_accessible" ], "dry_run_arg": "--dry-run", "id": "provision_new_cluster", "name": "Provision new libre-daoshi cluster (skip if already up)", "on_error": "Stop", "params": {}, "script": "steps/provision_new_cluster.nu" }, { "depends_on": [ "provision_new_cluster" ], "dry_run_arg": "--dry-run", "id": "restore_forgejo_from_radicle", "name": "Restore forgejo repositories from Radicle mirrors", "on_error": "Continue", "params": {}, "script": "steps/restore_forgejo_from_radicle.nu" }, { "depends_on": [ "provision_new_cluster" ], "dry_run_arg": "--dry-run", "id": "reissue_ci_keys", "name": "Reissue daoshi-CI NATS JWT (deploy+rollback scope)", "on_error": "Stop", "params": {}, "script": "steps/reissue_ci_keys.nu" }, { "depends_on": [ "provision_new_cluster" ], "dry_run_arg": "--dry-run", "id": "reconnect_radicle_seed", "name": "Reconnect Radicle seed node — sync daoshi governance ledgers", "on_error": "Continue", "params": {}, "script": "steps/reconnect_radicle_seed.nu" }, { "depends_on": [ "restore_forgejo_from_radicle", "reissue_ci_keys" ], "dry_run_arg": "--dry-run", "id": "verify_ci_pipeline", "name": "Verify CI pipeline connectivity (woodpecker → NATS → ops-controller)", "on_error": "Continue", "params": {}, "script": "steps/verify_ci_pipeline.nu" }, { "depends_on": [ "verify_ci_pipeline", "reconnect_radicle_seed" ], "dry_run_arg": "--dry-run", "id": "emit_audit", "name": "Emit dr-daoshi-recovered audit event", "on_error": "Continue", "params": { "event_type": "dr_daoshi_recovered" }, "script": "steps/emit_audit.nu" } ], "success_criteria": [ "New libre-daoshi cluster is up with all components running", "forgejo repositories restored from Radicle mirrors", "daoshi-CI NATS JWT reissued with deploy+rollback scope only", "Radicle seed synced daoshi governance ledgers from peers", "woodpecker can trigger a test build and publish OCI image to wuji zot", "ops history shows dr-daoshi-recovered audit event" ], "version": 1 }