{
  "adr_refs": [
    "adr-037",
    "adr-038",
    "adr-039"
  ],
  "description": "Disaster recovery for catastrophic loss of libre-wuji. Provisions a new cluster, restores zot from S3, reconnects to existing Radicle governance ledgers, replays any stranded audit events. Governance ledgers survive in Radicle; only the runtime is lost.",
  "emit_audit": true,
  "id": "dr_wuji_lost",
  "name": "DR — Libre-Wuji Cluster Lost",
  "params": [
    {
      "default_val": "libre-wuji",
      "description": "Target workspace (must be libre-wuji)",
      "name": "workspace",
      "required": true
    },
    {
      "default_val": "",
      "description": "hcloud context name for cluster provisioning",
      "name": "hetzner_context",
      "required": true
    },
    {
      "default_val": "libre-wuji-zot",
      "description": "S3 bucket name for zot restore",
      "name": "s3_bucket",
      "required": true
    },
    {
      "default_val": "",
      "description": "ops-vm SSH host (for Radicle sync)",
      "name": "ops_vm_host",
      "required": true
    },
    {
      "default_val": "",
      "description": "NATS account server URL for NATS reconnection",
      "name": "nats_account_url",
      "required": false
    },
    {
      "default_val": "false",
      "description": "Skip cluster provision if new cluster already up",
      "name": "skip_provision",
      "required": false
    }
  ],
  "preconditions": [
    "Radicle governance ledgers (policy-libre-wuji, desired-libre-wuji, state-libre-wuji) are accessible from ops-vm or Radicle peers",
    "S3 bucket for zot (libre-wuji-zot) is intact with versioning enabled",
    "ops-vm is operational and keeper-daemon is running",
    "Hetzner API credentials are available for cluster reprovisioning",
    "NATS cluster config (account server URL, credentials) is documented in ops runbook"
  ],
  "rollback_strategy": "manual",
  "steps": [
    {
      "depends_on": [],
      "dry_run_arg": "--dry-run",
      "id": "verify_radicle_ledgers_accessible",
      "name": "Verify Radicle governance ledgers are accessible",
      "on_error": "Stop",
      "params": {},
      "script": "steps/verify_radicle_ledgers_accessible.nu"
    },
    {
      "depends_on": [
        "verify_radicle_ledgers_accessible"
      ],
      "dry_run_arg": "--dry-run",
      "id": "provision_new_cluster",
      "name": "Provision new libre-wuji cluster (skip if already up)",
      "on_error": "Stop",
      "params": {},
      "script": "steps/provision_new_cluster.nu"
    },
    {
      "depends_on": [
        "provision_new_cluster"
      ],
      "dry_run_arg": "--dry-run",
      "id": "restore_zot_from_s3",
      "name": "Restore zot registry pointing to existing S3 bucket",
      "on_error": "Stop",
      "params": {},
      "script": "steps/restore_zot_from_s3.nu"
    },
    {
      "depends_on": [
        "provision_new_cluster"
      ],
      "dry_run_arg": "--dry-run",
      "id": "reconnect_radicle_seed",
      "name": "Reconnect Radicle seed node — sync governance ledgers",
      "on_error": "Stop",
      "params": {},
      "script": "steps/reconnect_radicle_seed.nu"
    },
    {
      "depends_on": [
        "restore_zot_from_s3",
        "reconnect_radicle_seed"
      ],
      "dry_run_arg": "--dry-run",
      "id": "verify_ops_controller",
      "name": "Verify ops-controller is running and NATS-connected",
      "on_error": "Continue",
      "params": {},
      "script": "steps/verify_ops_controller.nu"
    },
    {
      "depends_on": [
        "verify_ops_controller"
      ],
      "dry_run_arg": "--dry-run",
      "id": "replay_stranded_audit_events",
      "name": "Replay stranded audit events from Radicle state ledger",
      "on_error": "Continue",
      "params": {},
      "script": "steps/replay_stranded_audit_events.nu"
    },
    {
      "depends_on": [
        "replay_stranded_audit_events"
      ],
      "dry_run_arg": "--dry-run",
      "id": "emit_audit",
      "name": "Emit dr-wuji-recovered audit event",
      "on_error": "Continue",
      "params": {
        "event_type": "dr_wuji_recovered"
      },
      "script": "steps/emit_audit.nu"
    }
  ],
  "success_criteria": [
    "New libre-wuji cluster is up with all components running",
    "zot registry serving images from existing S3 bucket (no data loss)",
    "Radicle seed synced policy-libre-wuji, desired-libre-wuji, state-libre-wuji from peers",
    "ops-controller connected to NATS and processing ops from queue",
    "Any stranded audit events replayed into SurrealDB mirror",
    "ops history shows dr-wuji-recovered audit event"
  ],
  "version": 1
}