146 lines
4.5 KiB
JSON
146 lines
4.5 KiB
JSON
{
|
|
"adr_refs": [
|
|
"adr-037",
|
|
"adr-038",
|
|
"adr-039"
|
|
],
|
|
"description": "Disaster recovery for catastrophic loss of libre-wuji. Provisions a new cluster, restores zot from S3, reconnects to existing Radicle governance ledgers, replays any stranded audit events. Governance ledgers survive in Radicle; only the runtime is lost.",
|
|
"emit_audit": true,
|
|
"id": "dr_wuji_lost",
|
|
"name": "DR — Libre-Wuji Cluster Lost",
|
|
"params": [
|
|
{
|
|
"default_val": "libre-wuji",
|
|
"description": "Target workspace (must be libre-wuji)",
|
|
"name": "workspace",
|
|
"required": true
|
|
},
|
|
{
|
|
"default_val": "",
|
|
"description": "hcloud context name for cluster provisioning",
|
|
"name": "hetzner_context",
|
|
"required": true
|
|
},
|
|
{
|
|
"default_val": "libre-wuji-zot",
|
|
"description": "S3 bucket name for zot restore",
|
|
"name": "s3_bucket",
|
|
"required": true
|
|
},
|
|
{
|
|
"default_val": "",
|
|
"description": "ops-vm SSH host (for Radicle sync)",
|
|
"name": "ops_vm_host",
|
|
"required": true
|
|
},
|
|
{
|
|
"default_val": "",
|
|
"description": "NATS account server URL for NATS reconnection",
|
|
"name": "nats_account_url",
|
|
"required": false
|
|
},
|
|
{
|
|
"default_val": "false",
|
|
"description": "Skip cluster provision if new cluster already up",
|
|
"name": "skip_provision",
|
|
"required": false
|
|
}
|
|
],
|
|
"preconditions": [
|
|
"Radicle governance ledgers (policy-libre-wuji, desired-libre-wuji, state-libre-wuji) are accessible from ops-vm or Radicle peers",
|
|
"S3 bucket for zot (libre-wuji-zot) is intact with versioning enabled",
|
|
"ops-vm is operational and keeper-daemon is running",
|
|
"Hetzner API credentials are available for cluster reprovisioning",
|
|
"NATS cluster config (account server URL, credentials) is documented in ops runbook"
|
|
],
|
|
"rollback_strategy": "manual",
|
|
"steps": [
|
|
{
|
|
"depends_on": [],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "verify_radicle_ledgers_accessible",
|
|
"name": "Verify Radicle governance ledgers are accessible",
|
|
"on_error": "Stop",
|
|
"params": {},
|
|
"script": "steps/verify_radicle_ledgers_accessible.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"verify_radicle_ledgers_accessible"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "provision_new_cluster",
|
|
"name": "Provision new libre-wuji cluster (skip if already up)",
|
|
"on_error": "Stop",
|
|
"params": {},
|
|
"script": "steps/provision_new_cluster.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"provision_new_cluster"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "restore_zot_from_s3",
|
|
"name": "Restore zot registry pointing to existing S3 bucket",
|
|
"on_error": "Stop",
|
|
"params": {},
|
|
"script": "steps/restore_zot_from_s3.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"provision_new_cluster"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "reconnect_radicle_seed",
|
|
"name": "Reconnect Radicle seed node — sync governance ledgers",
|
|
"on_error": "Stop",
|
|
"params": {},
|
|
"script": "steps/reconnect_radicle_seed.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"restore_zot_from_s3",
|
|
"reconnect_radicle_seed"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "verify_ops_controller",
|
|
"name": "Verify ops-controller is running and NATS-connected",
|
|
"on_error": "Continue",
|
|
"params": {},
|
|
"script": "steps/verify_ops_controller.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"verify_ops_controller"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "replay_stranded_audit_events",
|
|
"name": "Replay stranded audit events from Radicle state ledger",
|
|
"on_error": "Continue",
|
|
"params": {},
|
|
"script": "steps/replay_stranded_audit_events.nu"
|
|
},
|
|
{
|
|
"depends_on": [
|
|
"replay_stranded_audit_events"
|
|
],
|
|
"dry_run_arg": "--dry-run",
|
|
"id": "emit_audit",
|
|
"name": "Emit dr-wuji-recovered audit event",
|
|
"on_error": "Continue",
|
|
"params": {
|
|
"event_type": "dr_wuji_recovered"
|
|
},
|
|
"script": "steps/emit_audit.nu"
|
|
}
|
|
],
|
|
"success_criteria": [
|
|
"New libre-wuji cluster is up with all components running",
|
|
"zot registry serving images from existing S3 bucket (no data loss)",
|
|
"Radicle seed synced policy-libre-wuji, desired-libre-wuji, state-libre-wuji from peers",
|
|
"ops-controller connected to NATS and processing ops from queue",
|
|
"Any stranded audit events replayed into SurrealDB mirror",
|
|
"ops history shows dr-wuji-recovered audit event"
|
|
],
|
|
"version": 1
|
|
}
|