99 lines
4.4 KiB
Text
99 lines
4.4 KiB
Text
let pb = import "schemas/lib/playbook.ncl" in
|
|
|
|
pb.make_playbook {
|
|
id = "dr_wuji_lost",
|
|
name = "DR — Libre-Wuji Cluster Lost",
|
|
description = "Disaster recovery for catastrophic loss of libre-wuji. Provisions a new cluster, restores zot from S3, reconnects to existing Radicle governance ledgers, replays any stranded audit events. Governance ledgers survive in Radicle; only the runtime is lost.",
|
|
version = 1,
|
|
|
|
preconditions = [
|
|
"Radicle governance ledgers (policy-libre-wuji, desired-libre-wuji, state-libre-wuji) are accessible from ops-vm or Radicle peers",
|
|
"S3 bucket for zot (libre-wuji-zot) is intact with versioning enabled",
|
|
"ops-vm is operational and keeper-daemon is running",
|
|
"Hetzner API credentials are available for cluster reprovisioning",
|
|
"NATS cluster config (account server URL, credentials) is documented in ops runbook",
|
|
],
|
|
|
|
params = [
|
|
{ name = "workspace", description = "Target workspace (must be libre-wuji)", required = true, default_val = "libre-wuji" },
|
|
{ name = "hetzner_context", description = "hcloud context name for cluster provisioning", required = true, default_val = "" },
|
|
{ name = "s3_bucket", description = "S3 bucket name for zot restore", required = true, default_val = "libre-wuji-zot" },
|
|
{ name = "ops_vm_host", description = "ops-vm SSH host (for Radicle sync)", required = true, default_val = "" },
|
|
{ name = "nats_account_url", description = "NATS account server URL for NATS reconnection", required = false, default_val = "" },
|
|
{ name = "skip_provision", description = "Skip cluster provision if new cluster already up", required = false, default_val = "false" },
|
|
],
|
|
|
|
steps = [
|
|
{
|
|
id = "verify_radicle_ledgers_accessible",
|
|
name = "Verify Radicle governance ledgers are accessible",
|
|
script = "steps/verify_radicle_ledgers_accessible.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Stop,
|
|
depends_on = [],
|
|
},
|
|
{
|
|
id = "provision_new_cluster",
|
|
name = "Provision new libre-wuji cluster (skip if already up)",
|
|
script = "steps/provision_new_cluster.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Stop,
|
|
depends_on = ["verify_radicle_ledgers_accessible"],
|
|
},
|
|
{
|
|
id = "restore_zot_from_s3",
|
|
name = "Restore zot registry pointing to existing S3 bucket",
|
|
script = "steps/restore_zot_from_s3.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Stop,
|
|
depends_on = ["provision_new_cluster"],
|
|
},
|
|
{
|
|
id = "reconnect_radicle_seed",
|
|
name = "Reconnect Radicle seed node — sync governance ledgers",
|
|
script = "steps/reconnect_radicle_seed.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Stop,
|
|
depends_on = ["provision_new_cluster"],
|
|
},
|
|
{
|
|
id = "verify_ops_controller",
|
|
name = "Verify ops-controller is running and NATS-connected",
|
|
script = "steps/verify_ops_controller.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Continue,
|
|
depends_on = ["restore_zot_from_s3", "reconnect_radicle_seed"],
|
|
},
|
|
{
|
|
id = "replay_stranded_audit_events",
|
|
name = "Replay stranded audit events from Radicle state ledger",
|
|
script = "steps/replay_stranded_audit_events.nu",
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Continue,
|
|
depends_on = ["verify_ops_controller"],
|
|
},
|
|
{
|
|
id = "emit_audit",
|
|
name = "Emit dr-wuji-recovered audit event",
|
|
script = "steps/emit_audit.nu",
|
|
params = { event_type = "dr_wuji_recovered" },
|
|
dry_run_arg = "--dry-run",
|
|
on_error = 'Continue,
|
|
depends_on = ["replay_stranded_audit_events"],
|
|
},
|
|
],
|
|
|
|
rollback_strategy = 'manual,
|
|
|
|
success_criteria = [
|
|
"New libre-wuji cluster is up with all components running",
|
|
"zot registry serving images from existing S3 bucket (no data loss)",
|
|
"Radicle seed synced policy-libre-wuji, desired-libre-wuji, state-libre-wuji from peers",
|
|
"ops-controller connected to NATS and processing ops from queue",
|
|
"Any stranded audit events replayed into SurrealDB mirror",
|
|
"ops history shows dr-wuji-recovered audit event",
|
|
],
|
|
|
|
emit_audit = true,
|
|
adr_refs = ["adr-037", "adr-038", "adr-039"],
|
|
}
|