99 lines
4.2 KiB
Text
99 lines
4.2 KiB
Text
|
let pb = import "schemas/lib/playbook.ncl" in
|
||
|
|
|
||
|
|
pb.make_playbook {
|
||
|
|
id = "dr_daoshi_lost",
|
||
|
|
name = "DR — Libre-Daoshi Cluster Lost",
|
||
|
|
description = "Disaster recovery for catastrophic loss of libre-daoshi (CI/build cluster). Provisions a new daoshi cluster, restores forgejo repos from Radicle mirrors, reissues daoshi-CI NATS keys, reconnects Radicle seed, and verifies CI pipeline. Because daoshi is declared replaceable in ADR-038, data loss risk is limited to in-flight builds.",
|
||
|
|
version = 1,
|
||
|
|
|
||
|
|
preconditions = [
|
||
|
|
"libre-wuji (wuji NATS, zot, ops-controller) is operational",
|
||
|
|
"Radicle governance ledgers for libre-daoshi are accessible from peers",
|
||
|
|
"ops-vm keeper-daemon is running",
|
||
|
|
"Hetzner API credentials available for cluster reprovisioning",
|
||
|
|
"Forgejo repos are mirrored in Radicle (or last-known-good repo archives exist in zot)",
|
||
|
|
],
|
||
|
|
|
||
|
|
params = [
|
||
|
|
{ name = "workspace", description = "Target workspace (must be libre-daoshi)", required = true, default_val = "libre-daoshi" },
|
||
|
|
{ name = "hetzner_context", description = "hcloud context name for cluster provisioning", required = true, default_val = "" },
|
||
|
|
{ name = "wuji_nats_url", description = "NATS URL for libre-wuji (connectivity check)", required = true, default_val = "" },
|
||
|
|
{ name = "ops_vm_host", description = "ops-vm SSH host (for CI key reissuance)", required = true, default_val = "" },
|
||
|
|
{ name = "skip_provision", description = "Skip cluster provision if new cluster already up", required = false, default_val = "false" },
|
||
|
|
],
|
||
|
|
|
||
|
|
steps = [
|
||
|
|
{
|
||
|
|
id = "verify_wuji_accessible",
|
||
|
|
name = "Verify libre-wuji (NATS + zot) is reachable",
|
||
|
|
script = "steps/verify_wuji_accessible.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Stop,
|
||
|
|
depends_on = [],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "provision_new_cluster",
|
||
|
|
name = "Provision new libre-daoshi cluster (skip if already up)",
|
||
|
|
script = "steps/provision_new_cluster.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Stop,
|
||
|
|
depends_on = ["verify_wuji_accessible"],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "restore_forgejo_from_radicle",
|
||
|
|
name = "Restore forgejo repositories from Radicle mirrors",
|
||
|
|
script = "steps/restore_forgejo_from_radicle.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Continue,
|
||
|
|
depends_on = ["provision_new_cluster"],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "reissue_ci_keys",
|
||
|
|
name = "Reissue daoshi-CI NATS JWT (deploy+rollback scope)",
|
||
|
|
script = "steps/reissue_ci_keys.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Stop,
|
||
|
|
depends_on = ["provision_new_cluster"],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "reconnect_radicle_seed",
|
||
|
|
name = "Reconnect Radicle seed node — sync daoshi governance ledgers",
|
||
|
|
script = "steps/reconnect_radicle_seed.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Continue,
|
||
|
|
depends_on = ["provision_new_cluster"],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "verify_ci_pipeline",
|
||
|
|
name = "Verify CI pipeline connectivity (woodpecker → NATS → ops-controller)",
|
||
|
|
script = "steps/verify_ci_pipeline.nu",
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Continue,
|
||
|
|
depends_on = ["restore_forgejo_from_radicle", "reissue_ci_keys"],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
id = "emit_audit",
|
||
|
|
name = "Emit dr-daoshi-recovered audit event",
|
||
|
|
script = "steps/emit_audit.nu",
|
||
|
|
params = { event_type = "dr_daoshi_recovered" },
|
||
|
|
dry_run_arg = "--dry-run",
|
||
|
|
on_error = 'Continue,
|
||
|
|
depends_on = ["verify_ci_pipeline", "reconnect_radicle_seed"],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
|
||
|
|
rollback_strategy = 'manual,
|
||
|
|
|
||
|
|
success_criteria = [
|
||
|
|
"New libre-daoshi cluster is up with all components running",
|
||
|
|
"forgejo repositories restored from Radicle mirrors",
|
||
|
|
"daoshi-CI NATS JWT reissued with deploy+rollback scope only",
|
||
|
|
"Radicle seed synced daoshi governance ledgers from peers",
|
||
|
|
"woodpecker can trigger a test build and publish OCI image to wuji zot",
|
||
|
|
"ops history shows dr-daoshi-recovered audit event",
|
||
|
|
],
|
||
|
|
|
||
|
|
emit_audit = true,
|
||
|
|
adr_refs = ["adr-037", "adr-038", "adr-039"],
|
||
|
|
}
|