provisioning-catalog/playbooks/bootstrap_initial/steps/deploy_ops_vm.nu

#!/usr/bin/env nu

# Deploy ops-vm on Hetzner ARM64 as keeper signing plane.
# Runs from M4 Mac — M4 has direct SSH access to ops-vm public IP.
# ops-vm is on the wuwei private network and can reach:
#   - NATS NodePort at wuji-cp:34222
#   - zot  NodePort at wuji-cp:35000  (image push during bootstrap)
# During bootstrap ops-vm compiles ops-controller + audit-mirror natively
# and pushes them to zot. Builds move to libre-daoshi once CI is up.

def wait-for-ssh [host: string, max_attempts: int]: nothing -> bool {
    (seq 1 $max_attempts) | reduce --fold false { |_, was_up|
        if $was_up { true } else {
            let r = do { ^ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=4 $"root@($host)" 'echo ready' } | complete
            if $r.exit_code != 0 { sleep 5sec }
            $r.exit_code == 0
        }
    }
}

def main [
    --hetzner-context:     string = ""
    --server-name:         string = "ops-vm"
    --server-type:         string = "cax21"
    --location:            string = "fsn1"
    --network:             string = "wuwei"
    --ssh-key-name:        string = ""
    --keeper-key:          string = ""
    --nats-host:           string = "10.0.8.20"
    --nats-port:           int    = 34222
    --zot-host:            string = "10.0.8.20"
    --zot-port:            int    = 35000
    --provisioning-source: string = ""
    --dry-run
]: nothing -> nothing {
    let hcloud_ctx = if ($hetzner_context | is-not-empty) {
        $hetzner_context
    } else {
        $env | get -o PLAYBOOK_PARAM_HETZNER_CONTEXT | default ""
    }
    let kpr_key = if ($keeper_key | is-not-empty) {
        $keeper_key
    } else {
        $env | get -o PLAYBOOK_PARAM_KEEPER_KEY | default ""
    }
    let src_root = if ($provisioning_source | is-not-empty) {
        $provisioning_source
    } else {
        $env | get -o PROVISIONING_ROOT | default ($env.FILE_PWD | path join ".." ".." ".." | path expand)
    }

    if $dry_run {
        print $"[dry-run] hcloud create: name=($server_name) type=($server_type) location=($location) network=($network)"
        print "[dry-run] install Rust, build ops-controller + audit-mirror natively, push to zot"
        return
    }

    if ($hcloud_ctx | is-empty) {
        error make { msg: "--hetzner-context required (or PLAYBOOK_PARAM_HETZNER_CONTEXT)" }
    }
    if ($kpr_key | is-empty) or not ($kpr_key | path exists) {
        error make { msg: $"keeper-key not found: ($kpr_key)" }
    }

    # ── Provision VM ──────────────────────────────────────────────────────────
    print $"── provision ($server_name) — Hetzner ($server_type) ($location) ──"
    let describe_r = do { with-env { HCLOUD_CONTEXT: $hcloud_ctx } { ^hcloud server describe $server_name --output json } } | complete

    let ops_vm_ip = if $describe_r.exit_code == 0 {
        print $"  ($server_name) already exists — skipping creation"
        $describe_r.stdout | from json | get public_net.ipv4.ip? | default ""
    } else {
        let create_r = if ($ssh_key_name | is-not-empty) {
            do { with-env { HCLOUD_CONTEXT: $hcloud_ctx } { ^hcloud server create --name $server_name --type $server_type --location $location --image debian-12 --network $network --ssh-key $ssh_key_name --output json } } | complete
        } else {
            do { with-env { HCLOUD_CONTEXT: $hcloud_ctx } { ^hcloud server create --name $server_name --type $server_type --location $location --image debian-12 --network $network --output json } } | complete
        }
        if $create_r.exit_code != 0 {
            error make { msg: $"hcloud server create failed: ($create_r.stderr | str trim)" }
        }
        let ip = $create_r.stdout | from json | get server.public_net.ipv4.ip? | default ""
        print $"  created ($server_name) at ($ip)"
        $ip
    }

    if ($ops_vm_ip | is-empty) {
        error make { msg: "could not determine ops-vm public IP" }
    }

    # ── Wait for SSH ──────────────────────────────────────────────────────────
    print "── waiting for SSH (max 150s) ──"
    if not (wait-for-ssh $ops_vm_ip 30) {
        error make { msg: "ops-vm did not become SSH-reachable within 150s" }
    }
    print $"  SSH ready at ($ops_vm_ip)"

    # ── Install Rust toolchain ────────────────────────────────────────────────
    print "── install Rust + build deps on ops-vm ──"
    let rust_script = '#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y --no-install-recommends curl git build-essential pkg-config libssl-dev ca-certificates buildah
if ! command -v cargo >/dev/null 2>&1; then
    curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
fi
source /root/.cargo/env
cargo --version && rustc --version
'
    $rust_script | save --force /tmp/_opsvm_rust.sh
    do { ^scp -o StrictHostKeyChecking=no /tmp/_opsvm_rust.sh $"root@($ops_vm_ip):/tmp/rust_setup.sh" } | complete | ignore
    let rust_r = do { ^ssh -o StrictHostKeyChecking=no $"root@($ops_vm_ip)" 'bash /tmp/rust_setup.sh' } | complete
    if $rust_r.exit_code != 0 {
        error make { msg: $"Rust install failed: ($rust_r.stderr | str trim)" }
    }
    print $rust_r.stdout

    # ── Rsync crate sources from M4 ───────────────────────────────────────────
    print "── rsync crate sources to ops-vm ──"
    let crates_dir = $src_root | path join "provisioning" "platform" "crates"
    if not ($crates_dir | path exists) {
        error make { msg: $"crates directory not found: ($crates_dir)" }
    }
    let rsync_r = do { ^rsync -az --delete --exclude "target/" --exclude ".git/" $"($crates_dir)/" $"root@($ops_vm_ip):/root/crates/" } | complete
    if $rsync_r.exit_code != 0 {
        error make { msg: $"rsync failed: ($rsync_r.stderr | str trim)" }
    }
    print "  sources synced to /root/crates/"

    # ── Native ARM64 build ────────────────────────────────────────────────────
    print "── cargo build --release (native ARM64) on ops-vm ──"
    let build_r = do {
        ^ssh $"root@($ops_vm_ip)" 'source /root/.cargo/env && cd /root/crates && cargo build --release -p ops-controller -p audit-mirror && ls -lh target/release/ops-controller target/release/audit-mirror'
    } | complete
    if $build_r.exit_code != 0 {
        error make { msg: $"cargo build failed: ($build_r.stderr | str trim)" }
    }
    print $build_r.stdout

    # ── Package + push images to zot (via wuwei private network) ─────────────
    print $"── push images to zot ($zot_host):($zot_port) ──"
    let push_lines = [
        "#!/bin/bash"
        "set -euo pipefail"
        $"ZOT=($zot_host):($zot_port)"
        'for svc in ops-controller audit-mirror; do'
        '  echo "==> ${svc}"'
        '  ctr=$(buildah from scratch)'
        '  buildah add "${ctr}" "/root/crates/target/release/${svc}" "/usr/local/bin/${svc}"'
        '  buildah config --cmd "" "${ctr}"'
        '  buildah config --entrypoint "[\""/usr/local/bin/"${svc}"\"]" "${ctr}"'
        '  buildah push "${ctr}" "docker://${ZOT}/prvng/${svc}:latest"'
        '  buildah rm "${ctr}"'
        '  echo "pushed ${ZOT}/prvng/${svc}:latest"'
        'done'
    ]
    $push_lines | str join "\n" | save --force /tmp/_opsvm_push.sh
    do { ^scp -o StrictHostKeyChecking=no /tmp/_opsvm_push.sh $"root@($ops_vm_ip):/tmp/image_push.sh" } | complete | ignore
    let push_r = do { ^ssh $"root@($ops_vm_ip)" 'bash /tmp/image_push.sh' } | complete
    if $push_r.exit_code != 0 {
        print $"WARNING: image push errors: ($push_r.stderr | str trim)"
        print $"  Manual: run /tmp/image_push.sh on ops-vm once zot is reachable at ($zot_host):($zot_port)"
    } else {
        print $push_r.stdout
    }

    # ── Upload keeper signing key ──────────────────────────────────────────────
    print "── upload keeper signing key ──"
    do { ^ssh $"root@($ops_vm_ip)" 'mkdir -p /etc/keeper && chmod 700 /etc/keeper' } | complete | ignore
    let scp_r = do { ^scp $kpr_key $"root@($ops_vm_ip):/etc/keeper/signing.key" } | complete
    if $scp_r.exit_code != 0 {
        error make { msg: $"keeper key upload failed: ($scp_r.stderr | str trim)" }
    }
    do { ^ssh $"root@($ops_vm_ip)" 'chmod 600 /etc/keeper/signing.key' } | complete | ignore
    print "  keeper key installed at /etc/keeper/signing.key"

    print ""
    print "── ops-vm ready ────────────────────────────────────────────────"
    print $"  public IP:   ($ops_vm_ip)"
    print $"  NATS target: ($nats_host):($nats_port)"
    print $"  zot  target: ($zot_host):($zot_port)"
    print ""
    print "Next:"
    print "  1. Write /etc/keeper/keeper_policy.ncl (declarative-only Nickel)"
    print "  2. systemctl enable --now keeper-daemon"
    print "  3. Install radicle-seed, join wuji gossip network"
    print "  4. Run bootstrap_radicle_governance step"
}