178 lines
7.1 KiB
Text
178 lines
7.1 KiB
Text
#!/usr/bin/env nu
|
|
# Build a golden BuildKit runner snapshot from Debian 12 ARM64 base.
|
|
# Idempotent: re-running with the same date tag is a no-op when the snapshot already exists.
|
|
#
|
|
# Requires: hcloud CLI authenticated (HCLOUD_TOKEN set), SSH agent with orchestrator key.
|
|
# SSH key coupling: the orchestrator pubkey baked into the golden image is the one injected
|
|
# via --ssh-key at server creation. Key rotation requires a fresh golden image build — document
|
|
# in bootstrap playbook and verify fingerprint before declaring an image good.
|
|
|
|
use std log
|
|
|
|
const GOLDEN_IMAGE_PREFIX = "buildkit-runner-golden"
|
|
const CLOUD_INIT_REL_PATH = ["..","cloud-init","runner.yaml"]
|
|
const VERIFY_TOOLS = ["buildctl","sccache","nu"]
|
|
|
|
def main [
|
|
--location: string = "fsn1"
|
|
--server_type: string = "cax21"
|
|
--ssh_key: string = ""
|
|
--date_tag: string = ""
|
|
--cloud_init: string = ""
|
|
]: nothing -> nothing {
|
|
let tag = if ($date_tag | is-empty) {
|
|
date now | format date "%Y-%m-%d"
|
|
} else {
|
|
$date_tag
|
|
}
|
|
|
|
let snapshot_name = $"($GOLDEN_IMAGE_PREFIX):($tag)"
|
|
let seed_name = $"buildkit-runner-seed-($tag)"
|
|
|
|
# Resolve ssh_key name to filesystem path for SSH commands.
|
|
# Hetzner key names use dashes; look in ~/.ssh/ for the matching private key.
|
|
let ssh_key_path = if ($ssh_key | is-not-empty) {
|
|
let candidate = ($"~/.ssh/($ssh_key)" | path expand)
|
|
if ($candidate | path exists) { $candidate } else { "" }
|
|
} else { "" }
|
|
|
|
let cloud_init_path = if ($cloud_init | is-empty) {
|
|
$env.FILE_PWD | path join ...$CLOUD_INIT_REL_PATH | path expand
|
|
} else {
|
|
$cloud_init | path expand
|
|
}
|
|
|
|
if not ($cloud_init_path | path exists) {
|
|
error make { msg: $"cloud-init file not found: ($cloud_init_path)" }
|
|
}
|
|
|
|
log info $"Target snapshot: ($snapshot_name)"
|
|
|
|
# Idempotency check — skip if today's snapshot already exists
|
|
let existing_out = do {
|
|
^hcloud image list --type snapshot --selector $"app=($GOLDEN_IMAGE_PREFIX)" --output json
|
|
} | complete
|
|
if $existing_out.exit_code != 0 {
|
|
error make { msg: $"hcloud image list failed:\n($existing_out.stderr)" }
|
|
}
|
|
let images = $existing_out.stdout | from json
|
|
let already_exists = $images | where { ($in | get -o labels | default {} | get -o buildkit_tag | default "") == $tag }
|
|
if ($already_exists | length) > 0 {
|
|
log info $"Snapshot ($snapshot_name) already exists — no-op."
|
|
return
|
|
}
|
|
|
|
# Create seed server
|
|
log info $"Creating seed server ($seed_name) at ($location) type ($server_type)..."
|
|
let create_args = (
|
|
["server","create","--name",$seed_name,"--image","debian-12","--type",$server_type,
|
|
"--location",$location,"--user-data-from-file",$cloud_init_path,
|
|
"--label",$"app=($GOLDEN_IMAGE_PREFIX)"]
|
|
| if ($ssh_key | is-not-empty) { append ["--ssh-key",$ssh_key] } else { $in }
|
|
)
|
|
let create_out = do { ^hcloud ...$create_args } | complete
|
|
if $create_out.exit_code != 0 {
|
|
error make { msg: $"hcloud server create failed:\n($create_out.stderr)" }
|
|
}
|
|
|
|
# Retrieve server details by name
|
|
let describe_out = do { ^hcloud server describe $seed_name --output json } | complete
|
|
if $describe_out.exit_code != 0 {
|
|
error make { msg: $"hcloud server describe failed:\n($describe_out.stderr)" }
|
|
}
|
|
let server = $describe_out.stdout | from json
|
|
let server_id = $server.id
|
|
let server_ip = $server.public_net.ipv4.ip
|
|
|
|
log info $"Seed server id=($server_id) ip=($server_ip) — waiting for running state..."
|
|
_wait_for_status $server_id "running" 300
|
|
|
|
log info "Waiting for SSH and cloud-init completion..."
|
|
_wait_for_ssh $server_ip 420 $ssh_key_path
|
|
|
|
log info "Verifying tools..."
|
|
_verify_tools $server_ip $ssh_key_path
|
|
|
|
# Snapshot the seed server
|
|
log info $"Creating snapshot '($snapshot_name)'..."
|
|
let snap_out = do {
|
|
^hcloud server create-image $server_id --type snapshot --description $snapshot_name --label $"app=($GOLDEN_IMAGE_PREFIX)" --label $"buildkit_tag=($tag)"
|
|
} | complete
|
|
if $snap_out.exit_code != 0 {
|
|
_cleanup_server $server_id
|
|
error make { msg: $"Snapshot creation failed:\n($snap_out.stderr)" }
|
|
}
|
|
|
|
_cleanup_server $server_id
|
|
|
|
let snap_list = do {
|
|
^hcloud image list --type snapshot --selector $"buildkit_tag=($tag)" --output json
|
|
} | complete
|
|
let snap_id = if $snap_list.exit_code == 0 {
|
|
$snap_list.stdout | from json | get -o 0 | get -o id | default "unknown"
|
|
} else { "unknown" }
|
|
log info $"Golden image ($snapshot_name) ready. id=($snap_id)"
|
|
}
|
|
|
|
def _wait_for_status [server_id: int, target_status: string, timeout_sec: int]: nothing -> nothing {
|
|
let deadline = (date now) + ($timeout_sec * 1sec)
|
|
loop {
|
|
let r = do { ^hcloud server describe $server_id --output json } | complete
|
|
if $r.exit_code == 0 {
|
|
let status = $r.stdout | from json | get status
|
|
if $status == $target_status { return }
|
|
}
|
|
if (date now) > $deadline {
|
|
error make { msg: $"Timeout: server ($server_id) did not reach status '($target_status)'" }
|
|
}
|
|
sleep 10sec
|
|
}
|
|
}
|
|
|
|
def _wait_for_ssh [host: string, timeout_sec: int, ssh_key: string = ""]: nothing -> nothing {
|
|
let deadline = (date now) + ($timeout_sec * 1sec)
|
|
let key_args = if ($ssh_key | is-not-empty) { ["-i" $ssh_key] } else { [] }
|
|
loop {
|
|
let r = do {
|
|
^ssh ...$key_args -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes $"root@($host)" "
|
|
if [ -f /var/lib/cloud/instance/boot-finished ]; then
|
|
grep -iq error /var/log/cloud-init.log && echo error || echo done
|
|
else
|
|
echo waiting
|
|
fi
|
|
"
|
|
} | complete
|
|
if $r.exit_code == 0 {
|
|
let out = ($r.stdout | str trim)
|
|
if $out == "done" { return }
|
|
if $out == "error" {
|
|
error make { msg: $"cloud-init reported errors on ($host) — check /var/log/cloud-init.log" }
|
|
}
|
|
}
|
|
if (date now) > $deadline {
|
|
error make { msg: $"Timeout waiting for SSH + cloud-init on ($host)" }
|
|
}
|
|
sleep 15sec
|
|
}
|
|
}
|
|
|
|
def _verify_tools [host: string, ssh_key: string = ""]: nothing -> nothing {
|
|
let key_args = if ($ssh_key | is-not-empty) { ["-i" $ssh_key] } else { [] }
|
|
for tool in $VERIFY_TOOLS {
|
|
let r = do {
|
|
^ssh ...$key_args -o StrictHostKeyChecking=no -o BatchMode=yes $"root@($host)" $"command -v ($tool) && ($tool) --version 2>&1 | head -1"
|
|
} | complete
|
|
if $r.exit_code != 0 {
|
|
error make { msg: $"Tool '($tool)' not found on runner — cloud-init may have failed.\nSSH output: ($r.stdout)\n($r.stderr)" }
|
|
}
|
|
log info $" ($tool): ($r.stdout | str trim)"
|
|
}
|
|
}
|
|
|
|
def _cleanup_server [server_id: int]: nothing -> nothing {
|
|
log info $"Destroying seed server ($server_id)..."
|
|
let r = do { ^hcloud server delete $server_id } | complete
|
|
if $r.exit_code != 0 {
|
|
log warning $"Server delete returned non-zero (server may need manual cleanup): ($r.stderr)"
|
|
}
|
|
}
|