provisioning-core/nulib/cli/op.nu

1059 lines
45 KiB
Text

#!/usr/bin/env nu
# Op governance — workspace operation tracking with jj + Radicle + restic.
# Each op is a DAG node: pre-snapshot → execution → post-snapshot.
# Rollback creates a new forward op pointing to a previous snapshot.
export-env {
let lib_dirs_raw = ($env.NU_LIB_DIRS? | default "")
let current_lib_dirs = if ($lib_dirs_raw | type) == "string" {
if ($lib_dirs_raw | is-empty) { [] } else { ($lib_dirs_raw | split row ":") }
} else {
$lib_dirs_raw
}
let dynamic = ($env.PROVISIONING? | default "" | path join "core" "nulib")
$env.NU_LIB_DIRS = ([
"/opt/provisioning/core/nulib"
"/usr/local/provisioning/core/nulib"
] | append $current_lib_dirs | append (if ($dynamic | is-not-empty) { [$dynamic] } else { [] }))
}
use tools/nickel/process.nu [ncl-eval, ncl-eval-soft, default-ncl-paths]
use primitives/io/logging.nu [is-debug-enabled]
# ── helpers ──────────────────────────────────────────────────────────────────
def op-err [msg: string]: nothing -> nothing {
if (is-debug-enabled) {
error make { msg: $msg }
} else {
error make --unspanned { msg: $msg }
}
}
# Walk up from cwd looking for config/provisioning.ncl; fall back to cwd.
def op-ws-root [explicit: string]: nothing -> string {
if ($explicit | is-not-empty) { return $explicit }
mut dir = $env.PWD
loop {
if ($dir | path join "config" "provisioning.ncl" | path exists) { return $dir }
let parent = ($dir | path dirname)
if $parent == $dir { break }
$dir = $parent
}
$env.PWD
}
def op-load-config [ws_root: string]: nothing -> record {
let cfg_path = $ws_root | path join "config" "provisioning.ncl"
if not ($cfg_path | path exists) {
op-err $"No config/provisioning.ncl in ($ws_root)"
}
let workspace_cfg = ncl-eval $cfg_path (default-ncl-paths $ws_root)
# Merge ops config from infra settings.ncl — infra owns its archive/retention config.
let infra = $workspace_cfg.current_infra? | default ""
let infra_ops = if ($infra | is-not-empty) {
let settings_path = $ws_root | path join "infra" $infra "settings.ncl"
if ($settings_path | path exists) {
let settings = ncl-eval-soft $settings_path (default-ncl-paths $ws_root) {}
$settings.settings?.ops? | default {}
} else { {} }
} else { {} }
if ($infra_ops | is-empty) {
$workspace_cfg
} else {
$workspace_cfg | insert ops $infra_ops
}
}
# Radicle NID when available; local:{user} fallback.
def op-get-actor []: nothing -> record {
let r = do { ^rad self --nid } | complete
let user = $env.USER? | default "system"
let nid = if $r.exit_code == 0 { $r.stdout | str trim } else { $"local:($user)" }
{ nid: $nid, identity: $user, source: "cli" }
}
# {nid-short}:{uuid}
def op-gen-id [nid: string]: nothing -> string {
let uuid_r = do { ^uuidgen } | complete
let uuid = if $uuid_r.exit_code == 0 {
$uuid_r.stdout | str trim | str downcase
} else {
random chars --length 32
}
let short = if ($nid | str starts-with "local:") {
$nid | str replace "local:" "" | str substring 0..6
} else {
$nid | str substring 0..7
}
$"($short):($uuid)"
}
def op-now []: nothing -> string {
let r = do { ^date -u +"%Y-%m-%dT%H:%M:%SZ" } | complete
if $r.exit_code == 0 { $r.stdout | str trim } else { "" }
}
# Resolve the ops directory for a workspace: infra/{current_infra}/ops/
# Falls back to ws_root/ops if cfg is not available (init path).
def op-ops-dir [ws_root: string, cfg: record = {}]: nothing -> string {
let infra = $cfg.current_infra? | default ""
if ($infra | is-not-empty) {
$ws_root | path join "infra" $infra "ops"
} else {
$ws_root | path join "infra" (
ls ($ws_root | path join "infra") | where type == "dir" | get name | each {|p| $p | path basename} | get 0? | default ""
) "ops"
}
}
# Abort if a running op exists for this component (unless --force).
def op-check-lock [ws_root: string, component: string]: nothing -> nothing {
let ops_dir = op-ops-dir $ws_root
if not ($ops_dir | path exists) { return }
let running = (
ls $ops_dir
| where type == "dir"
| each {|e|
let p = $e.name | path join "op.json"
if ($p | path exists) {
let d = open $p
if ($d.status? == "running") and ($d.component? == $component) {
$d.id? | default "?"
} else { null }
} else { null }
}
| compact
)
if ($running | length) > 0 {
op-err $"Concurrent op running for '($component)': ($running | str join ', '). Use --force to override."
}
}
# Capture component config + server states for snapshot.
def op-snapshot [ws_root: string, component: string, infra: string, targets: list<string>]: nothing -> record {
let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl"
let comp_cfg = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {}
let state_path = $ws_root | path join ".provisioning-state.json"
let state_data = if ($state_path | path exists) { open $state_path } else { {} }
{
timestamp: (op-now),
component: $component,
infra: $infra,
targets: $targets,
component_config: $comp_cfg,
server_states: ($targets | each {|t|
{ server: $t, state: ($state_data | get -o $t | default {}) }
}),
}
}
# Create a jj change and return the short change ID (empty string if jj unavailable).
def op-jj-new [message: string]: nothing -> string {
let r = do { ^jj new --message $message } | complete
if $r.exit_code != 0 { return "" }
let id_r = do { ^jj log --no-graph --template "change_id.short(8)" --revisions "@" } | complete
if $id_r.exit_code == 0 { $id_r.stdout | str trim } else { "" }
}
def op-jj-commit [message: string]: nothing -> nothing {
let r = do { ^jj commit --message $message } | complete
if $r.exit_code != 0 {
print $"(ansi yellow)warn(ansi reset): jj commit failed — ($r.stderr | str trim)"
}
}
# Load the backup provider record from catalog/providers/backup/{tool}/nickel/provider.ncl.
# Load archive credentials from infra/{current_infra}/secrets/ops.sops.yaml.
# Uses the age key at infra/{current_infra}/.kage.
# Always reads from SOPS — never from the caller's env — to avoid credential bleed.
def op-load-archive-creds [ws_root: string, cfg: record]: nothing -> record {
let infra = $cfg.current_infra? | default ""
if ($infra | is-empty) {
print $"(ansi yellow)warn(ansi reset): current_infra not set in config — cannot load archive credentials"
return {}
}
let sops_path = $ws_root | path join "infra" $infra "secrets" "ops.sops.yaml"
let kage_path = $ws_root | path join "infra" $infra ".kage"
if not ($sops_path | path exists) {
print $"(ansi yellow)warn(ansi reset): ops secrets not found at ($sops_path)"
return {}
}
let r = if ($kage_path | path exists) {
do { with-env { SOPS_AGE_KEY_FILE: $kage_path } { ^sops --decrypt $sops_path } } | complete
} else {
do { ^sops --decrypt $sops_path } | complete
}
if $r.exit_code != 0 {
print $"(ansi yellow)warn(ansi reset): failed to decrypt ($sops_path) — ($r.stderr | str trim)"
return {}
}
$r.stdout | from yaml
}
def op-load-archive-provider [tool: string]: nothing -> record {
let prov = $env.PROVISIONING? | default ""
if ($prov | is-empty) {
print $"(ansi yellow)warn(ansi reset): PROVISIONING not set — cannot load archive provider"
return {}
}
let ncl_path = $prov | path join "catalog" "providers" "backup" $tool "nickel" "provider.ncl"
if not ($ncl_path | path exists) {
print $"(ansi yellow)warn(ansi reset): backup provider not found: ($ncl_path)"
return {}
}
let prov_root = $env.PROVISIONING? | default ""
let data = ncl-eval-soft $ncl_path (default-ncl-paths $prov_root) {}
$data.provider? | default {}
}
# Ensure required keys are present in the creds record loaded from SOPS.
def op-archive-check-env [provider: record, creds: record]: nothing -> bool {
let required = $provider.env?.required? | default []
let missing = $required | where {|v| ($creds | get -o $v | default "" | is-empty) }
if ($missing | length) > 0 {
print $"(ansi yellow)warn(ansi reset): archive provider '($provider.name?)' missing credentials: ($missing | str join ', ') — check infra/*/secrets/ops.sops.yaml"
return false
}
true
}
# Connect provider to its repository if connection.required = true (kopia model).
# Returns true if connected or not required; false on failure.
def op-archive-connect [provider: record, ws_root: string, cfg: record, creds: record]: nothing -> bool {
let conn = $provider.connection? | default {}
if not ($conn.required? | default false) { return true }
let bin = $provider.binary? | default ""
let status_sub = $conn.status_subcmd? | default ""
let connect_sub = $conn.connect_subcmd? | default ""
let cfg_file = (op-ops-dir $ws_root $cfg) | path join ($conn.state_file? | default ".kopia-config")
let cfg_args = ["--config-file" $cfg_file]
if ($status_sub | is-not-empty) {
let status_r = do { with-env $creds { ^$bin ...$cfg_args ...($status_sub | split words) } } | complete
if $status_r.exit_code == 0 { return true }
}
if ($connect_sub | is-empty) { return false }
let bucket = $cfg.ops?.archive?.bucket? | default ""
let endpoint = $cfg.ops?.archive?.endpoint? | default ""
let prefix = $cfg.ops?.archive?.prefix? | default "ops"
let s3f = $conn.s3_flags? | default {}
if ($bucket | is-empty) {
print $"(ansi yellow)warn(ansi reset): ops.archive.bucket not set — cannot connect ($bin)"
return false
}
let s3_args = (
[($s3f.bucket? | default "--bucket") $bucket]
++ (if ($endpoint | is-not-empty) { [($s3f.endpoint? | default "--endpoint") $endpoint] } else { [] })
++ [($s3f.prefix? | default "--prefix") $prefix]
)
let r = do { with-env $creds { ^$bin ...$cfg_args ...($connect_sub | split words) ...$s3_args } } | complete
if $r.exit_code != 0 {
print $"(ansi yellow)warn(ansi reset): ($bin) connect failed — ($r.stderr | str trim)"
return false
}
true
}
# Execute archive backup using provider definition. Returns snapshot ID or "".
def op-archive-backup [
op_dir: string
tags: list<string>
workspace_tag: string
retention: record
provider: record
ws_root: string
cfg: record
]: nothing -> string {
if ($provider | is-empty) { return "" }
let creds = op-load-archive-creds $ws_root $cfg
if ($creds | is-empty) { return "" }
if not (op-archive-check-env $provider $creds) { return "" }
if not (op-archive-connect $provider $ws_root $cfg $creds) { return "" }
let bin = $provider.binary? | default ""
let cmd = $provider.commands?.backup? | default {}
let repo = $creds.RESTIC_REPOSITORY? | default ""
let cfg_args = if ($provider.connection?.required? | default false) {
let cfg_file = (op-ops-dir $ws_root $cfg) | path join ($provider.connection?.state_file? | default ".kopia-config")
["--config-file" $cfg_file]
} else { [] }
let repo_args = if ($cmd.repo_flag? | is-not-empty) and ($repo | is-not-empty) {
[$cmd.repo_flag $repo]
} else { [] }
let tag_flag = $cmd.tag_flag? | default ""
let tag_args = if ($tag_flag | is-not-empty) {
$tags | each {|t| [$tag_flag $t] } | flatten
} else { [] }
let all_args = $cfg_args ++ ($cmd.subcmd | split words) ++ $repo_args ++ $tag_args ++ [$op_dir]
let r = do { with-env $creds { ^$bin ...$all_args } } | complete
if $r.exit_code != 0 {
print $"(ansi yellow)warn(ansi reset): ($bin) backup failed — ($r.stderr | str trim)"
return ""
}
let id_regex = $cmd.snapshot_id_regex? | default ""
let snap_id = if ($id_regex | is-not-empty) {
$r.stdout | lines
| each {|l| $l | parse --regex $id_regex | get 0? | get id? | default "" }
| where {|v| $v | is-not-empty }
| get 0? | default ""
} else { "" }
let ops_dir = op-ops-dir $ws_root $cfg
op-archive-forget $provider $op_dir $workspace_tag $retention $ops_dir $creds
$snap_id
}
# Apply retention policy using provider definition (tag-scoped where supported).
def op-archive-forget [
provider: record
source: string
workspace_tag: string
retention: record
ops_dir: string
creds: record
]: nothing -> nothing {
let bin = $provider.binary? | default ""
let forget = $provider.commands?.forget? | default {}
let keep_last = ($retention.keep_last? | default 50 | into string)
let keep_monthly = ($retention.keep_monthly? | default 12 | into string)
let keep_yearly = ($retention.keep_yearly? | default 3 | into string)
let cfg_args = if ($provider.connection?.required? | default false) {
let cfg_file = $ops_dir | path join ($provider.connection?.state_file? | default ".kopia-config")
["--config-file" $cfg_file]
} else { [] }
let policy_sub = $forget.policy_subcmd? | default ""
if ($policy_sub | is-not-empty) {
let kf = $forget.policy_keep_flags? | default {}
let policy_args = (
$cfg_args
++ ($policy_sub | split words)
++ [$source]
++ [($kf.keep_latest? | default "--keep-latest") $keep_last]
++ [($kf.keep_monthly? | default "--keep-monthly") $keep_monthly]
++ [($kf.keep_annual? | default "--keep-annual") $keep_yearly]
)
do { with-env $creds { ^$bin ...$policy_args } } | complete | ignore
}
let repo = $creds.RESTIC_REPOSITORY? | default ""
let repo_args = if ($forget.repo_flag? | is-not-empty) and ($repo | is-not-empty) {
[$forget.repo_flag $repo]
} else { [] }
let tag_args = if ($forget.tag_flag? | is-not-empty) and ($workspace_tag | is-not-empty) {
[$forget.tag_flag $workspace_tag]
} else { [] }
let keep_args = if ($forget.keep_last_flag? | is-not-empty) {
[($forget.keep_last_flag) $keep_last
($forget.keep_monthly_flag? | default "") $keep_monthly
($forget.keep_yearly_flag? | default "") $keep_yearly]
| where {|v| $v | is-not-empty }
} else { [] }
let extra = $forget.extra_flags? | default []
let forget_args = $cfg_args ++ ($forget.subcmd | split words) ++ $repo_args ++ $tag_args ++ $keep_args ++ $extra
do { with-env $creds { ^$bin ...$forget_args } } | complete | ignore
}
# Resolve target hostnames for a component.
# Priority: component NCL target/targets field > enabled servers from servers.ncl.
def op-load-targets [ws_root: string, component: string, infra: string]: nothing -> list<string> {
let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl"
let comp = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {}
let inner = $comp | get -o $component | default {}
let targets_field = $inner | get -o targets | default []
let target_field = $inner | get -o target | default ""
if ($targets_field | length) > 0 { return $targets_field }
if ($target_field | is-not-empty) { return [$target_field] }
let srv_ncl = $ws_root | path join "infra" $infra "servers.ncl"
if not ($srv_ncl | path exists) { return [] }
let srv_data = ncl-eval-soft $srv_ncl (default-ncl-paths $ws_root) { servers: [] }
($srv_data.servers? | default [])
| where { |s| ($s.enabled? | default true) and not ($s.not_use? | default false) }
| each { |s| $s.hostname? | default "" }
| where { |h| $h | is-not-empty }
}
# ── constraint evaluator ──────────────────────────────────────────────────────
# Load op_constraints from a component NCL definition (optional field).
def op-load-component-constraints [ws_root: string, component: string, infra: string]: nothing -> record {
let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl"
let comp = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {}
$comp | get -o op_constraints | default { pre: [], on_failure: [] }
}
# Evaluate a single pre-constraint. Returns true on pass, false on fail.
def op-eval-constraint [ws_root: string, constraint: record]: nothing -> bool {
let kind = $constraint.kind? | default ""
let resource = $constraint.resource? | default ""
let prov = $env.PROVISIONING? | default "prvng"
match $kind {
"backup" => {
print $" constraint: backup ($resource)"
let r = do { ^$prov component backup $resource } | complete
$r.exit_code == 0
}
"health_check" => {
print $" constraint: health_check ($resource)"
let r = do { ^$prov component health $resource } | complete
$r.exit_code == 0
}
"dry_run_check" => {
print $" constraint: dry_run_check ($resource)"
let r = do { ^$prov component install $resource --check } | complete
$r.exit_code == 0
}
"concurrent_lock" => {
op-check-lock $ws_root $resource
true
}
_ => {
print $" constraint: unknown kind '($kind)' — skipping"
true
}
}
}
# Evaluate a single recovery action (on_failure).
def op-eval-recovery-action [ws_root: string, action: record]: nothing -> nothing {
let kind = $action.kind? | default ""
let resource = $action.resource? | default ""
let prov = $env.PROVISIONING? | default "prvng"
match $kind {
"restore" => {
print $" recovery: restore ($resource)"
let r = do { ^$prov component restore $resource } | complete
if $r.exit_code != 0 {
print $" (ansi red)recovery failed(ansi reset): ($r.stderr | str trim)"
}
}
"backup" => {
print $" recovery: backup ($resource) — post-failure snapshot"
do { ^$prov component backup $resource } | complete | ignore
}
_ => {
print $" recovery: unknown kind '($kind)' — skipping"
}
}
}
# Evaluate all pre-constraints. Fails the op if any constraint fails.
def op-eval-pre [ws_root: string, constraints: list]: nothing -> nothing {
if ($constraints | length) == 0 { return }
print $"evaluating ($constraints | length) pre-constraint(s)..."
for c in $constraints {
let passed = op-eval-constraint $ws_root $c
if not $passed {
let kind = $c.kind? | default "unknown"
let resource = $c.resource? | default ""
op-err $"Pre-constraint failed: ($kind) on ($resource)"
}
}
}
# Run all on_failure recovery actions.
def op-eval-recovery [ws_root: string, actions: list]: nothing -> nothing {
if ($actions | length) == 0 { return }
print $"running ($actions | length) recovery action(s)..."
for a in $actions {
op-eval-recovery-action $ws_root $a
}
}
# Publish committed op to Radicle network (best-effort, skipped if rad unavailable).
def op-rad-sync []: nothing -> string {
let r = do { ^rad sync } | complete
if $r.exit_code == 0 {
let rid_r = do { ^rad self --rid } | complete
if $rid_r.exit_code == 0 { $rid_r.stdout | str trim } else { "" }
} else {
let err = $r.stderr | str trim
let silent = (
($err | str contains "not a Radicle repository") or
($err | str contains "node is not running") or
($err | str contains "connection refused") or
($err | str contains "failed to connect") or
($err | str contains "not connected")
)
if not $silent and (is-debug-enabled) {
print $"(ansi yellow)warn(ansi reset): rad sync failed — ($err)"
}
""
}
}
# ── subcommands ───────────────────────────────────────────────────────────────
def op-cmd-start [
args: list<string>
workspace: string
force: bool
]: nothing -> nothing {
let component = $args | get 0? | default ""
let operation = $args | get 1? | default ""
let intent = $args | get 2? | default ""
if ($component | is-empty) { op-err "Usage: provisioning op start <component> <operation> <intent>" }
if ($operation | is-empty) { op-err "Usage: provisioning op start <component> <operation> <intent>" }
if ($intent | is-empty) { op-err "Usage: provisioning op start <component> <operation> <intent>" }
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default")
if not $force { op-check-lock $ws_root $component }
# Load constraints from component definition (op_constraints field, if present)
let comp_constraints = op-load-component-constraints $ws_root $component $infra
let pre_constraints = $comp_constraints.pre? | default []
# Evaluate pre-constraints before doing anything else
# On failure this exits with error — no op record is created
op-eval-pre $ws_root $pre_constraints
let actor = op-get-actor
let op_id = op-gen-id $actor.nid
let ops_dir = op-ops-dir $ws_root $cfg
let op_dir = $ops_dir | path join $op_id
mkdir $op_dir
let started_at = op-now
let targets = op-load-targets $ws_root $component $infra
# Pre-snapshot (taken after pre-constraints pass — any backup is already done)
let snap = op-snapshot $ws_root $component $infra $targets
let pre_rel = $"infra/($infra)/ops/($op_id)/pre.json"
$snap | to json --indent 2 | save --force ($op_dir | path join "pre.json")
# jj change
let jj_change = op-jj-new $"op:($op_id) ($operation) ($component)"
# Op record
{
id: $op_id,
actor: $actor,
intent: $intent,
workspace: ($cfg.workspace? | default ""),
component: $component,
operation: $operation,
targets: $targets,
constraints: $comp_constraints,
snapshots: { pre: $pre_rel, post: null },
artifacts: { archive_snapshot: null, bundles: [] },
lineage: { parent_op: null, rollback_of: null },
status: "running",
jj_change: (if ($jj_change | is-empty) { null } else { $jj_change }),
radicle_rid: null,
started_at: $started_at,
ended_at: null,
} | to json --indent 2 | save --force ($op_dir | path join "op.json")
print $"op started ($op_id)"
print $" component : ($component) operation: ($operation)"
print $" workspace : ($ws_root)"
if ($jj_change | is-not-empty) { print $" jj change : ($jj_change)" }
print $" intent : ($intent)"
}
def op-cmd-finish [
args: list<string>
workspace: string
]: nothing -> nothing {
let op_id = $args | get 0? | default ""
let status = $args | get 1? | default "success"
if ($op_id | is-empty) { op-err "Usage: provisioning op finish <op-id> [success|failed|cancelled]" }
if $status not-in ["success" "failed" "cancelled"] {
op-err $"Invalid status '($status)' — must be: success, failed, cancelled"
}
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default")
let op_dir = (op-ops-dir $ws_root $cfg) | path join $op_id
let op_file = $op_dir | path join "op.json"
if not ($op_file | path exists) { op-err $"Op not found: ($op_id)" }
let rec = open $op_file
let ended_at = op-now
# On failure: run on_failure recovery actions before anything else
if $status == "failed" {
let recovery = $rec.constraints? | default {} | get -o on_failure | default []
op-eval-recovery $ws_root $recovery
}
# Post-snapshot (only on success)
let post_rel = if $status == "success" {
let snap = op-snapshot $ws_root $rec.component $infra ($rec.targets? | default [])
$snap | to json --indent 2 | save --force ($op_dir | path join "post.json")
$"infra/($infra)/ops/($op_id)/post.json"
} else { null }
# archive backup via provider
let workspace_name = $cfg.workspace? | default ""
let retention = $cfg.ops?.retention? | default { keep_last: 50, keep_monthly: 12, keep_yearly: 3 }
let workspace_tag = $"workspace=($workspace_name)"
let tool = $cfg.ops?.archive?.tool? | default "restic"
# ops.archive.enabled = false skips op-archive entirely (empty provider →
# op-archive-backup returns "" with no warning). Default true is backward
# compatible. Use to turn off archival when no archive target is configured.
let provider = if ($cfg.ops?.archive?.enabled? | default true) { op-load-archive-provider $tool } else { {} }
let tags = [
$"op-id=($op_id)",
$"author=($rec.actor.identity)",
$"nid=($rec.actor.nid | str substring 0..12)",
$"component=($rec.component)",
$"operation=($rec.operation)",
$workspace_tag,
$"status=($status)",
]
let snap_id = op-archive-backup $op_dir $tags $workspace_tag $retention $provider $ws_root $cfg
# Finalize op record
let updated = $rec | merge {
status: $status,
ended_at: $ended_at,
snapshots: ($rec.snapshots | merge { post: $post_rel }),
artifacts: ($rec.artifacts | merge {
archive_snapshot: (if ($snap_id | is-empty) { null } else { $snap_id })
}),
}
$updated | to json --indent 2 | save --force $op_file
# jj commit — only when jj was active at op-start (jj_change non-empty)
let icon = if $status == "success" { "✓" } else { "✗" }
if ($rec.jj_change? | is-not-empty) {
op-jj-commit $"op:($op_id) ($rec.operation) ($rec.component) ($icon)"
}
# Radicle sync (best-effort)
let rad_rid = op-rad-sync
if ($rad_rid | is-not-empty) {
let final = (open $op_file) | merge { radicle_rid: $rad_rid }
$final | to json --indent 2 | save --force $op_file
print $" radicle : ($rad_rid)"
}
print $"op finished ($op_id) status: ($status)"
if ($snap_id | is-not-empty) { print $" archive : ($snap_id)" }
}
def op-cmd-log [
args: list<string>
workspace: string
]: nothing -> nothing {
let component_filter = $args | get 0? | default ""
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let ops_dir = op-ops-dir $ws_root $cfg
if not ($ops_dir | path exists) {
print "No ops recorded yet. Run: provisioning op start <component> <operation> <intent>"
return
}
let records = (
ls $ops_dir
| where type == "dir"
| sort-by modified --reverse
| each {|e|
let p = $e.name | path join "op.json"
if ($p | path exists) { open $p } else { null }
}
| compact
| where { |r|
if ($component_filter | is-empty) { true } else { $r.component? == $component_filter }
}
)
if ($records | length) == 0 {
let suffix = if ($component_filter | is-not-empty) { $" for component '($component_filter)'" } else { "" }
print $"No ops found($suffix)."
return
}
$records | select id component operation status started_at ended_at | table
}
# Field-level diff between two records — shows added/removed/changed keys with values.
def op-diff-records [pre: record, post: record]: nothing -> nothing {
let all_keys = (($pre | columns) ++ ($post | columns)) | uniq | sort
let changes = ($all_keys | each {|k|
let pv = $pre | get -o $k
let nv = $post | get -o $k
if $pv == null and $nv != null {
{ change: "added", field: $k, before: "—", after: ($nv | to nuon) }
} else if $pv != null and $nv == null {
{ change: "removed", field: $k, before: ($pv | to nuon), after: "—" }
} else if $pv != $nv {
{ change: "changed", field: $k, before: ($pv | to nuon), after: ($nv | to nuon) }
} else {
null
}
} | compact)
if ($changes | length) == 0 {
print " (no changes)"
} else {
for c in $changes {
print $" ($c.change) ($c.field) ($c.before) → ($c.after)"
}
}
}
def op-cmd-show [
args: list<string>
workspace: string
]: nothing -> nothing {
let op_id = $args | get 0? | default ""
if ($op_id | is-empty) { op-err "Usage: provisioning op show <op-id>" }
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let op_file = (op-ops-dir $ws_root $cfg) | path join $op_id "op.json"
if not ($op_file | path exists) { op-err $"Op not found: ($op_id)" }
let rec = open $op_file
print $"id : ($rec.id)"
print $"component : ($rec.component) operation: ($rec.operation)"
print $"intent : ($rec.intent)"
print $"actor : ($rec.actor.identity) nid: ($rec.actor.nid)"
print $"status : ($rec.status)"
print $"started : ($rec.started_at)"
print $"ended : ($rec.ended_at? | default "-")"
if ($rec.jj_change? | is-not-empty) { print $"jj change : ($rec.jj_change)" }
if ($rec.radicle_rid? | is-not-empty) { print $"radicle : ($rec.radicle_rid)" }
if ($rec.artifacts.archive_snapshot? | is-not-empty) {
print $"archive : ($rec.artifacts.archive_snapshot)"
}
if ($rec.targets? | default [] | length) > 0 {
print $"targets : ($rec.targets | str join ', ')"
}
let lineage = $rec.lineage? | default {}
if ($lineage.parent_op? | is-not-empty) { print $"parent : ($lineage.parent_op)" }
if ($lineage.rollback_of? | is-not-empty) { print $"rollback : undoes ($lineage.rollback_of)" }
let pre_path = $ws_root | path join ($rec.snapshots.pre)
let post_path = if ($rec.snapshots.post? | is-not-empty) {
$ws_root | path join $rec.snapshots.post
} else { "" }
if ($pre_path | path exists) and ($post_path | is-not-empty) and ($post_path | path exists) {
print ""
print "── snapshot diff — component_config ───────────"
let pre = open $pre_path | get component_config? | default {}
let post = open $post_path | get component_config? | default {}
op-diff-records $pre $post
print ""
print "── snapshot diff — server_states ──────────────"
let pre_states = (open $pre_path | get server_states? | default [] | each {|s| {($s.server): $s.state} } | reduce -f {} {|a, b| $a | merge $b })
let post_states = (open $post_path | get server_states? | default [] | each {|s| {($s.server): $s.state} } | reduce -f {} {|a, b| $a | merge $b })
op-diff-records $pre_states $post_states
} else if ($pre_path | path exists) {
print ""
print "── pre-snapshot (no post — op in progress or failed) ─"
let pre = open $pre_path | get component_config? | default {}
print $" component_config keys: ($pre | columns | str join ', ')"
}
}
def op-cmd-rollback [
args: list<string>
workspace: string
]: nothing -> nothing {
let target_id = $args | get 0? | default ""
if ($target_id | is-empty) { op-err "Usage: provisioning op rollback <op-id>" }
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default")
let ops_dir = op-ops-dir $ws_root $cfg
let op_file = $ops_dir | path join $target_id "op.json"
if not ($op_file | path exists) { op-err $"Op not found: ($target_id)" }
let target_rec = open $op_file
let pre_path = $ws_root | path join ($target_rec.snapshots.pre)
if not ($pre_path | path exists) { op-err $"Pre-snapshot missing for op ($target_id)" }
let actor = op-get-actor
let op_id = op-gen-id $actor.nid
let op_dir = $ops_dir | path join $op_id
mkdir $op_dir
let started_at = op-now
let pre_rel = $"infra/($infra)/ops/($op_id)/pre.json"
# The rollback's pre-snapshot is the current state (before restoring)
let cfg = op-load-config $ws_root
let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default")
let snap = op-snapshot $ws_root $target_rec.component $infra ($target_rec.targets? | default [])
$snap | to json --indent 2 | save --force ($op_dir | path join "pre.json")
# Restore: copy target op's pre.json as the new desired state for the operator to apply
let restore_path = $op_dir | path join "restore-target.json"
cp $pre_path $restore_path
let jj_change = op-jj-new $"op:($op_id) rollback ($target_rec.component) (undoes ($target_id))"
{
id: $op_id,
actor: $actor,
intent: $"Rollback ($target_rec.component) to state before op ($target_id)",
workspace: ($cfg.workspace? | default ""),
component: $target_rec.component,
operation: "rollback",
targets: ($target_rec.targets? | default []),
constraints: { pre: [], on_failure: [] },
snapshots: { pre: $pre_rel, post: null },
artifacts: { archive_snapshot: null, bundles: [] },
lineage: { parent_op: $target_id, rollback_of: $target_id },
status: "running",
jj_change: (if ($jj_change | is-empty) { null } else { $jj_change }),
radicle_rid: null,
started_at: $started_at,
ended_at: null,
} | to json --indent 2 | save --force ($op_dir | path join "op.json")
print $"rollback op started ($op_id)"
print $" restoring : ($target_rec.component) to snapshot from op ($target_id)"
print $" target : ($restore_path)"
print $" run 'provisioning component ($target_rec.operation) ($target_rec.component)' to apply"
print $" then run : provisioning op finish ($op_id) success"
}
def op-cmd-init [workspace: string]: nothing -> nothing {
let ws_root = op-ws-root $workspace
let cfg = op-load-config $ws_root
let ops_dir = op-ops-dir $ws_root $cfg
# ── 1. ops/ directory ────────────────────────────────────────────────────
if not ($ops_dir | path exists) {
mkdir $ops_dir
print $" [+] ($ops_dir | path relative-to $ws_root) created"
} else {
print $" [✓] ($ops_dir | path relative-to $ws_root) already exists"
}
# ── 2. jj init (skip if .jj/ already present) ───────────────────────────
let jj_dir = $ws_root | path join ".jj"
if not ($jj_dir | path exists) {
let r = do { ^jj git init --colocate $ws_root } | complete
if $r.exit_code != 0 {
op-err $"jj git init failed: ($r.stderr | str trim)"
}
print " [+] jj initialized"
} else {
print " [✓] jj already initialized"
}
# ── 3. Fix git HEAD (jj git init on empty repos points HEAD to refs/jj/root) ──
let head_file = $ws_root | path join ".git" "HEAD"
if ($head_file | path exists) {
let head = open --raw $head_file | str trim
if $head == "ref: refs/jj/root" {
let r = do { ^git -C $ws_root symbolic-ref HEAD refs/heads/main } | complete
if $r.exit_code == 0 {
print " [+] git HEAD fixed: refs/heads/main"
} else {
print $" (ansi yellow)warn(ansi reset): could not fix HEAD — ($r.stderr | str trim)"
}
}
}
# ── 4. Initial jj commit (rad init requires ≥1 git commit on the branch) ─
let git_log_r = do { ^git -C $ws_root log --oneline -1 } | complete
let git_has_commits = ($git_log_r.exit_code == 0 and ($git_log_r.stdout | str trim | is-not-empty))
if not $git_has_commits {
let r = do { cd $ws_root; ^jj commit --message "chore: initialize op governance" } | complete
if $r.exit_code == 0 {
let exp = do { cd $ws_root; ^jj git export } | complete
if $exp.exit_code == 0 {
print " [+] initial commit created and exported to git"
}
} else {
print $" (ansi yellow)warn(ansi reset): jj commit failed — ($r.stderr | str trim)"
}
} else {
print " [✓] git commits already exist"
}
# ── 5. Radicle init (skip if already registered — rad stores state in git config, not .rad/) ──
let rad_check = do { cd $ws_root; ^rad . } | complete
if $rad_check.exit_code != 0 {
let ws_name = $cfg.workspace? | default ($ws_root | path basename)
# Read description from .ontology/manifest.ncl if present.
let manifest_path = $ws_root | path join ".ontology" "manifest.ncl"
let ws_desc = if ($manifest_path | path exists) {
let m = ncl-eval-soft $manifest_path (default-ncl-paths $ws_root) {}
$m.description? | default ""
} else { "" }
let ws_desc_trunc = if ($ws_desc | str length) > 200 { $"($ws_desc | str substring 0..200)..." } else { $ws_desc }
let desc_args = if ($ws_desc_trunc | is-not-empty) { ["--description" $ws_desc_trunc] } else { [] }
let desc_hint = if ($ws_desc_trunc | is-not-empty) { $" --description \"($ws_desc_trunc | str substring 0..60)...\"" } else { "" }
# Detect locked keystore via node status before attempting rad init.
let node_r = do { ^rad node status } | complete
let node_stopped = ($node_r.stdout | str contains "stopped") or ($node_r.stderr | str contains "stopped")
if $node_stopped {
print ""
print $" (ansi red)[!](ansi reset) Radicle keystore is locked — node is not running."
print $" Run first: RAD_PASSPHRASE=<passphrase> rad node start"
print $" Then retry: just init"
print ""
return
}
# Try with --default-branch (rad ≥1.8); fall back without it for older builds.
let r1 = do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --default-branch main --no-confirm } | complete
let r1 = if ($r1.exit_code != 0 and not ($r1.stderr | str contains "Passphrase")) {
do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --default-branch main } | complete
} else { $r1 }
let r = if ($r1.exit_code != 0 and ($r1.stderr | str contains "unknown") and not ($r1.stderr | str contains "Passphrase")) {
do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --no-confirm } | complete
} else { $r1 }
if $r.exit_code == 0 {
let rid_r = do { cd $ws_root; ^rad . } | complete
let rid = if $rid_r.exit_code == 0 { $rid_r.stdout | str trim } else { "?" }
print $" [+] Radicle initialized — RID: ($rid)"
} else if ($r.stderr | str contains "Passphrase") {
print ""
print $" (ansi red)[!](ansi reset) Radicle keystore is locked."
print $" Run first: RAD_PASSPHRASE=<passphrase> rad node start"
print $" Then retry: just init"
print ""
return
} else {
let err_out = ([$r.stderr $r.stdout] | each { str trim } | where { is-not-empty } | str join " | ")
print $" (ansi yellow)warn(ansi reset): rad init failed — ($err_out)"
print $" Run: cd ($ws_root) && rad init --name ($ws_name)($desc_hint) --public --default-branch main"
}
} else {
let rid = $rad_check.stdout | str trim
print $" [✓] Radicle already initialized — RID: ($rid)"
}
# ── 6. .gitignore entries ────────────────────────────────────────────────
let gitignore = $ws_root | path join ".gitignore"
let gi_raw = if ($gitignore | path exists) { open --raw $gitignore } else { "" }
mut gi_additions = ""
if not ($gi_raw | str contains ".ops-archive/") {
$gi_additions = $gi_additions + "\n# ops restic/kopia archive repo (S3-backed, not committed)\n.ops-archive/\n"
}
if not ($gi_raw | str contains ".kopia-config") {
let kopia_rel = ($ops_dir | path relative-to $ws_root) | path join ".kopia-config"
let nl = (char nl)
$gi_additions = $gi_additions + $"($nl)# kopia per-workspace connection state($nl)($kopia_rel)($nl)"
}
if ($gi_additions | is-not-empty) {
$gi_additions | save --append $gitignore
print " [+] .gitignore updated (archive paths)"
} else {
print " [✓] .gitignore already covers archive paths"
}
# ── 7. Archive provider connect (kopia only — lazy for restic) ───────────
let tool = $cfg.ops?.archive?.tool? | default "restic"
if $tool == "kopia" {
let kopia_cfg = $ops_dir | path join ".kopia-config"
let status_r = do { ^kopia --config-file $kopia_cfg repository status } | complete
if $status_r.exit_code == 0 {
print " [✓] kopia repository already connected"
do { ^kopia --config-file $kopia_cfg policy set $ops_dir --keep-latest 50 --keep-monthly 12 --keep-annual 3 } | complete | ignore
} else {
print " [i] kopia: run 'kopia repository connect s3 ...' then re-run init to set retention policy"
}
}
let bucket = $cfg.ops?.archive?.bucket? | default "<bucket>"
let endpoint = $cfg.ops?.archive?.endpoint? | default "<endpoint>"
let prefix = $cfg.ops?.archive?.prefix? | default "ops"
let archive_hint = if $tool == "kopia" {
$"credentials loaded from infra secrets — connect: kopia repository connect s3 --bucket ($bucket) --endpoint ($endpoint) --prefix ($prefix)"
} else {
$"credentials loaded from infra secrets — repo: s3:($endpoint)/($bucket)/($prefix)"
}
print ""
print "op governance ready."
print $" archive ($tool) ($archive_hint)"
print " start an op : provisioning op start <component> <operation> <intent>"
}
def op-help []: nothing -> nothing {
print "Op Governance — workspace operation tracking"
print "============================================="
print ""
print "Usage: provisioning op <subcommand> [args]"
print ""
print "Subcommands:"
print " init Initialize jj + Radicle + ops/ in workspace"
print " start <component> <op> <intent> Begin an op — captures pre-snapshot, opens jj change"
print " finish <op-id> [status] Close an op — post-snapshot, archive (restic/kopia), jj commit"
print " log [component] Show op history"
print " show <op-id> Show op details + snapshot diff"
print " rollback <op-id> Create rollback op restoring pre-snapshot of target"
print ""
print "Flags:"
print " --workspace (-w) <path> Workspace root (default: auto-detect from cwd)"
print " --force (-f) Bypass concurrent op lock (start only)"
print " --debug (-x) Verbose errors"
print ""
print "Examples:"
print " provisioning op init"
print " provisioning op start postgresql install \"update to v16\""
print " provisioning op finish z6mk1234:abc... success"
print " provisioning op log postgresql"
print " provisioning op rollback z6mk1234:abc..."
}
# ── entry ─────────────────────────────────────────────────────────────────────
def main [
...args: string
--workspace (-w): string = ""
--force (-f)
--debug (-x)
]: nothing -> nothing {
if $debug { $env.PROVISIONING_DEBUG = "true" }
let rest = if (($args | length) > 0) and (($args | first) in ["op", "o"]) {
$args | skip 1
} else {
$args
}
let sub = $rest | get 0? | default ""
let tail = $rest | skip 1
match $sub {
"init" => { op-cmd-init $workspace }
"start" => { op-cmd-start $tail $workspace $force }
"finish" => { op-cmd-finish $tail $workspace }
"log" => { op-cmd-log $tail $workspace }
"show" => { op-cmd-show $tail $workspace }
"rollback" => { op-cmd-rollback $tail $workspace }
_ => { op-help }
}
}