#!/usr/bin/env nu # Op governance — workspace operation tracking with jj + Radicle + restic. # Each op is a DAG node: pre-snapshot → execution → post-snapshot. # Rollback creates a new forward op pointing to a previous snapshot. export-env { let lib_dirs_raw = ($env.NU_LIB_DIRS? | default "") let current_lib_dirs = if ($lib_dirs_raw | type) == "string" { if ($lib_dirs_raw | is-empty) { [] } else { ($lib_dirs_raw | split row ":") } } else { $lib_dirs_raw } let dynamic = ($env.PROVISIONING? | default "" | path join "core" "nulib") $env.NU_LIB_DIRS = ([ "/opt/provisioning/core/nulib" "/usr/local/provisioning/core/nulib" ] | append $current_lib_dirs | append (if ($dynamic | is-not-empty) { [$dynamic] } else { [] })) } use tools/nickel/process.nu [ncl-eval, ncl-eval-soft, default-ncl-paths] use primitives/io/logging.nu [is-debug-enabled] # ── helpers ────────────────────────────────────────────────────────────────── def op-err [msg: string]: nothing -> nothing { if (is-debug-enabled) { error make { msg: $msg } } else { error make --unspanned { msg: $msg } } } # Walk up from cwd looking for config/provisioning.ncl; fall back to cwd. def op-ws-root [explicit: string]: nothing -> string { if ($explicit | is-not-empty) { return $explicit } mut dir = $env.PWD loop { if ($dir | path join "config" "provisioning.ncl" | path exists) { return $dir } let parent = ($dir | path dirname) if $parent == $dir { break } $dir = $parent } $env.PWD } def op-load-config [ws_root: string]: nothing -> record { let cfg_path = $ws_root | path join "config" "provisioning.ncl" if not ($cfg_path | path exists) { op-err $"No config/provisioning.ncl in ($ws_root)" } let workspace_cfg = ncl-eval $cfg_path (default-ncl-paths $ws_root) # Merge ops config from infra settings.ncl — infra owns its archive/retention config. let infra = $workspace_cfg.current_infra? | default "" let infra_ops = if ($infra | is-not-empty) { let settings_path = $ws_root | path join "infra" $infra "settings.ncl" if ($settings_path | path exists) { let settings = ncl-eval-soft $settings_path (default-ncl-paths $ws_root) {} $settings.settings?.ops? | default {} } else { {} } } else { {} } if ($infra_ops | is-empty) { $workspace_cfg } else { $workspace_cfg | insert ops $infra_ops } } # Radicle NID when available; local:{user} fallback. def op-get-actor []: nothing -> record { let r = do { ^rad self --nid } | complete let user = $env.USER? | default "system" let nid = if $r.exit_code == 0 { $r.stdout | str trim } else { $"local:($user)" } { nid: $nid, identity: $user, source: "cli" } } # {nid-short}:{uuid} def op-gen-id [nid: string]: nothing -> string { let uuid_r = do { ^uuidgen } | complete let uuid = if $uuid_r.exit_code == 0 { $uuid_r.stdout | str trim | str downcase } else { random chars --length 32 } let short = if ($nid | str starts-with "local:") { $nid | str replace "local:" "" | str substring 0..6 } else { $nid | str substring 0..7 } $"($short):($uuid)" } def op-now []: nothing -> string { let r = do { ^date -u +"%Y-%m-%dT%H:%M:%SZ" } | complete if $r.exit_code == 0 { $r.stdout | str trim } else { "" } } # Resolve the ops directory for a workspace: infra/{current_infra}/ops/ # Falls back to ws_root/ops if cfg is not available (init path). def op-ops-dir [ws_root: string, cfg: record = {}]: nothing -> string { let infra = $cfg.current_infra? | default "" if ($infra | is-not-empty) { $ws_root | path join "infra" $infra "ops" } else { $ws_root | path join "infra" ( ls ($ws_root | path join "infra") | where type == "dir" | get name | each {|p| $p | path basename} | get 0? | default "" ) "ops" } } # Abort if a running op exists for this component (unless --force). def op-check-lock [ws_root: string, component: string]: nothing -> nothing { let ops_dir = op-ops-dir $ws_root if not ($ops_dir | path exists) { return } let running = ( ls $ops_dir | where type == "dir" | each {|e| let p = $e.name | path join "op.json" if ($p | path exists) { let d = open $p if ($d.status? == "running") and ($d.component? == $component) { $d.id? | default "?" } else { null } } else { null } } | compact ) if ($running | length) > 0 { op-err $"Concurrent op running for '($component)': ($running | str join ', '). Use --force to override." } } # Capture component config + server states for snapshot. def op-snapshot [ws_root: string, component: string, infra: string, targets: list]: nothing -> record { let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl" let comp_cfg = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {} let state_path = $ws_root | path join ".provisioning-state.json" let state_data = if ($state_path | path exists) { open $state_path } else { {} } { timestamp: (op-now), component: $component, infra: $infra, targets: $targets, component_config: $comp_cfg, server_states: ($targets | each {|t| { server: $t, state: ($state_data | get -o $t | default {}) } }), } } # Create a jj change and return the short change ID (empty string if jj unavailable). def op-jj-new [message: string]: nothing -> string { let r = do { ^jj new --message $message } | complete if $r.exit_code != 0 { return "" } let id_r = do { ^jj log --no-graph --template "change_id.short(8)" --revisions "@" } | complete if $id_r.exit_code == 0 { $id_r.stdout | str trim } else { "" } } def op-jj-commit [message: string]: nothing -> nothing { let r = do { ^jj commit --message $message } | complete if $r.exit_code != 0 { print $"(ansi yellow)warn(ansi reset): jj commit failed — ($r.stderr | str trim)" } } # Load the backup provider record from catalog/providers/backup/{tool}/nickel/provider.ncl. # Load archive credentials from infra/{current_infra}/secrets/ops.sops.yaml. # Uses the age key at infra/{current_infra}/.kage. # Always reads from SOPS — never from the caller's env — to avoid credential bleed. def op-load-archive-creds [ws_root: string, cfg: record]: nothing -> record { let infra = $cfg.current_infra? | default "" if ($infra | is-empty) { print $"(ansi yellow)warn(ansi reset): current_infra not set in config — cannot load archive credentials" return {} } let sops_path = $ws_root | path join "infra" $infra "secrets" "ops.sops.yaml" let kage_path = $ws_root | path join "infra" $infra ".kage" if not ($sops_path | path exists) { print $"(ansi yellow)warn(ansi reset): ops secrets not found at ($sops_path)" return {} } let r = if ($kage_path | path exists) { do { with-env { SOPS_AGE_KEY_FILE: $kage_path } { ^sops --decrypt $sops_path } } | complete } else { do { ^sops --decrypt $sops_path } | complete } if $r.exit_code != 0 { print $"(ansi yellow)warn(ansi reset): failed to decrypt ($sops_path) — ($r.stderr | str trim)" return {} } $r.stdout | from yaml } def op-load-archive-provider [tool: string]: nothing -> record { let prov = $env.PROVISIONING? | default "" if ($prov | is-empty) { print $"(ansi yellow)warn(ansi reset): PROVISIONING not set — cannot load archive provider" return {} } let ncl_path = $prov | path join "catalog" "providers" "backup" $tool "nickel" "provider.ncl" if not ($ncl_path | path exists) { print $"(ansi yellow)warn(ansi reset): backup provider not found: ($ncl_path)" return {} } let prov_root = $env.PROVISIONING? | default "" let data = ncl-eval-soft $ncl_path (default-ncl-paths $prov_root) {} $data.provider? | default {} } # Ensure required keys are present in the creds record loaded from SOPS. def op-archive-check-env [provider: record, creds: record]: nothing -> bool { let required = $provider.env?.required? | default [] let missing = $required | where {|v| ($creds | get -o $v | default "" | is-empty) } if ($missing | length) > 0 { print $"(ansi yellow)warn(ansi reset): archive provider '($provider.name?)' missing credentials: ($missing | str join ', ') — check infra/*/secrets/ops.sops.yaml" return false } true } # Connect provider to its repository if connection.required = true (kopia model). # Returns true if connected or not required; false on failure. def op-archive-connect [provider: record, ws_root: string, cfg: record, creds: record]: nothing -> bool { let conn = $provider.connection? | default {} if not ($conn.required? | default false) { return true } let bin = $provider.binary? | default "" let status_sub = $conn.status_subcmd? | default "" let connect_sub = $conn.connect_subcmd? | default "" let cfg_file = (op-ops-dir $ws_root $cfg) | path join ($conn.state_file? | default ".kopia-config") let cfg_args = ["--config-file" $cfg_file] if ($status_sub | is-not-empty) { let status_r = do { with-env $creds { ^$bin ...$cfg_args ...($status_sub | split words) } } | complete if $status_r.exit_code == 0 { return true } } if ($connect_sub | is-empty) { return false } let bucket = $cfg.ops?.archive?.bucket? | default "" let endpoint = $cfg.ops?.archive?.endpoint? | default "" let prefix = $cfg.ops?.archive?.prefix? | default "ops" let s3f = $conn.s3_flags? | default {} if ($bucket | is-empty) { print $"(ansi yellow)warn(ansi reset): ops.archive.bucket not set — cannot connect ($bin)" return false } let s3_args = ( [($s3f.bucket? | default "--bucket") $bucket] ++ (if ($endpoint | is-not-empty) { [($s3f.endpoint? | default "--endpoint") $endpoint] } else { [] }) ++ [($s3f.prefix? | default "--prefix") $prefix] ) let r = do { with-env $creds { ^$bin ...$cfg_args ...($connect_sub | split words) ...$s3_args } } | complete if $r.exit_code != 0 { print $"(ansi yellow)warn(ansi reset): ($bin) connect failed — ($r.stderr | str trim)" return false } true } # Execute archive backup using provider definition. Returns snapshot ID or "". def op-archive-backup [ op_dir: string tags: list workspace_tag: string retention: record provider: record ws_root: string cfg: record ]: nothing -> string { if ($provider | is-empty) { return "" } let creds = op-load-archive-creds $ws_root $cfg if ($creds | is-empty) { return "" } if not (op-archive-check-env $provider $creds) { return "" } if not (op-archive-connect $provider $ws_root $cfg $creds) { return "" } let bin = $provider.binary? | default "" let cmd = $provider.commands?.backup? | default {} let repo = $creds.RESTIC_REPOSITORY? | default "" let cfg_args = if ($provider.connection?.required? | default false) { let cfg_file = (op-ops-dir $ws_root $cfg) | path join ($provider.connection?.state_file? | default ".kopia-config") ["--config-file" $cfg_file] } else { [] } let repo_args = if ($cmd.repo_flag? | is-not-empty) and ($repo | is-not-empty) { [$cmd.repo_flag $repo] } else { [] } let tag_flag = $cmd.tag_flag? | default "" let tag_args = if ($tag_flag | is-not-empty) { $tags | each {|t| [$tag_flag $t] } | flatten } else { [] } let all_args = $cfg_args ++ ($cmd.subcmd | split words) ++ $repo_args ++ $tag_args ++ [$op_dir] let r = do { with-env $creds { ^$bin ...$all_args } } | complete if $r.exit_code != 0 { print $"(ansi yellow)warn(ansi reset): ($bin) backup failed — ($r.stderr | str trim)" return "" } let id_regex = $cmd.snapshot_id_regex? | default "" let snap_id = if ($id_regex | is-not-empty) { $r.stdout | lines | each {|l| $l | parse --regex $id_regex | get 0? | get id? | default "" } | where {|v| $v | is-not-empty } | get 0? | default "" } else { "" } let ops_dir = op-ops-dir $ws_root $cfg op-archive-forget $provider $op_dir $workspace_tag $retention $ops_dir $creds $snap_id } # Apply retention policy using provider definition (tag-scoped where supported). def op-archive-forget [ provider: record source: string workspace_tag: string retention: record ops_dir: string creds: record ]: nothing -> nothing { let bin = $provider.binary? | default "" let forget = $provider.commands?.forget? | default {} let keep_last = ($retention.keep_last? | default 50 | into string) let keep_monthly = ($retention.keep_monthly? | default 12 | into string) let keep_yearly = ($retention.keep_yearly? | default 3 | into string) let cfg_args = if ($provider.connection?.required? | default false) { let cfg_file = $ops_dir | path join ($provider.connection?.state_file? | default ".kopia-config") ["--config-file" $cfg_file] } else { [] } let policy_sub = $forget.policy_subcmd? | default "" if ($policy_sub | is-not-empty) { let kf = $forget.policy_keep_flags? | default {} let policy_args = ( $cfg_args ++ ($policy_sub | split words) ++ [$source] ++ [($kf.keep_latest? | default "--keep-latest") $keep_last] ++ [($kf.keep_monthly? | default "--keep-monthly") $keep_monthly] ++ [($kf.keep_annual? | default "--keep-annual") $keep_yearly] ) do { with-env $creds { ^$bin ...$policy_args } } | complete | ignore } let repo = $creds.RESTIC_REPOSITORY? | default "" let repo_args = if ($forget.repo_flag? | is-not-empty) and ($repo | is-not-empty) { [$forget.repo_flag $repo] } else { [] } let tag_args = if ($forget.tag_flag? | is-not-empty) and ($workspace_tag | is-not-empty) { [$forget.tag_flag $workspace_tag] } else { [] } let keep_args = if ($forget.keep_last_flag? | is-not-empty) { [($forget.keep_last_flag) $keep_last ($forget.keep_monthly_flag? | default "") $keep_monthly ($forget.keep_yearly_flag? | default "") $keep_yearly] | where {|v| $v | is-not-empty } } else { [] } let extra = $forget.extra_flags? | default [] let forget_args = $cfg_args ++ ($forget.subcmd | split words) ++ $repo_args ++ $tag_args ++ $keep_args ++ $extra do { with-env $creds { ^$bin ...$forget_args } } | complete | ignore } # Resolve target hostnames for a component. # Priority: component NCL target/targets field > enabled servers from servers.ncl. def op-load-targets [ws_root: string, component: string, infra: string]: nothing -> list { let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl" let comp = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {} let inner = $comp | get -o $component | default {} let targets_field = $inner | get -o targets | default [] let target_field = $inner | get -o target | default "" if ($targets_field | length) > 0 { return $targets_field } if ($target_field | is-not-empty) { return [$target_field] } let srv_ncl = $ws_root | path join "infra" $infra "servers.ncl" if not ($srv_ncl | path exists) { return [] } let srv_data = ncl-eval-soft $srv_ncl (default-ncl-paths $ws_root) { servers: [] } ($srv_data.servers? | default []) | where { |s| ($s.enabled? | default true) and not ($s.not_use? | default false) } | each { |s| $s.hostname? | default "" } | where { |h| $h | is-not-empty } } # ── constraint evaluator ────────────────────────────────────────────────────── # Load op_constraints from a component NCL definition (optional field). def op-load-component-constraints [ws_root: string, component: string, infra: string]: nothing -> record { let comp_ncl = $ws_root | path join "infra" $infra "components" $"($component).ncl" let comp = ncl-eval-soft $comp_ncl (default-ncl-paths $ws_root) {} $comp | get -o op_constraints | default { pre: [], on_failure: [] } } # Evaluate a single pre-constraint. Returns true on pass, false on fail. def op-eval-constraint [ws_root: string, constraint: record]: nothing -> bool { let kind = $constraint.kind? | default "" let resource = $constraint.resource? | default "" let prov = $env.PROVISIONING? | default "prvng" match $kind { "backup" => { print $" constraint: backup ($resource)" let r = do { ^$prov component backup $resource } | complete $r.exit_code == 0 } "health_check" => { print $" constraint: health_check ($resource)" let r = do { ^$prov component health $resource } | complete $r.exit_code == 0 } "dry_run_check" => { print $" constraint: dry_run_check ($resource)" let r = do { ^$prov component install $resource --check } | complete $r.exit_code == 0 } "concurrent_lock" => { op-check-lock $ws_root $resource true } _ => { print $" constraint: unknown kind '($kind)' — skipping" true } } } # Evaluate a single recovery action (on_failure). def op-eval-recovery-action [ws_root: string, action: record]: nothing -> nothing { let kind = $action.kind? | default "" let resource = $action.resource? | default "" let prov = $env.PROVISIONING? | default "prvng" match $kind { "restore" => { print $" recovery: restore ($resource)" let r = do { ^$prov component restore $resource } | complete if $r.exit_code != 0 { print $" (ansi red)recovery failed(ansi reset): ($r.stderr | str trim)" } } "backup" => { print $" recovery: backup ($resource) — post-failure snapshot" do { ^$prov component backup $resource } | complete | ignore } _ => { print $" recovery: unknown kind '($kind)' — skipping" } } } # Evaluate all pre-constraints. Fails the op if any constraint fails. def op-eval-pre [ws_root: string, constraints: list]: nothing -> nothing { if ($constraints | length) == 0 { return } print $"evaluating ($constraints | length) pre-constraint(s)..." for c in $constraints { let passed = op-eval-constraint $ws_root $c if not $passed { let kind = $c.kind? | default "unknown" let resource = $c.resource? | default "" op-err $"Pre-constraint failed: ($kind) on ($resource)" } } } # Run all on_failure recovery actions. def op-eval-recovery [ws_root: string, actions: list]: nothing -> nothing { if ($actions | length) == 0 { return } print $"running ($actions | length) recovery action(s)..." for a in $actions { op-eval-recovery-action $ws_root $a } } # Publish committed op to Radicle network (best-effort, skipped if rad unavailable). def op-rad-sync []: nothing -> string { let r = do { ^rad sync } | complete if $r.exit_code == 0 { let rid_r = do { ^rad self --rid } | complete if $rid_r.exit_code == 0 { $rid_r.stdout | str trim } else { "" } } else { let err = $r.stderr | str trim let silent = ( ($err | str contains "not a Radicle repository") or ($err | str contains "node is not running") or ($err | str contains "connection refused") or ($err | str contains "failed to connect") or ($err | str contains "not connected") ) if not $silent and (is-debug-enabled) { print $"(ansi yellow)warn(ansi reset): rad sync failed — ($err)" } "" } } # ── subcommands ─────────────────────────────────────────────────────────────── def op-cmd-start [ args: list workspace: string force: bool ]: nothing -> nothing { let component = $args | get 0? | default "" let operation = $args | get 1? | default "" let intent = $args | get 2? | default "" if ($component | is-empty) { op-err "Usage: provisioning op start " } if ($operation | is-empty) { op-err "Usage: provisioning op start " } if ($intent | is-empty) { op-err "Usage: provisioning op start " } let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default") if not $force { op-check-lock $ws_root $component } # Load constraints from component definition (op_constraints field, if present) let comp_constraints = op-load-component-constraints $ws_root $component $infra let pre_constraints = $comp_constraints.pre? | default [] # Evaluate pre-constraints before doing anything else # On failure this exits with error — no op record is created op-eval-pre $ws_root $pre_constraints let actor = op-get-actor let op_id = op-gen-id $actor.nid let ops_dir = op-ops-dir $ws_root $cfg let op_dir = $ops_dir | path join $op_id mkdir $op_dir let started_at = op-now let targets = op-load-targets $ws_root $component $infra # Pre-snapshot (taken after pre-constraints pass — any backup is already done) let snap = op-snapshot $ws_root $component $infra $targets let pre_rel = $"infra/($infra)/ops/($op_id)/pre.json" $snap | to json --indent 2 | save --force ($op_dir | path join "pre.json") # jj change let jj_change = op-jj-new $"op:($op_id) ($operation) ($component)" # Op record { id: $op_id, actor: $actor, intent: $intent, workspace: ($cfg.workspace? | default ""), component: $component, operation: $operation, targets: $targets, constraints: $comp_constraints, snapshots: { pre: $pre_rel, post: null }, artifacts: { archive_snapshot: null, bundles: [] }, lineage: { parent_op: null, rollback_of: null }, status: "running", jj_change: (if ($jj_change | is-empty) { null } else { $jj_change }), radicle_rid: null, started_at: $started_at, ended_at: null, } | to json --indent 2 | save --force ($op_dir | path join "op.json") print $"op started ($op_id)" print $" component : ($component) operation: ($operation)" print $" workspace : ($ws_root)" if ($jj_change | is-not-empty) { print $" jj change : ($jj_change)" } print $" intent : ($intent)" } def op-cmd-finish [ args: list workspace: string ]: nothing -> nothing { let op_id = $args | get 0? | default "" let status = $args | get 1? | default "success" if ($op_id | is-empty) { op-err "Usage: provisioning op finish [success|failed|cancelled]" } if $status not-in ["success" "failed" "cancelled"] { op-err $"Invalid status '($status)' — must be: success, failed, cancelled" } let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default") let op_dir = (op-ops-dir $ws_root $cfg) | path join $op_id let op_file = $op_dir | path join "op.json" if not ($op_file | path exists) { op-err $"Op not found: ($op_id)" } let rec = open $op_file let ended_at = op-now # On failure: run on_failure recovery actions before anything else if $status == "failed" { let recovery = $rec.constraints? | default {} | get -o on_failure | default [] op-eval-recovery $ws_root $recovery } # Post-snapshot (only on success) let post_rel = if $status == "success" { let snap = op-snapshot $ws_root $rec.component $infra ($rec.targets? | default []) $snap | to json --indent 2 | save --force ($op_dir | path join "post.json") $"infra/($infra)/ops/($op_id)/post.json" } else { null } # archive backup via provider let workspace_name = $cfg.workspace? | default "" let retention = $cfg.ops?.retention? | default { keep_last: 50, keep_monthly: 12, keep_yearly: 3 } let workspace_tag = $"workspace=($workspace_name)" let tool = $cfg.ops?.archive?.tool? | default "restic" # ops.archive.enabled = false skips op-archive entirely (empty provider → # op-archive-backup returns "" with no warning). Default true is backward # compatible. Use to turn off archival when no archive target is configured. let provider = if ($cfg.ops?.archive?.enabled? | default true) { op-load-archive-provider $tool } else { {} } let tags = [ $"op-id=($op_id)", $"author=($rec.actor.identity)", $"nid=($rec.actor.nid | str substring 0..12)", $"component=($rec.component)", $"operation=($rec.operation)", $workspace_tag, $"status=($status)", ] let snap_id = op-archive-backup $op_dir $tags $workspace_tag $retention $provider $ws_root $cfg # Finalize op record let updated = $rec | merge { status: $status, ended_at: $ended_at, snapshots: ($rec.snapshots | merge { post: $post_rel }), artifacts: ($rec.artifacts | merge { archive_snapshot: (if ($snap_id | is-empty) { null } else { $snap_id }) }), } $updated | to json --indent 2 | save --force $op_file # jj commit — only when jj was active at op-start (jj_change non-empty) let icon = if $status == "success" { "✓" } else { "✗" } if ($rec.jj_change? | is-not-empty) { op-jj-commit $"op:($op_id) ($rec.operation) ($rec.component) ($icon)" } # Radicle sync (best-effort) let rad_rid = op-rad-sync if ($rad_rid | is-not-empty) { let final = (open $op_file) | merge { radicle_rid: $rad_rid } $final | to json --indent 2 | save --force $op_file print $" radicle : ($rad_rid)" } print $"op finished ($op_id) status: ($status)" if ($snap_id | is-not-empty) { print $" archive : ($snap_id)" } } def op-cmd-log [ args: list workspace: string ]: nothing -> nothing { let component_filter = $args | get 0? | default "" let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let ops_dir = op-ops-dir $ws_root $cfg if not ($ops_dir | path exists) { print "No ops recorded yet. Run: provisioning op start " return } let records = ( ls $ops_dir | where type == "dir" | sort-by modified --reverse | each {|e| let p = $e.name | path join "op.json" if ($p | path exists) { open $p } else { null } } | compact | where { |r| if ($component_filter | is-empty) { true } else { $r.component? == $component_filter } } ) if ($records | length) == 0 { let suffix = if ($component_filter | is-not-empty) { $" for component '($component_filter)'" } else { "" } print $"No ops found($suffix)." return } $records | select id component operation status started_at ended_at | table } # Field-level diff between two records — shows added/removed/changed keys with values. def op-diff-records [pre: record, post: record]: nothing -> nothing { let all_keys = (($pre | columns) ++ ($post | columns)) | uniq | sort let changes = ($all_keys | each {|k| let pv = $pre | get -o $k let nv = $post | get -o $k if $pv == null and $nv != null { { change: "added", field: $k, before: "—", after: ($nv | to nuon) } } else if $pv != null and $nv == null { { change: "removed", field: $k, before: ($pv | to nuon), after: "—" } } else if $pv != $nv { { change: "changed", field: $k, before: ($pv | to nuon), after: ($nv | to nuon) } } else { null } } | compact) if ($changes | length) == 0 { print " (no changes)" } else { for c in $changes { print $" ($c.change) ($c.field) ($c.before) → ($c.after)" } } } def op-cmd-show [ args: list workspace: string ]: nothing -> nothing { let op_id = $args | get 0? | default "" if ($op_id | is-empty) { op-err "Usage: provisioning op show " } let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let op_file = (op-ops-dir $ws_root $cfg) | path join $op_id "op.json" if not ($op_file | path exists) { op-err $"Op not found: ($op_id)" } let rec = open $op_file print $"id : ($rec.id)" print $"component : ($rec.component) operation: ($rec.operation)" print $"intent : ($rec.intent)" print $"actor : ($rec.actor.identity) nid: ($rec.actor.nid)" print $"status : ($rec.status)" print $"started : ($rec.started_at)" print $"ended : ($rec.ended_at? | default "-")" if ($rec.jj_change? | is-not-empty) { print $"jj change : ($rec.jj_change)" } if ($rec.radicle_rid? | is-not-empty) { print $"radicle : ($rec.radicle_rid)" } if ($rec.artifacts.archive_snapshot? | is-not-empty) { print $"archive : ($rec.artifacts.archive_snapshot)" } if ($rec.targets? | default [] | length) > 0 { print $"targets : ($rec.targets | str join ', ')" } let lineage = $rec.lineage? | default {} if ($lineage.parent_op? | is-not-empty) { print $"parent : ($lineage.parent_op)" } if ($lineage.rollback_of? | is-not-empty) { print $"rollback : undoes ($lineage.rollback_of)" } let pre_path = $ws_root | path join ($rec.snapshots.pre) let post_path = if ($rec.snapshots.post? | is-not-empty) { $ws_root | path join $rec.snapshots.post } else { "" } if ($pre_path | path exists) and ($post_path | is-not-empty) and ($post_path | path exists) { print "" print "── snapshot diff — component_config ───────────" let pre = open $pre_path | get component_config? | default {} let post = open $post_path | get component_config? | default {} op-diff-records $pre $post print "" print "── snapshot diff — server_states ──────────────" let pre_states = (open $pre_path | get server_states? | default [] | each {|s| {($s.server): $s.state} } | reduce -f {} {|a, b| $a | merge $b }) let post_states = (open $post_path | get server_states? | default [] | each {|s| {($s.server): $s.state} } | reduce -f {} {|a, b| $a | merge $b }) op-diff-records $pre_states $post_states } else if ($pre_path | path exists) { print "" print "── pre-snapshot (no post — op in progress or failed) ─" let pre = open $pre_path | get component_config? | default {} print $" component_config keys: ($pre | columns | str join ', ')" } } def op-cmd-rollback [ args: list workspace: string ]: nothing -> nothing { let target_id = $args | get 0? | default "" if ($target_id | is-empty) { op-err "Usage: provisioning op rollback " } let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default") let ops_dir = op-ops-dir $ws_root $cfg let op_file = $ops_dir | path join $target_id "op.json" if not ($op_file | path exists) { op-err $"Op not found: ($target_id)" } let target_rec = open $op_file let pre_path = $ws_root | path join ($target_rec.snapshots.pre) if not ($pre_path | path exists) { op-err $"Pre-snapshot missing for op ($target_id)" } let actor = op-get-actor let op_id = op-gen-id $actor.nid let op_dir = $ops_dir | path join $op_id mkdir $op_dir let started_at = op-now let pre_rel = $"infra/($infra)/ops/($op_id)/pre.json" # The rollback's pre-snapshot is the current state (before restoring) let cfg = op-load-config $ws_root let infra = $cfg.current_infra? | default ($cfg.workspace? | default "default") let snap = op-snapshot $ws_root $target_rec.component $infra ($target_rec.targets? | default []) $snap | to json --indent 2 | save --force ($op_dir | path join "pre.json") # Restore: copy target op's pre.json as the new desired state for the operator to apply let restore_path = $op_dir | path join "restore-target.json" cp $pre_path $restore_path let jj_change = op-jj-new $"op:($op_id) rollback ($target_rec.component) (undoes ($target_id))" { id: $op_id, actor: $actor, intent: $"Rollback ($target_rec.component) to state before op ($target_id)", workspace: ($cfg.workspace? | default ""), component: $target_rec.component, operation: "rollback", targets: ($target_rec.targets? | default []), constraints: { pre: [], on_failure: [] }, snapshots: { pre: $pre_rel, post: null }, artifacts: { archive_snapshot: null, bundles: [] }, lineage: { parent_op: $target_id, rollback_of: $target_id }, status: "running", jj_change: (if ($jj_change | is-empty) { null } else { $jj_change }), radicle_rid: null, started_at: $started_at, ended_at: null, } | to json --indent 2 | save --force ($op_dir | path join "op.json") print $"rollback op started ($op_id)" print $" restoring : ($target_rec.component) to snapshot from op ($target_id)" print $" target : ($restore_path)" print $" run 'provisioning component ($target_rec.operation) ($target_rec.component)' to apply" print $" then run : provisioning op finish ($op_id) success" } def op-cmd-init [workspace: string]: nothing -> nothing { let ws_root = op-ws-root $workspace let cfg = op-load-config $ws_root let ops_dir = op-ops-dir $ws_root $cfg # ── 1. ops/ directory ──────────────────────────────────────────────────── if not ($ops_dir | path exists) { mkdir $ops_dir print $" [+] ($ops_dir | path relative-to $ws_root) created" } else { print $" [✓] ($ops_dir | path relative-to $ws_root) already exists" } # ── 2. jj init (skip if .jj/ already present) ─────────────────────────── let jj_dir = $ws_root | path join ".jj" if not ($jj_dir | path exists) { let r = do { ^jj git init --colocate $ws_root } | complete if $r.exit_code != 0 { op-err $"jj git init failed: ($r.stderr | str trim)" } print " [+] jj initialized" } else { print " [✓] jj already initialized" } # ── 3. Fix git HEAD (jj git init on empty repos points HEAD to refs/jj/root) ── let head_file = $ws_root | path join ".git" "HEAD" if ($head_file | path exists) { let head = open --raw $head_file | str trim if $head == "ref: refs/jj/root" { let r = do { ^git -C $ws_root symbolic-ref HEAD refs/heads/main } | complete if $r.exit_code == 0 { print " [+] git HEAD fixed: refs/heads/main" } else { print $" (ansi yellow)warn(ansi reset): could not fix HEAD — ($r.stderr | str trim)" } } } # ── 4. Initial jj commit (rad init requires ≥1 git commit on the branch) ─ let git_log_r = do { ^git -C $ws_root log --oneline -1 } | complete let git_has_commits = ($git_log_r.exit_code == 0 and ($git_log_r.stdout | str trim | is-not-empty)) if not $git_has_commits { let r = do { cd $ws_root; ^jj commit --message "chore: initialize op governance" } | complete if $r.exit_code == 0 { let exp = do { cd $ws_root; ^jj git export } | complete if $exp.exit_code == 0 { print " [+] initial commit created and exported to git" } } else { print $" (ansi yellow)warn(ansi reset): jj commit failed — ($r.stderr | str trim)" } } else { print " [✓] git commits already exist" } # ── 5. Radicle init (skip if already registered — rad stores state in git config, not .rad/) ── let rad_check = do { cd $ws_root; ^rad . } | complete if $rad_check.exit_code != 0 { let ws_name = $cfg.workspace? | default ($ws_root | path basename) # Read description from .ontology/manifest.ncl if present. let manifest_path = $ws_root | path join ".ontology" "manifest.ncl" let ws_desc = if ($manifest_path | path exists) { let m = ncl-eval-soft $manifest_path (default-ncl-paths $ws_root) {} $m.description? | default "" } else { "" } let ws_desc_trunc = if ($ws_desc | str length) > 200 { $"($ws_desc | str substring 0..200)..." } else { $ws_desc } let desc_args = if ($ws_desc_trunc | is-not-empty) { ["--description" $ws_desc_trunc] } else { [] } let desc_hint = if ($ws_desc_trunc | is-not-empty) { $" --description \"($ws_desc_trunc | str substring 0..60)...\"" } else { "" } # Detect locked keystore via node status before attempting rad init. let node_r = do { ^rad node status } | complete let node_stopped = ($node_r.stdout | str contains "stopped") or ($node_r.stderr | str contains "stopped") if $node_stopped { print "" print $" (ansi red)[!](ansi reset) Radicle keystore is locked — node is not running." print $" Run first: RAD_PASSPHRASE= rad node start" print $" Then retry: just init" print "" return } # Try with --default-branch (rad ≥1.8); fall back without it for older builds. let r1 = do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --default-branch main --no-confirm } | complete let r1 = if ($r1.exit_code != 0 and not ($r1.stderr | str contains "Passphrase")) { do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --default-branch main } | complete } else { $r1 } let r = if ($r1.exit_code != 0 and ($r1.stderr | str contains "unknown") and not ($r1.stderr | str contains "Passphrase")) { do { cd $ws_root; ^rad init --name $ws_name ...$desc_args --public --no-confirm } | complete } else { $r1 } if $r.exit_code == 0 { let rid_r = do { cd $ws_root; ^rad . } | complete let rid = if $rid_r.exit_code == 0 { $rid_r.stdout | str trim } else { "?" } print $" [+] Radicle initialized — RID: ($rid)" } else if ($r.stderr | str contains "Passphrase") { print "" print $" (ansi red)[!](ansi reset) Radicle keystore is locked." print $" Run first: RAD_PASSPHRASE= rad node start" print $" Then retry: just init" print "" return } else { let err_out = ([$r.stderr $r.stdout] | each { str trim } | where { is-not-empty } | str join " | ") print $" (ansi yellow)warn(ansi reset): rad init failed — ($err_out)" print $" Run: cd ($ws_root) && rad init --name ($ws_name)($desc_hint) --public --default-branch main" } } else { let rid = $rad_check.stdout | str trim print $" [✓] Radicle already initialized — RID: ($rid)" } # ── 6. .gitignore entries ──────────────────────────────────────────────── let gitignore = $ws_root | path join ".gitignore" let gi_raw = if ($gitignore | path exists) { open --raw $gitignore } else { "" } mut gi_additions = "" if not ($gi_raw | str contains ".ops-archive/") { $gi_additions = $gi_additions + "\n# ops restic/kopia archive repo (S3-backed, not committed)\n.ops-archive/\n" } if not ($gi_raw | str contains ".kopia-config") { let kopia_rel = ($ops_dir | path relative-to $ws_root) | path join ".kopia-config" let nl = (char nl) $gi_additions = $gi_additions + $"($nl)# kopia per-workspace connection state($nl)($kopia_rel)($nl)" } if ($gi_additions | is-not-empty) { $gi_additions | save --append $gitignore print " [+] .gitignore updated (archive paths)" } else { print " [✓] .gitignore already covers archive paths" } # ── 7. Archive provider connect (kopia only — lazy for restic) ─────────── let tool = $cfg.ops?.archive?.tool? | default "restic" if $tool == "kopia" { let kopia_cfg = $ops_dir | path join ".kopia-config" let status_r = do { ^kopia --config-file $kopia_cfg repository status } | complete if $status_r.exit_code == 0 { print " [✓] kopia repository already connected" do { ^kopia --config-file $kopia_cfg policy set $ops_dir --keep-latest 50 --keep-monthly 12 --keep-annual 3 } | complete | ignore } else { print " [i] kopia: run 'kopia repository connect s3 ...' then re-run init to set retention policy" } } let bucket = $cfg.ops?.archive?.bucket? | default "" let endpoint = $cfg.ops?.archive?.endpoint? | default "" let prefix = $cfg.ops?.archive?.prefix? | default "ops" let archive_hint = if $tool == "kopia" { $"credentials loaded from infra secrets — connect: kopia repository connect s3 --bucket ($bucket) --endpoint ($endpoint) --prefix ($prefix)" } else { $"credentials loaded from infra secrets — repo: s3:($endpoint)/($bucket)/($prefix)" } print "" print "op governance ready." print $" archive ($tool) ($archive_hint)" print " start an op : provisioning op start " } def op-help []: nothing -> nothing { print "Op Governance — workspace operation tracking" print "=============================================" print "" print "Usage: provisioning op [args]" print "" print "Subcommands:" print " init Initialize jj + Radicle + ops/ in workspace" print " start Begin an op — captures pre-snapshot, opens jj change" print " finish [status] Close an op — post-snapshot, archive (restic/kopia), jj commit" print " log [component] Show op history" print " show Show op details + snapshot diff" print " rollback Create rollback op restoring pre-snapshot of target" print "" print "Flags:" print " --workspace (-w) Workspace root (default: auto-detect from cwd)" print " --force (-f) Bypass concurrent op lock (start only)" print " --debug (-x) Verbose errors" print "" print "Examples:" print " provisioning op init" print " provisioning op start postgresql install \"update to v16\"" print " provisioning op finish z6mk1234:abc... success" print " provisioning op log postgresql" print " provisioning op rollback z6mk1234:abc..." } # ── entry ───────────────────────────────────────────────────────────────────── def main [ ...args: string --workspace (-w): string = "" --force (-f) --debug (-x) ]: nothing -> nothing { if $debug { $env.PROVISIONING_DEBUG = "true" } let rest = if (($args | length) > 0) and (($args | first) in ["op", "o"]) { $args | skip 1 } else { $args } let sub = $rest | get 0? | default "" let tail = $rest | skip 1 match $sub { "init" => { op-cmd-init $workspace } "start" => { op-cmd-start $tail $workspace $force } "finish" => { op-cmd-finish $tail $workspace } "log" => { op-cmd-log $tail $workspace } "show" => { op-cmd-show $tail $workspace } "rollback" => { op-cmd-rollback $tail $workspace } _ => { op-help } } }