prvng_core/nulib/taskservs/run.nu

use std
use ../lib_provisioning/config/accessor.nu *
use ../lib_provisioning/utils/logging.nu [is-debug-enabled]
use ../lib_provisioning/utils/error.nu [throw-error]
use ../lib_provisioning/utils/ssh.nu [scp_to, ssh_cmd]
#use utils.nu taskserv_get_file
#use utils/templates.nu on_template_path

def make_cmd_env_temp [
  defs: record
  taskserv_env_path: string
  wk_vars: string
] {
  let tmp_sh      = (mktemp --tmpdir-path $taskserv_env_path --suffix ".sh")
  let cmd_env_temp = ($taskserv_env_path | path join $"cmd_env_(($tmp_sh | path basename))")
  mv $tmp_sh $cmd_env_temp  # rename the mktemp file; avoids leaving tmp.*.sh side-effect
  let nu_lib_dirs = ($env.NU_LIB_DIRS? | default [] | if ($in | describe) == "string" { $in | split row ":" } else { $in } | str join ":")
  ($"export PROVISIONING_VARS=($wk_vars)\nexport PROVISIONING_DEBUG=((is-debug-enabled))\n" +
    $"export NU_LIB_DIRS=($nu_lib_dirs)\n" +
    $"export NU_LOG_LEVEL=($env.NU_LOG_LEVEL? | default '')\n" +
    $"export PROVISIONING_RESOURCES=((get-provisioning-resources))\n" +
    $"export PROVISIONING_SETTINGS_SRC=($defs.settings.src)\nexport PROVISIONING_SETTINGS_SRC_PATH=($defs.settings.src_path)\n" +
    $"export PROVISIONING_KLOUD=($defs.settings | get -o infra | default ($defs.settings.infra_path | path basename))\nexport PROVISIONING_KLOUD_PATH=($defs.settings.infra_path)\n" +
    $"export PROVISIONING_USE_SOPS=((get-provisioning-use-sops))\nexport PROVISIONING_WK_ENV_PATH=($taskserv_env_path)\n" +
    $"export PROVISIONING_WORKSPACES=($defs.settings.src_path | path dirname)\nexport CURRENT_WORKSPACE=($defs.settings.src_path | path basename)\n" +
    $"export SOPS_AGE_KEY_FILE=($env.SOPS_AGE_KEY_FILE? | default '')\nexport PROVISIONING_KAGE=($env.PROVISIONING_KAGE? | default '')\n" +
    $"export SOPS_AGE_RECIPIENTS=($env.SOPS_AGE_RECIPIENTS? | default '')\n"
  ) | save --force $cmd_env_temp
  if (is-debug-enabled) { _print $"cmd_env_temp: ($cmd_env_temp)" }
  $cmd_env_temp
}
def run_cmd [
  cmd_name: string
  title: string
  where: string
  defs: record
  taskserv_env_path: string
  wk_vars: string
] {
  _print (
    $"($title) for (_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " +
    $"($defs.server.hostname) ($defs.pos.server) ..."
  )
  let runner = (grep "^#!" ($taskserv_env_path | path join $cmd_name) | str trim)
  let run_ops = if (is-debug-enabled) { if ($runner | str contains "bash" ) { "-x" } else { "" } } else { "" }
  let cmd_run_file = make_cmd_env_temp $defs $taskserv_env_path $wk_vars
  if ($cmd_run_file | path exists) and ($wk_vars | path exists) {
    if ($runner | str ends-with "bash" ) {
        $"($run_ops) ($taskserv_env_path | path join $cmd_name) ($wk_vars) ($defs.pos.server) ($defs.pos.taskserv) (^pwd)" | save --append $cmd_run_file
    } else if ($runner | str ends-with "nu" ) {
        $"($env.NU) ($env.NU_ARGS? | default '') ($taskserv_env_path | path join $cmd_name)"  | save --append $cmd_run_file
    } else {
        $"($taskserv_env_path | path join $cmd_name) ($wk_vars)" | save --append $cmd_run_file
    }
    let run_res = (^bash $cmd_run_file | complete)
    if $run_res.exit_code != 0 {
      (throw-error $"🛑 Error server ($defs.server.hostname) taskserv ($defs.taskserv.name)
        ($taskserv_env_path)/($cmd_name) with ($wk_vars) ($defs.pos.server) ($defs.pos.taskserv) (^pwd)"
        $"($run_res.stdout)\n($run_res.stderr)\n"
        $where --span (metadata $run_res).span)
      exit 1
    }
    if (is-debug-enabled) {
      if ($run_res.stdout | is-not-empty) { _print $"($run_res.stdout)" }
      if ($run_res.stderr | is-not-empty) { _print $"($run_res.stderr)" }
    } else {
      rm -f $cmd_run_file
      rm -f ($taskserv_env_path | path join "prepare")
    }
  }
}
export def run_taskserv_library [
    defs: record
    taskserv_path: string
    taskserv_env_path: string
    wk_vars: string
] {

  if not ($taskserv_path | path exists) { return false }
  let prov_resources_path = ($defs.settings.data | get -o prov_resources_path | default "" | str replace  "~" $env.HOME)
  let taskserv_server_name = $defs.server.hostname
  rm -rf ...(glob ($taskserv_env_path | path join "*.ncl")) ($taskserv_env_path |path join "nickel")
  mkdir ($taskserv_env_path | path join "nickel")

  let err_out = ($taskserv_env_path | path join (mktemp --tmpdir-path $taskserv_env_path --suffix ".err" | path basename))
  let nickel_temp = ($taskserv_env_path | path join "nickel"| path join (mktemp --tmpdir-path $taskserv_env_path --suffix ".ncl" | path basename))

  let wk_format = if (get-provisioning-wk-format) == "json" { "json" } else { "yaml" }
  let taskserv_settings = ($defs.settings.data | get -o taskservs | default {} | get -o $defs.taskserv.name | default {})
  # merge order: static schema settings overlay defs defaults, but runtime-set fields
  # (cmd_task, profile) from $defs.taskserv must always win — they carry handler intent
  # (e.g. cmd_task="reinstall" for reset ops) that the schema default would overwrite.
  let merged_taskserv   = ($defs.taskserv | merge $taskserv_settings)
  let runtime_overrides = ($defs.taskserv | select -o cmd_task profile)
  let wk_data = {
    defs: $defs.settings.data,
    pos: $defs.pos,
    server: $defs.server,
    taskserv: ($merged_taskserv | merge $runtime_overrides)
  }
  if $wk_format == "json" {
    $wk_data | to json | save --force $wk_vars
  } else {
    $wk_data | to yaml | save --force $wk_vars
  }
  # Pre-compute Nickel template paths so we can gate the import on actual file presence
  let nickel_taskserv_path = if ($taskserv_path | path join "nickel" | path join $"($defs.taskserv.name).ncl" | path exists) {
    ($taskserv_path | path join "nickel" | path join $"($defs.taskserv.name).ncl")
  } else if ($taskserv_path | path dirname | path join "nickel" | path join $"($defs.taskserv.name).ncl" | path exists) {
    ($taskserv_path | path dirname | path join "nickel" | path join $"($defs.taskserv.name).ncl")
  } else if ($taskserv_path | path dirname | path join "default" | path join "nickel" | path join $"($defs.taskserv.name).ncl" | path exists) {
    ($taskserv_path | path dirname | path join "default" | path join "nickel" | path join $"($defs.taskserv.name).ncl")
  } else { "" }
  let nickel_taskserv_profile_path = if ($taskserv_path | path join "nickel" | path join $"($defs.taskserv.profile).ncl" | path exists) {
    ($taskserv_path | path join "nickel" | path join $"($defs.taskserv.profile).ncl")
  } else if ($taskserv_path | path dirname | path join "nickel" | path join $"($defs.taskserv.profile).ncl" | path exists) {
    ($taskserv_path | path dirname | path join "nickel" | path join $"($defs.taskserv.profile).ncl")
  } else if ($taskserv_path | path dirname | path join "default" | path join "nickel" | path join $"($defs.taskserv.profile).ncl" | path exists) {
    ($taskserv_path | path dirname | path join "default" | path join "nickel" | path join $"($defs.taskserv.profile).ncl")
  } else { "" }
  let has_ncl_files = ($nickel_taskserv_path != "" or $nickel_taskserv_profile_path != "")

  if (get-use-nickel) and $has_ncl_files {
    if (which nickel | is-empty) {
      _print $"❗(_ansi red_bold)nickel binary not found(_ansi reset) — install nickel or set PROVISIONING_USE_NICKEL=false to skip"
      return false
    }
    cd $defs.settings.infra_path
    if ($nickel_temp | path exists) { rm -f $nickel_temp }
    let res = (^nickel import -m $wk_format $wk_vars -o $nickel_temp | complete)
    if $res.exit_code != 0 {
      _print $"❗Nickel import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found "
      _print $res.stdout
      rm -f $nickel_temp
      cd $env.PWD
      return false
    }
    # Very important! Remove external block for import and re-format it
    open $nickel_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $nickel_temp
    let res = (^nickel fmt $nickel_temp | complete)
    if $nickel_taskserv_path != "" and ($nickel_taskserv_path | path exists) {
      if (is-debug-enabled) {
        _print $"adding task name: ($defs.taskserv.name) ->  ($nickel_taskserv_path)"
      }
      cat $nickel_taskserv_path | save --append $nickel_temp
    }
    if $nickel_taskserv_profile_path != "" and ($nickel_taskserv_profile_path | path exists) {
      if (is-debug-enabled) {
        _print $"adding task profile: ($defs.taskserv.profile) ->  ($nickel_taskserv_profile_path)"
      }
      cat $nickel_taskserv_profile_path | save --append $nickel_temp
    }
    let keys_path_config = (get-keys-path)
    if $keys_path_config != "" {
      #use sops on_sops
      let keys_path = ($defs.settings.src_path | path join $keys_path_config)
      if not ($keys_path | path exists) {
        if (is-debug-enabled) {
          _print $"❗Error KEYS_PATH (_ansi red_bold)($keys_path)(_ansi reset) found "
        } else {
          _print $"❗Error (_ansi red_bold)KEYS_PATH(_ansi reset) not found "
        }
        return false
      }
      (on_sops d $keys_path) | save --append $nickel_temp
      let nickel_defined_taskserv_path = if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl")
      } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname |  path join $"($defs.taskserv.profile).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl")
      } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).ncl")
      } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).ncl")
      } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile |  path join $"($defs.taskserv.name).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile |  path join $"($defs.taskserv.name).ncl")
      } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).ncl" | path exists ) {
        ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).ncl")
      } else { "" }
      if $nickel_defined_taskserv_path != "" and ($nickel_defined_taskserv_path | path exists) {
        if (is-debug-enabled) {
          _print $"adding defs taskserv: ($nickel_defined_taskserv_path)"
        }
        cat $nickel_defined_taskserv_path | save --append $nickel_temp
      }
      let res = (^nickel $nickel_temp -o $wk_vars | complete)
      if $res.exit_code != 0 {
        _print $"❗Nickel errors (_ansi red_bold)($nickel_temp)(_ansi reset) found "
        _print $res.stdout
        _print $res.stderr
        rm -f $wk_vars
        cd $env.PWD
        return false
      }
      rm -f $nickel_temp $err_out
    } else if ( $defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).yaml" | path exists) {
      cat ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).yaml") | tee { save -a $wk_vars } | ignore
    }
    cd $env.PWD
  }
  (open -r $wk_vars | str replace --all "NOW" $env.NOW | save -f $wk_vars)
  if $defs.taskserv_install_mode == "library" {
    let taskserv_data = (open $wk_vars)
    let quiet = if (is-debug-enabled) { false } else { true }
    if $taskserv_data.taskserv? != null and $taskserv_data.taskserv.copy_paths? != null {
      #use utils/files.nu *
      for it in $taskserv_data.taskserv.copy_paths {
        let it_list = ($it | split row "|" | default [])
        let cp_source = ($it_list | get -o 0 | default "")
        let cp_target = ($it_list | get -o 1 | default "")
        if ($cp_source | path exists) {
          copy_prov_files $cp_source "." ($taskserv_env_path | path join $cp_target) false $quiet
        } else if ($prov_resources_path | path join $cp_source | path exists) {
          copy_prov_files $prov_resources_path $cp_source ($taskserv_env_path | path join $cp_target) false $quiet
        } else if ($"($prov_resources_path)/($cp_source)" | path exists) {
          copy_prov_file ($prov_resources_path | path join $cp_source) ($taskserv_env_path | path join $cp_target) $quiet
        }
      }
    }
  }
  rm -f ($taskserv_env_path | path join "nickel") ...(glob $"($taskserv_env_path)/*.ncl")
  on_template_path $taskserv_env_path $wk_vars true true
  let env_file = ($taskserv_env_path | path join $"env-($defs.taskserv.name)")
  if ($env_file | path exists) {
    let env_content = (open -r $env_file
      | lines
      | each {|l| $l | str replace --all "\t" "" | str trim --left }
      | where {|l| ($l | is-not-empty) }
      | str join "\n")
    $env_content | save -f $env_file
  }
  if ($taskserv_env_path | path join "prepare" | path exists) {
    run_cmd "prepare" "prepare" "run_taskserv_library" $defs $taskserv_env_path $wk_vars
    if ($taskserv_env_path | path join "resources" | path exists) {
      on_template_path ($taskserv_env_path | path join "resources") $wk_vars false true
    }
  }
  if not (is-debug-enabled) {
    rm -f ...(glob $"($taskserv_env_path)/*.j2") $err_out $nickel_temp
  }
  true
}
export def run_taskserv [
    defs: record
    taskserv_path: string
    env_path: string
] {
  if  not ($taskserv_path | path exists) { return false }
  let prov_resources_path = ($defs.settings.data | get -o prov_resources_path | default "" | str replace  "~" $env.HOME)
  let taskserv_server_name = $defs.server.hostname

  let str_created_taskservs_dirpath = ($defs.settings.data | get -o created_taskservs_dirpath | default "/tmp" |
    str replace "~" $env.HOME | str replace "NOW" $env.NOW | str replace "./" $"($defs.settings.src_path)/")
  let created_taskservs_dirpath = if ($str_created_taskservs_dirpath | str starts-with "/" ) { $str_created_taskservs_dirpath } else { $defs.settings.src_path | path join $str_created_taskservs_dirpath }
  if not ( $created_taskservs_dirpath | path exists) { ^mkdir -p $created_taskservs_dirpath }

  let str_taskserv_env_path = if  $defs.taskserv_install_mode == "server" { $"($env_path)_($defs.taskserv_install_mode)" } else { $env_path }
  let taskserv_env_path = if ($str_taskserv_env_path | str starts-with "/" ) { $str_taskserv_env_path } else { $defs.settings.src_path | path join $str_taskserv_env_path }
  if not ( $taskserv_env_path | path exists) { ^mkdir -p $taskserv_env_path }

  (^cp -pr ...(glob ($taskserv_path | path join "*")) $taskserv_env_path)
  rm -rf ...(glob ($taskserv_env_path | path join "*.ncl"))  ($taskserv_env_path | path join "nickel")

  let wk_vars = ($created_taskservs_dirpath | path join $"($defs.server.hostname).yaml")
  let j2_files = (glob ($taskserv_env_path | path join "*.j2"))

  let res = if $defs.taskserv_install_mode == "library" or ($j2_files | is-not-empty) {
    (run_taskserv_library $defs $taskserv_path $taskserv_env_path $wk_vars)
  } else { true }
  if not $res {
    if not (is-debug-enabled) { rm -f $wk_vars }
    return $res
  }
  let err_out = ($env_path | path join (mktemp --tmpdir-path $env_path --suffix ".err") | path basename)
  let tar_ops = if (is-debug-enabled) { "v" } else { "" }
  let bash_ops = if (is-debug-enabled) { "bash -x" } else { "" }

  # Inject common.sh into every bundle so install scripts can source standard helpers
  let common_sh_src = (get-taskservs-path | path join "common.sh")
  if ($common_sh_src | path exists) {
    cp $common_sh_src ($taskserv_env_path | path join "common.sh")
  }

  # Remove local-only build artefacts and non-essential dirs before bundling.
  # _cri/ contains supplementary CRI configs not referenced by install scripts.
  # prepare is a local Nu script (already removed in non-debug, safe to force-rm here).
  # tmp.*.sh and cmd_env_*.sh are build-time side-effects that must never reach the server.
  rm -f ($taskserv_env_path | path join "prepare")
  rm -rf ($taskserv_env_path | path join "_cri")
  for pat in ["tmp.*.sh", "cmd_env_*.sh", "tmp.*.err"] {
    let matches = (glob $"($taskserv_env_path)/($pat)")
    if ($matches | is-not-empty) { rm -f ...$matches }
  }

  let res_tar = (with-env { COPYFILE_DISABLE: "1" } { ^tar -C $taskserv_env_path $"-c($tar_ops)zmf" (["/tmp" $"($defs.taskserv.name).tar.gz"] | path join) . | complete })
  if $res_tar.exit_code != 0 {
    _print (
      $"🛑 Error (_ansi red_bold)tar taskserv(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset)" +
      $"  taskserv (_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) ($taskserv_env_path) -> (['/tmp' $'($defs.taskserv.name).tar.gz'] | path join)"
    )
    return false
  }
  if $defs.check {
    if not (is-debug-enabled) {
      rm -f $wk_vars
      if $err_out != "" { rm -f $err_out }
      rm -rf ...(glob $"($taskserv_env_path)/*.ncl") ($taskserv_env_path | path join "nickel")
    }
    return true
  }
  let is_local = if $defs.taskserv_install_mode == "local" {
    true  # local mode: always run on provisioning machine regardless of IP
  } else if $nu.os-info.name == "macos" {
    (do -i { ^ifconfig } | default "" | str contains $"($defs.ip)")
  } else {
    (do -i { ^ip addr } | default "" | str contains $"($defs.ip)")
  }
  if $is_local and not (is-debug-check-enabled) {
    if $defs.taskserv_install_mode ==  "getfile" {
      if (taskserv_get_file $defs.settings $defs.taskserv $defs.server $defs.ip true true) { return false }
      return true
    }
    rm -rf (["/tmp" $defs.taskserv.name ] | path join)
    mkdir (["/tmp" $defs.taskserv.name ] | path join)
    cd (["/tmp" $defs.taskserv.name ] | path join)
    ^tar $"x($tar_ops)zmf" (["/tmp" $"($defs.taskserv.name).tar.gz"] | path join)
    let cmd_task = ($defs.taskserv.cmd_task? | default "install")
    # local mode: no sudo — tool (k0sctl etc.) manages its own auth
    let script = $"./install-($defs.taskserv.name).sh"
    let res_run = if $defs.taskserv_install_mode == "local" {
      if (is-debug-enabled) {
        (do { ^bash -x $script $cmd_task } | complete)
      } else {
        (do { ^bash $script $cmd_task } | complete)
      }
    } else {
      if (is-debug-enabled) {
        (do { ^sudo bash -x $script $cmd_task } | complete)
      } else {
        (do { ^sudo bash $script $cmd_task } | complete)
      }
    }
    if $res_run.exit_code != 0 {
      (throw-error $"🛑 Error server ($defs.server.hostname) taskserv ($defs.taskserv.name)
          ./install-($defs.taskserv.name).sh ($defs.pos.server) ($defs.pos.taskserv) (^pwd)"
          $"($res_run.stdout)\n($res_run.stderr)"
          "run_taskserv_library" --span (metadata $res_run).span)
      exit 1
    }
    cd /tmp
    rm -fr (["/tmp" $"($defs.taskserv.name).tar.gz"] | path join) (["/tmp" $"($defs.taskserv.name)"] | path join)
  } else {
    if $defs.taskserv_install_mode == "getfile" {
      if (taskserv_get_file $defs.settings $defs.taskserv $defs.server $defs.ip true false) { return false }
      return true
    }
    if not (is-debug-check-enabled) {
      #use ssh.nu *
      let scp_list: list<string> = ([] | append $"/tmp/($defs.taskserv.name).tar.gz")
      if not (scp_to $defs.settings $defs.server $scp_list "/tmp" $defs.ip) {
        _print (
          $"🛑 Error (_ansi red_bold)ssh_to(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " +
          $"  taskserv (_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) /tmp/($defs.taskserv.name).tar.gz"
        )
        return false
      }
      #    $"rm -rf /tmp/($defs.taskserv.name); mkdir -p /tmp/($defs.taskserv.name) ;" +
      let run_ops = if (is-debug-enabled) { "bash -x" } else { "" }
      let cmd_task = ($defs.taskserv.cmd_task? | default "install")
      let cmd = (
          $"rm -rf /tmp/($defs.taskserv.name); mkdir -p /tmp/($defs.taskserv.name) ;" +
          $" cd /tmp/($defs.taskserv.name) ; sudo tar x($tar_ops)zmf /tmp/($defs.taskserv.name).tar.gz &&" +
          $" sudo ($run_ops) ./install-($defs.taskserv.name).sh ($cmd_task)"
      )
      if not (ssh_cmd $defs.settings $defs.server false $cmd $defs.ip) {
        _print (
          $"🛑 Error (_ansi red_bold)ssh_cmd(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " +
          $"  taskserv (_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) install_($defs.taskserv.name).sh"
        )
        return false
      }
      if not (is-debug-enabled) {
        let rm_cmd = $"sudo rm -f /tmp/($defs.taskserv.name).tar.gz; sudo rm -rf /tmp/($defs.taskserv.name)"
        let _res = (ssh_cmd $defs.settings $defs.server false $rm_cmd $defs.ip)
        rm -f $"/tmp/($defs.taskserv.name).tar.gz"
      }
    }
  }
  if ($taskserv_path | path join "postrun" | path exists ) {
    cp ($taskserv_path | path join "postrun") ($taskserv_env_path | path join "postrun")
    run_cmd "postrun" "PostRune" "run_taskserv_library" $defs $taskserv_env_path $wk_vars
  }
  if not (is-debug-enabled) {
    rm -f $wk_vars
    if $err_out != "" { rm -f $err_out }
    rm -rf ...(glob $"($taskserv_env_path)/*.ncl") ($taskserv_env_path | path join join "nickel")
  }
  true
}