prvng_core/nulib/servers/utils.nu

# Provider middleware now available through lib_provisioning
# REMOVED: use lib_provisioning * - causes circular import
use ssh.nu *
use ../lib_provisioning/utils/ssh.nu ssh_cmd
use ../lib_provisioning/utils/settings.nu get_file_format
use ../lib_provisioning/secrets/lib.nu encrypt_secret
use ../lib_provisioning/config/accessor.nu *
use ../../../extensions/providers/prov_lib/middleware.nu [mw_query_servers]
use ../lib_provisioning/utils/nickel_processor.nu [ncl-eval-soft]

# Read FSM dimension states from the workspace ontology via ontoref.
# Returns a flat record: {dimension_id: current_state, ...}. Fails gracefully → empty record.
export def read_fsm_states [ws_root: string]: nothing -> record {
    if ($ws_root | is-empty) { return {} }
    let onto_path = ($ws_root | path join ".ontology" "state.ncl")
    if not ($onto_path | path exists) { return {} }

    let result = (do {
        cd $ws_root
        ^ontoref describe state --format json
    } | complete)

    if $result.exit_code != 0 or ($result.stdout | str trim | is-empty) { return {} }

    let dims = ($result.stdout | from json | get -o dimensions | default [])
    $dims | reduce -f {} {|d acc|
        $acc | insert $d.id { current: ($d.current_state? | default "?"), desired: ($d.desired_state? | default "?") }
    }
}

# Derive the FSM dimension id for a server from its taskservs and loaded FSM states.
# Convention: taskserv "k0s" → dimension "k0s-status". Returns the first match, or "".
export def server_fsm_dimension [server: record, fsm_states: record]: nothing -> string {
    $server.server_taskservs? | default []
    | each {|ts| $"($ts.name)-status"}
    | where {|did| ($fsm_states | get -o $did | default null) != null}
    | first | default ""
}

# Format protection flags as a short human-readable string.
# Servers: "del+rbld" | "del" | "—". Networks/FIPs: "del" | "—".
export def format_protection [prot: any]: nothing -> string {
    if $prot == null { return "—" }
    let d = ($prot | get -o delete  | default false)
    let r = ($prot | get -o rebuild | default false)
    match [$d, $r] {
        [true,  true]  => "del+rbld"
        [true,  false] => "del"
        [false, true]  => "rbld"
        _              => "—"
    }
}

# Read taskserv runtime states from the infra's .provisioning-state.ncl via nickel export.
# Returns a record keyed by hostname → {taskserv_name → state_string}. Fails gracefully → {}.
# infra_dir: full path to the infra subdirectory (e.g. .../infra/sgoyol)
export def read_infra_taskserv_states [infra_dir: string]: nothing -> record {
    if ($infra_dir | is-empty) or not ($infra_dir | path exists) { return {} }
    let state_path = ($infra_dir | path join ".provisioning-state.ncl")
    if not ($state_path | path exists) { return {} }

    let parsed = (ncl-eval-soft $state_path [] {})
    if ($parsed | is-empty) { return {} }
    let servers = ($parsed | get -o servers | default {})

    $servers | items {|hostname srv|
        let ts = ($srv | get -o taskservs | default {})
        let taskserv_states = $ts | items {|name t|
            { name: $name, state: ($t | get -o state | default "unknown"), operation: ($t | get -o operation | default "") }
        }
        { key: $hostname, value: $taskserv_states }
    } | reduce -f {} {|it acc| $acc | insert $it.key $it.value}
}

# Display servers information in table format with live provider status
export def mw_servers_info [
    settings: record
] {
    let servers = if ($settings | get data? | is-not-empty) {
        ($settings.data | get servers? | default [])
    } else if ($settings | get servers? | is-not-empty) {
        ($settings | get servers? | default [])
    } else {
        []
    }

    if ($servers | is-empty) { return [] }

    # Query live status from provider (fails gracefully — returns [] on error)
    let live_data = (
        do --ignore-errors { mw_query_servers $settings "" "" }
        | default []
    )

    # Query floating IPs once; build server_id -> [ip, ...] map.
    # hcloud floating-ip list -o json returns [{id, name, ip, server: <int|null>}, ...]
    let fip_by_server_id = (
        do {
            let res = (do { ^hcloud floating-ip list -o json } | complete)
            if $res.exit_code == 0 and ($res.stdout | str trim | is-not-empty) {
                let fips = ($res.stdout | from json)
                $fips
                | where {|f| ($f.server? | default null) != null}
                | group-by {|f| $f.server | into string}
                | items {|sid fip_list|
                    { key: $sid, value: ($fip_list | each {|f| $f.ip} | str join ", ") }
                }
                | reduce -f {} {|it acc| $acc | insert $it.key $it.value}
            } else { {} }
        }
        | default {}
    )

    let ws_root = ($settings | get -o infra_path | default "" | path dirname)
    let fsm_states = (read_fsm_states $ws_root)

    let safe_live_data = if ($live_data | describe | str starts-with "list") { $live_data } else { [] }
    $servers | each {|server|
        let live = ($safe_live_data | where {|l| $l.name == $server.hostname} | first | default null)
        let status   = if $live != null { $live.status? | default "unknown" } else { "—" }
        let pub_ip   = if $live != null { $live.public_net?.ipv4?.ip? | default "" } else { "" }
        let priv_ip  = $server.networking?.private_ip? | default ""
        let location = if $live != null { $live.datacenter?.location?.name? | default ($server.location? | default "") } else { $server.location? | default "" }
        let srv_id   = if $live != null { $live.id? | default null } else { null }
        let fip_ip   = if $srv_id != null { $fip_by_server_id | get -o ($srv_id | into string) | default "" } else { "" }

        let dim_id    = (server_fsm_dimension $server $fsm_states)
        let fsm_entry = if ($dim_id | is-not-empty) { $fsm_states | get -o $dim_id | default null } else { null }
        let fsm_state = if $fsm_entry != null { $"($fsm_entry.current)/($fsm_entry.desired)" } else { "—" }

        let raw_prot  = if $live != null { $live | get -o protection | default null } else { null }
        let protection = (format_protection $raw_prot)

        {
            hostname:    $server.hostname
            type:        ($server.server_type? | default "")
            location:    $location
            status:      $status
            public_ip:   $pub_ip
            private_ip:  $priv_ip
            floating_ip: $fip_ip
            fsm_state:   $fsm_state
            protection:  $protection
            provider:    $server.provider
        }
    }
}

export def on_server [
    settings: record   # Settings record
    check: bool        # Only check mode no servers will be created
    wait: bool         # Wait for creation
    outfile?: string   # Out file for creation
    hostname?: string  # Server hostname in settings
    serverpos?: int    # Server position in settings
] {
    # _check_settings
    let match_hostname = if $hostname != null {
        $hostname
    } else if $serverpos != null  {
        let total = $settings.data.servers | length
        let pos = if $serverpos == 0 {
            _print $"Use number form 1 to ($total)"
            $serverpos
        } else if $serverpos <= $total {
            $serverpos - 1
        } else {
            (throw-error $"🛑 server pos" $"($serverpos) from ($total) servers"
                "on_create" --span (metadata $serverpos).span)
            exit 1
        }
        ($settings.data.servers | get $pos).hostname
    }
    if $check {
        $settings.data.servers | enumerate | each { |it|
            if $match_hostname == null or $it.item.hostname == $match_hostname  {
                on_create_server $it.item $it.index true $outfile
            }
        }
    } else {
        $settings.data.servers | enumerate | par-each { |it|
            if $match_hostname == null or $it.item.hostname == $match_hostname  {
                on_create_server $it.item $it.index false $outfile
            }
        }
    }
}

export def wait_for_server [
    server_pos: int
    server: record
    settings: record
    ip: string
    --quiet
] {
    if $ip == "" { return false }
    mut num = 0
    let liveness_port = (if $server.liveness_port? != null { $server.liveness_port }  else { 22 } | into int)
    let val_timeout = if $server.running_timeout? != null { $server.running_timeout } else { 0 }
    let wait = if $server.running_wait? != null { $server.running_wait } else { 10 }
    let wait_duration = ($"($wait)sec"| into duration)

    if not $quiet {
        _print (
            $"wait for server (_ansi blue_bold)($server.hostname)(_ansi reset) state " +
            $"(_ansi yellow_bold)started(_ansi reset) (_ansi default_dimmed)until ($val_timeout)secs check every ($wait)sec(_ansi reset)"
        )
    }

    while true {
        let status = (mw_server_is_running $server false)
        #let res = (run-external --redirect-combine "nc" "-zv" "-w" 1 $ip $liveness_port | complete)
        #if $res.exit_code == 0 {
        if $status and (port_scan $ip $liveness_port 1) {
            if not $quiet {
                _print $"done in ($num)secs "
            }
            break
        } else if $val_timeout > 0 and $num > $val_timeout {
            _print ($"\n🛑 (_ansi red)Timeout(_ansi reset) ($val_timeout) (_ansi blue)($server.hostname)(_ansi reset)" +
                $"(_ansi blue_bold)($ip)(_ansi reset) at ($liveness_port) (_ansi red_bold)failed(_ansi reset) "
            )
            #print $"\n($res.stdout)"
            return false
        } else {
            $num = $num + $wait
            if not $quiet {
                print -n $"(_ansi blue_bold) 🌥 (_ansi reset)"
            }
            sleep $wait_duration
        }
    }

    if not $quiet {
        _print (
            $"(_ansi blue)($server.hostname)(_ansi reset) at (_ansi blue_bold)($ip)(_ansi reset) " +
            $"port ($liveness_port)  (_ansi green_bold)ready(_ansi reset) "
        )
    }
    true
}
export def on_server_template [
    server_template: string
    server: record
    index: int
    check: bool
    only_make: bool
    wait: bool
    settings: record
    outfile?: string
] {
    if $server.provider == local { return true }
    if not ( $server_template | path exists )  {
        _print $"($server_template) not found for ($server.hostname) [($index)]"
        return false
    }
    let suffix = if ($server_template | str contains "storage") {
        "storage"
    } else {
        "server"
    }
    #use utils/templates.nu run_from_template
    #mut create_result = false
    let duration = timeit {
        let wk_file = $"($settings.wk_path)/($server.hostname)_($suffix)_cmd"
        # Test: Force JSON format
        let wk_vars = $"($settings.wk_path)/($server.hostname)_($suffix)_vars.json"
        let run_file = $"($settings.wk_path)/on_($server.hostname)_($suffix)_run.sh"
        rm --force $wk_file $wk_vars $run_file
        let data_settings = if $suffix == "storage" {
            ($settings.data | merge { wk_file: $wk_file, now: (date now), server_pos: $index, storage_pos: 0, provisioning_vers: ("1.0.4" | str replace "null" ""),
            wait: $wait, server: $server })
        } else {
            let filtered_provider = ($settings.providers | where {|it| $it.provider == $server.provider})
            let provider_settings = if ($filtered_provider | is-empty) { {} } else { $filtered_provider | first | get settings? | default {} }
            ($settings.data | merge { wk_file: $wk_file, now: (date now), serverpos: $index,  provisioning_vers: ("1.0.4" | str replace "null" ""),
            wait: $wait, provider: $provider_settings,
            server: $server })
        }
        # Test: Force JSON to isolate YAML parsing issue
        $data_settings |  to json  | save --force $wk_vars

        # if (get-provisioning-wk-format) == "json" {
        #     $data_settings |  to json  | save --force $wk_vars
        # } else {
        #     # Generate YAML preserving {{variable}} template patterns (no longer need $variable quoting)
        #     $data_settings | to yaml | save --force $wk_vars
        # }
        let res = if $only_make and $check {
            (run_from_template $server_template $wk_vars $run_file $outfile --only_make --check_mode)
        } else if $only_make {
            (run_from_template $server_template $wk_vars $run_file $outfile --only_make)
        } else if $check {
            (run_from_template $server_template $wk_vars $run_file $outfile --check_mode)
        } else {
            (run_from_template $server_template $wk_vars $run_file $outfile)
        }

        if $res {
            # IMPORTANT: Capture rendered script BEFORE cleanup for orchestrator transmission
            # The script is what orchestrator will execute, not parameters
            if (not $only_make) {
                let rendered_content = (open -r $run_file)
                $env.LAST_RENDERED_SCRIPT = $rendered_content
                $env.LAST_TEMPLATE_PATH = $server_template
                $env.LAST_TEMPLATE_CONTEXT = ($data_settings | to json | from json)
            }

            if (is-debug-enabled) == false { rm --force $wk_file $wk_vars $run_file }
            _print $"(_ansi green_bold)($server.hostname)(_ansi reset) (_ansi green)successfully(_ansi reset)"
        } else {
            _print $"(_ansi red)Failed(_ansi reset) (_ansi green_bold)($server.hostname)(_ansi reset)"
        }
    }
    let text_duration = if not $check { $"in (_ansi blue_bold)($duration)(_ansi reset)" } else { "" }
    _print $"Done run template (_ansi blue_italic)($server_template | path basename | str replace ".j2" "" )(_ansi reset) for (_ansi green_bold)($server.hostname)(_ansi reset) ($text_duration)"
    true # $create_result
}
export def servers_selector [
    settings: record
    ip_type: string
    is_for_task: bool
] {
    if (get-provisioning-out | is-not-empty) or (get-provisioning-no-terminal) { return ""}
    mut servers_pick_lists = []
    if not (is-debug-check-enabled) {
        #use ssh.nu *
        for server in $settings.data.servers  {
            let ip = (mw_get_ip $settings $server $ip_type false | default "")
            if $ip == "" {
                _print $"🛑 No IP ($ip_type) found for (_ansi green_bold)($server.hostname)(_ansi reset) "
                continue
            }
            let ssh_id = (server_ssh_id $server)
            let ssh_addr = (server_ssh_addr $settings $server)
            $servers_pick_lists = ($servers_pick_lists | append { name: $server.hostname,
                id: $ssh_id, addr: $ssh_addr
            })
        }
    }
    let msg_sel = if $is_for_task {
        "Select one server"
    } else {
        "To connect to a server select one"
    }
    if ($servers_pick_lists | length) == 0 { return "" }
    let selection = if ($servers_pick_lists | length) > 1 {
        _print $"(_ansi default_dimmed)($msg_sel) \(use arrows and press [enter] or [esc] to cancel\):(_ansi reset)"
        ($servers_pick_lists | each {|it| _print $"($it.name) -> ($it.addr)"})
        let pos_select = ($servers_pick_lists | each {|it| $"($it.name) -> ($it.addr)"} |input list --index)
        if $pos_select == null { return null }
        let selection = if ($servers_pick_lists | length) > $pos_select { $servers_pick_lists | get $pos_select } else { null }
        if not $is_for_task  {
            _print $"\nFor (_ansi green_bold)($selection.name)(_ansi reset) server use:"
        }
        $selection
    } else {
        let selection = if ($servers_pick_lists | is-empty) { null } else { $servers_pick_lists | first }
        if not $is_for_task  {
            _print $"\n(_ansi default_dimmed)To connect to server (_ansi reset)(_ansi green_bold)($selection.name)(_ansi reset) use:"
        }
        $selection
    }
    if not $is_for_task  {
        let id = ($selection | get id? | default "")
        if ($id | is-not-empty) {
            show_clip_to $"ssh -i($id) ($selection.addr)" true
        }
    }
    $selection
}
def add_item_price [
    server: record
    already_created: bool
    resource: string
    item: string
    price: record
    host_color: string
] {
    let str_price_monthly = if $price.month < 10 { $" ($price.month)" } else { $"($price.month)" }
    let price_monthly = if ($str_price_monthly | str contains ".") { $str_price_monthly } else { $"($str_price_monthly).0"}
    if (get-provisioning-out | is-empty)  {
        {
            host: $"(_ansi $host_color)($server.hostname)(_ansi reset)",
            item: $"(_ansi default_bold)($item)(_ansi reset)",
            resource: $"(_ansi blue_bold)($resource)(_ansi reset)",
            prov: $"(_ansi default_bold)($server.provider)(_ansi reset)",
            zone: $"(_ansi default_bold)($server.zone)(_ansi reset)",
            unit: $" ($price.unit_info | default '') "
            hour: $"(_ansi default_bold)  ($price.hour | fill -a left -c '0' -w 7 | str replace '.' ',') €  (_ansi reset)",
            day: $"(_ansi default_bold)  ($price.day | math round -p 4 | fill -a left -c '0' -w 7 | str replace '.' ',') €  (_ansi reset)",
            month: $"(_ansi default_bold)  ($price_monthly | fill -a left -c '0' -w 7 | str replace '.' ',' | str replace  ',0000' '') € (_ansi reset)",
            created: $already_created,
        }
    } else {
        {
            host: $server.hostname,
            item: $item,
            resource: $resource,
            prov: $server.provider,
            zone: $server.zone,
            unit: ($price.unit_info | default ""),
            hour: $"($price.hour | fill -a left -c '0' -w 7 | str replace '.' ',') €",
            day: $"($price.day | math round -p 4 | fill -a left -c '0' -w 7 | str replace '.' ',') €",
            month: $"($price_monthly | fill -a left -c '0' -w 7 | str replace '.' ',' | str replace  ',0000' '') €",
            created: $already_created,
        }
    }
}
export def servers_walk_by_costs [
    settings: record   # Settings record
    match_hostname: string
    check: bool        # Only check mode no servers will be created
    return_no_exists: bool
    outfile?: string
] {
    if $outfile != null  { set-provisioning-no-terminal true }
    if $outfile == null {
        _print $"\n (_ansi cyan)($settings.data | get main_title? | default "")(_ansi reset) prices"
    }
    mut total_month = 0
    mut total_hour = 0
    mut total_day = 0
    mut table_items = []
    let total_color  = { fg: '#ffff00' bg: '#0000ff' attr: b }

    # Phase 1 Optimization: Pre-load all provider data upfront
    # Collect unique providers from servers that match the hostname filter
    let target_servers = if $match_hostname != null and $match_hostname != "" {
        ($settings.data.servers | where {|s| $s.hostname == $match_hostname})
    } else {
        $settings.data.servers
    }

    let unique_providers = ($target_servers | each {|s| $s.provider} | uniq)

    # Load all provider pricing data upfront (leverages existing file-based cache)
    mut infra_servers = {}
    for provider in $unique_providers {
        # Get first server with this provider to use as reference
        let ref_server = ($target_servers | where {|s| $s.provider == $provider} | get 0)
        $infra_servers = ($infra_servers | merge {
            $provider: (mw_load_infra_servers_info $settings $ref_server false)
        })
    }

    # Main pricing calculation loop
    for server in $target_servers {
        let item_raw = (mw_get_infra_item $server $settings $infra_servers false)
        let item = { item: $item_raw, target: "server" }
        if ($item | get item? | default null | is-empty) { continue }
        #if  (is-debug-check-enabled) { _print ($item | table -e)}

        let already_created = (mw_server_exists $server false)
        let host_color = if $already_created { "green_bold" } else { "red" }

        # Phase 3 Optimization: Use batched price calculation
        let price = (mw_get_all_infra_prices $server $item false)
        let str_server_plan = if ($server.reqplan? != null ) {
            $"($server.reqplan.cores | default 1)xCPU-(($server.reqplan.memory | default 1024) / 1024)GB ($server.plan)"
        } else { $server.plan }
        if ($price.hour > 0 or $price.month > 0) {
            $total_month += $price.month
            $total_hour += $price.hour
            $total_day += ($price.day)
            $table_items = ($table_items | append (add_item_price $server $already_created $str_server_plan "server" $price $host_color))
        }
        for it in ($server | get storages? | default [] | enumerate) {
            let storage = $it.item
            let storage_item = { item: (mw_get_infra_storage $server $settings $infra_servers false), target: "storage", src: $it }
            if ($storage_item | get item? | default null | is-empty) { continue }
            #if  (is-debug-check-enabled) { _print ($storage_item | table -e)}
            let storage_size = if (($storage | get parts? | default null) != null and (($storage | get parts? | default []) | length) > 0) {
                ($storage | get parts? | default [] | each {|part| $part | get size? | default 0} | math sum)
            } else {
                ($storage | get size? | default 0)
            }
            if $storage_size > 0 {
                let storage_targets = if (($storage | get parts? | default null) != null and (($storage | get parts? | default []) | length) > 0) {
                    ($storage | get parts? | default [] | each {|part| $part | get mount_path? | default ""} | str join " - ")
                } else {
                    ($storage | get mount_path? | default "")
                }
                # Phase 3 Optimization: Use batched price calculation for storage
                let base_price = (mw_get_all_infra_prices $server $storage_item false)
                let store_price = {
                    month: (($base_price.month * $storage_size) | math round -p 4),
                    day: (($base_price.day * $storage_size) | math round -p 4),
                    hour: (($base_price.hour * $storage_size) | math round -p 4),
                    unit_info: $base_price.unit_info
                }
                if ($store_price.hour > 0 or $store_price.month > 0) {
                    $total_month += $store_price.month
                    $total_hour += $store_price.hour
                    $total_day += ($store_price.day)
                    $table_items = ($table_items | append (add_item_price $server $already_created $"($storage_size) Gb ($storage_targets)" "store" $store_price $host_color))
                }
            }
        }
        if not $check {
            let already_created = (mw_server_exists $server false)
            if not ($already_created) {
                if $return_no_exists {
                    return { status: false, error: $"($server.hostname) not created" }
                #   _print $"(_ansi red_bold)($server.hostname)(_ansi reset) not created"
                }
            }
        }
        #{ status: true, error: "" }
    }
    if (get-provisioning-out | is-empty)  {
        $table_items = ($table_items | append { host: "", item: "",  resource: "", prov: "", zone: "", unit: "", month: "", day: "", hour: "", created: ""})
    }
    # Format total_month: pad to 7 chars, use comma as decimal separator, remove trailing zeros
    let str_total_month = ($"($total_month | math round -p 4)" | fill -a left -c '0' -w 7 | str replace '.' ',' | str replace -r ',0+$' '' | str replace -r ',$' '')
    if (get-provisioning-out | is-empty)  {
        $table_items = ($table_items | append {
            host: $"(_ansi --escape $total_color)   TOTAL  (_ansi reset)",
            item: $"(_ansi default_bold) (_ansi reset)",
            resource: $"(_ansi default_bold) (_ansi reset)",
            prov: $"(_ansi default_bold) (_ansi reset)",
            zone: $"(_ansi default_bold) (_ansi reset)",
            unit: $"(_ansi default_bold) (_ansi reset)",
            hour: $"(_ansi --escape $total_color)  ($"($total_hour | math round -p 4)" | fill -a left -c '0' -w 7 | str replace '.' ',' | str replace -r ',0+$' '' | str replace -r ',$' '') €  (_ansi reset)",
            day:  $"(_ansi --escape $total_color)  ($"($total_day | math round -p 4)" | fill -a left -c '0' -w 7 | str replace '.' ',' | str replace -r ',0+$' '' | str replace -r ',$' '') €  (_ansi reset)",
            month:$"(_ansi --escape $total_color)  ($str_total_month) €  (_ansi reset)"
            created: $"(_ansi default_bold) (_ansi reset)",
        })
    } else {
        $table_items = ($table_items | append {
            host: "TOTAL",
            item: "",
            resource: "",
            prov: "",
            zone: "",
            unit: "",
            hour: $"(_ansi --escape $total_color)($total_hour | math round -p 4 | fill -a left -c '0' -w 7 | str replace '.' ',') €(_ansi reset)",
            day:  $"(_ansi --escape $total_color)($total_day | math round -p 4 | fill -a left -c '0' -w 7 | str replace '.' ',') €(_ansi reset)",
            month:$"(_ansi --escape $total_color)($str_total_month | str replace '.' ',' | str replace  ',0000' '') €(_ansi reset)"
            created: false,
        })
    }
    if $outfile != null  {
        if ($outfile == "stdout") {
            return  $table_items
        } else if ($outfile | str ends-with ".json") {
            $table_items | to json | save --force $outfile
        } else if ($outfile | str ends-with ".yaml") {
            $table_items | to yaml | save --force $outfile
        } else if ($outfile | str ends-with ".csv") {
            $table_items | to csv | save --force $outfile
        } else if ($outfile | str ends-with ".table") {
            $table_items | table -e | save --force $outfile
        } else {
            $table_items | to text | save --force $outfile
        }
        set-provisioning-no-terminal false
        _print  $"Prices saved in (_ansi cyan_bold)($outfile)(_ansi reset) "
    } else {
        set-provisioning-no-terminal false
        match (get-provisioning-out)  {
            "json" => { _print ($table_items | to json) "json" "result" "table" },
            "yaml" => { _print ($table_items | to yaml) "yaml" "result" "table" },
            _ => { _print ($table_items | table -i false) },
        }
    }
}
export def wait_for_servers [
    settings: record
    check: bool
    ip_type: string = "public"
] {
    mut server_pos = 0
    mut has_errors = false
    for srvr in $settings.data.servers  {
        $server_pos += 1
        let ip = if (is-debug-check-enabled) or $check {
            "127.0.0.1"
        } else {
            let curr_ip = (mw_get_ip $settings $srvr $ip_type false | default "")
            if $curr_ip == "" {
                _print $"🛑 No IP ($ip_type) found for (_ansi green_bold)($srvr.hostname)(_ansi reset) ($server_pos) "
                $has_errors = true
            continue
            }
            #use utils.nu wait_for_server
            if not (wait_for_server $server_pos $srvr $settings $curr_ip) {
                _print $"🛑 server ($srvr.hostname) ($curr_ip)  (_ansi red_bold)not in running state(_ansi reset)"
                $has_errors = true
                continue
            }
        }
        _print $"on (_ansi green_bold)($srvr.hostname)(_ansi reset) ($ip)"
    }
    $has_errors
}
export def provider_data_cache [
    settings: record
    --outfile (-o): string  # Output file
] {
    mut cache_already_loaded = []
    for server in ($settings.data.servers? | default []) {
        _print $"server (_ansi green)($server.hostname)(_ansi reset) on (_ansi blue)($server.provider)(_ansi reset)"
        if ($cache_already_loaded | where {|it| $it == $server.provider} |length) > 0 { continue } else { $cache_already_loaded = ($cache_already_loaded | append $server.provider)}
        let provider_path = (get_provider_data_path $settings $server)
        #use ../lib_provisioning/utils/settings.nu load_provider_env
        let data = (load_provider_env $settings $server $provider_path)
        if ($data | is-empty) {
            _print $"❗server ($server.hostname) no data in cache path found ($provider_path | path basename)"
            exit
        }
        let outfile_path = if ($outfile | is-not-empty) { ($outfile | path dirname | path join $"($server.provider)_($outfile | path basename)") } else { "" }
        if ($outfile_path | is-not-empty ) {
            let out_extension = (get_file_format $outfile_path)
            if $out_extension == "json" {
                ($data | to json | save --force $outfile_path)
            } else {
                ($data | to yaml | save --force $outfile_path)
            }
            if ($outfile_path | path exists)  {
            _print $"✅ (_ansi green_bold)($server.provider)(_ansi reset) (_ansi cyan_bold)cache settings(_ansi reset) saved in (_ansi yellow_bold)($outfile_path)(_ansi reset)"
            _print $"To create a (_ansi purple)nickel(_ansi reset) for (_ansi cyan)defs(_ansi reset) file use:"
            let k_file_path = $"($outfile_path | str replace $'.($out_extension)' '').ncl"
            ^nickel import ($outfile_path) -o ($k_file_path) --force
            ^sed -i '1,4d;s/^{/_data = {/' $k_file_path
            '{ main = _data.main, priv = _data.priv }' | tee {save -a $k_file_path} | ignore
            let res = ( ^nickel $k_file_path | complete)
            if $res.exit_code == 0 {
                $res.stdout | save $"($k_file_path).yaml" --force
                ^nickel import $"($k_file_path).yaml" -o ($k_file_path) --force
                ^sed -i '1,4d;s/^{/_data = {/' $k_file_path
                let content = (open $k_file_path --raw)
                let comment = $"# ($server.provider)" + " environment settings, if not set will be autogenerated in 'provider_path' (data/" + $server.provider + "_cache.yaml)"
                let from_scratch = (mw_start_cache_info $settings  $server)
                ($"# Info: Nickel Settings created by (get-provisioning-name)\n# Date: (date now | format date '%Y-%m-%d %H:%M:%S')\n\n" +
                $"($comment)\n($from_scratch)" +
                $"# Use a command like: '(get-provisioning-name) server cache -o /tmp/data.yaml' to genereate '/tmp/($server.provider)_data.ncl' for 'defs' settings\n" +
                $"# then you can move genereated '/tmp/($server.provider)_data.ncl' to '/defs/($server.provider)_data.ncl'  \n\n" +
                $"import ($server.provider)_prov\n" +
                ($content | str replace '_data = {' $"($server.provider)_prov.Provision_($server.provider) {") | save $k_file_path --force
                )
                let result = (encrypt_secret $k_file_path --quiet)
                if ($result | is-not-empty) { ($result | save --force $k_file_path) }
                _print $"(_ansi purple)nickel(_ansi reset) for (_ansi cyan)defs(_ansi reset) file has been created at (_ansi green)($k_file_path)(_ansi reset)"
            }
            #show_clip_to $"nickel import ($outfile_path) -o ($k_file_path) --force ; sed -i '1,4d;s/^{/_data = {/' ($k_file_path) ; echo '{ main = _data.main, priv = _data.priv }' >> ($k_file_path)" true
            }
        } else {
            let cmd = (get-file-viewer)
            if $cmd != "bat" { _print $"(_ansi magenta_bold)----------------------------------------------------------------------------------------------------------------(_ansi reset)"}
            if (get-provisioning-wk-format) == "json" {
                ($data | to json | run-external $cmd -)
            } else {
                ($data | to yaml | run-external $cmd -)
            }
            if $cmd != "bat" { _print $"(_ansi magenta_bold)----------------------------------------------------------------------------------------------------------------(_ansi reset)"}
        }
    }
}

export def find_server [
    item: string
    servers: list,
    out: string,
] {
    if ($item | parse --regex '^[0-9]' | length) > 0 {
        let pos = ($item | into int)
        if ($pos >= ($servers | length)) {
            if ($out | is-empty)  { _print $"No server index ($pos) found "}
            {}
        } else {
            let idx = ($item | into int)
            if ($idx >= 0 and $idx < ($servers | length)) {
                $servers | get $idx
            } else {
                {}
            }
        }
    } else {
        let filtered = ($servers | where {|s| ($s | get hostname? | default "") == $item})
        if ($filtered | is-empty) { {} } else { $filtered | first }
    }
}
export def find_serversdefs [
    settings: record
] {
    let src_path = ($settings | get src_path? | default "")
    mut defs = []
    for it in ($settings | get data? | default {} | get servers_paths? | default []) {
        let name = ($it| str replace "/" "_")
        let it_path = if ($it | str ends-with ".ncl") { $it } else { $"($it).ncl" }
        let path_def = ($src_path | path join $it_path )
        let defs_srvs = if ($path_def | path exists ) {
            (open -r $path_def)
        } else { "" }
            $defs = ($defs | append {
                name: $name, path_def: $it_path, def: $defs_srvs, defaults: ""
            }
        )
    }
    let defaults_path = (get-config-base-path | path join "nickel" | path join "defaults.ncl")
    let defaults = if ($defaults_path   | path exists) {
        (open -r $defaults_path | default "")
    } else { "" }
    let path_main = (get-config-base-path | path join "nickel" | path join "server.ncl")
    let main = if ($path_main | path exists) {
        (open -r $path_main | default "")
    } else { "" }
    let prov_defs = if (get-providers-path | is-empty) {
        {
            defs_providers: [],
            providers: [],
        }
    } else {
        let providers_list = (ls -s  (get-providers-path) | where {|it| (
            ($it.name | str starts-with "_") == false
                and (get-providers-path | path join $it.name | path type) == "dir"
                and (get-providers-path | path join $it.name | path join "templates" | path exists)
            )
        })
        let defs_providers = ($providers_list | each {|it|
            let it_path = ($src_path| path join "defs")
            let defaults = if ($it_path | path join $"($it.name)_defaults.ncl" | path exists) {
                (open -r ($it_path | path join $"($it.name)_defaults.ncl"))
            } else { "" }
            let def = if ($it_path | path join "servers.ncl" | path exists) {
                (open -r ($it_path | path join "servers.ncl"))
            } else { "" }
            {
                name: $it.name, path_def: $it_path, def: $def, defaults: $defaults
            }
        } | default [])
        let providers = ($providers_list | each {|it|
            let it_path = (get-providers-path | path join $it.name | path join "nickel")
            let defaults = if ($it_path | path join $"defaults_($it.name).ncl" | path exists) {
                (open -r ($it_path | path join $"defaults_($it.name).ncl"))
            } else { "" }
            let def = if ($it_path | path join $"server_($it.name).ncl" | path exists) {
                (open -r ($it_path | path join $"server_($it.name).ncl"))
            } else { "" }
            {
                name: $it.name, path_def: $it_path, def: $def, defaults: $defaults
            }
        } | default [])
        {
            defs_providers: $defs_providers,
            providers: $providers,
        }
    }
    {
        defaults: $defaults,
        path_main: $path_main,
        main: $main,
        providers: $prov_defs.providers,
        defs_providers: $prov_defs.defs_providers,
        defs: $defs
    }
}
export def find_provgendefs [
] {
    let prov_defs = if (get-providers-path | is-empty) {
        {
            defs_providers: [],
        }
    } else {
        let providers_list = (ls -s  (get-providers-path) | where {|it| (
            ($it.name | str starts-with "_") == false
                and (get-providers-path | path join $it.name | path type) == "dir"
                and (get-providers-path | path join $it.name | path join "templates" | path exists)
            )
        })
        mut provdefs = []
        for it in $providers_list {
            let it_defs_path = (get-providers-path | path join $it.name
                | path join (get-provisioning-generate-dirpath)
                | path join (get-provisioning-generate-defsfile))
            if ($it_defs_path | path exists) {
                $provdefs = ($provdefs | append { name: $it.name, defs: (open $it_defs_path) })
            }
        }
        $provdefs
    }
    $prov_defs
}