diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..b5a200e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,37 @@ +#!/usr/bin/env nu + +use toolkit.nu fmt + +fmt # --check --verbose + +# ADR-025: Block root star-imports of lib_provisioning / main_provisioning. +# A line matching `use lib_provisioning *` or `use main_provisioning *` at the +# start of a file (top-level) reintroduces the transitive parse cost this +# refactor was designed to eliminate. All imports must be selective. +let staged = (git diff --cached --name-only | lines | where { str ends-with ".nu" }) + +if ($staged | length) > 0 { + let violations = ( + $staged + | each {|f| + let hits = ( + do { git show $":($f)" } | complete + | if $in.exit_code == 0 { $in.stdout } else { "" } + | lines + | enumerate + | where { $it.item | str starts-with "use lib_provisioning *" or $it.item | str starts-with "use main_provisioning *" } + | each {|row| $" ($f):($row.index + 1): ($row.item | str trim)"} + ) + $hits + } + | flatten + ) + + if ($violations | length) > 0 { + print "โŒ ADR-025 star-import violation โ€” selective imports required:" + for v in $violations { print $v } + print "" + print "Replace `use lib_provisioning *` with explicit `use lib_provisioning/path/to/module.nu [sym1 sym2]`" + exit 1 + } +} diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..1861437 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,6 @@ +#!/usr/bin/env nu + +use toolkit.nu [fmt, clippy] + +fmt --check --verbose +#clippy --verbose diff --git a/.githooks/toolkit.nu b/.githooks/toolkit.nu new file mode 100644 index 0000000..8983736 --- /dev/null +++ b/.githooks/toolkit.nu @@ -0,0 +1,674 @@ +# this module regroups a bunch of development tools to make the development +# process easier for anyone. +# +# the main purpose of `toolkit` is to offer an easy to use interface for the +# developer during a PR cycle, namely to (**1**) format the source base, +# (**2**) catch classical flaws in the new changes with *clippy* and (**3**) +# make sure all the tests pass. + +const toolkit_dir = path self . + +# check standard code formatting and apply the changes +export def fmt [ + --check # do not apply the format changes, only check the syntax + --verbose # print extra information about the command's progress +] { + if $verbose { + print $"running ('toolkit fmt' | pretty-format-command)" + } + + if $check { + let result = (do { ^cargo fmt --all -- --check } | complete) + if $result.exit_code != 0 { + error make --unspanned { + msg: $"\nplease run ('toolkit fmt' | pretty-format-command) to fix formatting!" + } + } + } else { + ^cargo fmt --all + } +} + +# check that you're using the standard code style +# +# > it is important to make `clippy` happy :relieved: +export def clippy [ + --verbose # print extra information about the command's progress + --features: list # the list of features to run *Clippy* on +] { + if $verbose { + print $"running ('toolkit clippy' | pretty-format-command)" + } + + # If changing these settings also change CI settings in .github/workflows/ci.yml + let result1 = (do { + ^cargo clippy + --workspace + --exclude nu_plugin_* + --features ($features | default [] | str join ",") + -- + -D warnings + -D clippy::unwrap_used + -D clippy::unchecked_duration_subtraction + } | complete) + + if $result1.exit_code != 0 { + error make --unspanned { + msg: $"\nplease fix the above ('clippy' | pretty-format-command) errors before continuing!" + } + } + + if $verbose { + print $"running ('toolkit clippy' | pretty-format-command) on tests" + } + # In tests we don't have to deny unwrap + let result2 = (do { + ^cargo clippy + --tests + --workspace + --exclude nu_plugin_* + --features ($features | default [] | str join ",") + -- + -D warnings + } | complete) + + if $result2.exit_code != 0 { + error make --unspanned { + msg: $"\nplease fix the above ('clippy' | pretty-format-command) errors before continuing!" + } + } + + if $verbose { + print $"running ('toolkit clippy' | pretty-format-command) on plugins" + } + let result3 = (do { + ^cargo clippy + --package nu_plugin_* + -- + -D warnings + -D clippy::unwrap_used + -D clippy::unchecked_duration_subtraction + } | complete) + + if $result3.exit_code != 0 { + error make --unspanned { + msg: $"\nplease fix the above ('clippy' | pretty-format-command) errors before continuing!" + } + } +} + +# check that all the tests pass +export def test [ + --fast # use the "nextext" `cargo` subcommand to speed up the tests (see [`cargo-nextest`](https://nexte.st/) and [`nextest-rs/nextest`](https://github.com/nextest-rs/nextest)) + --features: list # the list of features to run the tests on + --workspace # run the *Clippy* command on the whole workspace (overrides `--features`) +] { + if $fast { + if $workspace { + ^cargo nextest run --all + } else { + ^cargo nextest run --features ($features | default [] | str join ",") + } + } else { + if $workspace { + ^cargo test --workspace + } else { + ^cargo test --features ($features | default [] | str join ",") + } + } +} + +# run the tests for the standard library +export def "test stdlib" [ + --extra-args: string = '' +] { + ^cargo run -- --no-config-file -c $" + use crates/nu-std/testing.nu + testing run-tests --path crates/nu-std ($extra_args) + " +} + +# formats the pipe input inside backticks, dimmed and italic, as a pretty command +def pretty-format-command [] { + $"`(ansi default_dimmed)(ansi default_italic)($in)(ansi reset)`" +} + +# return a report about the check stage +# +# - fmt comes first +# - then clippy +# - and finally the tests +# +# without any option, `report` will return an empty report. +# otherwise, the truth values will be incremental, following +# the order above. +def report [ + --fail-fmt + --fail-clippy + --fail-test + --fail-test-stdlib + --no-fail +] { + [fmt clippy test "test stdlib"] + | wrap stage + | merge ( + if $no_fail { [true true true true] } + else if $fail_fmt { [false null null null] } + else if $fail_clippy { [true false null null] } + else if $fail_test { [true true false null] } + else if $fail_test_stdlib { [true true true false] } + else { [null null null null] } + | wrap success + ) + | upsert emoji {|it| + if ($it.success == null) { + ":black_circle:" + } else if $it.success { + ":green_circle:" + } else { + ":red_circle:" + } + } + | each {|it| + $"- ($it.emoji) `toolkit ($it.stage)`" + } + | to text +} + +# run all the necessary checks and tests to submit a perfect PR +# +# # Example +# let us say we apply a change that +# - breaks the formatting, e.g. with extra newlines everywhere +# - makes clippy sad, e.g. by adding unnecessary string conversions with `.to_string()` +# - breaks the tests by output bad string data from a data structure conversion +# +# > the following diff breaks all of the three checks! +# > ```diff +# > diff --git a/crates/nu-command/src/formats/to/nuon.rs b/crates/nu-command/src/formats/to/nuon.rs +# > index abe34c054..927d6a3de 100644 +# > --- a/crates/nu-command/src/formats/to/nuon.rs +# > +++ b/crates/nu-command/src/formats/to/nuon.rs +# > @@ -131,7 +131,8 @@ pub fn value_to_string(v: &Value, span: Span) -> Result { +# > } +# > }) +# > .collect(); +# > - let headers_output = headers.join(", "); +# > + let headers_output = headers.join(&format!("x {}", "") +# > + .to_string()); +# > +# > let mut table_output = vec![]; +# > for val in vals { +# > ``` +# +# > **Note** +# > at every stage, the `toolkit check pr` will return a report of the few stages being run. +# +# - we run the toolkit once and it fails... +# ```nushell +# >_ toolkit check pr +# running `toolkit fmt` +# Diff in /home/amtoine/.local/share/git/store/github.com/amtoine/nushell/crates/nu-command/src/formats/to/nuon.rs at line 131: +# } +# }) +# .collect(); +# - let headers_output = headers.join(&format!("x {}", "") +# - .to_string()); +# + let headers_output = headers.join(&format!("x {}", "").to_string()); +# +# let mut table_output = vec![]; +# for val in vals { +# +# please run toolkit fmt to fix the formatting +# ``` +# - we run `toolkit fmt` as proposed and rerun the toolkit... to see clippy is sad... +# ```nushell +# running `toolkit fmt` +# running `toolkit clippy` +# ... +# error: redundant clone +# --> crates/nu-command/src/formats/to/nuon.rs:134:71 +# | +# 134 | let headers_output = headers.join(&format!("x {}", "").to_string()); +# | ^^^^^^^^^^^^ help: remove this +# | +# note: this value is dropped without further use +# --> crates/nu-command/src/formats/to/nuon.rs:134:52 +# | +# 134 | let headers_output = headers.join(&format!("x {}", "").to_string()); +# | ^^^^^^^^^^^^^^^^^^^ +# = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#redundant_clone +# = note: `-D clippy::redundant-clone` implied by `-D warnings` +# +# error: could not compile `nu-command` due to previous error +# ``` +# - we remove the useless `.to_string()`, and in that cases, the whole format is useless, only `"x "` is useful! +# but now the tests do not pass :sob: +# ```nushell +# running `toolkit fmt` +# running `toolkit clippy` +# ... +# running `toolkit test` +# ... +# failures: +# commands::insert::insert_uses_enumerate_index +# commands::merge::multi_row_table_overwrite +# commands::merge::single_row_table_no_overwrite +# commands::merge::single_row_table_overwrite +# commands::update::update_uses_enumerate_index +# commands::upsert::upsert_uses_enumerate_index_inserting +# commands::upsert::upsert_uses_enumerate_index_updating +# commands::where_::where_uses_enumerate_index +# format_conversions::nuon::does_not_quote_strings_unnecessarily +# format_conversions::nuon::to_nuon_table +# ``` +# - finally let's fix the tests by removing the `x`, essentially removing the whole diff we applied at the top! +# +# now the whole `toolkit check pr` passes! :tada: +export def "check pr" [ + --fast # use the "nextext" `cargo` subcommand to speed up the tests (see [`cargo-nextest`](https://nexte.st/) and [`nextest-rs/nextest`](https://github.com/nextest-rs/nextest)) + --features: list # the list of features to check the current PR on +] { + $env.NU_TEST_LOCALE_OVERRIDE = 'en_US.utf8' + $env.LANG = 'en_US.UTF-8' + $env.LANGUAGE = 'en' + + let fmt_result = (do { fmt --check --verbose } | complete) + if $fmt_result.exit_code != 0 { + return (report --fail-fmt) + } + + let clippy_result = (do { clippy --features $features --verbose } | complete) + if $clippy_result.exit_code != 0 { + return (report --fail-clippy) + } + + print $"running ('toolkit test' | pretty-format-command)" + let test_result = (do { + if $fast { + if ($features | is-empty) { + test --workspace --fast + } else { + test --features $features --fast + } + } else { + if ($features | is-empty) { + test --workspace + } else { + test --features $features + } + } + } | complete) + + if $test_result.exit_code != 0 { + return (report --fail-test) + } + + print $"running ('toolkit test stdlib' | pretty-format-command)" + let stdlib_result = (do { test stdlib } | complete) + if $stdlib_result.exit_code != 0 { + return (report --fail-test-stdlib) + } + + report --no-fail +} + +# run Nushell from source with a right indicator +export def run [] { + ^cargo run -- ...[ + -e "$env.PROMPT_COMMAND_RIGHT = $'(ansi magenta_reverse)trying Nushell inside Cargo(ansi reset)'" + ] +} + +# set up git hooks to run: +# - `toolkit fmt --check --verbose` on `git commit` +# - `toolkit fmt --check --verbose` and `toolkit clippy --verbose` on `git push` +export def setup-git-hooks [] { + print "This command will change your local git configuration and hence modify your development workflow. Are you sure you want to continue? [y]" + if (input) == "y" { + print $"running ('toolkit setup-git-hooks' | pretty-format-command)" + git config --local core.hooksPath .githooks + } else { + print $"aborting ('toolkit setup-git-hooks' | pretty-format-command)" + } +} + +def build-nushell [features: string] { + print $'(char nl)Building nushell' + print '----------------------------' + + ^cargo build --features $features --locked +} + +def build-plugin [] { + let plugin = $in + + print $'(char nl)Building ($plugin)' + print '----------------------------' + + cd $"crates/($plugin)" + ^cargo build +} + +# build Nushell and plugins with some features +export def build [ + ...features: string@"nu-complete list features" # a space-separated list of feature to install with Nushell + --all # build all plugins with Nushell +] { + build-nushell ($features | default [] | str join ",") + + if not $all { + return + } + + let plugins = [ + nu_plugin_inc, + nu_plugin_gstat, + nu_plugin_query, + nu_plugin_polars, + nu_plugin_example, + nu_plugin_custom_values, + nu_plugin_formats, + ] + + for plugin in $plugins { + $plugin | build-plugin + } +} + +def "nu-complete list features" [] { + open Cargo.toml | get features | transpose feature dependencies | get feature +} + +def install-plugin [] { + let plugin = $in + + print $'(char nl)Installing ($plugin)' + print '----------------------------' + + ^cargo install --path $"crates/($plugin)" +} + +# install Nushell and features you want +export def install [ + ...features: string@"nu-complete list features" # a space-separated list of feature to install with Nushell + --all # install all plugins with Nushell +] { + touch crates/nu-cmd-lang/build.rs # needed to make sure `version` has the correct `commit_hash` + ^cargo install --path . --features ($features | default [] | str join ",") --locked --force + if not $all { + return + } + + let plugins = [ + nu_plugin_inc, + nu_plugin_gstat, + nu_plugin_query, + nu_plugin_polars, + nu_plugin_example, + nu_plugin_custom_values, + nu_plugin_formats, + ] + + for plugin in $plugins { + $plugin | install-plugin + } +} + +def windows? [] { + $nu.os-info.name == windows +} + +# filter out files that end in .d +def keep-plugin-executables [] { + if (windows?) { where name ends-with '.exe' } else { where name !~ '\.d' } +} + +# add all installed plugins +export def "add plugins" [] { + let plugin_path = (which nu | get path.0 | path dirname) + let plugins = (ls $plugin_path | where name =~ nu_plugin | keep-plugin-executables | get name) + + if ($plugins | is-empty) { + print $"no plugins found in ($plugin_path)..." + return + } + + for plugin in $plugins { + let plugin_result = (do { + print $"> plugin add ($plugin)" + plugin add $plugin + } | complete) + if $plugin_result.exit_code != 0 { + print -e $"(ansi rb)Failed to add ($plugin):\n($plugin_result.stderr)(ansi reset)" + } + } + + print $"\n(ansi gb)plugins registered, please restart nushell(ansi reset)" +} + +def compute-coverage [] { + print "Setting up environment variables for coverage" + # Enable LLVM coverage tracking through environment variables + # show env outputs .ini/.toml style description of the variables + # In order to use from toml, we need to make sure our string literals are single quoted + # This is especially important when running on Windows since "C:\blah" is treated as an escape + ^cargo llvm-cov show-env | str replace (char dq) (char sq) -a | from toml | load-env + + print "Cleaning up coverage data" + ^cargo llvm-cov clean --workspace + + print "Building with workspace and profile=ci" + # Apparently we need to explicitly build the necessary parts + # using the `--profile=ci` is basically `debug` build with unnecessary symbols stripped + # leads to smaller binaries and potential savings when compiling and running + ^cargo build --workspace --profile=ci + + print "Running tests with --workspace and profile=ci" + ^cargo test --workspace --profile=ci + + # You need to provide the used profile to find the raw data + print "Generating coverage report as lcov.info" + ^cargo llvm-cov report --lcov --output-path lcov.info --profile=ci +} + +# Script to generate coverage locally +# +# Output: `lcov.info` file +# +# Relies on `cargo-llvm-cov`. Install via `cargo install cargo-llvm-cov` +# https://github.com/taiki-e/cargo-llvm-cov +# +# You probably have to run `cargo llvm-cov clean` once manually, +# as you have to confirm to install additional tooling for your rustup toolchain. +# Else the script might stall waiting for your `y` +# +# Some of the internal tests rely on the exact cargo profile +# (This is somewhat criminal itself) +# but we have to signal to the tests that we use the `ci` `--profile` +# +# Manual gathering of coverage to catch invocation of the `nu` binary. +# This is relevant for tests using the `nu!` macro from `nu-test-support` +# see: https://github.com/taiki-e/cargo-llvm-cov#get-coverage-of-external-tests +# +# To display the coverage in your editor see: +# +# - https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters +# - https://github.com/umaumax/vim-lcov +# - https://github.com/andythigpen/nvim-coverage (probably needs some additional config) +export def cov [] { + let start = (date now) + $env.NUSHELL_CARGO_PROFILE = "ci" + + compute-coverage + + let end = (date now) + print $"Coverage generation took ($end - $start)." +} + +# Benchmark a target revision (default: current branch) against a reference revision (default: main branch) +# +# Results are saved in a `./tango` directory +# Ensure you have `cargo-export` installed to generate separate artifacts for each branch. +export def benchmark-compare [ + target?: string # which branch to compare (default: current branch) + reference?: string # the reference to compare against (default: main branch) +] { + let reference = $reference | default "main" + let current = git branch --show-current + let target = $target | default $current + + print $'-- Benchmarking ($target) against ($reference)' + + let export_dir = $env.PWD | path join "tango" + let ref_bin_dir = $export_dir | path join bin $reference + let tgt_bin_dir = $export_dir | path join bin $target + + # benchmark the target revision + print $'-- Running benchmarks for ($target)' + git checkout $target + ^cargo export $tgt_bin_dir -- bench + + # benchmark the comparison reference revision + print $'-- Running benchmarks for ($reference)' + git checkout $reference + ^cargo export $ref_bin_dir -- bench + + # return back to the whatever revision before benchmarking + print '-- Done' + git checkout $current + + # report results + let reference_bin = $ref_bin_dir | path join benchmarks + let target_bin = $tgt_bin_dir | path join benchmarks + ^$target_bin compare $reference_bin -o -s 50 --dump ($export_dir | path join samples) +} + +# Benchmark the current branch and logs the result in `./tango/samples` +# +# Results are saved in a `./tango` directory +# Ensure you have `cargo-export` installed to generate separate artifacts for each branch. +export def benchmark-log [ + target?: string # which branch to compare (default: current branch) +] { + let current = git branch --show-current + let target = $target | default $current + print $'-- Benchmarking ($target)' + + let export_dir = $env.PWD | path join "tango" + let bin_dir = ($export_dir | path join bin $target) + + # benchmark the target revision + if $target != $current { + git checkout $target + } + ^cargo export $bin_dir -- bench + + # return back to the whatever revision before benchmarking + print '-- Done' + if $target != $current { + git checkout $current + } + + # report results + let bench_bin = ($bin_dir | path join benchmarks) + ^$bench_bin compare -o -s 50 --dump ($export_dir | path join samples) +} + +# Build all Windows archives and MSIs for release manually +# +# This builds std and full distributions for both aarch64 and x86_64. +# +# You need to have the cross-compilers for MSVC installed (see Visual Studio). +# If compiling on x86_64, you need ARM64 compilers and libs too, and vice versa. +export def 'release-pkg windows' [ + --artifacts-dir="artifacts" # Where to copy the final msi and zip files to +] { + $env.RUSTFLAGS = "" + $env.CARGO_TARGET_DIR = "" + hide-env RUSTFLAGS + hide-env CARGO_TARGET_DIR + $env.OS = "windows-latest" + $env.GITHUB_WORKSPACE = ("." | path expand) + $env.GITHUB_OUTPUT = ("./output/out.txt" | path expand) + let version = (open Cargo.toml | get package.version) + mkdir $artifacts_dir + for target in ["aarch64" "x86_64"] { + $env.TARGET = $target ++ "-pc-windows-msvc" + + rm -rf output + _EXTRA_=bin nu .github/workflows/release-pkg.nu + cp $"output/nu-($version)-($target)-pc-windows-msvc.zip" $artifacts_dir + + rm -rf output + _EXTRA_=msi nu .github/workflows/release-pkg.nu + cp $"target/wix/nu-($version)-($target)-pc-windows-msvc.msi" $artifacts_dir + } +} + +# these crates should compile for wasm +const wasm_compatible_crates = [ + "nu-cmd-base", + "nu-cmd-extra", + "nu-cmd-lang", + "nu-color-config", + "nu-command", + "nu-derive-value", + "nu-engine", + "nu-glob", + "nu-json", + "nu-parser", + "nu-path", + "nu-pretty-hex", + "nu-protocol", + "nu-std", + "nu-system", + "nu-table", + "nu-term-grid", + "nu-utils", + "nuon" +] + +def "prep wasm" [] { + ^rustup target add wasm32-unknown-unknown +} + +# build crates for wasm +export def "build wasm" [] { + prep wasm + + for crate in $wasm_compatible_crates { + print $'(char nl)Building ($crate) for wasm' + print '----------------------------' + ( + ^cargo build + -p $crate + --target wasm32-unknown-unknown + --no-default-features + ) + } +} + +# make sure no api is used that doesn't work with wasm +export def "clippy wasm" [] { + prep wasm + + $env.CLIPPY_CONF_DIR = $toolkit_dir | path join clippy wasm + + for crate in $wasm_compatible_crates { + print $'(char nl)Checking ($crate) for wasm' + print '----------------------------' + ( + ^cargo clippy + -p $crate + --target wasm32-unknown-unknown + --no-default-features + -- + -D warnings + -D clippy::unwrap_used + -D clippy::unchecked_duration_subtraction + ) + } +} + +export def main [] { help toolkit } diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e60bc4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +.internal.git/ +.p +.claude +.vscode +.shellcheckrc +.coder +.migration +.zed +# ai_demo.nu +CLAUDE.md +.cache +.coder +.wrks +ROOT +OLD +old-config +plugins/nushell-plugins +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ +# Encryption keys and related files (CRITICAL - NEVER COMMIT) +.k.backup +*.key.backup + +config.*.toml +config.*back + +# where book is written +_book + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +node_modules/ + +**/output.css +**/input.css + +# Environment files +.env +.env.local +.env.production +.env.development +.env.staging + +# Keep example files +!.env.example + +# Configuration files (may contain sensitive data) +config.prod.toml +config.production.toml +config.local.toml +config.*.local.toml + +# Keep example configuration files +!config.toml +!config.dev.toml +!config.example.toml + +# Log files +logs/ +*.log + +# TLS certificates and keys +certs/ +*.pem +*.crt +*.key +*.p12 +*.pfx + +# Database files +*.db +*.sqlite +*.sqlite3 + +# Backup files +*.bak +*.backup +*.tmp +*~ + +# Encryption and security related files +*.encrypted +*.enc +secrets/ +private/ +security/ + +# Configuration backups that may contain secrets +config.*.backup +config.backup.* + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db +# Documentation build output +book-output/ +# Generated setup report +SETUP_COMPLETE.md + +.coder/ +.claude/ diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000..b1d73e6 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,96 @@ +// Markdownlint-cli2 Configuration +// Documentation quality enforcement aligned with CLAUDE.md guidelines +// See: https://github.com/igorshubovych/markdownlint-cli2 + +{ + "config": { + "default": true, + + // Headings - enforce proper hierarchy + "MD001": false, // heading-increment (relaxed - allow flexibility) + "MD026": { "punctuation": ".,;:!?" }, // heading-punctuation + + // Lists - enforce consistency + "MD004": { "style": "consistent" }, // ul-style (consistent list markers) + "MD005": false, // inconsistent-indentation (relaxed) + "MD007": { "indent": 2 }, // ul-indent + "MD029": false, // ol-prefix (allow flexible list numbering) + "MD030": { "ul_single": 1, "ol_single": 1, "ul_multi": 1, "ol_multi": 1 }, + + // Code blocks - fenced only + "MD046": { "style": "fenced" }, // code-block-style + + // Formatting - strict whitespace + "MD009": true, // no-hard-tabs + "MD010": true, // hard-tabs + "MD011": true, // reversed-link-syntax + "MD018": true, // no-missing-space-atx + "MD019": true, // no-multiple-space-atx + "MD020": true, // no-missing-space-closed-atx + "MD021": true, // no-multiple-space-closed-atx + "MD023": true, // heading-starts-line + "MD027": true, // no-multiple-spaces-blockquote + "MD037": true, // no-space-in-emphasis + "MD039": true, // no-space-in-links + + // Trailing content + "MD012": false, // no-multiple-blanks (relaxed - allow formatting space) + "MD024": false, // no-duplicate-heading (too strict for docs) + "MD028": false, // no-blanks-blockquote (relaxed) + "MD047": true, // single-trailing-newline + + // Links and references + "MD034": true, // no-bare-urls (links must be formatted) + "MD040": true, // fenced-code-language (code blocks need language) + "MD042": true, // no-empty-links + + // HTML - allow for documentation formatting and images + "MD033": { "allowed_elements": ["br", "hr", "details", "summary", "p", "img"] }, + + // Line length - relaxed for technical documentation + "MD013": { + "line_length": 150, + "heading_line_length": 150, + "code_block_line_length": 150, + "code_blocks": true, + "tables": true, + "headers": true, + "headers_line_length": 150, + "strict": false, + "stern": false + }, + + // Images + "MD045": true, // image-alt-text + + // Disable rules that conflict with relaxed style + "MD003": false, // consistent-indentation + "MD041": false, // first-line-heading + "MD025": false, // single-h1 / multiple-top-level-headings + "MD022": false, // blanks-around-headings (flexible spacing) + "MD032": false, // blanks-around-lists (flexible spacing) + "MD035": false, // hr-style (consistent) + "MD036": false, // no-emphasis-as-heading + "MD044": false // proper-names + }, + + // Documentation patterns + "globs": [ + "docs/**/*.md", + "!docs/node_modules/**", + "!docs/build/**" + ], + + // Ignore build artifacts, external content, and operational directories + "ignores": [ + "node_modules/**", + "target/**", + ".git/**", + "build/**", + "dist/**", + ".coder/**", + ".claude/**", + ".wrks/**", + ".vale/**" + ] +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b358285 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,176 @@ +# Pre-commit Framework Configuration +# Generated by dev-system/ci +# Configures git pre-commit hooks for Rust projects + +repos: + # ============================================================================ + # Rust Hooks (COMMENTED OUT - Not used in this repo) + # ============================================================================ + # - repo: local + # hooks: + # - id: rust-fmt + # name: Rust formatting (cargo +nightly fmt) + # entry: bash -c 'cargo +nightly fmt --all -- --check' + # language: system + # types: [rust] + # pass_filenames: false + # stages: [pre-commit] + # + # - id: rust-clippy + # name: Rust linting (cargo clippy) + # entry: bash -c 'cargo clippy --all-targets -- -D warnings' + # language: system + # types: [rust] + # pass_filenames: false + # stages: [pre-commit] + # + # - id: rust-test + # name: Rust tests + # entry: bash -c 'cargo test --workspace' + # language: system + # types: [rust] + # pass_filenames: false + # stages: [pre-push] + # + # - id: cargo-deny + # name: Cargo deny (licenses & advisories) + # entry: bash -c 'cargo deny check licenses advisories' + # language: system + # pass_filenames: false + # stages: [pre-push] + + # ============================================================================ + # Nushell Hooks (ACTIVE) + # ============================================================================ + - repo: local + hooks: + - id: nushell-check + name: Nushell validation (nu --ide-check) + entry: >- + bash -c 'for f in $(git diff --cached --name-only --diff-filter=ACM | grep "\.nu$"); do + echo "Checking: $f"; nu --ide-check 100 "$f" || exit 1; done' + language: system + types: [file] + files: \.nu$ + pass_filenames: false + stages: [pre-commit] + + # ============================================================================ + # Nickel Hooks (ACTIVE) + # ============================================================================ + - repo: local + hooks: + - id: nickel-typecheck + name: Nickel type checking + entry: >- + bash -c 'export NICKEL_IMPORT_PATH="../:."; for f in $(git diff --cached --name-only --diff-filter=ACM | grep "\.ncl$"); do + echo "Checking: $f"; nickel typecheck "$f" || exit 1; done' + language: system + types: [file] + files: \.ncl$ + pass_filenames: false + stages: [pre-commit] + + # ============================================================================ + # Bash Hooks (optional - enable if using Bash) + # ============================================================================ + # - repo: local + # hooks: + # - id: shellcheck + # name: Shellcheck (bash linting) + # entry: shellcheck + # language: system + # types: [shell] + # stages: [commit] + # + # - id: shfmt + # name: Shell script formatting + # entry: bash -c 'shfmt -i 2 -d' + # language: system + # types: [shell] + # stages: [commit] + + # ============================================================================ + # Markdown Hooks (ACTIVE) + # ============================================================================ + - repo: local + hooks: + - id: markdownlint + name: Markdown linting (markdownlint-cli2) + entry: markdownlint-cli2 + language: system + types: [markdown] + stages: [pre-commit] + + # CRITICAL: markdownlint-cli2 MD040 only checks opening fences for language. + # It does NOT catch malformed closing fences (e.g., ```plaintext) - CommonMark violation. + # This hook is ESSENTIAL to prevent malformed closing fences from entering the repo. + # See: .markdownlint-cli2.jsonc line 22-24 for details. + - id: check-malformed-fences + name: Check malformed closing fences (CommonMark) + entry: bash -c 'cd .. && nu scripts/check-malformed-fences.nu $(git diff --cached --name-only --diff-filter=ACM | grep "\.md$" | grep -v ".coder/" | grep -v ".claude/" | grep -v "old_config/" | tr "\n" " ")' + language: system + types: [markdown] + pass_filenames: false + stages: [pre-commit] + exclude: ^\.coder/|^\.claude/|^old_config/ + + # ============================================================================ + # General Pre-commit Hooks + # ============================================================================ + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files + args: ['--maxkb=1000'] + + - id: check-case-conflict + + - id: check-merge-conflict + + - id: check-toml + + # - id: check-yaml + # exclude: ^\.woodpecker/ + + - id: end-of-file-fixer + exclude: ^(\.coder/|\.wrks/|\.claude/) + + - id: trailing-whitespace + exclude: \.md$|^(\.coder/|\.wrks/|\.claude/) + + - id: mixed-line-ending + exclude: ^(\.coder/|\.wrks/|\.claude/) + + # ============================================================================ + # ADR-026 nulib structural constraints (M6) + # ============================================================================ + - repo: local + hooks: + - id: nulib-no-relative-imports-new-tree + name: "ADR-026: no relative imports in new nulib layers" + entry: bash -c 'found=$(git diff --cached --name-only | grep "nulib/(cli|orchestration|platform|domain|tools|primitives|extensions)/" | xargs grep -l "^use \.\." 2>/dev/null); [ -z "$found" ] && exit 0; echo "Relative imports found in new tree:"; echo "$found"; exit 1' + language: system + pass_filenames: false + stages: [pre-commit] + + - id: nulib-single-nickel-export + name: "ADR-026: ^nickel export only in tools/nickel/process.nu" + entry: bash -c 'found=$(git diff --cached --name-only | grep "nulib/.*\.nu$" | grep -v "nulib/tools/nickel/process.nu" | xargs grep -l "\\^nickel export" 2>/dev/null); [ -z "$found" ] && exit 0; echo "^nickel export outside tools/nickel/process.nu:"; echo "$found"; exit 1' + language: system + pass_filenames: false + stages: [pre-commit] + + - id: nulib-no-surrealdb + name: "ADR-026: no surrealdb calls in nulib" + entry: bash -c 'found=$(git diff --cached --name-only | grep "nulib/.*\.nu$" | xargs grep -l "surrealdb\|surreal\b" 2>/dev/null); [ -z "$found" ] && exit 0; echo "surrealdb in nulib:"; echo "$found"; exit 1' + language: system + pass_filenames: false + stages: [pre-commit] + + - id: nulib-no-star-imports-new-tree + name: "ADR-026: no star imports (use X *) in new nulib layers" + entry: bash -c 'found=$(git diff --cached --name-only | grep "nulib/(cli|orchestration|platform|domain|tools|primitives|extensions)/.*\.nu$" | xargs grep -lP "^use \S+ \*$" 2>/dev/null | grep -v "mod\.nu"); [ -z "$found" ] && exit 0; echo "Star imports in new tree (outside mod.nu):"; echo "$found"; exit 1' + language: system + pass_filenames: false + stages: [pre-commit] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..caf6853 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,381 @@ +# Provisioning Core - Changelog + +**Date**: 2026-04-17 +**Repository**: provisioning/core +**Status**: Nickel IaC (PRIMARY) + +--- + +## ๐Ÿ“‹ Summary + +Major refactor: three-layer DAG architecture with workspace composition, Unified +Component Architecture (components + workflows + capabilities), Nickel-backed +commands-registry with JSON cache for fast CLI startup, consolidated platform +service manager, and completed Nushell 0.110/0.112 migration (no try/catch, no +bash redirections). TTY stack moved from `shlib/` into `cli/tty-*`. Numerous new +domain modules: `dag`, `components`, `workflow` engine, `images` lifecycle, +workspace state/sync, ontoref queries, FIP handler. + +--- + +## ๐Ÿ”„ Latest Release (2026-04-17) + +### Three-Layer DAG Architecture + +**Scope**: Workspace composition as a DAG with formula_id::task_name namespacing, +health gates, conditions, and NATS subject emission. + +**New files**: +- `nulib/main_provisioning/dag.nu` โ€” `dag show/validate/export` (DOT/JSON/Mermaid) +- `nulib/lib_provisioning/config/loader/dag.nu` โ€” DAG config loader +- `nulib/taskservs/dag-executor.nu` โ€” taskserv-level DAG execution helper + +**Related**: ADR-020 (extension capability declarations), ADR-021 (workspace +composition DAG). Orchestrator consumes composition via +`WorkspaceComposition::into_workflow` and emits NATS events. + +### Unified Component Architecture + +**Scope**: Components + workflows + capabilities as first-class citizens +(libre-daoshi plan, blocks A-H complete). + +**New files**: +- `nulib/components/mod.nu` โ€” component dispatch module +- `nulib/main_provisioning/components.nu` โ€” `validate capabilities/components`, + `component list/info` +- `nulib/main_provisioning/workflow.nu` โ€” full workflow engine: run/list/status/ + validate, topological sort, NATS event emission (+605 lines) +- `nulib/main_provisioning/extensions.nu` โ€” `extensions capabilities/graph` +- `nulib/main_provisioning/ontoref-queries.nu` โ€” on+re-aware CLI queries + (describe component/databases/namespace/storage/workflow) + +### Commands-Registry & Fast-Path Dispatch + +**Scope**: Eliminate Nu startup cost on every `prvng` invocation. + +**New files**: +- `nulib/commands-registry.ncl` โ€” Nickel command catalog (314 lines) +- `nulib/lib_provisioning/utils/command-registry.nu` โ€” registry accessor +- `nulib/scripts/validate-command.nu` โ€” cache-aware command validator + +**Behavior**: `cli/provisioning` reads the JSON cache at +`~/.cache/provisioning/commands-registry.json`, rebuilt automatically via +`nickel export` when the `.ncl` source is newer. Single-char aliases +(`s`, `t`, `c`, `e`, `w`, `j`, `b`, `o`, `a`) are expanded in bash before +dispatch. `nulib/main_provisioning/ADDING_COMMANDS.md` documents the four-step +procedure for new commands. + +### Platform Service Manager + +**New files**: +- `nulib/lib_provisioning/platform/service-manager.nu` (+573 lines) +- `nulib/lib_provisioning/platform/startup.nu` (+611 lines) +- `nulib/lib_provisioning/utils/service-check.nu` (+255 lines) + +**Refactored**: `platform/autostart.nu`, `platform/bootstrap.nu`, +`platform/health.nu`, `platform/target.nu` โ€” unified lifecycle, health probes, +and autostart logic. + +### Nushell 0.112.2 Migration + +**Scope**: Project-wide refactor driven by `scripts/refactor-try-catch.nu` and +`scripts/refactor-try-catch-simplified.nu` to reach Nushell 0.112.2 compliance. + +**Enforced**: +- No `try/catch` โ€” all use `do { } | complete` +- No bash redirections (`2>&1`, `2>/dev/null`) +- External commands prefixed with `^` +- Parenthesized pipelines in `if` +- Type signatures: `def f [x: string]: nothing -> record { }` + +### TTY Stack Replacement + +**Removed**: `shlib/README.md`, `shlib/auth-login-tty.sh`, +`shlib/mfa-enroll-tty.sh`, `shlib/setup-wizard-tty.sh`. + +**Replaced by**: +- `cli/tty-dispatch.sh` (+86 lines) โ€” TTY-safe command dispatcher +- `cli/tty-filter.sh` (+137 lines) โ€” command filter +- `cli/tty-commands.conf` โ€” TTY command manifest + +### New Domain Modules + +- `nulib/images/` โ€” golden image lifecycle (`create`, `delete`, `list`, `state`, + `update`, `watch`) +- `nulib/workspace/state.nu` (+641 lines) โ€” workspace state model +- `nulib/workspace/sync.nu` (+148 lines) โ€” workspace synchronization +- `nulib/main_provisioning/bootstrap.nu` โ€” platform bootstrap +- `nulib/main_provisioning/cluster-deploy.nu` โ€” component/taskserv dispatch +- `nulib/main_provisioning/fip.nu` (+421 lines) โ€” floating IP handler +- `nulib/main_provisioning/state.nu` โ€” state command +- `nulib/main_provisioning/commands/state.nu`, `commands/build.nu`, + `commands/integrations/auth.nu`, `commands/utilities/alias.nu` +- `nulib/main_provisioning/commands/platform.nu` โ€” major expansion (+874 lines) + +### Config Loader Overhaul + +- `lib_provisioning/config/loader/core.nu` โ€” slimmed (โˆ’759 lines of legacy paths) +- `lib_provisioning/config/cache/core.nu` โ€” refactored (โˆ’454 lines of dead paths) +- `lib_provisioning/config/cache/nickel.nu` โ€” simplified +- Removed: `lib_provisioning/config/loaders/file_loader.nu` (โˆ’330 lines) +- Added: `config/accessor-minimal.nu`, `config/accessor/functions.nu` helpers + +### Scripts & Tooling + +- `nulib/scripts/` โ€” query-* family (clusters/infra/providers/servers/taskservs/ + workspace-info), validate-command, validate-config +- `scripts/auto-refactor-priority.nu`, `scripts/batch-refactor.sh` +- `scripts/build-nixos-image-remote.sh`, `scripts/deploy-cp-server.sh` + +### CLI Modular Subcommands + +New top-level Nu modules referenced by the bash dispatcher: +`provisioning-batch.nu`, `provisioning-bootstrap.nu`, `provisioning-cluster.nu`, +`provisioning-component.nu`, `provisioning-extension.nu`, `provisioning-job.nu`, +`provisioning-platform.nu`, `provisioning-server.nu`, `provisioning-state.nu`, +`provisioning-status.nu`, `provisioning-taskserv.nu`, `provisioning-volume.nu`, +`provisioning-workflow.nu`. + +### Tests + +- `nulib/tests/test_workspace_state.nu` (+351 lines) +- Updates to `test_oci_registry.nu`, `test_services.nu` + +### Statistics + +| Area | Files | Lines +/โˆ’ | +| ---- | ----- | --------- | +| DAG + Components + Workflows | 8 | +1800 / โˆ’50 | +| Commands-registry + dispatch | 6 | +900 / โˆ’200 | +| Platform service manager | 5 | +1700 / โˆ’300 | +| Config loader/cache refactor | 10 | +400 / โˆ’1500 | +| TTY replacement | 4 | +250 / โˆ’515 | +| New subcommand modules | 13 | +1700 / โˆ’0 | +| **Total staged** | **242 files** | **+21949 / โˆ’6012** | + +--- + +## ๐Ÿ”„ Previous Release (2026-01-14) + +### Terminology Migration: Cluster โ†’ Taskserv + +**Scope**: Complete refactoring across nulib/ modules to standardize on taskserv nomenclature + +**Files Updated**: +- `nulib/clusters/handlers.nu` - Handler signature updates, ANSI formatting improvements +- `nulib/clusters/run.nu` - Function parameter and path updates (+326 lines modified) +- `nulib/clusters/utils.nu` - Utility function updates (+144 lines modified) +- `nulib/clusters/discover.nu` - Discovery module refactoring +- `nulib/clusters/load.nu` - Configuration loader updates +- `nulib/ai/query_processor.nu` - AI integration updates +- `nulib/api/routes.nu` - API routing adjustments +- `nulib/api/server.nu` - Server module updates +- `.pre-commit-config.yaml` - Pre-commit hook updates + +**Changes**: +- Updated function parameters: `server_cluster_path` โ†’ `server_taskserv_path` +- Updated record fields: `defs.cluster.name` โ†’ `defs.taskserv.name` +- Enhanced output formatting with consistent ANSI styling (yellow_bold, default_dimmed, purple_bold) +- Improved function documentation and import organization +- Pre-commit configuration refinements + +**Rationale**: Taskserv better reflects the service-oriented nature of infrastructure components and improves semantic clarity throughout the codebase. + +### i18n/Localization System + +**New Feature**: Fluent i18n integration for internationalized help system + +**Implementation**: +- `nulib/main_provisioning/help_system_fluent.nu` - Fluent-based i18n framework +- Active locale detection from `LANG` environment variable +- Fallback to English (en-US) for missing translations +- Fluent catalog parsing: `locale/{locale}/help.ftl` +- Locale format conversion: `es_ES.UTF-8` โ†’ `es-ES` + +**Features**: +- Automatic locale detection from system LANG +- Fluent catalog format support for translations +- Graceful fallback mechanism +- Category-based color formatting (infrastructure, orchestration, development, etc.) +- Tab-separated help column formatting + +--- + +## ๐Ÿ“‹ Version History + +### v1.0.10 (Previous Release) +- Stable release with Nickel IaC support +- Base version with core CLI and library system + +### v1.0.11 (Current - 2026-01-14) +- **Cluster โ†’ Taskserv** terminology migration +- **Fluent i18n** system documentation +- Enhanced ANSI output formatting + +--- + +## ๐Ÿ“ Changes by Directory + +### cli/ directory + +**Major Updates (586 lines added to provisioning)** + +- Expanded CLI command implementations (+590 lines) +- Enhanced tools installation system (tools-install: +163 lines) +- Improved install script for Nushell environment (install_nu.sh: +31 lines) +- Better CLI routing and command validation +- Help system enhancements for Nickel-aware commands +- Support for Nickel schema evaluation and validation + +### nulib/ directory + +**Nushell libraries - Nickel-first architecture** + +**Config System** +- `config/loader.nu` - Nickel schema loading and evaluation +- `config/accessor.nu` - Accessor patterns for Nickel fields +- `config/cache/` - Cache system optimized for Nickel evaluation + +**AI & Documentation** +- `ai/README.md` - Nickel IaC patterns +- `ai/info_about.md` - Nickel-focused documentation +- `ai/lib.nu` - AI integration for Nickel schema analysis + +**Extension System** +- `extensions/QUICKSTART.md` - Nickel extension quickstart (+50 lines) +- `extensions/README.md` - Extension system for Nickel (+63 lines) +- `extensions/loader_oci.nu` - OCI registry loader (minor updates) + +**Infrastructure & Validation** +- `infra_validator/rules_engine.nu` - Validation rules for Nickel schemas +- `infra_validator/validator.nu` - Schema validation support +- `loader-minimal.nu` - Minimal loader for lightweight deployments + +**Clusters & Workflows** +- `clusters/discover.nu`, `clusters/load.nu`, `clusters/run.nu` - Cluster operations updated +- Plugin definitions updated for Nickel integration (+28-38 lines) + +**Documentation** +- `SERVICE_MANAGEMENT_SUMMARY.md` - Expanded service documentation (+90 lines) +- `gitea/IMPLEMENTATION_SUMMARY.md` - Gitea integration guide (+89 lines) +- Extension and validation quickstarts and README updates + +### plugins/ directory + +Nushell plugins for performance optimization + +**Sub-repositories:** + +- `nushell-plugins/` - Multiple Nushell plugins + - `_nu_plugin_inquire/` - Interactive form plugin + - `api_nu_plugin_nickel/` - Nickel integration plugin + - Additional plugin implementations + +**Plugin Documentation:** + +- Build summaries +- Installation guides +- Configuration examples +- Test documentation +- Fix and limitation reports + +### scripts/ directory + +Utility scripts for system operations + +- Build scripts +- Installation scripts +- Testing scripts +- Development utilities +- Infrastructure scripts + +### services/ directory + +Service definitions and configurations + +- Service descriptions +- Service management + +### forminquire/ directory (ARCHIVED) + +**Status**: DEPRECATED - Archived to `.coder/archive/forminquire/` + +**Replacement**: TypeDialog forms (`.typedialog/provisioning/`) + +- Legacy: Jinja2-based form system +- Archived: 2025-01-09 +- Replaced by: TypeDialog with bash wrappers for TTY-safe input + +### Additional Files + +- `README.md` - Core system documentation +- `versions.ncl` - Version definitions +- `.gitignore` - Git ignore patterns +- `nickel.mod` / `nickel.mod.lock` - Nickel module definitions +- `.githooks/` - Git hooks for development + +--- + +## ๐Ÿ“Š Change Statistics + +| Category | Files | Lines Added | Lines Removed | Status | +| -------- | ----- | ----------- | ------------- | ------ | +| CLI | 3 | 780+ | 30+ | Major update | +| Config System | 15+ | 300+ | 200+ | Refactored | +| AI/Docs | 8+ | 350+ | 100+ | Enhanced | +| Extensions | 5+ | 150+ | 50+ | Updated | +| Infrastructure | 8+ | 100+ | 70+ | Updated | +| Clusters/Workflows | 5+ | 80+ | 30+ | Enhanced | +| **Total** | **60+ files** | **1700+ lines** | **500+ lines** | **Complete** | + +--- + +## โœจ Key Areas + +### CLI System + +- Command implementations with Nickel support +- Tools installation system +- Nushell environment setup +- Nickel schema evaluation commands +- Error messages and help text +- Nickel type checking and validation + +### Config System + +- **Nickel-first loader**: Schema evaluation via config/loader.nu +- **Optimized caching**: Nickel evaluation cache patterns +- **Field accessors**: Nickel record manipulation +- **Schema validation**: Type-safe configuration loading + +### AI & Documentation + +- AI integration for Nickel IaC +- Extension development guides +- Service management documentation + +### Extensions & Infrastructure + +- OCI registry loader optimization +- Schema-aware extension system +- Infrastructure validation for Nickel definitions +- Cluster discovery and operations enhanced + +--- + +## ๐ŸŽฏ Current Features + +- **Nickel IaC**: Type-safe infrastructure definitions +- **CLI System**: Unified command interface with 80+ shortcuts +- **Provider Abstraction**: Cloud-agnostic operations +- **Config System**: Hierarchical configuration with 476+ accessors +- **Workflow Engine**: Batch operations with dependency resolution +- **Validation**: Schema-aware infrastructure validation +- **AI Integration**: Schema-driven configuration generation + +--- + +**Status**: Production +**Date**: 2026-01-14 +**Repository**: provisioning/core +**Version**: 1.0.11 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..670d007 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,109 @@ +# Code of Conduct + +## Our Pledge + +We, as members, contributors, and leaders, pledge to make participation in our project and community +a harassment-free experience for everyone, regardless of: + +- Age +- Body size +- Visible or invisible disability +- Ethnicity +- Sex characteristics +- Gender identity and expression +- Level of experience +- Education +- Socioeconomic status +- Nationality +- Personal appearance +- Race +- Caste +- Color +- Religion +- Sexual identity and orientation + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by mistakes +- Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery +- Trolling, insulting, or derogatory comments +- Personal or political attacks +- Public or private harassment +- Publishing others' private information (doxing) +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying and enforcing our standards of acceptable behavior +and will take appropriate corrective action in response to unacceptable behavior. + +Maintainers have the right and responsibility to: + +- Remove, edit, or reject comments, commits, code, and other contributions +- Ban contributors for behavior they deem inappropriate, threatening, or harmful + +## Scope + +This Code of Conduct applies to: + +- All community spaces (GitHub, forums, chat, events, etc.) +- Official project channels and representations +- Interactions between community members related to the project + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to project maintainers: + +- Email: [project contact] +- GitHub: Private security advisory +- Issues: Report with `conduct` label (public discussions only) + +All complaints will be reviewed and investigated promptly and fairly. + +### Enforcement Guidelines + +**1. Correction** + +- Community impact: Use of inappropriate language or unwelcoming behavior +- Action: Private written warning with explanation and clarity on impact +- Consequence: Warning and no further violations + +**2. Warning** + +- Community impact: Violation through single incident or series of actions +- Action: Written warning with severity consequences for continued behavior +- Consequence: Suspension from community interaction + +**3. Temporary Ban** + +- Community impact: Serious violation of standards +- Action: Temporary ban from community interaction +- Consequence: Revocation of ban after reflection period + +**4. Permanent Ban** + +- Community impact: Pattern of violating community standards +- Action: Permanent ban from community interaction + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1. + +For answers to common questions about this code of conduct, see the FAQ at . + +--- + +**Thank you for being part of our community!** + +We believe in creating a welcoming and inclusive space where everyone can contribute their best work. Together, we make this project better. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..92ec1e9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,131 @@ +# Contributing to provisioning + +Thank you for your interest in contributing! This document provides guidelines and instructions for contributing to this project. + +## Code of Conduct + +This project adheres to a Code of Conduct. By participating, you are expected to uphold this code. +Please see [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for details. + +## Getting Started + +### Prerequisites + +- Rust 1.70+ (if project uses Rust) +- NuShell (if project uses Nushell scripts) +- Git + +### Development Setup + +1. Fork the repository +2. Clone your fork: `git clone https://repo.jesusperez.pro/jesus/provisioning` +3. Add upstream: `git remote add upstream https://repo.jesusperez.pro/jesus/provisioning` +4. Create a branch: `git checkout -b feature/your-feature` + +## Development Workflow + +### Before You Code + +- Check existing issues and pull requests to avoid duplication +- Create an issue to discuss major changes before implementing +- Assign yourself to let others know you're working on it + +### Code Standards + +#### Rust + +- Run `cargo fmt --all` before committing +- All code must pass `cargo clippy -- -D warnings` +- Write tests for new functionality +- Maintain 100% documentation coverage for public APIs + +#### Nushell + +- Validate scripts with `nu --ide-check 100 script.nu` +- Follow consistent naming conventions +- Use type hints where applicable + +#### Nickel + +- Type check schemas with `nickel typecheck` +- Document schema fields with comments +- Test schema validation + +### Commit Guidelines + +- Write clear, descriptive commit messages +- Reference issues with `Fixes #123` or `Related to #123` +- Keep commits focused on a single concern +- Use imperative mood: "Add feature" not "Added feature" + +### Testing + +All changes must include tests: + +```bash +# Run all tests +cargo test --workspace + +# Run with coverage +cargo llvm-cov --all-features --lcov + +# Run locally before pushing +just ci-full +``` + +### Pull Request Process + +1. Update documentation for any changed functionality +2. Add tests for new code +3. Ensure all CI checks pass +4. Request review from maintainers +5. Be responsive to feedback and iterate quickly + +## Review Process + +- Maintainers will review your PR within 3-5 business days +- Feedback is constructive and meant to improve the code +- All discussions should be respectful and professional +- Once approved, maintainers will merge the PR + +## Reporting Bugs + +Found a bug? Please file an issue with: + +- **Title**: Clear, descriptive title +- **Description**: What happened and what you expected +- **Steps to reproduce**: Minimal reproducible example +- **Environment**: OS, Rust version, etc. +- **Screenshots**: If applicable + +## Suggesting Enhancements + +Have an idea? Please file an issue with: + +- **Title**: Clear feature title +- **Description**: What, why, and how +- **Use cases**: Real-world scenarios where this would help +- **Alternative approaches**: If you've considered any + +## Documentation + +- Keep README.md up to date +- Document public APIs with rustdoc comments +- Add examples for non-obvious functionality +- Update CHANGELOG.md with your changes + +## Release Process + +Maintainers handle releases following semantic versioning: + +- MAJOR: Breaking changes +- MINOR: New features (backward compatible) +- PATCH: Bug fixes + +## Questions + +- Check existing documentation and issues +- Ask in discussions or open an issue +- Join our community channels + +Thank you for contributing! diff --git a/cli/.gitkeep b/cli/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..d298fff --- /dev/null +++ b/cli/README.md @@ -0,0 +1,467 @@ +# Provisioning CLI - Flow-Aware TTY Command Management + +## Architecture Overview + +The provisioning wrapper (`provisioning/core/cli/provisioning`) is a **flow controller** that manages three execution paths for command handling: + +1. **Standalone TTY** - Interactive commands that exit after execution +2. **Pipeline TTY** - Interactive commands that output for piping to other commands +3. **Regular** - Standard Nushell command processing + +This design enables: +- Interactive commands (TTY input) without blocking Nushell +- Inter-command piping of TTY output to subsequent commands +- Same-command flow (TTY input โ†’ Nushell processing in one execution) +- Daemon optimization for non-interactive commands + +## How Flow Management Works + +### Execution Flow + +```text +User Command: provisioning + โ†“ +Bash wrapper (provisioning) + โ†“ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Phase 1: TTY Command Detection โ”‚ +โ”‚ - Read tty-commands.conf registry โ”‚ +โ”‚ - Match command pattern โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ†“ + โ”œโ”€โ†’ Not a TTY command โ†’ Continue to Nushell (normal processing) + โ”‚ + โ””โ”€โ†’ TTY command found โ†’ Check flow type + โ†“ + โ”œโ”€โ†’ flow=exit โ†’ Execute wrapper, exit immediately + โ”œโ”€โ†’ flow=pipe โ†’ Execute wrapper, output to stdout, exit (allows piping) + โ””โ”€โ†’ flow=continue โ†’ Execute wrapper, capture output, continue to Nushell + ($env.TTY_OUTPUT available in Nushell) +``` + +### Flow Types Explained + +#### 1. Standalone TTY Commands (flow=exit) + +**Use case**: Interactive forms, setup wizards, authentication dialogs + +**Example**: `provisioning setup wizard` + +**Flow**: + +```bash +Bash wrapper โ†’ TTY filter detects "setup wizard" โ†’ flow=exit + โ†“ +Execute wrapper: core/shlib/setup-wizard-tty.sh + โ†“ +User interaction (TypeDialog form) + โ†“ +Exit wrapper โ†’ Exit bash wrapper + โ†“ +Never reaches Nushell +``` + +**Registry entry**: + +```bash +"setup wizard" "core/shlib/setup-wizard-tty.sh" "exit" +``` + +#### 2. Pipeline TTY Commands (flow=pipe) + +**Use case**: Getting user input to pipe to another command + +**Example**: `provisioning auth get-key | provisioning deploy --api-key-stdin` + +**Flow**: + +```bash +Bash wrapper โ†’ TTY filter detects "auth get-key" โ†’ flow=pipe + โ†“ +Execute wrapper: core/shlib/auth-get-key-tty.sh + โ†“ +User provides API key via TTY prompt + โ†“ +Wrapper outputs API key to stdout + โ†“ +Exit wrapper (process exits, pipe has captured output) + โ†“ +Next command receives API key from stdin +``` + +**Registry entry**: + +```bash +"auth get-key" "core/shlib/auth-get-key-tty.sh" "pipe" +``` + +**Wrapper requirements** (flow=pipe): +- Must output result to stdout +- Output must be newline-terminated +- Exit with proper code (0=success, non-zero=error) + +#### 3. Continue-to-Nushell TTY Commands (flow=continue) + +**Use case**: TTY input that needs further processing in Nushell + +**Example**: `provisioning auth integrate --provider azure` + +**Flow**: + +```bash +Bash wrapper โ†’ TTY filter detects "auth integrate" โ†’ flow=continue + โ†“ +Execute wrapper: core/shlib/auth-integrate-tty.sh + โ†“ +User provides credentials via TTY prompt + โ†“ +Wrapper outputs credentials (usually JSON) to stdout + โ†“ +Filter CAPTURES output to $TTY_OUTPUT environment variable + โ†“ +Set $env.PROVISIONING_BYPASS_DAEMON=true (skip daemon) + โ†“ +Return 0 WITHOUT EXITING (continue to Nushell) + โ†“ +Nushell dispatcher receives both: + - CLI args: --provider azure + - TTY output: $env.TTY_OUTPUT (credentials JSON) + โ†“ +Nushell script processes both, completes integration +``` + +**Registry entry**: + +```bash +"auth integrate" "core/shlib/auth-integrate-tty.sh" "continue" +``` + +**Wrapper requirements** (flow=continue): +- Must output result to stdout (usually JSON for structured data) +- Exit with proper code (0=success, non-zero=error) + +**Nushell script requirements** (receives flow=continue output): + +```nushell +export def "provisioning auth integrate" [--provider: string] { + # Check if TTY output exists (guard pattern) + let tty_output = ($env.TTY_OUTPUT? | default "") + if ($tty_output | is-empty) { + error make {msg: "No credentials provided via TTY"} + } + + # Parse TTY output (credentials) + let credentials = ($tty_output | from json) + + # Use both TTY input ($credentials) and CLI args ($provider) + # Complete integration logic... + + # Clear sensitive data after use + hide-env TTY_OUTPUT +} +``` + +#### 4. Regular Commands + +**Use case**: Standard provisioning operations + +**Example**: `provisioning server list` + +**Flow**: + +```bash +Bash wrapper โ†’ TTY filter checks registry โ†’ Not found โ†’ Return 1 + โ†“ +Continue to normal processing: + - Fast-path checks (help, workspace, env, etc.) + - Daemon check (if applicable) + - Nushell dispatcher +``` + +## Registry Format + +**File**: `provisioning/core/cli/tty-commands.conf` + +**Three-field format**: `"PATTERN" "WRAPPER_PATH" "FLOW_TYPE"` + +```bash +# Exact command match (e.g., "setup wizard" matches "provisioning setup wizard") +"setup wizard" "core/shlib/setup-wizard-tty.sh" "exit" + +# Paths are relative to $PROVISIONING +"auth get-key" "core/shlib/auth-get-key-tty.sh" "pipe" + +# Flow types: exit | pipe | continue +"auth integrate" "core/shlib/auth-integrate-tty.sh" "continue" +``` + +### Flow Type Decision Matrix + +| Interaction | Flow Type | Example | +| ----------- | --------- | ------- | +| Interactive form, no output needed | `exit` | Setup wizard, auth login | +| User input โ†’ pipe to next command | `pipe` | API key for piping to deploy | +| User input โ†’ same-command Nushell processing | `continue` | Credentials for integration | + +## Adding New TTY Commands + +### Step 1: Create Wrapper Script + +Create wrapper in `provisioning/core/shlib/`: + +```bash +#!/bin/bash +set -euo pipefail + +main() { + local input + + # Get input from user + read -rsp "Prompt: " input + echo # Newline + + # For flow=pipe: output to stdout + # For flow=continue: output to stdout (will be captured by filter) + echo "$input" + + return 0 +} + +main "$@" +``` + +Make it executable: + +```bash +chmod +x provisioning/core/shlib/your-wrapper-tty.sh +``` + +### Step 2: Add Registry Entry + +Edit `provisioning/core/cli/tty-commands.conf`: + +```bash +# Standalone TTY +"your command" "core/shlib/your-wrapper-tty.sh" "exit" + +# Pipeline TTY +"get something" "core/shlib/get-something-tty.sh" "pipe" + +# Continue-to-Nushell TTY +"setup something" "core/shlib/setup-something-tty.sh" "continue" +``` + +### Step 3: No Wrapper Modifications Required + +The provisioning wrapper automatically: +- Reads registry +- Matches command pattern +- Routes based on flow type +- Handles all three flows + +**No need to modify provisioning wrapper for new commands!** + +## Wrapper Script Requirements + +### For All Wrappers + +- **Shebang**: `#!/bin/bash` +- **Safety**: `set -euo pipefail` +- **Arguments**: Accept `"${@}"` from wrapper +- **Exit codes**: 0=success, non-zero=error +- **Validation**: `shellcheck` passes without warnings + +### For flow=exit Wrappers + +- Complete all interaction in wrapper +- Exit with proper code (0=success, non-zero=error) +- Output shown directly to user (from wrapper) + +### For flow=pipe Wrappers + +- Get input from user (TTY) +- Output result to stdout +- Output must be newline-terminated +- Exit with proper code (0=success, non-zero=error) + +### For flow=continue Wrappers + +- Get input from user (TTY) +- Output result to stdout (usually JSON) +- Exit with proper code (0=success, non-zero=error) +- Filter captures output โ†’ $TTY_OUTPUT +- Nushell script reads $env.TTY_OUTPUT + +## Environment Variables + +### Exported by Filter (flow=continue only) + +- **`$TTY_OUTPUT`**: Captured output from wrapper (available in Nushell as `$env.TTY_OUTPUT`) +- **`$PROVISIONING_BYPASS_DAEMON`**: Set to "true" to skip daemon (flow=continue automatically sets this) +- **`$TTY_WRAPPER_EXECUTED`**: Set to "true" when TTY wrapper was executed + +### Usage in Nushell + +```nushell +# Access TTY output in Nushell script +export def "provisioning auth integrate" [--provider: string] { + let tty_output = ($env.TTY_OUTPUT? | default "") + + # Parse if JSON + let creds = ($tty_output | from json) + + # Use both TTY output and CLI args + integration-logic $provider $creds + + # Clear after use (security) + hide-env TTY_OUTPUT +} +``` + +## Daemon Interaction + +The flow filter intelligently manages daemon usage: + +### For flow=exit and flow=pipe +- โœ… **Daemon can be used** - No stdin required +- No output needs to be captured and passed to Nushell +- Daemon optimization available (~100ms startup improvement) + +### For flow=continue +- โŒ **Daemon MUST be bypassed** - stdin required for TTY interaction +- `PROVISIONING_BYPASS_DAEMON=true` automatically set by filter +- Direct Nushell execution (preserves stdin for TTY) +- Zero overhead (same as non-daemon path) + +## Testing TTY Commands + +### Test Standalone (flow=exit) + +```bash +provisioning setup wizard +# Expected: TypeDialog form, user interaction, exits +``` + +### Test Pipeline (flow=pipe) + +```bash +provisioning auth get-key | wc -c +# Expected: Prompts for API key, outputs to pipe +``` + +### Test Continue (flow=continue) + +```bash +provisioning auth integrate --provider azure +# Expected: Prompts for credentials, passes to Nushell with $env.TTY_OUTPUT +``` + +### Test Regular Command + +```bash +provisioning server list +# Expected: Normal Nushell processing +``` + +## Troubleshooting + +### Command Not Executed +- **Check**: Is command in tty-commands.conf? +- **Check**: Does pattern exactly match command? +- **Check**: Is wrapper path correct and executable? + +### Wrapper Not Found +- **Error message**: `Warning: TTY wrapper not found or not executable: /path/to/wrapper` +- **Check**: File exists at `$PROVISIONING/wrapper-path` +- **Check**: File is executable: `chmod +x wrapper-path` + +### Output Not Piping (flow=pipe) +- **Check**: Wrapper outputs to stdout (not stderr) +- **Check**: Output is newline-terminated: `echo "output"` +- **Check**: No daemon interference (PROVISIONING_BYPASS_DAEMON not set) + +### Nushell Not Receiving Output (flow=continue) +- **Check**: `$env.TTY_OUTPUT` accessible in Nushell: `echo $env.TTY_OUTPUT` +- **Check**: Output format (usually JSON): `echo $env.TTY_OUTPUT | from json` +- **Check**: Wrapper exits with 0: `echo $?` + +## Implementation Details + +### Filter Location and Function + +**File**: `provisioning/core/cli/tty-filter.sh` +**Function**: `filter_tty_command()` +**Lines**: ~104 (includes documentation and three flow paths) + +### Integration in Wrapper + +**File**: `provisioning/core/cli/provisioning` +**Lines**: ~20 (sources filter, calls function, continues to Nushell) + +### Registry Parsing + +- **File**: `provisioning/core/cli/tty-commands.conf` +- **Method**: Line-by-line bash read (no jq dependency) +- **Format**: Three-field bash array (bash-compatible) +- **Sections**: Organized by flow type for clarity + +## Performance Implications + +### startup time +- **flow=exit/pipe**: Daemon available for startup optimization (~100ms improvement) +- **flow=continue**: Daemon bypassed (stdin needed), ~500ms traditional path +- **Regular commands**: Normal daemon/non-daemon path selection + +### Memory +- **flow=continue**: Wrapper output stored in `$TTY_OUTPUT` environment variable +- Typical size: < 1KB (credentials, keys, etc.) +- Cleared after Nushell processing (or via `hide-env`) + +## Security Considerations + +### Sensitive Data in $TTY_OUTPUT + +- **Credentials** captured in `$TTY_OUTPUT` +- **Nushell scripts should clear after use**: `hide-env TTY_OUTPUT` +- **Wrapper output may be logged**: Use standard Unix conventions (hide passwords from output) + +### Wrapper Location Restriction + +- Wrappers should be in `provisioning/core/shlib/` or `provisioning/scripts/` +- Registry reads only wrappers from these trusted locations +- Pattern validation prevents arbitrary script execution + +### No Shell Injection + +- All variables quoted: `"$variable"` +- No eval or command substitution with user input +- Pattern matching uses exact string match (no regex) + +## Related Files + +- **Filter**: `provisioning/core/cli/tty-filter.sh` +- **Registry**: `provisioning/core/cli/tty-commands.conf` +- **Wrapper**: `provisioning/core/cli/provisioning` +- **Example wrappers**: `provisioning/core/shlib/auth-get-key-tty.sh`, `provisioning/core/shlib/auth-integrate-tty.sh` + +## Key Insights + +The provisioning wrapper is not just a pass-through - it's a **flow controller** that: + +1. **Detects TTY requirements** (registry matching) +2. **Manages execution paths** (three flows: exit, pipe, continue) +3. **Controls exit behavior** (standalone vs pipeline vs same-command) +4. **Enables inter-command piping** (TTY output to pipes) +5. **Supports Nushell integration** (TTYโ†’Nushell continuation) +6. **Optimizes with daemon** (skip when stdin needed) + +This solves: +- "el tema no es sรณlo un filter" โ†’ โœ… Flow controller with three execution paths +- "cรณmo gestionar el flow por medio del provisioning command" โ†’ โœ… Registry + flow types +- "usamos tty para input de una API key, se lo pasamos a un script de nushell" โ†’ โœ… Pipeline + continue flows + +--- + +**Version**: 1.0.0 +**Last Updated**: January 2026 +**Status**: โœ… Production Ready diff --git a/cli/cache b/cli/cache new file mode 100755 index 0000000..b690260 --- /dev/null +++ b/cli/cache @@ -0,0 +1,130 @@ +#!/usr/bin/env nu +# Cache management CLI - minimal wrapper for cache operations +# Works without requiring an active workspace + +def main [...args: string] { + use ../nulib/lib_provisioning/config/cache/simple-cache.nu * + + # Default to "status" if no args + let args = if ($args | is-empty) { ["status"] } else { $args } + + # Parse command + let command = if ($args | length) > 0 { $args | get 0 } else { "status" } + let sub_args = if ($args | length) > 1 { $args | skip 1 } else { [] } + + match $command { + "status" => { + print "" + cache-status + print "" + } + + "config" => { + let sub_cmd = if ($sub_args | length) > 0 { $sub_args | get 0 } else { "show" } + match $sub_cmd { + "show" => { + print "" + let config = (get-cache-config) + print "Cache Configuration:" + print $" enabled: ($config | get --optional enabled | default true)" + print $" ttl_final_config: ($config | get --optional ttl_final_config | default 300)s" + print $" ttl_kcl: ($config | get --optional ttl_kcl | default 1800)s" + print $" ttl_sops: ($config | get --optional ttl_sops | default 900)s" + print "" + } + "get" => { + if ($sub_args | length) > 1 { + let setting = $sub_args | get 1 + let value = (cache-config-get $setting) + if $value != null { + print $"($setting) = ($value)" + } else { + print $"Setting not found: ($setting)" + } + } else { + print "โŒ cache config get requires a setting" + print "Usage: cache config get " + exit 1 + } + } + "set" => { + if ($sub_args | length) > 2 { + let setting = $sub_args | get 1 + let value = ($sub_args | skip 2 | str join " ") + # Convert value to appropriate type + let converted_value = ( + if $value == "1" or $value == "yes" or $value == "on" { true } + else if $value == "0" or $value == "no" or $value == "off" { false } + else { $value } + ) + cache-config-set $setting $converted_value + # Display the actual value stored + let display_value = if $converted_value == true { "true" } else if $converted_value == false { "false" } else { $value } + print $"โœ“ Set ($setting) = ($display_value)" + } else { + print "โŒ cache config set requires setting and value" + print "Usage: cache config set " + print " For boolean: use 0/no/off for false, 1/yes/on for true" + exit 1 + } + } + _ => { + print $"โŒ Unknown cache config command: ($sub_cmd)" + print "Available: show, get, set" + exit 1 + } + } + } + + "clear" => { + let cache_type = if ($sub_args | length) > 0 { $sub_args | get 0 } else { "all" } + cache-clear $cache_type + print $"โœ“ Cleared cache: ($cache_type)" + } + + "list" => { + let cache_type = if ($sub_args | length) > 0 { $sub_args | get 0 } else { "*" } + let items = (cache-list $cache_type) + if ($items | length) > 0 { + print $"Cache items \(type: ($cache_type)\):" + $items | each { |item| print $" ($item)" } + } else { + print "No cache items found" + } + } + + "help" | "--help" | "-h" => { + print " +Cache Management Commands: + + cache status # Show cache status and statistics + cache config show # Show cache configuration + cache config get # Get specific cache setting + cache config set # Set cache setting + cache clear [type] # Clear cache (default: all) + cache list [type] # List cached items (default: all) + cache help # Show this help message + +Available settings (for get/set): + enabled - Cache enabled (true/false) + ttl_final_config - TTL for final config (seconds) + ttl_kcl - TTL for KCL compilation (seconds) + ttl_sops - TTL for SOPS decryption (seconds) + +Examples: + cache status + cache config get ttl_final_config + cache config set ttl_final_config 600 + cache config set enabled false + cache clear kcl + cache list +" + } + + _ => { + print $"โŒ Unknown command: ($command)" + print "Use 'cache help' for available commands" + exit 1 + } + } +} diff --git a/cli/cfssl-install.sh b/cli/cfssl-install.sh new file mode 100755 index 0000000..2dd530a --- /dev/null +++ b/cli/cfssl-install.sh @@ -0,0 +1,17 @@ +#!/bin/bash +VERSION="1.6.4" + +# shellcheck disable=SC2006 +OS=$(uname | tr '[:upper:]' '[:lower:]') +ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" + +wget https://github.com/cloudflare/cfssl/releases/download/v${VERSION}/cfssl_${VERSION}_${OS}_${ARCH} +if [ -r "cfssl_${VERSION}_${OS}_${ARCH}" ] ; then + chmod +x "cfssl_${VERSION}_${OS}_${ARCH}" + sudo mv "cfssl_${VERSION}_${OS}_${ARCH}" /usr/local/bin/cfssl +fi +wget https://github.com/cloudflare/cfssl/releases/download/v${VERSION}/cfssljson_${VERSION}_${OS}_${ARCH} +if [ -r "cfssljson_${VERSION}_${OS}_${ARCH}" ] ; then + chmod +x "cfssljson_${VERSION}_${OS}_${ARCH}" + sudo mv "cfssljson_${VERSION}_${OS}_${ARCH}" /usr/local/bin/cfssljson +fi diff --git a/cli/install_config.sh b/cli/install_config.sh new file mode 100755 index 0000000..2570ecb --- /dev/null +++ b/cli/install_config.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Info: Script to install Provisioning config +# Author: JesusPerezLorenzo +# Release: 1.0.4 +# Date: 15-04-2024 + + +NU_FILES=" +core/nulib/libremote.nu +core/nulib/lib_provisioning/setup/config.nu +" + +WK_FILE=/tmp/make_config_provisioning.nu + +[ -r "$WK_FILE" ] && rm -f "$WK_FILE" + +set -o allexport +## shellcheck disable=SC1090 +[ -n "$PROVISIONING_ENV" ] && [ -r "$PROVISIONING_ENV" ] && source "$PROVISIONING_ENV" +set +o allexport + +export NU=$(type -P nu) +[ -z "$NU" ] && echo "Nu shell not found" && exit 1 + +export PROVISIONING=${PROVISIONING:-/usr/local/provisioning} +export PROVISIONING_DEBUG=false + +for it in $NU_FILES +do + [ -r "$PROVISIONING/$it" ] && cat $PROVISIONING/$it >> $WK_FILE +done + +echo " +install_config \"reset\" --context +" >> $WK_FILE + +NU_ARGS="" +CMD_ARGS="" + +DEFAULT_CONTEXT_TEMPLATE="default_context.yaml" +case "$(uname | tr '[:upper:]' '[:lower:]')" in + linux) PROVISIONING_USER_CONFIG="$HOME/.config/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/.config/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; + darwin) PROVISIONING_USER_CONFIG="$HOME/Library/Application\ Support/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/Library/Application\ Support/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; + *) PROVISIONING_USER_CONFIG="$HOME/.config/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/.config/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; +esac + +[ -d "$PROVISIONING_USER_CONFIG" ] && rm -r "$PROVISIONING_USER_CONFIG" +[ -r "$PROVISIONING_CONTEXT_PATH" ] && rm -f "$PROVISIONING_CONTEXT_PATH" + +nu $NU_ARGS $WK_FILE $CMD_ARGS + +rm -f $WK_FILE diff --git a/cli/install_nu.sh b/cli/install_nu.sh new file mode 100755 index 0000000..4d2bffc --- /dev/null +++ b/cli/install_nu.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Info: Script to instal NUSHELL for Provisioning +# Author: JesusPerezLorenzo +# Release: 1.0.5 +# Date: 8-03-2024 + +test_runner() { + echo -e "\nTest installation ... " + RUNNER_PATH=$(type -P $RUNNER) + [ -z "$RUNNER_PATH" ] && echo "๐Ÿ›‘ Error $RUNNER not found in PATH ! " && exit 1 + if $RUNNER ; then + echo -e "\nโœ… Installation completed successfully ! Use \"$RUNNER\"" + else + echo -e "\n๐Ÿ›‘ Error $RUNNER ! Review installation " && exit 1 + fi +} +register_plugins() { + local source=$1 + local warn=$2 + [ ! -d "$source" ] && echo "๐Ÿ›‘ Error path $source is not a directory" && exit 1 + [ -z "$(ls $source/nu_plugin_* 2> /dev/null)" ] && echo "๐Ÿ›‘ Error no 'nu_plugin_*' found in $source to register" && exit 1 + echo -e "Nushell $NU_VERSION plugins registration \n" + if [ -n "$warn" ] ; then + echo -e $"โ—Warning: Be sure Nushell plugins are compiled for same Nushell version $NU_VERSION\n otherwise will probably not work and will break installation !\n" + fi + for plugin in ${source}/nu_plugin_* + do + if $source/nu -c "register \"${plugin}\" " 2>/dev/null ; then + echo -en "$(basename $plugin)" + if [[ "$plugin" == *_notifications ]] ; then + echo -e " registred " + else + echo -e "\t\t registred " + fi + fi + done + + # Install nu_plugin_tera if available + if command -v cargo >/dev/null 2>&1; then + echo -e "Installing nu_plugin_tera..." + if cargo install nu_plugin_tera; then + if $source/nu -c "register ~/.cargo/bin/nu_plugin_tera" 2>/dev/null; then + echo -e "nu_plugin_tera\t\t registred" + else + echo -e "โ— Failed to register nu_plugin_tera" + fi + else + echo -e "โ— Failed to install nu_plugin_tera" + fi + else + echo -e "โ— Cargo not found - nu_plugin_tera not installed" + fi +} + +# Check Nickel configuration language installation +check_nickel_installation() { + if command -v nickel >/dev/null 2>&1; then + nickel_version=$(nickel --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + echo -e "Nickel\t\t\t already installed (version $nickel_version)" + return 0 + else + echo -e "โš ๏ธ Nickel not found - Optional but recommended for config rendering" + echo -e " Install via: \$PROVISIONING/core/cli/tools-install nickel" + echo -e " Recommended method: nix profile install nixpkgs#nickel" + echo -e " (Pre-built binaries have Nix library dependencies)" + echo -e " https://nickel-lang.org/getting-started" + return 1 + fi +} + +install_mode() { + local mode=$1 + case "$mode" in + ui| desktop) + if cp $PROVISIONING_MODELS_SRC/plugins_defs.nu $PROVISIONING_MODELS_TARGET/plugins_defs.nu ; then + echo "Mode $mode installed" + fi + ;; + *) + NC_PATH=$(type -P nc) + if [ -z "$NC_PATH" ] ; then + echo "'nc' command not found in PATH. Install 'nc' (netcat) command." + exit 1 + fi + if cp $PROVISIONING_MODELS_SRC/no_plugins_defs.nu $PROVISIONING_MODELS_TARGET/plugins_defs.nu ; then + echo "Mode 'no plugins' installed" + fi + esac +} +install_from_url() { + local target_path=$1 + local lib_mode + local url_source + local download_path + local download_url + local tar_file + + [ ! -d "$target_path" ] && echo "๐Ÿ›‘ Error path $target_path is not a directory" && exit 1 + lib_mode=$(grep NU_LIB $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') + url_source=$(grep NU_SOURCE $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') + download_path="nu-${NU_VERSION}-${ARCH_ORG}-${OS}" + case "$OS" in + linux) download_path="nu-${NU_VERSION}-${ARCH_ORG}-unknown-${OS}-gnu" + ;; + esac + download_url="$url_source/${NU_VERSION}/$download_path.tar.gz" + tar_file=$download_path.tar.gz + echo -e "Nushell $NU_VERSION downloading ..." + if ! curl -sSfL $download_url -o $tar_file ; then + echo "๐Ÿ›‘ Error download $download_url " && exit 1 + return 1 + fi + echo -e "Nushell $NU_VERSION extracting ..." + if ! tar xzf $tar_file ; then + echo "๐Ÿ›‘ Error download $download_url " && exit 1 + return 1 + fi + rm -f $tar_file + if [ ! -d "$download_path" ] ; then + echo "๐Ÿ›‘ Error $download_path not found " && exit 1 + return 1 + fi + echo -e "Nushell $NU_VERSION installing ..." + if [ -r "$download_path/nu" ] ; then + chmod +x $download_path/nu + if ! sudo cp $download_path/nu $target_path ; then + echo "๐Ÿ›‘ Error installing \"nu\" in $target_path" + rm -rf $download_path + return 1 + fi + fi + rm -rf $download_path + echo "โœ… Nushell and installed in $target_path" + [[ ! "$PATH" =~ $target_path ]] && echo "โ— Warning: \"$target_path\" is not in your PATH for $(basename $SHELL) ! Fix your PATH settings " + echo "" + # TDOO install plguins via cargo ?? + # TODO a NU version without PLUGINS + # register_plugins $target_path +} + +install_from_local() { + local source=$1 + local target=$2 + local tmpdir + + [ ! -d "$target" ] && echo "๐Ÿ›‘ Error path $target is not a directory" && exit 1 + [ ! -r "$source/nu.gz" ] && echo "๐Ÿ›‘ Error command 'nu' not found in $source/nu.gz" && exit 1 + + echo -e "Nushell $NU_VERSION self installation guarantees consistency with plugins and settings \n" + tmpdir=$(mktemp -d) + cp $source/*gz $tmpdir + for file in $tmpdir/*gz ; do gunzip $file ; done + if ! sudo mv $tmpdir/* $target ; then + echo -e "๐Ÿ›‘ Errors to install Nushell and plugins in \"${target}\"" + rm -rf $tmpdir + return 1 + fi + rm -rf $tmpdir + echo "โœ… Nushell and plugins installed in $target" + [[ ! "$PATH" =~ $target ]] && echo "โ— Warning: \"$target\" is not in your PATH for $(basename $SHELL) ! Fix your PATH settings " + echo "" + register_plugins $target +} + +message_install() { + local ask=$1 + local msg + local answer + [ -r "$PROVISIONING/resources/ascii.txt" ] && cat "$PROVISIONING/resources/ascii.txt" && echo "" + if [ -z "$NU" ] ; then + echo -e "๐Ÿ›‘ Nushell $NU_VERSION not installed is mandatory for \"${RUNNER}\"" + echo -e "Check PATH or https://www.nushell.sh/book/installation.html with version $NU_VERSION" + else + echo -e "Nushell $NU_VERSION update for \"${RUNNER}\"" + fi + echo "" + if [ -n "$ask" ] && [ -d "$(dirname $0)/nu/${ARCH}-${OS}" ] ; then + echo -en "Install Nushell $(uname -m) $(uname) in \"$INSTALL_PATH\" now (yes/no) ? : " + read -r answer + if [ "$answer" != "yes" ] && [ "$answer" != "y" ] ; then + return 1 + fi + fi + if [ -d "$(dirname $0)/nu/${ARCH}-${OS}" ] ; then + install_from_local $(dirname $0)/nu/${ARCH}-${OS} $INSTALL_PATH + install_mode "ui" + else + install_from_url $INSTALL_PATH + install_mode "" + fi + echo "" + echo -e "Checking optional configuration languages..." + check_nickel_installation +} + +set +o errexit +set +o pipefail + +RUNNER="provisioning" +export NU=$(type -P nu) + +[ -n "$PROVISIONING_ENV" ] && [ -r "$PROVISIONING_ENV" ] && source "$PROVISIONING_ENV" +[ -r "../env-provisioning" ] && source ../env-provisioning +[ -r "env-provisioning" ] && source ./env-provisioning +#[ -r ".env" ] && source .env set +set +o allexport + +if [ -n "$1" ] && [ -d "$1" ] && [ -d "$1/core" ] ; then + export PROVISIONING=$1 +else + export PROVISIONING=${PROVISIONING:-/usr/local/provisioning} +fi + +TASK=${1:-check} +shift +if [ "$TASK" == "mode" ] && [ -n "$1" ] ; then + INSTALL_MODE=$1 + shift +else + INSTALL_MODE="ui" +fi + +ASK_MESSAGE="ask" +[ -n "$1" ] && [ "$1" == "no-ask" ] && ASK_MESSAGE="" && shift +[ -n "$1" ] && [ "$1" == "mode-ui" ] && INSTALL_MODE="ui" && shift +[ -n "$1" ] && [[ "$1" == mode-* ]] && INSTALL_MODE="" && shift + +INSTALL_PATH=${1:-/usr/local/bin} + +NU_VERSION=$(grep NU_VERSION $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') +#ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" +ARCH="$(uname -m | sed -e 's/amd64/x86_64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" +ARCH_ORG="$(uname -m | tr '[:upper:]' '[:lower:]')" +OS="$(uname | tr '[:upper:]' '[:lower:]')" + +PROVISIONING_MODELS_SRC=$PROVISIONING/core/nulib/models +PROVISIONING_MODELS_TARGET=$PROVISIONING/core/nulib/lib_provisioning + +USAGE="$(basename $0) [install | reinstall | mode | check] no-ask mode-?? " +case $TASK in + install) + message_install $ASK_MESSAGE + ;; + reinstall | update) + INSTALL_PATH=$(dirname $NU) + if message_install ; then + test_runner + fi + ;; + mode) + install_mode $INSTALL_MODE + ;; + check) + $PROVISIONING/core/bin/tools-install check nu + ;; + help|-h) + echo "$USAGE" + ;; + *) echo "Option $TASK not defined" +esac diff --git a/cli/module-loader b/cli/module-loader new file mode 100755 index 0000000..5f5a4a6 --- /dev/null +++ b/cli/module-loader @@ -0,0 +1,981 @@ +#!/usr/bin/env nu + +# Enhanced Module Loader CLI +# Unified CLI for discovering and loading taskservs, providers, and clusters +# Includes template and layer support from enhanced version + +use ../nulib/taskservs/discover.nu * +use ../nulib/taskservs/load.nu * +use ../nulib/providers/discover.nu * +use ../nulib/providers/load.nu * +use ../nulib/clusters/discover.nu * +use ../nulib/clusters/load.nu * +use ../nulib/lib_provisioning/module_loader.nu * +use ../nulib/lib_provisioning/config/accessor.nu config-get + +# Main module loader command with enhanced features +def main [subcommand?: string] { + if ($subcommand | is-empty) { + print_enhanced_help + return + } + + match $subcommand { + "help" => print_enhanced_help + "discover" => print_discover_help + "load" => print_load_help + "list" => print_list_help + "unload" => print_unload_help + "init" => print_init_help + "validate" => print_validate_help + "info" => print_info_help + "template" => print_template_help + "layer" => print_layer_help + "override" => print_override_help + _ => { + print $"Unknown command: ($subcommand)" + print_enhanced_help + } + } +} + +# === DISCOVERY COMMANDS === + +# Discover available modules +export def "main discover" [ + type: string, # Module type: taskservs, providers, clusters + query?: string, # Search query + --format: string = "table", # Output format: table, yaml, json, names + --category: string = "", # Filter by category (for taskservs) + --group: string = "" # Filter by group (for taskservs) +] { + match $type { + "taskservs" => { + let taskservs = if ($query | is-empty) { + discover-taskservs + } else { + search-taskservs $query + } + + let filtered = if ($category | is-empty) and ($group | is-empty) { + $taskservs + } else if not ($category | is-empty) { + $taskservs | where group == $category + } else if not ($group | is-empty) { + $taskservs | where group == $group + } else { + $taskservs + } + + format_output $filtered $format + } + "providers" => { + print "Provider discovery not implemented yet" + } + "clusters" => { + print "Cluster discovery not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + print "Available types: taskservs, providers, clusters" + } + } +} + +# Sync Nickel dependencies for infrastructure workspace +export def "main sync" [ + infra: string, # Infrastructure name or path + --manifest: string = "providers.manifest.yaml", # Manifest file name + --show-modules # Show module info after sync +] { + # Resolve infrastructure path + let infra_path = if ($infra | path exists) { + $infra + } else { + # Try workspace path + let workspace_path = $"workspace/infra/($infra)" + if ($workspace_path | path exists) { + $workspace_path + } else { + print $"โŒ Infrastructure not found: ($infra)" + return + } + } + + # Sync Nickel dependencies using library function + sync-nickel-dependencies $infra_path --manifest $manifest + + # Show Nickel module info if requested + if $show_modules { + print "" + print "๐Ÿ“‹ Nickel Modules:" + let modules_dir = (get-config-value "nickel" "modules_dir") + let modules_path = ($infra_path | path join $modules_dir) + + if ($modules_path | path exists) { + ls $modules_path | each {|entry| + print $" โ€ข ($entry.name | path basename) โ†’ ($entry.name)" + } + } + } +} + +# === LOAD/UNLOAD COMMANDS === + +# Load modules into workspace +export def "main load" [ + type: string, # Module type: taskservs, providers, clusters + workspace: string, # Workspace path + ...modules: string, # Module names to load + --layer: string = "workspace", # Layer to load into: workspace, infra + --validate # Validate after loading + --force (-f) # Force overwrite existing files +] { + if ($modules | is-empty) { + print $"No modules specified for loading" + return + } + + print $"Loading ($modules | length) ($type) into ($workspace) at layer ($layer)" + + match $type { + "taskservs" | "providers" | "clusters" | "workflows" => { + load_extension_to_workspace $type $workspace $modules $layer $force + } + _ => { + print $"Unknown module type: ($type)" + } + } + + if $validate { + main validate $workspace + } +} + +# Enhanced load with template support +export def "main load enhanced" [ + type: string, # Module type + workspace: string, # Workspace path + infra: string, # Infrastructure name + modules: list, # Module names + --layer: string = "workspace", # Target layer + --template-base # Use template as base +] { + print $"๐Ÿš€ Enhanced loading ($modules | length) ($type) for infra ($infra)" + + for module in $modules { + print $" ๐Ÿ“ฆ Loading ($module)..." + + # Check if template exists for this module + let template_path = $"provisioning/workspace/templates/taskservs/*/($module).k" + let has_template = (glob $template_path | length) > 0 + + if $has_template and $template_base { + print $" โœ“ Using template base for ($module)" + # Template-based loading would go here + } else { + print $" โœ“ Direct loading for ($module)" + # Direct loading + } + } + + print "โœ… Enhanced loading completed" +} + +# Unload module from workspace +export def "main unload" [ + type: string, # Module type + workspace: string, # Workspace path + module: string, # Module name to unload + --layer: string = "workspace" # Layer to unload from +] { + print $"Unloading ($module) from ($workspace) at layer ($layer)" + + match $type { + "taskservs" => { + unload_taskserv_from_workspace $workspace $module $layer + } + "providers" => { + print "Provider unloading not implemented yet" + } + "clusters" => { + print "Cluster unloading not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + } + } +} + +# === LIST COMMANDS === + +# List modules in workspace +export def "main list" [ + type: string, # Module type + workspace: string, # Workspace path + --layer: string = "all", # Layer to list: workspace, infra, all + --format: string = "table" # Output format +] { + print $"Listing ($type) in ($workspace) for layer ($layer)" + + match $type { + "taskservs" => { + list_workspace_taskservs $workspace $layer $format + } + "providers" => { + print "Provider listing not implemented yet" + } + "clusters" => { + print "Cluster listing not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + } + } +} + +# === TEMPLATE COMMANDS === + +# List available templates +export def "main template list" [ + --template-type: string = "all", # Template type: taskservs, providers, servers, clusters + --format: string = "table" # Output format +] { + print $"๐Ÿ“‹ Available templates type: ($template_type)" + + let template_base = "provisioning/workspace/templates" + + match $template_type { + "taskservs" | "all" => { + let taskserv_templates = if (($template_base | path join "taskservs") | path exists) { + glob ($template_base | path join "taskservs" "*" "*.k") + | each { |path| + let category = ($path | path dirname | path basename) + let name = ($path | path basename | str replace ".k" "") + { type: "taskserv", category: $category, name: $name, path: $path } + } + } else { [] } + + format_output $taskserv_templates $format + } + "providers" => { + print "Provider templates not implemented yet" + } + "servers" => { + let server_templates = if (($template_base | path join "servers") | path exists) { + ls ($template_base | path join "servers") | get name + | each { |path| { type: "server", name: ($path | path basename), path: $path } } + } else { [] } + + format_output $server_templates $format + } + _ => { + print $"Unknown template type: ($template_type)" + } + } +} + +# Extract template from existing infrastructure +export def "main template extract" [ + source_infra: string, # Source infrastructure path + template_name: string, # Name for the new template + --type: string = "taskserv", # Template type + --output: string = "provisioning/workspace/templates" # Output directory +] { + print $"๐Ÿ“ค Extracting template ($template_name) from ($source_infra)" + + # Implementation would analyze the source infra and create template + print "Template extraction not yet implemented" +} + +# Apply template to infrastructure +export def "main template apply" [ + template_name: string, # Template to apply + target_infra: string, # Target infrastructure + --override-file: string = "", # Override file path + --dry-run # Show what would be done +] { + if $dry_run { + print $"๐Ÿ” [DRY RUN] Would apply template ($template_name) to ($target_infra)" + } else { + print $"๐Ÿ“ฅ Applying template ($template_name) to ($target_infra)" + } + + # Implementation would apply template with overrides + print "Template application not yet implemented" +} + +# === LAYER COMMANDS === + +# Show layer information +export def "main layer show" [ + workspace: string, # Workspace path + --module: string = "", # Specific module to show + --type: string = "taskservs" # Module type +] { + print $"๐Ÿ“Š Layer information for ($workspace)" + + if not ($module | is-empty) { + # Use existing layer utilities + try { + nu -c $"use provisioning/workspace/tools/layer-utils.nu *; test_layer_resolution ($module) ($workspace) upcloud" + } catch { + print $"Could not test layer resolution for ($module)" + } + } else { + print "Showing overall layer structure..." + try { + nu -c "use provisioning/workspace/tools/layer-utils.nu *; show_layer_stats" + } catch { + print "Could not show layer statistics" + } + } +} + +# Test layer resolution +export def "main layer test" [ + module: string, # Module to test + workspace: string, # Workspace/infra name + provider: string = "upcloud" # Provider for testing +] { + print $"๐Ÿงช Testing layer resolution: ($module) in ($workspace) with ($provider)" + + try { + nu -c $"use provisioning/workspace/tools/layer-utils.nu *; test_layer_resolution ($module) ($workspace) ($provider)" + } catch { + print $"โŒ Layer resolution test failed for ($module)" + } +} + +# === OVERRIDE COMMANDS === + +# Create configuration override +export def "main override create" [ + type: string, # Type: taskservs, providers, clusters + infra: string, # Infrastructure name + module: string, # Module name + --from: string = "", # Template to base override on + --layer: string = "infra" # Layer for override +] { + print $"โš™๏ธ Creating override for ($module) in ($infra) at layer ($layer)" + + let override_path = match $layer { + "infra" => $"workspace/infra/($infra)/overrides/($module).k" + "workspace" => $"provisioning/workspace/templates/($type)/($module).k" + _ => { + print $"Unknown layer: ($layer)" + return + } + } + + print $"๐Ÿ“ Override will be created at: ($override_path)" + + if not ($from | is-empty) { + print $"๐Ÿ“‹ Based on template: ($from)" + } + + # Create directory if needed + mkdir ($override_path | path dirname) + + # Create basic override file + let content = if not ($from | is-empty) { + $"# Override for ($module) in ($infra) +# Based on template: ($from) + +import ($type).*.($module).ncl.($module) as base +import provisioning.workspace.templates.($type).($from) as template + +# Infrastructure-specific overrides +($module)_($infra)_override: base.($module | str capitalize) = template.($from)_template { + # Add your overrides here + # Example: + # replicas = 3 + # resources.memory = \"1Gi\" +} +" + } else { + $"# Override for ($module) in ($infra) + +import ($type).*.($module).ncl.($module) as base + +# Infrastructure-specific overrides +($module)_($infra)_override: base.($module | str capitalize) = base.($module)_config { + # Add your overrides here + # Example: + # replicas = 3 + # resources.memory = \"1Gi\" +} +" + } + + $content | save $override_path + print $"โœ… Override created: ($override_path)" +} + +# === WORKSPACE MANAGEMENT === + +# Initialize workspace with modules +export def "main init" [ + workspace: string, # Workspace path + --modules: list = [], # Initial modules to load + --template: string = "", # Workspace template + --provider: string = "upcloud" # Default provider +] { + print $"๐Ÿš€ Initializing workspace: ($workspace)" + + # Create workspace structure + let workspace_dirs = [ + $"($workspace)/config" + $"($workspace)/taskservs" + $"($workspace)/overrides" + $"($workspace)/defs" + $"($workspace)/clusters" + ] + + for dir in $workspace_dirs { + mkdir $dir + print $" ๐Ÿ“ Created: ($dir)" + } + + # Create basic configuration + let config_content = $"# Workspace configuration for ($workspace) +# Provider: ($provider) +# Initialized: (date now) + +provider = "($provider)" +workspace = "($workspace)" +" + $config_content | save $"($workspace)/config/workspace.toml" + print $" ๐Ÿ“„ Created: ($workspace)/config/workspace.toml" + + # Load initial modules + if ($modules | length) > 0 { + print $"๐Ÿ“ฆ Loading initial modules: (($modules | str join ', '))" + main load taskservs $workspace ...$modules + } + + print $"โœ… Workspace ($workspace) initialized successfully" +} + +# Validate workspace integrity +export def "main validate" [workspace: string] { + print $"๐Ÿ” Validating workspace: ($workspace)" + + let required_dirs = ["config", "taskservs", "overrides", "defs"] + mut validation_errors = [] + + for dir in $required_dirs { + let full_path = ($workspace | path join $dir) + if not ($full_path | path exists) { + $validation_errors = ($validation_errors | append $"Missing directory: ($full_path)") + } + } + + # Check configuration file + let config_file = ($workspace | path join "config" "workspace.toml") + if not ($config_file | path exists) { + $validation_errors = ($validation_errors | append $"Missing configuration: ($config_file)") + } + + # Report results + if ($validation_errors | is-empty) { + print "โœ… Workspace validation passed" + return true + } else { + print "โŒ Workspace validation failed:" + for error in $validation_errors { + print $" โ€ข ($error)" + } + return false + } +} + +# Show workspace information +export def "main info" [workspace: string] { + print $"๐Ÿ“Š Workspace Information: ($workspace)" + + if not (($workspace | path join "config" "workspace.toml") | path exists) { + print "โŒ Workspace not found or not initialized" + return + } + + # Show basic info + let config = try { open ($workspace | path join "config" "workspace.toml") | from toml } catch { {} } + + print $" Provider: (($config.provider? | default 'unknown'))" + print $" Path: ($workspace)" + + # Count modules + let taskserv_count = try { + ls ($workspace | path join "taskservs") | length + } catch { 0 } + + let override_count = try { + ls ($workspace | path join "overrides") | length + } catch { 0 } + + print $" Task Services: ($taskserv_count)" + print $" Overrides: ($override_count)" + + # Show recent activity + let recent_files = try { + ls $workspace | where type == file | sort-by modified | last 3 | get name + } catch { [] } + + if ($recent_files | length) > 0 { + print " Recent activity:" + for file in $recent_files { + print $" โ€ข ($file | path basename)" + } + } +} + +# === HELPER FUNCTIONS === + +# Generic extension loading function (taskservs, providers, clusters, workflows) +def load_extension_to_workspace [ + extension_type: string, # taskservs, providers, clusters, workflows + workspace: string, + modules: list, + layer: string, + force: bool = false +] { + # Get extension-specific info function based on type + let get_info_fn = match $extension_type { + "taskservs" => { |name| get-taskserv-info $name } + "providers" => { |name| get-provider-info $name } + "clusters" => { |name| get-cluster-info $name } + _ => { |name| {name: $name, group: "", type: $extension_type} } + } + + # Get source path from config + let source_base_path = (config-get $"paths.($extension_type)" | path expand) + + # Get template base path from config + let provisioning_base = (config-get "paths.base" | path expand) + let template_base_path = ($provisioning_base | path join "workspace" "templates" $extension_type) + + for module in $modules { + print $" ๐Ÿ“ฆ Loading ($extension_type): ($module)" + + # Get module info + let module_info = try { + do $get_info_fn $module + } catch { + print $" โŒ Module not found: ($module)" + continue + } + + print $" โœ“ Found: ($module_info.name) (($module_info.group? | default ""))" + + # Resolve workspace paths + let workspace_abs = ($workspace | path expand) + let workspace_root = if ($workspace_abs | str contains "/infra/") { + let parts = ($workspace_abs | split row "/infra/") + $parts.0 + } else { + $workspace_abs + } + + # Build source path (handle optional group, "root" means no category) + let group_path = ($module_info.group? | default "") + let group_path = if ($group_path == "root") { "" } else { $group_path } + let source_module_path = if ($group_path | is-not-empty) { + $source_base_path | path join $group_path $module + } else { + $source_base_path | path join $module + } + + # STEP 1: Copy schemas to workspace/.{extension_type} + let target_schemas_dir = ($workspace_root | path join $".($extension_type)") + let target_module_path = if ($group_path | is-not-empty) { + $target_schemas_dir | path join $group_path $module + } else { + $target_schemas_dir | path join $module + } + + # Config file directory + let config_dir = ($workspace_abs | path join $extension_type) + let config_file_path = ($config_dir | path join $"($module).k") + + # Check if already loaded + if ($config_file_path | path exists) and ($target_module_path | path exists) { + if not $force { + print $" โœ… Module already loaded: ($module)" + print $" Config: ($config_file_path)" + print $" Source: ($target_module_path)" + print $" ๐Ÿ’ก Use --force to overwrite existing files" + continue + } else { + print $" ๐Ÿ”„ Overwriting existing module: ($module)" + } + } + + # Copy schemas from system extensions to workspace + let parent_dir = ($target_module_path | path dirname) + mkdir $parent_dir + + if ($source_module_path | path exists) { + print $" ๐Ÿ“ฆ Copying schemas to workspace .($extension_type)..." + print $" From: ($source_module_path)" + print $" To: ($target_module_path)" + + if ($target_module_path | path exists) { + rm -rf $target_module_path + } + + cp -r $source_module_path $parent_dir + print $" โœ“ Schemas copied to workspace .($extension_type)/" + + # STEP 2a: Update individual module's nickel.mod with correct workspace paths + # Calculate relative paths based on categorization depth + let provisioning_path = if ($group_path | is-not-empty) { + # Categorized: .{ext}/{category}/{module}/nickel/ -> ../../../../.nickel/packages/provisioning + "../../../../.nickel/packages/provisioning" + } else { + # Non-categorized: .{ext}/{module}/nickel/ -> ../../../.nickel/packages/provisioning + "../../../.nickel/packages/provisioning" + } + + let parent_path = if ($group_path | is-not-empty) { + # Categorized: .{ext}/{category}/{module}/nickel/ -> ../../.. + "../../.." + } else { + # Non-categorized: .{ext}/{module}/nickel/ -> ../.. + "../.." + } + + # Update the module's nickel.mod file with workspace-relative paths + let module_nickel_mod_path = ($target_module_path | path join "nickel" "nickel.mod") + if ($module_nickel_mod_path | path exists) { + print $" ๐Ÿ”ง Updating module nickel.mod with workspace paths" + let module_nickel_mod_content = $"[package] +name = \"($module)\" +edition = \"v0.11.3\" +version = \"0.0.1\" + +[dependencies] +provisioning = { path = \"($provisioning_path)\", version = \"0.0.1\" } +($extension_type) = { path = \"($parent_path)\", version = \"0.1.0\" } +" + $module_nickel_mod_content | save -f $module_nickel_mod_path + print $" โœ“ Updated nickel.mod: ($module_nickel_mod_path)" + } + } else { + print $" โš ๏ธ Warning: Source not found at ($source_module_path)" + } + + # STEP 2b: Create nickel.mod in workspace/.{extension_type} + let extension_nickel_mod = ($target_schemas_dir | path join "nickel.mod") + if not ($extension_nickel_mod | path exists) { + print $" ๐Ÿ“ฆ Creating nickel.mod for .($extension_type) package" + let nickel_mod_content = $"[package] +name = \"($extension_type)\" +edition = \"v0.11.3\" +version = \"0.1.0\" +description = \"Workspace-level ($extension_type) schemas\" +" + $nickel_mod_content | save $extension_nickel_mod + } + + # Ensure config directory exists + mkdir $config_dir + + # STEP 4: Generate config from template + let template_path = if ($group_path | is-not-empty) { + $template_base_path | path join $group_path $"($module).k" + } else { + $template_base_path | path join $"($module).k" + } + + # Build import statement with "as {module}" alias + let import_stmt = if ($group_path | is-not-empty) { + $"import ($extension_type).($group_path).($module).ncl.($module) as ($module)" + } else { + $"import ($extension_type).($module).ncl.($module) as ($module)" + } + + # Get relative paths for comments + let workspace_name = ($workspace_root | path basename) + let relative_schema_path = if ($group_path | is-not-empty) { + $"($workspace_name)/.($extension_type)/($group_path)/($module)" + } else { + $"($workspace_name)/.($extension_type)/($module)" + } + + let config_content = if ($template_path | path exists) { + print $" ๐Ÿ“„ Using template from: ($template_path)" + let template_body = (open $template_path) + $"# Configuration for ($module) +# Workspace: ($workspace_name) +# Schemas from: ($relative_schema_path) +($import_stmt) + +($template_body)" + } else { + $"# Configuration for ($module) +# Workspace: ($workspace_name) +# Schemas from: ($relative_schema_path) +($import_stmt) + +# TODO: Configure your ($module) instance +# See available schemas at: ($relative_schema_path)/nickel/ +" + } + + $config_content | save -f $config_file_path + print $" โœ“ Config created: ($config_file_path)" + print $" ๐Ÿ“ Edit ($extension_type)/($module).k to configure settings" + + # STEP 3: Update infra nickel.mod + if ($workspace_abs | str contains "/infra/") { + let nickel_mod_path = ($workspace_abs | path join "nickel.mod") + if ($nickel_mod_path | path exists) { + let nickel_mod_content = (open $nickel_mod_path) + if not ($nickel_mod_content | str contains $"($extension_type) =") { + print $" ๐Ÿ”ง Updating nickel.mod to include ($extension_type) dependency" + let new_dependency = $"\n# Workspace-level ($extension_type) \(shared across infras\)\n($extension_type) = { path = \"../../.($extension_type)\" }\n" + $"($nickel_mod_content)($new_dependency)" | save -f $nickel_mod_path + } + } + } + } +} + +# Unload taskserv from workspace +def unload_taskserv_from_workspace [workspace: string, module: string, layer: string] { + let target_path = match $layer { + "workspace" => ($workspace | path join "taskservs" $"($module).k") + "infra" => ($workspace | path join "overrides" $"($module).k") + _ => ($workspace | path join "taskservs" $"($module).k") + } + + if ($target_path | path exists) { + rm $target_path + print $" โœ“ Removed: ($target_path)" + } else { + print $" โŒ Not found: ($target_path)" + } +} + +# List workspace taskservs +def list_workspace_taskservs [workspace: string, layer: string, format: string] { + let paths = match $layer { + "workspace" => [($workspace | path join "taskservs")] + "infra" => [($workspace | path join "overrides")] + "all" => [($workspace | path join "taskservs"), ($workspace | path join "overrides")] + _ => [($workspace | path join "taskservs")] + } + + mut all_taskservs = [] + + for path in $paths { + if ($path | path exists) { + let taskservs = ls $path + | where type == file + | where name =~ '\\.k$' + | each { |file| + { + name: ($file.name | path basename | str replace ".k" "") + layer: ($path | path basename) + path: $file.name + modified: $file.modified + } + } + $all_taskservs = ($all_taskservs | append $taskservs) + } + } + + format_output $all_taskservs $format +} + +# Format output based on requested format +def format_output [data: any, format: string] { + match $format { + "json" => ($data | to json) + "yaml" => ($data | to yaml) + "names" => ($data | get name | str join "\n") + "table" | _ => ($data | table) + } +} + +# === HELP FUNCTIONS === + +def print_enhanced_help [] { + print "Enhanced Module Loader CLI - Discovery and loading with template support" + print "" + print "Usage: module-loader [options]" + print "" + print "CORE COMMANDS:" + print " discover [query] [--format ] [--category ] - Discover available modules" + print " sync [--manifest ] [--show-modules] - Sync Nickel dependencies for infrastructure" + print " load [--layer ] - Load modules into workspace" + print " list [--layer ] - List loaded modules" + print " unload [--layer ] - Unload module from workspace" + print "" + print "WORKSPACE COMMANDS:" + print " init [--modules ] [--template ] - Initialize workspace" + print " validate - Validate workspace integrity" + print " info - Show workspace information" + print "" + print "TEMPLATE COMMANDS:" + print " template list [--type ] [--format ] - List available templates" + print " template extract [--type ] - Extract template from infra" + print " template apply