#!/usr/bin/env nu # Distribution testing tool - tests generated distributions for functionality # # Tests: # - Binary execution and basic functionality # - Core library loading and imports # - Configuration system # - Template rendering # - Integration tests use std log def main [ --dist-dir: string = "dist" # Distribution directory to test --test-types: string = "all" # Test types: all, basic, integration, performance --platform: string = "" # Target platform (auto-detect if empty) --temp-workspace: string = "" # Temporary workspace for tests (auto-generated if empty) --cleanup = true # Cleanup test workspace after tests --verbose = false # Enable verbose logging --timeout: int = 300 # Test timeout in seconds ] { let dist_root = ($dist_dir | path expand) let platform_detected = if $platform == "" { detect_platform } else { $platform } let temp_workspace = if $temp_workspace == "" { $env.TMPDIR | path join $"provisioning-test-(random uuid)" } else { $temp_workspace | path expand } let test_config = { dist_dir: $dist_root test_types: ($test_types | split row "," | each { str trim }) platform: $platform_detected temp_workspace: $temp_workspace cleanup: $cleanup verbose: $verbose timeout: $timeout } log info $"Starting distribution testing with config: ($test_config)" # Validate distribution directory if not ($dist_root | path exists) { log error $"Distribution directory does not exist: ($dist_root)" exit 1 } # Create temporary workspace mkdir ($test_config.temp_workspace) # Run different test categories let test_categories = if "all" in $test_config.test_types { ["basic", "integration", "performance"] } else { $test_config.test_types } let test_results = run_test_categories $test_categories $test_config let result = (do { # Generate test report let test_report = generate_test_report $test_results $test_config # Cleanup if requested if $test_config.cleanup { rm -rf ($test_config.temp_workspace) log info "Cleaned up test workspace" } let summary = { total_categories: ($test_results | length) successful_categories: ($test_results | where status == "success" | length) failed_categories: ($test_results | where status == "failed" | length) test_config: $test_config results: $test_results report: $test_report } if $summary.failed_categories > 0 { log error $"Distribution testing completed with ($summary.failed_categories) failed categories" exit 1 } else { log info $"Distribution testing completed successfully - all ($summary.total_categories) categories passed" } $summary } | complete) if $result.exit_code != 0 { # Ensure cleanup even on error if $test_config.cleanup and ($test_config.temp_workspace | path exists) { rm -rf ($test_config.temp_workspace) } log error $"Distribution testing failed with exit code ($result.exit_code)" exit 1 } else { return $result.stdout } } # Run all test categories and collect results def run_test_categories [categories: list, test_config: record] { mut accumulated_results = [] for category in $categories { let category_result = match $category { "basic" => { run_basic_tests $test_config } "integration" => { run_integration_tests $test_config } "performance" => { run_performance_tests $test_config } _ => { log warning $"Unknown test category: ($category)" { category: $category, status: "skipped", reason: "unknown category" } } } $accumulated_results = ($accumulated_results | append $category_result) } $accumulated_results } # Detect current platform def detect_platform [] { match $nu.os-info.name { "linux" => "linux" "macos" => "macos" "windows" => "windows" _ => "unknown" } } # Run basic functionality tests def run_basic_tests [test_config: record] { log info "Running basic functionality tests..." let start_time = (date now) mut test_errors = [] mut tests_passed = 0 mut tests_total = 0 # Test 1: Platform binaries exist and execute let platform_test = test_platform_binaries $test_config $tests_total = $tests_total + 1 if $platform_test.status == "success" { $tests_passed = $tests_passed + 1 } else { $test_errors = ($test_errors | append $platform_test.errors) } # Test 2: Core bundle integrity let core_test = test_core_bundle $test_config $tests_total = $tests_total + 1 if $core_test.status == "success" { $tests_passed = $tests_passed + 1 } else { $test_errors = ($test_errors | append $core_test.errors) } # Test 3: Configuration system let config_test = test_configuration_system $test_config $tests_total = $tests_total + 1 if $config_test.status == "success" { $tests_passed = $tests_passed + 1 } else { $test_errors = ($test_errors | append $config_test.errors) } # Test 4: Basic CLI functionality let cli_test = test_basic_cli $test_config $tests_total = $tests_total + 1 if $cli_test.status == "success" { $tests_passed = $tests_passed + 1 } else { $test_errors = ($test_errors | append $cli_test.errors) } let success_rate = (($tests_passed | into float) / ($tests_total | into float) * 100) { category: "basic" status: (if ($test_errors | length) > 0 { "failed" } else { "success" }) tests_total: $tests_total tests_passed: $tests_passed success_rate: $success_rate duration: ((date now) - $start_time) errors: $test_errors } } # Test platform binaries def test_platform_binaries [test_config: record] { log info "Testing platform binaries..." let platform_dir = ($test_config.dist_dir | path join "platform") mut errors = [] if not ($platform_dir | path exists) { return { status: "failed" errors: [{ test: "platform_binaries", error: "platform directory not found" }] } } # Find platform binaries let binaries = (ls $platform_dir | where type == file | get name) if ($binaries | length) == 0 { return { status: "failed" errors: [{ test: "platform_binaries", error: "no platform binaries found" }] } } # Test each binary for binary in $binaries { let test_result = (do { # Test if binary is executable and runs without error (run-external $binary --help e>| null) | complete } | complete) if $test_result.exit_code != 0 { let parse_result = (do { $test_result.stdout | from json } | complete) let binary_result = if $parse_result.exit_code == 0 { $parse_result.stdout } else { {} } let exit_code = ($binary_result | get exit_code 1) $errors = ($errors | append { test: "binary_execution" binary: $binary error: $"Binary failed to execute with exit code ($exit_code)" }) } else { log info $"Binary test passed: ($binary)" } } return { status: (if ($errors | length) > 0 { "failed" } else { "success" }) tested_binaries: ($binaries | length) errors: $errors } } # Test core bundle def test_core_bundle [test_config: record] { log info "Testing core bundle..." let core_dir = ($test_config.dist_dir | path join "core") mut errors = [] if not ($core_dir | path exists) { return { status: "failed" errors: [{ test: "core_bundle", error: "core directory not found" }] } } # Test essential directories exist let essential_dirs = ["bin", "lib"] for dir in $essential_dirs { let dir_path = ($core_dir | path join $dir) if not ($dir_path | path exists) { $errors = ($errors | append { test: "core_structure" error: $"Essential directory missing: ($dir)" }) } } # Test provisioning CLI exists let provisioning_cli = ($core_dir | path join "bin" "provisioning") if not ($provisioning_cli | path exists) { $errors = ($errors | append { test: "provisioning_cli" error: "provisioning CLI not found" }) } else { # Test CLI execution let cli_test = (do { (run-external $provisioning_cli help e>| null) | complete } | complete) if $cli_test.exit_code != 0 { $errors = ($errors | append { test: "provisioning_cli" error: $"CLI execution failed with exit code ($cli_test.exit_code)" }) } } # Test core libraries exist let lib_dir = ($core_dir | path join "lib" "lib_provisioning") if not ($lib_dir | path exists) { $errors = ($errors | append { test: "core_libraries" error: "Core libraries directory not found" }) } return { status: (if ($errors | length) > 0 { "failed" } else { "success" }) errors: $errors } } # Test configuration system def test_configuration_system [test_config: record] { log info "Testing configuration system..." let config_dir = ($test_config.dist_dir | path join "config") mut errors = [] if not ($config_dir | path exists) { return { status: "failed" errors: [{ test: "configuration_system", error: "config directory not found" }] } } # Test default config exists let default_config = ($config_dir | path join "config.defaults.toml") if not ($default_config | path exists) { $errors = ($errors | append { test: "default_config" error: "config.defaults.toml not found" }) } else { # Test config parsing let config_result = (do { open $default_config } | complete) if $config_result.exit_code != 0 { $errors = ($errors | append { test: "config_parsing" error: $"Failed to parse default config with exit code ($config_result.exit_code)" }) } else { let config_data = $config_result.stdout log info $"Default config loaded successfully with ($config_data | columns | length) sections" } } return { status: (if ($errors | length) > 0 { "failed" } else { "success" }) errors: $errors } } # Test basic CLI functionality def test_basic_cli [test_config: record] { log info "Testing basic CLI functionality..." let provisioning_cli = ($test_config.dist_dir | path join "core" "bin" "provisioning") mut errors = [] if not ($provisioning_cli | path exists) { return { status: "failed" errors: [{ test: "basic_cli", error: "provisioning CLI not found" }] } } # Test basic commands let test_commands = ["version", "help", "env"] for cmd in $test_commands { let cmd_result = (do { (run-external $provisioning_cli $cmd e>| null) | complete } | complete) if $cmd_result.exit_code != 0 { $errors = ($errors | append { test: "cli_command" command: $cmd error: $"Command failed with exit code ($cmd_result.exit_code)" }) } else { log info $"CLI command test passed: ($cmd)" } } return { status: (if ($errors | length) > 0 { "failed" } else { "success" }) tested_commands: ($test_commands | length) errors: $errors } } # Run integration tests def run_integration_tests [test_config: record] { log info "Running integration tests..." let start_time = (date now) mut test_errors = [] # Integration tests would include: # - End-to-end workflow testing # - Multi-component interaction # - External dependency integration # Placeholder for now log info "Integration tests not implemented yet" { category: "integration" status: "success" tests_total: 0 tests_passed: 0 success_rate: 100.0 duration: ((date now) - $start_time) errors: [] } } # Run performance tests def run_performance_tests [test_config: record] { log info "Running performance tests..." let start_time = (date now) mut test_errors = [] # Performance tests would include: # - CLI response time benchmarks # - Memory usage tests # - Large configuration handling # Placeholder for now log info "Performance tests not implemented yet" { category: "performance" status: "success" tests_total: 0 tests_passed: 0 success_rate: 100.0 duration: ((date now) - $start_time) errors: [] } } # Generate test report def generate_test_report [results: list, test_config: record] { let total_tests = ($results | get tests_total | math sum) let total_passed = ($results | get tests_passed | math sum) let overall_success_rate = if $total_tests > 0 { (($total_passed | into float) / ($total_tests | into float) * 100) } else { 100.0 } let report = { timestamp: (date now) distribution_directory: $test_config.dist_dir platform: $test_config.platform total_categories: ($results | length) total_tests: $total_tests total_passed: $total_passed overall_success_rate: $overall_success_rate categories: $results all_errors: ($results | get errors | flatten) } # Save report to file let report_file = ($test_config.temp_workspace | path join "test-report.json") $report | to json | save $report_file log info $"Test report saved to: ($report_file)" return $report } # Show distribution info for testing def "main info" [dist_dir: string = "dist"] { let dist_root = ($dist_dir | path expand) if not ($dist_root | path exists) { log error $"Distribution directory does not exist: ($dist_root)" exit 1 } let platform_dir = ($dist_root | path join "platform") let core_dir = ($dist_root | path join "core") let config_dir = ($dist_root | path join "config") { distribution_directory: $dist_root platform: (detect_platform) structure: { platform: ($platform_dir | path exists) core: ($core_dir | path exists) config: ($config_dir | path exists) } platform_binaries: (if ($platform_dir | path exists) { ls $platform_dir | where type == file | get name } else { [] }) core_size: (if ($core_dir | path exists) { get_directory_size $core_dir } else { 0 }) config_files: (if ($config_dir | path exists) { glob ($config_dir + "/**/*.toml") | length } else { 0 }) } } # Get directory size helper def get_directory_size [dir: string] { if not ($dir | path exists) { return 0 } let size_result = (do { glob ($dir + "/**/*") | where { |f| ($f | path exists) and (($f | ls | get 0.type) == "file") } | each {|file| $file | ls | get 0.size } | math sum } | complete) if $size_result.exit_code != 0 { return 0 } else { let total_size = $size_result.stdout return ($total_size | if $in == null { 0 } else { $in }) } }