510 lines
16 KiB
Plaintext
510 lines
16 KiB
Plaintext
#!/usr/bin/env nu
|
|
|
|
# Distribution testing tool - tests generated distributions for functionality
|
|
#
|
|
# Tests:
|
|
# - Binary execution and basic functionality
|
|
# - Core library loading and imports
|
|
# - Configuration system
|
|
# - Template rendering
|
|
# - Integration tests
|
|
|
|
use std log
|
|
|
|
def main [
|
|
--dist-dir: string = "dist" # Distribution directory to test
|
|
--test-types: string = "all" # Test types: all, basic, integration, performance
|
|
--platform: string = "" # Target platform (auto-detect if empty)
|
|
--temp-workspace: string = "" # Temporary workspace for tests (auto-generated if empty)
|
|
--cleanup: bool = true # Cleanup test workspace after tests
|
|
--verbose: bool = false # Enable verbose logging
|
|
--timeout: int = 300 # Test timeout in seconds
|
|
] -> record {
|
|
|
|
let dist_root = ($dist_dir | path expand)
|
|
let platform_detected = if $platform == "" { detect_platform } else { $platform }
|
|
let temp_workspace = if $temp_workspace == "" {
|
|
$env.TMPDIR | path join $"provisioning-test-(random uuid)"
|
|
} else {
|
|
$temp_workspace | path expand
|
|
}
|
|
|
|
let test_config = {
|
|
dist_dir: $dist_root
|
|
test_types: ($test_types | split row "," | each { str trim })
|
|
platform: $platform_detected
|
|
temp_workspace: $temp_workspace
|
|
cleanup: $cleanup
|
|
verbose: $verbose
|
|
timeout: $timeout
|
|
}
|
|
|
|
log info $"Starting distribution testing with config: ($test_config)"
|
|
|
|
# Validate distribution directory
|
|
if not ($dist_root | path exists) {
|
|
log error $"Distribution directory does not exist: ($dist_root)"
|
|
exit 1
|
|
}
|
|
|
|
# Create temporary workspace
|
|
mkdir ($test_config.temp_workspace)
|
|
|
|
let test_results = []
|
|
|
|
try {
|
|
# Run different test categories
|
|
let test_categories = if "all" in $test_config.test_types {
|
|
["basic", "integration", "performance"]
|
|
} else {
|
|
$test_config.test_types
|
|
}
|
|
|
|
for category in $test_categories {
|
|
let category_result = match $category {
|
|
"basic" => { run_basic_tests $test_config }
|
|
"integration" => { run_integration_tests $test_config }
|
|
"performance" => { run_performance_tests $test_config }
|
|
_ => {
|
|
log warning $"Unknown test category: ($category)"
|
|
{ category: $category, status: "skipped", reason: "unknown category" }
|
|
}
|
|
}
|
|
let test_results = ($test_results | append $category_result)
|
|
}
|
|
|
|
# Generate test report
|
|
let test_report = generate_test_report $test_results $test_config
|
|
|
|
# Cleanup if requested
|
|
if $test_config.cleanup {
|
|
rm -rf ($test_config.temp_workspace)
|
|
log info "Cleaned up test workspace"
|
|
}
|
|
|
|
let summary = {
|
|
total_categories: ($test_results | length)
|
|
successful_categories: ($test_results | where status == "success" | length)
|
|
failed_categories: ($test_results | where status == "failed" | length)
|
|
test_config: $test_config
|
|
results: $test_results
|
|
report: $test_report
|
|
}
|
|
|
|
if $summary.failed_categories > 0 {
|
|
log error $"Distribution testing completed with ($summary.failed_categories) failed categories"
|
|
exit 1
|
|
} else {
|
|
log info $"Distribution testing completed successfully - all ($summary.total_categories) categories passed"
|
|
}
|
|
|
|
return $summary
|
|
|
|
} catch {|err|
|
|
# Ensure cleanup even on error
|
|
if $test_config.cleanup and ($test_config.temp_workspace | path exists) {
|
|
rm -rf ($test_config.temp_workspace)
|
|
}
|
|
|
|
log error $"Distribution testing failed: ($err.msg)"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
# Detect current platform
|
|
def detect_platform [] -> string {
|
|
match $nu.os-info.name {
|
|
"linux" => "linux"
|
|
"macos" => "macos"
|
|
"windows" => "windows"
|
|
_ => "unknown"
|
|
}
|
|
}
|
|
|
|
# Run basic functionality tests
|
|
def run_basic_tests [test_config: record] -> record {
|
|
log info "Running basic functionality tests..."
|
|
|
|
let start_time = (date now)
|
|
let mut test_errors = []
|
|
let mut tests_passed = 0
|
|
let mut tests_total = 0
|
|
|
|
# Test 1: Platform binaries exist and execute
|
|
let platform_test = test_platform_binaries $test_config
|
|
$tests_total = $tests_total + 1
|
|
if $platform_test.status == "success" {
|
|
$tests_passed = $tests_passed + 1
|
|
} else {
|
|
$test_errors = ($test_errors | append $platform_test.errors)
|
|
}
|
|
|
|
# Test 2: Core bundle integrity
|
|
let core_test = test_core_bundle $test_config
|
|
$tests_total = $tests_total + 1
|
|
if $core_test.status == "success" {
|
|
$tests_passed = $tests_passed + 1
|
|
} else {
|
|
$test_errors = ($test_errors | append $core_test.errors)
|
|
}
|
|
|
|
# Test 3: Configuration system
|
|
let config_test = test_configuration_system $test_config
|
|
$tests_total = $tests_total + 1
|
|
if $config_test.status == "success" {
|
|
$tests_passed = $tests_passed + 1
|
|
} else {
|
|
$test_errors = ($test_errors | append $config_test.errors)
|
|
}
|
|
|
|
# Test 4: Basic CLI functionality
|
|
let cli_test = test_basic_cli $test_config
|
|
$tests_total = $tests_total + 1
|
|
if $cli_test.status == "success" {
|
|
$tests_passed = $tests_passed + 1
|
|
} else {
|
|
$test_errors = ($test_errors | append $cli_test.errors)
|
|
}
|
|
|
|
let success_rate = (($tests_passed | into float) / ($tests_total | into float) * 100)
|
|
|
|
{
|
|
category: "basic"
|
|
status: (if ($test_errors | length) > 0 { "failed" } else { "success" })
|
|
tests_total: $tests_total
|
|
tests_passed: $tests_passed
|
|
success_rate: $success_rate
|
|
duration: ((date now) - $start_time)
|
|
errors: $test_errors
|
|
}
|
|
}
|
|
|
|
# Test platform binaries
|
|
def test_platform_binaries [test_config: record] -> record {
|
|
log info "Testing platform binaries..."
|
|
|
|
let platform_dir = ($test_config.dist_dir | path join "platform")
|
|
let mut errors = []
|
|
|
|
if not ($platform_dir | path exists) {
|
|
return {
|
|
status: "failed"
|
|
errors: [{ test: "platform_binaries", error: "platform directory not found" }]
|
|
}
|
|
}
|
|
|
|
# Find platform binaries
|
|
let binaries = (ls $platform_dir | where type == file | get name)
|
|
|
|
if ($binaries | length) == 0 {
|
|
return {
|
|
status: "failed"
|
|
errors: [{ test: "platform_binaries", error: "no platform binaries found" }]
|
|
}
|
|
}
|
|
|
|
# Test each binary
|
|
for binary in $binaries {
|
|
try {
|
|
# Test if binary is executable and runs without error
|
|
let test_result = (run-external --redirect-combine $binary --help | complete)
|
|
|
|
if $test_result.exit_code != 0 {
|
|
$errors = ($errors | append {
|
|
test: "binary_execution"
|
|
binary: $binary
|
|
error: $"Binary failed to execute: ($test_result.stderr)"
|
|
})
|
|
} else {
|
|
log info $"Binary test passed: ($binary)"
|
|
}
|
|
} catch {|err|
|
|
$errors = ($errors | append {
|
|
test: "binary_execution"
|
|
binary: $binary
|
|
error: $err.msg
|
|
})
|
|
}
|
|
}
|
|
|
|
return {
|
|
status: (if ($errors | length) > 0 { "failed" } else { "success" })
|
|
tested_binaries: ($binaries | length)
|
|
errors: $errors
|
|
}
|
|
}
|
|
|
|
# Test core bundle
|
|
def test_core_bundle [test_config: record] -> record {
|
|
log info "Testing core bundle..."
|
|
|
|
let core_dir = ($test_config.dist_dir | path join "core")
|
|
let mut errors = []
|
|
|
|
if not ($core_dir | path exists) {
|
|
return {
|
|
status: "failed"
|
|
errors: [{ test: "core_bundle", error: "core directory not found" }]
|
|
}
|
|
}
|
|
|
|
# Test essential directories exist
|
|
let essential_dirs = ["bin", "lib"]
|
|
for dir in $essential_dirs {
|
|
let dir_path = ($core_dir | path join $dir)
|
|
if not ($dir_path | path exists) {
|
|
$errors = ($errors | append {
|
|
test: "core_structure"
|
|
error: $"Essential directory missing: ($dir)"
|
|
})
|
|
}
|
|
}
|
|
|
|
# Test provisioning CLI exists
|
|
let provisioning_cli = ($core_dir | path join "bin" "provisioning")
|
|
if not ($provisioning_cli | path exists) {
|
|
$errors = ($errors | append {
|
|
test: "provisioning_cli"
|
|
error: "provisioning CLI not found"
|
|
})
|
|
} else {
|
|
# Test CLI execution
|
|
try {
|
|
let cli_test = (run-external --redirect-combine $provisioning_cli help | complete)
|
|
if $cli_test.exit_code != 0 {
|
|
$errors = ($errors | append {
|
|
test: "provisioning_cli"
|
|
error: $"CLI execution failed: ($cli_test.stderr)"
|
|
})
|
|
}
|
|
} catch {|err|
|
|
$errors = ($errors | append {
|
|
test: "provisioning_cli"
|
|
error: $err.msg
|
|
})
|
|
}
|
|
}
|
|
|
|
# Test core libraries exist
|
|
let lib_dir = ($core_dir | path join "lib" "lib_provisioning")
|
|
if not ($lib_dir | path exists) {
|
|
$errors = ($errors | append {
|
|
test: "core_libraries"
|
|
error: "Core libraries directory not found"
|
|
})
|
|
}
|
|
|
|
return {
|
|
status: (if ($errors | length) > 0 { "failed" } else { "success" })
|
|
errors: $errors
|
|
}
|
|
}
|
|
|
|
# Test configuration system
|
|
def test_configuration_system [test_config: record] -> record {
|
|
log info "Testing configuration system..."
|
|
|
|
let config_dir = ($test_config.dist_dir | path join "config")
|
|
let mut errors = []
|
|
|
|
if not ($config_dir | path exists) {
|
|
return {
|
|
status: "failed"
|
|
errors: [{ test: "configuration_system", error: "config directory not found" }]
|
|
}
|
|
}
|
|
|
|
# Test default config exists
|
|
let default_config = ($config_dir | path join "config.defaults.toml")
|
|
if not ($default_config | path exists) {
|
|
$errors = ($errors | append {
|
|
test: "default_config"
|
|
error: "config.defaults.toml not found"
|
|
})
|
|
} else {
|
|
# Test config parsing
|
|
try {
|
|
let config_data = (open $default_config)
|
|
log info $"Default config loaded successfully with ($config_data | columns | length) sections"
|
|
} catch {|err|
|
|
$errors = ($errors | append {
|
|
test: "config_parsing"
|
|
error: $"Failed to parse default config: ($err.msg)"
|
|
})
|
|
}
|
|
}
|
|
|
|
return {
|
|
status: (if ($errors | length) > 0 { "failed" } else { "success" })
|
|
errors: $errors
|
|
}
|
|
}
|
|
|
|
# Test basic CLI functionality
|
|
def test_basic_cli [test_config: record] -> record {
|
|
log info "Testing basic CLI functionality..."
|
|
|
|
let provisioning_cli = ($test_config.dist_dir | path join "core" "bin" "provisioning")
|
|
let mut errors = []
|
|
|
|
if not ($provisioning_cli | path exists) {
|
|
return {
|
|
status: "failed"
|
|
errors: [{ test: "basic_cli", error: "provisioning CLI not found" }]
|
|
}
|
|
}
|
|
|
|
# Test basic commands
|
|
let test_commands = ["version", "help", "env"]
|
|
|
|
for cmd in $test_commands {
|
|
try {
|
|
let cmd_result = (run-external --redirect-combine $provisioning_cli $cmd | complete)
|
|
if $cmd_result.exit_code != 0 {
|
|
$errors = ($errors | append {
|
|
test: "cli_command"
|
|
command: $cmd
|
|
error: $"Command failed: ($cmd_result.stderr)"
|
|
})
|
|
} else {
|
|
log info $"CLI command test passed: ($cmd)"
|
|
}
|
|
} catch {|err|
|
|
$errors = ($errors | append {
|
|
test: "cli_command"
|
|
command: $cmd
|
|
error: $err.msg
|
|
})
|
|
}
|
|
}
|
|
|
|
return {
|
|
status: (if ($errors | length) > 0 { "failed" } else { "success" })
|
|
tested_commands: ($test_commands | length)
|
|
errors: $errors
|
|
}
|
|
}
|
|
|
|
# Run integration tests
|
|
def run_integration_tests [test_config: record] -> record {
|
|
log info "Running integration tests..."
|
|
|
|
let start_time = (date now)
|
|
let mut test_errors = []
|
|
|
|
# Integration tests would include:
|
|
# - End-to-end workflow testing
|
|
# - Multi-component interaction
|
|
# - External dependency integration
|
|
|
|
# Placeholder for now
|
|
log info "Integration tests not implemented yet"
|
|
|
|
{
|
|
category: "integration"
|
|
status: "success"
|
|
tests_total: 0
|
|
tests_passed: 0
|
|
success_rate: 100.0
|
|
duration: ((date now) - $start_time)
|
|
errors: []
|
|
}
|
|
}
|
|
|
|
# Run performance tests
|
|
def run_performance_tests [test_config: record] -> record {
|
|
log info "Running performance tests..."
|
|
|
|
let start_time = (date now)
|
|
let mut test_errors = []
|
|
|
|
# Performance tests would include:
|
|
# - CLI response time benchmarks
|
|
# - Memory usage tests
|
|
# - Large configuration handling
|
|
|
|
# Placeholder for now
|
|
log info "Performance tests not implemented yet"
|
|
|
|
{
|
|
category: "performance"
|
|
status: "success"
|
|
tests_total: 0
|
|
tests_passed: 0
|
|
success_rate: 100.0
|
|
duration: ((date now) - $start_time)
|
|
errors: []
|
|
}
|
|
}
|
|
|
|
# Generate test report
|
|
def generate_test_report [results: list, test_config: record] -> record {
|
|
let total_tests = ($results | get tests_total | math sum)
|
|
let total_passed = ($results | get tests_passed | math sum)
|
|
let overall_success_rate = if $total_tests > 0 {
|
|
(($total_passed | into float) / ($total_tests | into float) * 100)
|
|
} else {
|
|
100.0
|
|
}
|
|
|
|
let report = {
|
|
timestamp: (date now)
|
|
distribution_directory: $test_config.dist_dir
|
|
platform: $test_config.platform
|
|
total_categories: ($results | length)
|
|
total_tests: $total_tests
|
|
total_passed: $total_passed
|
|
overall_success_rate: $overall_success_rate
|
|
categories: $results
|
|
all_errors: ($results | get errors | flatten)
|
|
}
|
|
|
|
# Save report to file
|
|
let report_file = ($test_config.temp_workspace | path join "test-report.json")
|
|
$report | to json | save $report_file
|
|
log info $"Test report saved to: ($report_file)"
|
|
|
|
return $report
|
|
}
|
|
|
|
# Show distribution info for testing
|
|
def "main info" [dist_dir: string = "dist"] {
|
|
let dist_root = ($dist_dir | path expand)
|
|
|
|
if not ($dist_root | path exists) {
|
|
log error $"Distribution directory does not exist: ($dist_root)"
|
|
exit 1
|
|
}
|
|
|
|
let platform_dir = ($dist_root | path join "platform")
|
|
let core_dir = ($dist_root | path join "core")
|
|
let config_dir = ($dist_root | path join "config")
|
|
|
|
{
|
|
distribution_directory: $dist_root
|
|
platform: (detect_platform)
|
|
structure: {
|
|
platform: ($platform_dir | path exists)
|
|
core: ($core_dir | path exists)
|
|
config: ($config_dir | path exists)
|
|
}
|
|
platform_binaries: (if ($platform_dir | path exists) { ls $platform_dir | where type == file | get name } else { [] })
|
|
core_size: (if ($core_dir | path exists) { get_directory_size $core_dir } else { 0 })
|
|
config_files: (if ($config_dir | path exists) { find $config_dir -name "*.toml" | length } else { 0 })
|
|
}
|
|
}
|
|
|
|
# Get directory size helper
|
|
def get_directory_size [dir: string] -> int {
|
|
if not ($dir | path exists) {
|
|
return 0
|
|
}
|
|
|
|
let total_size = try {
|
|
find $dir -type f | each {|file| ls $file | get 0.size } | math sum
|
|
} catch {
|
|
0
|
|
}
|
|
|
|
return ($total_size | if $in == null { 0 } else { $in })
|
|
} |