#!/usr/bin/env nu # Storage Migration Tool for Orchestrator # # This script provides a user-friendly interface for migrating data between # different storage backends using the Rust migration library. use std log # Migration configuration const ORCHESTRATOR_BIN = "./target/release/orchestrator" const DEFAULT_BATCH_SIZE = 100 const DEFAULT_MAX_RETRIES = 3 # Available storage types const STORAGE_TYPES = ["filesystem", "surrealdb-embedded", "surrealdb-server"] # Storage migration parameters def main [ --from (-f): string # Source storage type (filesystem, surrealdb-embedded, surrealdb-server) --to (-t): string # Target storage type (filesystem, surrealdb-embedded, surrealdb-server) --source-dir: string # Source data directory --target-dir: string # Target data directory --surrealdb-url: string # SurrealDB server URL (for server mode) --username: string # SurrealDB username (for server mode) --password: string # SurrealDB password (for server mode) --namespace: string = "orchestrator" # SurrealDB namespace --database: string = "tasks" # SurrealDB database --dry-run (-n) # Perform dry run without actual migration --no-backup # Skip backup creation --no-verify # Skip data integrity verification --batch-size: int = 100 # Batch size for migration operations --max-retries: int = 3 # Maximum retry attempts --continue-on-error # Continue migration on non-critical errors --status-filter: string # Filter tasks by status (comma-separated) --created-after: string # Filter tasks created after date (YYYY-MM-DD) --created-before: string # Filter tasks created before date (YYYY-MM-DD) --backup-path: string # Custom backup file path --verbose (-v) # Enable verbose logging --interactive (-i) # Run interactive migration wizard ] { # Initialize logging if $verbose { log info "Starting storage migration tool" } # Run interactive wizard if requested if $interactive { run_interactive_wizard return } # Validate required parameters if $from == null or $to == null { print_usage return } # Validate storage types validate_storage_types $from $to # Build migration config let config = build_migration_config { from: $from to: $to source_dir: $source_dir target_dir: $target_dir surrealdb_url: $surrealdb_url username: $username password: $password namespace: $namespace database: $database dry_run: $dry_run no_backup: $no_backup no_verify: $no_verify batch_size: $batch_size max_retries: $max_retries continue_on_error: $continue_on_error status_filter: $status_filter created_after: $created_after created_before: $created_before backup_path: $backup_path verbose: $verbose } # Execute migration execute_migration $config $verbose } # Interactive migration wizard def run_interactive_wizard [] { print "\nšŸš€ Storage Migration Wizard" print "==========================\n" # Get source storage configuration print "šŸ“ Source Storage Configuration" let source = get_storage_config "source" print "\nšŸ“ Target Storage Configuration" let target = get_storage_config "target" # Get migration options print "\nāš™ļø Migration Options" let dry_run = (["yes", "no"] | input list "Perform dry run? ") == "yes" let create_backup = (["yes", "no"] | input list "Create backup? ") == "yes" let verify_integrity = (["yes", "no"] | input list "Verify data integrity? ") == "yes" let batch_size = (input "Batch size for migration (default: 100): ") | if ($in | is-empty) { 100 } else { $in | into int } # Optional filters print "\nšŸ” Data Filters (optional)" let status_filter = input "Filter by task status (comma-separated, e.g., Pending,Failed): " let created_after = input "Filter tasks created after (YYYY-MM-DD): " let created_before = input "Filter tasks created before (YYYY-MM-DD): " # Confirmation print "\nšŸ“‹ Migration Summary" print $"From: ($source.type) @ ($source.config | get data_dir? | default 'N/A')" print $"To: ($target.type) @ ($target.config | get data_dir? | default 'N/A')" print $"Dry run: ($dry_run)" print $"Create backup: ($create_backup)" print $"Verify integrity: ($verify_integrity)" print $"Batch size: ($batch_size)" if not ($status_filter | is-empty) { print $"Status filter: ($status_filter)" } if not ($created_after | is-empty) { print $"Created after: ($created_after)" } if not ($created_before | is-empty) { print $"Created before: ($created_before)" } let confirm = (["yes", "no"] | input list "\nā–¶ļø Proceed with migration? ") == "yes" if not $confirm { print "āŒ Migration cancelled" return } # Build full configuration let config = { from: $source.type to: $target.type source_config: $source.config target_config: $target.config dry_run: $dry_run create_backup: $create_backup verify_integrity: $verify_integrity batch_size: $batch_size status_filter: $status_filter created_after: $created_after created_before: $created_before } # Execute migration execute_migration_interactive $config } # Get storage configuration interactively def get_storage_config [role: string] -> record { let storage_type = $STORAGE_TYPES | input list $"Select ($role) storage type: " match $storage_type { "filesystem" => { let data_dir = input $"($role | str capitalize) data directory: " { type: $storage_type, config: { data_dir: $data_dir } } } "surrealdb-embedded" => { let data_dir = input $"($role | str capitalize) data directory: " { type: $storage_type, config: { data_dir: $data_dir } } } "surrealdb-server" => { let url = input $"($role | str capitalize) SurrealDB server URL: " let username = input $"($role | str capitalize) username: " let password = input $"($role | str capitalize) password: " --sensitive let namespace = input $"($role | str capitalize) namespace (default: orchestrator): " | if ($in | is-empty) { "orchestrator" } else { $in } let database = input $"($role | str capitalize) database (default: tasks): " | if ($in | is-empty) { "tasks" } else { $in } { type: $storage_type, config: { url: $url, username: $username, password: $password, namespace: $namespace, database: $database } } } } } # Build migration configuration from parameters def build_migration_config [params: record] -> record { mut config = { source_type: $params.from target_type: $params.to options: { dry_run: ($params.dry_run? | default false) create_backup: (not ($params.no_backup? | default false)) verify_integrity: (not ($params.no_verify? | default false)) batch_size: ($params.batch_size? | default $DEFAULT_BATCH_SIZE) max_retries: ($params.max_retries? | default $DEFAULT_MAX_RETRIES) continue_on_error: ($params.continue_on_error? | default false) } } # Add source configuration match $params.from { "filesystem" => { if ($params.source_dir | is-empty) { error make { msg: "Source directory required for filesystem storage" } } $config = ($config | upsert source_config { storage_type: "filesystem" data_dir: $params.source_dir }) } "surrealdb-embedded" => { if ($params.source_dir | is-empty) { error make { msg: "Source directory required for SurrealDB embedded storage" } } $config = ($config | upsert source_config { storage_type: "surrealdb-embedded" data_dir: $params.source_dir surrealdb_namespace: ($params.namespace | default "orchestrator") surrealdb_database: ($params.database | default "tasks") }) } "surrealdb-server" => { if ($params.surrealdb_url | is-empty) or ($params.username | is-empty) or ($params.password | is-empty) { error make { msg: "URL, username, and password required for SurrealDB server storage" } } $config = ($config | upsert source_config { storage_type: "surrealdb-server" data_dir: "" surrealdb_url: $params.surrealdb_url surrealdb_username: $params.username surrealdb_password: $params.password surrealdb_namespace: ($params.namespace | default "orchestrator") surrealdb_database: ($params.database | default "tasks") }) } } # Add target configuration match $params.to { "filesystem" => { if ($params.target_dir | is-empty) { error make { msg: "Target directory required for filesystem storage" } } $config = ($config | upsert target_config { storage_type: "filesystem" data_dir: $params.target_dir }) } "surrealdb-embedded" => { if ($params.target_dir | is-empty) { error make { msg: "Target directory required for SurrealDB embedded storage" } } $config = ($config | upsert target_config { storage_type: "surrealdb-embedded" data_dir: $params.target_dir surrealdb_namespace: ($params.namespace | default "orchestrator") surrealdb_database: ($params.database | default "tasks") }) } "surrealdb-server" => { if ($params.surrealdb_url | is-empty) or ($params.username | is-empty) or ($params.password | is-empty) { error make { msg: "URL, username, and password required for SurrealDB server storage" } } $config = ($config | upsert target_config { storage_type: "surrealdb-server" data_dir: "" surrealdb_url: $params.surrealdb_url surrealdb_username: $params.username surrealdb_password: $params.password surrealdb_namespace: ($params.namespace | default "orchestrator") surrealdb_database: ($params.database | default "tasks") }) } } # Add optional filters if not ($params.status_filter | is-empty) { $config.options = ($config.options | upsert status_filter ($params.status_filter | split row ",")) } if not ($params.created_after | is-empty) { $config.options = ($config.options | upsert created_after $params.created_after) } if not ($params.created_before | is-empty) { $config.options = ($config.options | upsert created_before $params.created_before) } if not ($params.backup_path | is-empty) { $config.options = ($config.options | upsert backup_path $params.backup_path) } $config } # Execute migration using the Rust binary def execute_migration [config: record, verbose: bool = false] { print "\nšŸš€ Starting Storage Migration" print "==============================" if $verbose { log info $"Migration configuration: ($config)" } # Write temporary config file let config_file = $"/tmp/migration_config_(random uuid).json" $config | to json | save $config_file # Build orchestrator command mut cmd_args = [ "migrate" "--config-file" $config_file ] if $verbose { $cmd_args = ($cmd_args | append "--verbose") } # Execute migration try { let result = run-external $ORCHESTRATOR_BIN ...$cmd_args if $verbose { log info $"Migration completed: ($result)" } print "āœ… Migration completed successfully!" # Parse and display results if available if ($result | str contains "Migration Report") { print "\nšŸ“Š Migration Report" print "===================" print $result } } catch { print "āŒ Migration failed!" # Try to get error details from the binary let error_result = run-external $ORCHESTRATOR_BIN ...$cmd_args --dry-run print $"Error details: ($error_result)" } finally { # Clean up temporary config file rm -f $config_file } } # Execute interactive migration def execute_migration_interactive [config: record] { print "\nšŸš€ Executing Migration..." print "========================\n" # Create a simplified config for the binary let binary_config = { source_type: $config.from target_type: $config.to source_config: $config.source_config target_config: $config.target_config options: { dry_run: $config.dry_run create_backup: $config.create_backup verify_integrity: $config.verify_integrity batch_size: $config.batch_size } } # Add filters if present if ($config.status_filter? | is-not-empty) { $binary_config.options = ($binary_config.options | upsert status_filter ($config.status_filter | split row ",")) } # Write config and execute let config_file = $"/tmp/migration_config_(random uuid).json" $binary_config | to json | save $config_file try { # Real-time progress monitoring print "šŸ“Š Migration Progress:" print "=====================\n" let result = run-external $ORCHESTRATOR_BIN "migrate" "--config-file" $config_file "--progress" print "\nāœ… Migration completed successfully!" print $result } catch { print "\nāŒ Migration failed!" print "Check the logs for more details." } finally { rm -f $config_file } } # Validate storage types def validate_storage_types [from: string, to: string] { if $from not-in $STORAGE_TYPES { error make { msg: $"Invalid source storage type: ($from). Available types: ($STORAGE_TYPES | str join ', ')" } } if $to not-in $STORAGE_TYPES { error make { msg: $"Invalid target storage type: ($to). Available types: ($STORAGE_TYPES | str join ', ')" } } if $from == $to { print "āš ļø Warning: Source and target storage types are the same" } } # Print usage information def print_usage [] { print "\nšŸ“– Storage Migration Tool Usage" print "===============================" print "" print "Interactive mode:" print " ./migrate-storage.nu --interactive" print "" print "Direct migration:" print " ./migrate-storage.nu --from --to [OPTIONS]" print "" print "Examples:" print " # Filesystem to SurrealDB embedded" print " ./migrate-storage.nu --from filesystem --to surrealdb-embedded \\" print " --source-dir ./data --target-dir ./surrealdb-data" print "" print " # SurrealDB embedded to server" print " ./migrate-storage.nu --from surrealdb-embedded --to surrealdb-server \\" print " --source-dir ./surrealdb-data --surrealdb-url ws://localhost:8000 \\" print " --username admin --password secret" print "" print " # Dry run migration" print " ./migrate-storage.nu --from filesystem --to surrealdb-embedded \\" print " --source-dir ./data --target-dir ./new-data --dry-run" print "" print "Storage Types:" print " - filesystem: File-based JSON storage" print " - surrealdb-embedded: Embedded SurrealDB with RocksDB" print " - surrealdb-server: Remote SurrealDB server" print "" print "Options:" print " --dry-run Perform dry run without actual migration" print " --no-backup Skip backup creation" print " --no-verify Skip data integrity verification" print " --batch-size Batch size for migration operations (default: 100)" print " --max-retries Maximum retry attempts (default: 3)" print " --continue-on-error Continue migration on non-critical errors" print " --status-filter Filter tasks by status (comma-separated)" print " --created-after Filter tasks created after date (YYYY-MM-DD)" print " --created-before Filter tasks created before date (YYYY-MM-DD)" print " --backup-path

Custom backup file path" print " --verbose Enable verbose logging" print "" } # Show available commands def "migrate help" [] { print_usage } # List available storage types def "migrate list-types" [] { print "\nšŸ”§ Available Storage Types" print "==========================" for type in $STORAGE_TYPES { match $type { "filesystem" => { print $"šŸ“ ($type)" print " Description: File-based JSON storage" print " Required: --source-dir / --target-dir" print " Features: Simple, portable, human-readable" print "" } "surrealdb-embedded" => { print $"šŸ—ƒļø ($type)" print " Description: Embedded SurrealDB with RocksDB engine" print " Required: --source-dir / --target-dir" print " Features: High performance, ACID compliance, embedded" print "" } "surrealdb-server" => { print $"🌐 ($type)" print " Description: Remote SurrealDB server via WebSocket" print " Required: --surrealdb-url, --username, --password" print " Features: Scalable, distributed, real-time" print "" } } } } # Validate migration prerequisites def "migrate validate" [ --from: string # Source storage type --to: string # Target storage type --source-dir: string --target-dir: string --surrealdb-url: string --username: string --password: string ] { print "\nšŸ” Validating Migration Prerequisites" print "====================================" mut errors = [] mut warnings = [] # Validate storage types if $from not-in $STORAGE_TYPES { $errors = ($errors | append $"Invalid source type: ($from)") } if $to not-in $STORAGE_TYPES { $errors = ($errors | append $"Invalid target type: ($to)") } # Validate source configuration match $from { "filesystem" => { if ($source_dir | is-empty) { $errors = ($errors | append "Source directory required for filesystem") } else if not ($source_dir | path exists) { $warnings = ($warnings | append $"Source directory does not exist: ($source_dir)") } } "surrealdb-embedded" => { if ($source_dir | is-empty) { $errors = ($errors | append "Source directory required for SurrealDB embedded") } } "surrealdb-server" => { if ($surrealdb_url | is-empty) { $errors = ($errors | append "SurrealDB URL required for server mode") } if ($username | is-empty) { $errors = ($errors | append "Username required for SurrealDB server") } if ($password | is-empty) { $errors = ($errors | append "Password required for SurrealDB server") } } } # Validate target configuration match $to { "filesystem" => { if ($target_dir | is-empty) { $errors = ($errors | append "Target directory required for filesystem") } else { let parent_dir = ($target_dir | path dirname) if not ($parent_dir | path exists) { $warnings = ($warnings | append $"Target parent directory does not exist: ($parent_dir)") } } } "surrealdb-embedded" => { if ($target_dir | is-empty) { $errors = ($errors | append "Target directory required for SurrealDB embedded") } } "surrealdb-server" => { if ($surrealdb_url | is-empty) { $errors = ($errors | append "SurrealDB URL required for server mode") } if ($username | is-empty) { $errors = ($errors | append "Username required for SurrealDB server") } if ($password | is-empty) { $errors = ($errors | append "Password required for SurrealDB server") } } } # Check if orchestrator binary exists if not ($ORCHESTRATOR_BIN | path exists) { $warnings = ($warnings | append $"Orchestrator binary not found at ($ORCHESTRATOR_BIN). Build with: cargo build --release") } # Report results if ($errors | length) > 0 { print "āŒ Validation Errors:" for error in $errors { print $" • ($error)" } return } if ($warnings | length) > 0 { print "āš ļø Warnings:" for warning in $warnings { print $" • ($warning)" } print "" } print "āœ… Validation passed!" if $from == $to { print "ā„¹ļø Note: Source and target types are the same - ensure directories/connections differ" } } # Check migration status (if available) def "migrate status" [] { print "šŸ” Checking Migration Status..." try { let status = run-external $ORCHESTRATOR_BIN "migrate" "--status" print $status } catch { print "No active migrations found or binary not available" } }