#!/bin/bash # Database Migration Management Script # Advanced migration tools for schema evolution and data management set -e # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' NC='\033[0m' # No Color # Script directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" # Change to project root cd "$PROJECT_ROOT" # Migration configuration MIGRATIONS_DIR="migrations" MIGRATION_TABLE="__migrations" MIGRATION_LOCK_TABLE="__migration_locks" MIGRATION_TEMPLATE_DIR="migration_templates" ROLLBACK_DIR="rollbacks" # Logging functions log() { echo -e "${GREEN}[INFO]${NC} $1" } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1" } log_error() { echo -e "${RED}[ERROR]${NC} $1" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" } log_debug() { if [ "$DEBUG" = "true" ]; then echo -e "${CYAN}[DEBUG]${NC} $1" fi } print_header() { echo -e "${BLUE}${BOLD}=== $1 ===${NC}" } print_subheader() { echo -e "${CYAN}--- $1 ---${NC}" } print_usage() { echo "Database Migration Management Script" echo echo "Usage: $0 [options]" echo echo "Commands:" echo " status Show migration status" echo " pending List pending migrations" echo " applied List applied migrations" echo " migrate Run pending migrations" echo " rollback Rollback migrations" echo " create Create new migration" echo " generate Generate migration from schema diff" echo " validate Validate migration files" echo " dry-run Show what would be migrated" echo " force Force migration state" echo " repair Repair migration table" echo " baseline Set migration baseline" echo " history Show migration history" echo " schema-dump Dump current schema" echo " data-migrate Migrate data between schemas" echo " template Manage migration templates" echo echo "Options:" echo " --env ENV Environment (dev/prod) [default: dev]" echo " --version VERSION Target migration version" echo " --steps N Number of migration steps" echo " --name NAME Migration name (for create command)" echo " --type TYPE Migration type (schema/data/both) [default: schema]" echo " --table TABLE Target table name" echo " --template TEMPLATE Migration template name" echo " --dry-run Show changes without applying" echo " --force Force operation without confirmation" echo " --debug Enable debug output" echo " --quiet Suppress verbose output" echo " --batch-size N Batch size for data migrations [default: 1000]" echo " --timeout N Migration timeout in seconds [default: 300]" echo echo "Examples:" echo " $0 status # Show migration status" echo " $0 migrate # Run all pending migrations" echo " $0 migrate --version 003 # Migrate to specific version" echo " $0 rollback --steps 1 # Rollback last migration" echo " $0 create --name add_user_preferences # Create new migration" echo " $0 create --name migrate_users --type data # Create data migration" echo " $0 dry-run # Preview pending migrations" echo " $0 validate # Validate all migrations" echo " $0 baseline --version 001 # Set baseline version" echo echo "Migration Templates:" echo " create-table Create new table" echo " alter-table Modify existing table" echo " add-column Add column to table" echo " drop-column Drop column from table" echo " add-index Add database index" echo " add-constraint Add table constraint" echo " data-migration Migrate data between schemas" echo " seed-data Insert seed data" } # Check if .env file exists and load it load_env() { if [ ! -f ".env" ]; then log_error ".env file not found" echo "Please run the database setup script first:" echo " ./scripts/db-setup.sh setup" exit 1 fi # Load environment variables export $(grep -v '^#' .env | xargs) } # Parse database URL parse_database_url() { if [[ $DATABASE_URL == postgresql://* ]] || [[ $DATABASE_URL == postgres://* ]]; then DB_TYPE="postgresql" DB_HOST=$(echo $DATABASE_URL | sed -n 's/.*@\([^:]*\):.*/\1/p') DB_PORT=$(echo $DATABASE_URL | sed -n 's/.*:\([0-9]*\)\/.*/\1/p') DB_NAME=$(echo $DATABASE_URL | sed -n 's/.*\/\([^?]*\).*/\1/p') DB_USER=$(echo $DATABASE_URL | sed -n 's/.*\/\/\([^:]*\):.*/\1/p') DB_PASS=$(echo $DATABASE_URL | sed -n 's/.*:\/\/[^:]*:\([^@]*\)@.*/\1/p') elif [[ $DATABASE_URL == sqlite://* ]]; then DB_TYPE="sqlite" DB_FILE=$(echo $DATABASE_URL | sed 's/sqlite:\/\///') else log_error "Unsupported database URL format: $DATABASE_URL" exit 1 fi } # Execute SQL query execute_sql() { local query="$1" local capture_output="${2:-false}" log_debug "Executing SQL: $query" if [ "$DB_TYPE" = "postgresql" ]; then export PGPASSWORD="$DB_PASS" if [ "$capture_output" = "true" ]; then psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -t -A -c "$query" 2>/dev/null else psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>/dev/null fi unset PGPASSWORD elif [ "$DB_TYPE" = "sqlite" ]; then if [ "$capture_output" = "true" ]; then sqlite3 "$DB_FILE" "$query" 2>/dev/null else sqlite3 "$DB_FILE" "$query" 2>/dev/null fi fi } # Execute SQL file execute_sql_file() { local file="$1" local ignore_errors="${2:-false}" if [ ! -f "$file" ]; then log_error "SQL file not found: $file" return 1 fi log_debug "Executing SQL file: $file" if [ "$DB_TYPE" = "postgresql" ]; then export PGPASSWORD="$DB_PASS" if [ "$ignore_errors" = "true" ]; then psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$file" 2>/dev/null || true else psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$file" fi unset PGPASSWORD elif [ "$DB_TYPE" = "sqlite" ]; then if [ "$ignore_errors" = "true" ]; then sqlite3 "$DB_FILE" ".read $file" 2>/dev/null || true else sqlite3 "$DB_FILE" ".read $file" fi fi } # Initialize migration system init_migration_system() { log_debug "Initializing migration system" # Create migrations directory mkdir -p "$MIGRATIONS_DIR" mkdir -p "$ROLLBACK_DIR" mkdir -p "$MIGRATION_TEMPLATE_DIR" # Create migration tracking table if [ "$DB_TYPE" = "postgresql" ]; then execute_sql " CREATE TABLE IF NOT EXISTS $MIGRATION_TABLE ( id SERIAL PRIMARY KEY, version VARCHAR(50) NOT NULL UNIQUE, name VARCHAR(255) NOT NULL, type VARCHAR(20) NOT NULL DEFAULT 'schema', applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, applied_by VARCHAR(100) DEFAULT USER, execution_time_ms INTEGER DEFAULT 0, checksum VARCHAR(64), success BOOLEAN DEFAULT TRUE ); " >/dev/null 2>&1 execute_sql " CREATE TABLE IF NOT EXISTS $MIGRATION_LOCK_TABLE ( id INTEGER PRIMARY KEY DEFAULT 1, is_locked BOOLEAN DEFAULT FALSE, locked_by VARCHAR(100), locked_at TIMESTAMP, process_id INTEGER, CONSTRAINT single_lock CHECK (id = 1) ); " >/dev/null 2>&1 elif [ "$DB_TYPE" = "sqlite" ]; then execute_sql " CREATE TABLE IF NOT EXISTS $MIGRATION_TABLE ( id INTEGER PRIMARY KEY AUTOINCREMENT, version TEXT NOT NULL UNIQUE, name TEXT NOT NULL, type TEXT NOT NULL DEFAULT 'schema', applied_at DATETIME DEFAULT CURRENT_TIMESTAMP, applied_by TEXT DEFAULT 'system', execution_time_ms INTEGER DEFAULT 0, checksum TEXT, success BOOLEAN DEFAULT 1 ); " >/dev/null 2>&1 execute_sql " CREATE TABLE IF NOT EXISTS $MIGRATION_LOCK_TABLE ( id INTEGER PRIMARY KEY DEFAULT 1, is_locked BOOLEAN DEFAULT 0, locked_by TEXT, locked_at DATETIME, process_id INTEGER ); " >/dev/null 2>&1 fi # Insert initial lock record execute_sql "INSERT OR IGNORE INTO $MIGRATION_LOCK_TABLE (id, is_locked) VALUES (1, false);" >/dev/null 2>&1 } # Acquire migration lock acquire_migration_lock() { local process_id=$$ local lock_holder=$(whoami) log_debug "Acquiring migration lock" # Check if already locked local is_locked=$(execute_sql "SELECT is_locked FROM $MIGRATION_LOCK_TABLE WHERE id = 1;" true) if [ "$is_locked" = "true" ] || [ "$is_locked" = "1" ]; then local locked_by=$(execute_sql "SELECT locked_by FROM $MIGRATION_LOCK_TABLE WHERE id = 1;" true) local locked_at=$(execute_sql "SELECT locked_at FROM $MIGRATION_LOCK_TABLE WHERE id = 1;" true) log_error "Migration system is locked by $locked_by at $locked_at" return 1 fi # Acquire lock execute_sql " UPDATE $MIGRATION_LOCK_TABLE SET is_locked = true, locked_by = '$lock_holder', locked_at = CURRENT_TIMESTAMP, process_id = $process_id WHERE id = 1; " >/dev/null 2>&1 log_debug "Migration lock acquired by $lock_holder (PID: $process_id)" } # Release migration lock release_migration_lock() { log_debug "Releasing migration lock" execute_sql " UPDATE $MIGRATION_LOCK_TABLE SET is_locked = false, locked_by = NULL, locked_at = NULL, process_id = NULL WHERE id = 1; " >/dev/null 2>&1 } # Get migration files get_migration_files() { find "$MIGRATIONS_DIR" -name "*.sql" -type f | sort } # Get applied migrations get_applied_migrations() { execute_sql "SELECT version FROM $MIGRATION_TABLE ORDER BY version;" true } # Get pending migrations get_pending_migrations() { local applied_migrations=$(get_applied_migrations) local all_migrations=$(get_migration_files) for migration_file in $all_migrations; do local version=$(basename "$migration_file" .sql | cut -d'_' -f1) if ! echo "$applied_migrations" | grep -q "^$version$"; then echo "$migration_file" fi done } # Calculate file checksum calculate_checksum() { local file="$1" if command -v sha256sum >/dev/null 2>&1; then sha256sum "$file" | cut -d' ' -f1 elif command -v shasum >/dev/null 2>&1; then shasum -a 256 "$file" | cut -d' ' -f1 else # Fallback to md5 md5sum "$file" | cut -d' ' -f1 fi } # Show migration status show_migration_status() { print_header "Migration Status" local applied_count=$(execute_sql "SELECT COUNT(*) FROM $MIGRATION_TABLE;" true) local pending_migrations=$(get_pending_migrations) local pending_count=$(echo "$pending_migrations" | wc -l) if [ -z "$pending_migrations" ]; then pending_count=0 fi log "Applied migrations: $applied_count" log "Pending migrations: $pending_count" if [ "$applied_count" -gt "0" ]; then echo print_subheader "Last Applied Migration" if [ "$DB_TYPE" = "postgresql" ]; then execute_sql " SELECT version, name, applied_at, execution_time_ms FROM $MIGRATION_TABLE ORDER BY applied_at DESC LIMIT 1; " elif [ "$DB_TYPE" = "sqlite" ]; then execute_sql " SELECT version, name, applied_at, execution_time_ms FROM $MIGRATION_TABLE ORDER BY applied_at DESC LIMIT 1; " fi fi if [ "$pending_count" -gt "0" ]; then echo print_subheader "Pending Migrations" for migration in $pending_migrations; do local version=$(basename "$migration" .sql | cut -d'_' -f1) local name=$(basename "$migration" .sql | cut -d'_' -f2-) echo " $version - $name" done fi } # List applied migrations list_applied_migrations() { print_header "Applied Migrations" if [ "$DB_TYPE" = "postgresql" ]; then execute_sql " SELECT version, name, type, applied_at, applied_by, execution_time_ms || ' ms' as duration, CASE WHEN success THEN '✓' ELSE '✗' END as status FROM $MIGRATION_TABLE ORDER BY version; " elif [ "$DB_TYPE" = "sqlite" ]; then execute_sql " SELECT version, name, type, applied_at, applied_by, execution_time_ms || ' ms' as duration, CASE WHEN success THEN '✓' ELSE '✗' END as status FROM $MIGRATION_TABLE ORDER BY version; " fi } # List pending migrations list_pending_migrations() { print_header "Pending Migrations" local pending_migrations=$(get_pending_migrations) if [ -z "$pending_migrations" ]; then log_success "No pending migrations" return fi for migration in $pending_migrations; do local version=$(basename "$migration" .sql | cut -d'_' -f1) local name=$(basename "$migration" .sql | cut -d'_' -f2-) local size=$(du -h "$migration" | cut -f1) echo " $version - $name ($size)" done } # Run migrations run_migrations() { print_header "Running Migrations" local target_version="$1" local pending_migrations=$(get_pending_migrations) if [ -z "$pending_migrations" ]; then log_success "No pending migrations to run" return fi # Acquire lock if ! acquire_migration_lock; then exit 1 fi # Set up cleanup trap trap 'release_migration_lock; exit 1' INT TERM EXIT local migration_count=0 local success_count=0 for migration_file in $pending_migrations; do local version=$(basename "$migration_file" .sql | cut -d'_' -f1) local name=$(basename "$migration_file" .sql | cut -d'_' -f2-) # Check if we should stop at target version if [ -n "$target_version" ] && [ "$version" \> "$target_version" ]; then log "Stopping at target version $target_version" break fi ((migration_count++)) log "Running migration $version: $name" if [ "$DRY_RUN" = "true" ]; then echo "Would execute: $migration_file" continue fi local start_time=$(date +%s%3N) local success=true local checksum=$(calculate_checksum "$migration_file") # Execute migration if execute_sql_file "$migration_file"; then local end_time=$(date +%s%3N) local execution_time=$((end_time - start_time)) # Record successful migration execute_sql " INSERT INTO $MIGRATION_TABLE (version, name, type, execution_time_ms, checksum, success) VALUES ('$version', '$name', 'schema', $execution_time, '$checksum', true); " >/dev/null 2>&1 log_success "Migration $version completed in ${execution_time}ms" ((success_count++)) else local end_time=$(date +%s%3N) local execution_time=$((end_time - start_time)) # Record failed migration execute_sql " INSERT INTO $MIGRATION_TABLE (version, name, type, execution_time_ms, checksum, success) VALUES ('$version', '$name', 'schema', $execution_time, '$checksum', false); " >/dev/null 2>&1 log_error "Migration $version failed" success=false break fi done # Release lock release_migration_lock trap - INT TERM EXIT if [ "$DRY_RUN" = "true" ]; then log "Dry run completed. Would execute $migration_count migrations." else log "Migration run completed. $success_count/$migration_count migrations successful." fi } # Rollback migrations rollback_migrations() { print_header "Rolling Back Migrations" local steps="${1:-1}" if [ "$steps" -le 0 ]; then log_error "Invalid number of steps: $steps" return 1 fi # Get last N applied migrations local migrations_to_rollback if [ "$DB_TYPE" = "postgresql" ]; then migrations_to_rollback=$(execute_sql " SELECT version FROM $MIGRATION_TABLE WHERE success = true ORDER BY applied_at DESC LIMIT $steps; " true) elif [ "$DB_TYPE" = "sqlite" ]; then migrations_to_rollback=$(execute_sql " SELECT version FROM $MIGRATION_TABLE WHERE success = 1 ORDER BY applied_at DESC LIMIT $steps; " true) fi if [ -z "$migrations_to_rollback" ]; then log_warn "No migrations to rollback" return fi if [ "$FORCE" != "true" ]; then echo -n "This will rollback $steps migration(s). Continue? (y/N): " read -r confirm if [[ ! "$confirm" =~ ^[Yy]$ ]]; then log "Rollback cancelled" return fi fi # Acquire lock if ! acquire_migration_lock; then exit 1 fi # Set up cleanup trap trap 'release_migration_lock; exit 1' INT TERM EXIT local rollback_count=0 for version in $migrations_to_rollback; do local rollback_file="$ROLLBACK_DIR/rollback_${version}.sql" if [ -f "$rollback_file" ]; then log "Rolling back migration $version" if [ "$DRY_RUN" = "true" ]; then echo "Would execute rollback: $rollback_file" else if execute_sql_file "$rollback_file"; then # Remove from migration table execute_sql "DELETE FROM $MIGRATION_TABLE WHERE version = '$version';" >/dev/null 2>&1 log_success "Rollback $version completed" ((rollback_count++)) else log_error "Rollback $version failed" break fi fi else log_warn "Rollback file not found for migration $version: $rollback_file" log_warn "Manual rollback required" fi done # Release lock release_migration_lock trap - INT TERM EXIT if [ "$DRY_RUN" = "true" ]; then log "Dry run completed. Would rollback $rollback_count migrations." else log "Rollback completed. $rollback_count migrations rolled back." fi } # Create new migration create_migration() { local migration_name="$1" local migration_type="${2:-schema}" local template_name="$3" if [ -z "$migration_name" ]; then log_error "Migration name is required" return 1 fi # Generate version number local version=$(date +%Y%m%d%H%M%S) local migration_file="$MIGRATIONS_DIR/${version}_${migration_name}.sql" local rollback_file="$ROLLBACK_DIR/rollback_${version}.sql" log "Creating migration: $migration_file" # Create migration file from template if [ -n "$template_name" ] && [ -f "$MIGRATION_TEMPLATE_DIR/$template_name.sql" ]; then cp "$MIGRATION_TEMPLATE_DIR/$template_name.sql" "$migration_file" log "Created migration from template: $template_name" else # Create basic migration template cat > "$migration_file" << EOF -- Migration: $migration_name -- Type: $migration_type -- Created: $(date) -- Description: Add your migration description here -- Add your migration SQL here -- Example: -- CREATE TABLE example_table ( -- id SERIAL PRIMARY KEY, -- name VARCHAR(255) NOT NULL, -- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -- ); EOF fi # Create rollback file cat > "$rollback_file" << EOF -- Rollback: $migration_name -- Version: $version -- Created: $(date) -- Description: Add your rollback description here -- Add your rollback SQL here -- Example: -- DROP TABLE IF EXISTS example_table; EOF log_success "Migration files created:" log " Migration: $migration_file" log " Rollback: $rollback_file" log "" log "Next steps:" log " 1. Edit the migration file with your changes" log " 2. Edit the rollback file with reverse operations" log " 3. Run: $0 validate" log " 4. Run: $0 migrate" } # Validate migration files validate_migrations() { print_header "Validating Migrations" local migration_files=$(get_migration_files) local validation_errors=0 for migration_file in $migration_files; do local version=$(basename "$migration_file" .sql | cut -d'_' -f1) local name=$(basename "$migration_file" .sql | cut -d'_' -f2-) log_debug "Validating migration: $version - $name" # Check file exists and is readable if [ ! -r "$migration_file" ]; then log_error "Migration file not readable: $migration_file" ((validation_errors++)) continue fi # Check file is not empty if [ ! -s "$migration_file" ]; then log_warn "Migration file is empty: $migration_file" fi # Check for rollback file local rollback_file="$ROLLBACK_DIR/rollback_${version}.sql" if [ ! -f "$rollback_file" ]; then log_warn "Rollback file missing: $rollback_file" fi # Basic SQL syntax check (if possible) if [ "$DB_TYPE" = "postgresql" ] && command -v psql >/dev/null 2>&1; then # Try to parse SQL without executing export PGPASSWORD="$DB_PASS" if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration_file" --echo-queries --dry-run >/dev/null 2>&1; then log_warn "Potential SQL syntax issues in: $migration_file" fi unset PGPASSWORD fi done if [ $validation_errors -eq 0 ]; then log_success "All migrations validated successfully" else log_error "Found $validation_errors validation errors" return 1 fi } # Show what would be migrated (dry run) show_migration_preview() { print_header "Migration Preview (Dry Run)" local pending_migrations=$(get_pending_migrations) if [ -z "$pending_migrations" ]; then log_success "No pending migrations" return fi log "The following migrations would be executed:" echo for migration_file in $pending_migrations; do local version=$(basename "$migration_file" .sql | cut -d'_' -f1) local name=$(basename "$migration_file" .sql | cut -d'_' -f2-) print_subheader "Migration $version: $name" # Show first few lines of migration head -20 "$migration_file" | grep -v "^--" | grep -v "^$" | head -10 if [ $(wc -l < "$migration_file") -gt 20 ]; then echo " ... (truncated, $(wc -l < "$migration_file") total lines)" fi echo done } # Parse command line arguments COMMAND="" ENVIRONMENT="dev" VERSION="" STEPS="" MIGRATION_NAME="" MIGRATION_TYPE="schema" TABLE_NAME="" TEMPLATE_NAME="" DRY_RUN="false" FORCE="false" DEBUG="false" QUIET="false" BATCH_SIZE=1000 TIMEOUT=300 while [[ $# -gt 0 ]]; do case $1 in --env) ENVIRONMENT="$2" shift 2 ;; --version) VERSION="$2" shift 2 ;; --steps) STEPS="$2" shift 2 ;; --name) MIGRATION_NAME="$2" shift 2 ;; --type) MIGRATION_TYPE="$2" shift 2 ;; --table) TABLE_NAME="$2" shift 2 ;; --template) TEMPLATE_NAME="$2" shift 2 ;; --dry-run) DRY_RUN="true" shift ;; --force) FORCE="true" shift ;; --debug) DEBUG="true" shift ;; --quiet) QUIET="true" shift ;; --batch-size) BATCH_SIZE="$2" shift 2 ;; --timeout) TIMEOUT="$2" shift 2 ;; -h|--help) print_usage exit 0 ;; *) if [ -z "$COMMAND" ]; then COMMAND="$1" else log_error "Unknown option: $1" print_usage exit 1 fi shift ;; esac done # Set environment variable export ENVIRONMENT="$ENVIRONMENT" # Validate command if [ -z "$COMMAND" ]; then print_usage exit 1 fi # Check if we're in the right directory if [ ! -f "Cargo.toml" ]; then log_error "Please run this script from the project root directory" exit 1 fi # Load environment and parse database URL load_env parse_database_url # Initialize migration system init_migration_system # Execute command case "$COMMAND" in "status") show_migration_status ;; "pending") list_pending_migrations ;; "applied") list_applied_migrations ;; "migrate") run_migrations "$VERSION" ;; "rollback") rollback_migrations "${STEPS:-1}" ;; "create") create_migration "$MIGRATION_NAME" "$MIGRATION_TYPE" "$TEMPLATE_NAME" ;; "generate") log_warn "Schema diff generation not yet implemented" ;; "validate") validate_migrations ;; "dry-run") show_migration_preview ;; "force") log_warn "Force migration state not yet implemented" ;; "repair") log_warn "Migration table repair not yet implemented" ;; "baseline") log_warn "Migration baseline not yet implemented" ;; "history") list_applied_migrations ;; "schema-dump") log_warn "Schema dump not yet implemented" ;; "data-migrate") log_warn "Data migration not yet implemented" ;; "template") log_warn "Migration template management not yet implemented" ;; *) log_error "Unknown command: $COMMAND" print_usage exit 1 ;; esac