Rustelo/book/developers/features/migrations.md

Database Migrations

This guide covers database migrations in RUSTELO, including the database-agnostic migration system and how to migrate from legacy PostgreSQL-only code to the new multi-database architecture.

Overview

RUSTELO has been upgraded to use a database-agnostic architecture that supports both PostgreSQL and SQLite. This guide covers:

• Database migration patterns and best practices
• Migrating from PostgreSQL-only to database-agnostic code
• Writing database-specific migration files
• Handling data type differences between databases
• Testing migration strategies

Migration Architecture

Before (Legacy PostgreSQL-Only)

// Direct PostgreSQL dependency
use sqlx::PgPool;

// Repository tied to PostgreSQL
pub struct AuthRepository {
    pool: PgPool,
}

impl AuthRepository {
    pub fn new(pool: PgPool) -> Self {
        Self { pool }
    }

    pub async fn find_user(&self, id: Uuid) -> Result<Option<User>> {
        sqlx::query_as!(User, "SELECT * FROM users WHERE id = $1", id)
            .fetch_optional(&self.pool)
            .await
    }
}

After (Database-Agnostic)

// Database-agnostic abstractions
use crate::database::{DatabaseConnection, DatabaseType};

// Repository works with any database
pub struct AuthRepository {
    database: DatabaseConnection,
}

impl AuthRepository {
    pub fn new(database: DatabaseConnection) -> Self {
        Self { database }
    }

    pub async fn find_user(&self, id: Uuid) -> Result<Option<User>> {
        match self.database.database_type() {
            DatabaseType::PostgreSQL => self.find_user_postgres(id).await,
            DatabaseType::SQLite => self.find_user_sqlite(id).await,
        }
    }

    async fn find_user_postgres(&self, id: Uuid) -> Result<Option<User>> {
        let row = self.database
            .fetch_optional(
                "SELECT * FROM users WHERE id = $1",
                &[id.into()],
            )
            .await?;

        if let Some(row) = row {
            Ok(Some(User {
                id: row.get_uuid("id")?,
                email: row.get_string("email")?,
                username: row.get_string("username")?,
                created_at: row.get_datetime("created_at")?,
            }))
        } else {
            Ok(None)
        }
    }

    async fn find_user_sqlite(&self, id: Uuid) -> Result<Option<User>> {
        let row = self.database
            .fetch_optional(
                "SELECT * FROM users WHERE id = ?",
                &[id.to_string().into()],
            )
            .await?;

        if let Some(row) = row {
            Ok(Some(User {
                id: row.get_uuid("id")?,
                email: row.get_string("email")?,
                username: row.get_string("username")?,
                created_at: row.get_datetime("created_at")?,
            }))
        } else {
            Ok(None)
        }
    }
}

Migration Steps

1. Update Repository Constructors

Before:

let auth_repository = Arc::new(AuthRepository::new(pool.clone()));

After:

// For the new database-agnostic repositories
let auth_repository = Arc::new(database::auth::AuthRepository::new(database.clone()));

// Or create from pool
let auth_repository = Arc::new(database::auth::AuthRepository::from_pool(&database_pool));

2. Replace Direct SQL Queries

Before:

pub async fn create_user(&self, user: &User) -> Result<()> {
    sqlx::query!(
        "INSERT INTO users (id, email, username) VALUES ($1, $2, $3)",
        user.id,
        user.email,
        user.username
    )
    .execute(&self.pool)
    .await?;
    Ok(())
}

After:

pub async fn create_user(&self, user: &User) -> Result<()> {
    match self.database.database_type() {
        DatabaseType::PostgreSQL => self.create_user_postgres(user).await,
        DatabaseType::SQLite => self.create_user_sqlite(user).await,
    }
}

async fn create_user_postgres(&self, user: &User) -> Result<()> {
    self.database
        .execute(
            "INSERT INTO users (id, email, username) VALUES ($1, $2, $3)",
            &[
                user.id.into(),
                user.email.clone().into(),
                user.username.clone().into(),
            ],
        )
        .await?;
    Ok(())
}

async fn create_user_sqlite(&self, user: &User) -> Result<()> {
    self.database
        .execute(
            "INSERT INTO users (id, email, username) VALUES (?, ?, ?)",
            &[
                user.id.to_string().into(),
                user.email.clone().into(),
                user.username.clone().into(),
            ],
        )
        .await?;
    Ok(())
}

3. Update Row Processing

Before:

let row = sqlx::query_as::<_, UserRow>("SELECT * FROM users WHERE id = $1")
    .bind(id)
    .fetch_optional(&self.pool)
    .await?;

After:

let row = self.database
    .fetch_optional(
        "SELECT * FROM users WHERE id = $1", // PostgreSQL
        &[id.into()],
    )
    .await?;

if let Some(row) = row {
    let user = User {
        id: row.get_uuid("id")?,
        email: row.get_string("email")?,
        username: row.get_string("username")?,
        created_at: row.get_datetime("created_at")?,
    };
}

4. Database Initialization

Before:

let pool = PgPool::connect(&database_url).await?;
let auth_repository = AuthRepository::new(pool);

After:

let database_config = DatabaseConfig {
    url: database_url.to_string(),
    max_connections: 10,
    min_connections: 1,
    connect_timeout: Duration::from_secs(30),
    idle_timeout: Duration::from_secs(600),
    max_lifetime: Duration::from_secs(3600),
};

let database_pool = DatabasePool::new(&database_config).await?;
let database = Database::new(database_pool.clone());
let auth_repository = database::auth::AuthRepository::new(database.create_connection());

Database-Specific Considerations

PostgreSQL vs SQLite Differences

Feature	PostgreSQL	SQLite
Parameter Binding	$1, $2, $3	?, ?, ?
UUID Storage	Native UUID type	TEXT (string)
Timestamps	TIMESTAMP WITH TIME ZONE	TEXT (ISO 8601)
JSON	JSONB	TEXT (JSON string)
Arrays	Native arrays	TEXT (JSON array)
Boolean	BOOLEAN	INTEGER (0/1)

Handling Data Type Differences

// UUID handling
match self.database.database_type() {
    DatabaseType::PostgreSQL => {
        // PostgreSQL stores UUIDs natively
        params.push(user_id.into());
    }
    DatabaseType::SQLite => {
        // SQLite stores UUIDs as strings
        params.push(user_id.to_string().into());
    }
}

// Timestamp handling
match self.database.database_type() {
    DatabaseType::PostgreSQL => {
        // PostgreSQL stores timestamps natively
        params.push(created_at.into());
    }
    DatabaseType::SQLite => {
        // SQLite stores timestamps as ISO strings
        params.push(created_at.to_rfc3339().into());
    }
}

// JSON handling
match self.database.database_type() {
    DatabaseType::PostgreSQL => {
        // PostgreSQL supports native JSONB
        params.push(serde_json::to_value(&data)?.into());
    }
    DatabaseType::SQLite => {
        // SQLite stores JSON as text
        params.push(serde_json::to_string(&data)?.into());
    }
}

Database Migration Files

Migration File Structure

migrations/
├── 001_create_users/
│   ├── up_postgres.sql
│   ├── down_postgres.sql
│   ├── up_sqlite.sql
│   └── down_sqlite.sql
├── 002_add_2fa_support/
│   ├── up_postgres.sql
│   ├── down_postgres.sql
│   ├── up_sqlite.sql
│   └── down_sqlite.sql
└── 003_create_posts/
    ├── up_postgres.sql
    ├── down_postgres.sql
    ├── up_sqlite.sql
    └── down_sqlite.sql

Example Migration Files

PostgreSQL Migration (001_create_users/up_postgres.sql):

CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

CREATE TABLE users (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    email VARCHAR(255) UNIQUE NOT NULL,
    username VARCHAR(100) UNIQUE NOT NULL,
    password_hash VARCHAR(255) NOT NULL,
    email_verified BOOLEAN DEFAULT FALSE,
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);

CREATE INDEX idx_users_email ON users(email);
CREATE INDEX idx_users_username ON users(username);

SQLite Migration (001_create_users/up_sqlite.sql):

CREATE TABLE users (
    id TEXT PRIMARY KEY,
    email TEXT UNIQUE NOT NULL,
    username TEXT UNIQUE NOT NULL,
    password_hash TEXT NOT NULL,
    email_verified INTEGER DEFAULT 0,
    created_at TEXT DEFAULT (datetime('now')),
    updated_at TEXT DEFAULT (datetime('now'))
);

CREATE INDEX idx_users_email ON users(email);
CREATE INDEX idx_users_username ON users(username);

Running Migrations

# Run migrations for PostgreSQL
cargo run --bin migrate -- --database-url "postgresql://user:pass@localhost/db"

# Run migrations for SQLite
cargo run --bin migrate -- --database-url "sqlite//:database.db"

# Roll back migrations
cargo run --bin migrate -- --database-url "sqlite//:database.db" --rollback

# Check migration status
cargo run --bin migrate -- --database-url "sqlite//:database.db" --status

Migration Patterns

Pattern 1: Simple Repository Migration

// Old
pub struct MyRepository {
    pool: PgPool,
}

// New
pub struct MyRepository {
    database: DatabaseConnection,
}

impl MyRepository {
    pub fn new(database: DatabaseConnection) -> Self {
        Self { database }
    }

    // Add convenience constructor
    pub fn from_pool(pool: &DatabasePool) -> Self {
        let connection = DatabaseConnection::from_pool(pool);
        Self::new(connection)
    }
}

Pattern 2: Complex Query Migration

// Old
pub async fn complex_query(&self) -> Result<Vec<Item>> {
    sqlx::query_as!(
        Item,
        r#"
        SELECT i.*, u.username
        FROM items i
        JOIN users u ON i.user_id = u.id
        WHERE i.created_at > $1
        ORDER BY i.created_at DESC
        "#,
        since
    )
    .fetch_all(&self.pool)
    .await
}

// New
pub async fn complex_query(&self, since: DateTime<Utc>) -> Result<Vec<Item>> {
    match self.database.database_type() {
        DatabaseType::PostgreSQL => self.complex_query_postgres(since).await,
        DatabaseType::SQLite => self.complex_query_sqlite(since).await,
    }
}

async fn complex_query_postgres(&self, since: DateTime<Utc>) -> Result<Vec<Item>> {
    let rows = self.database
        .fetch_all(
            r#"
            SELECT i.id, i.name, i.user_id, i.created_at, u.username
            FROM items i
            JOIN users u ON i.user_id = u.id
            WHERE i.created_at > $1
            ORDER BY i.created_at DESC
            "#,
            &[since.into()],
        )
        .await?;

    self.rows_to_items(rows)
}

async fn complex_query_sqlite(&self, since: DateTime<Utc>) -> Result<Vec<Item>> {
    let rows = self.database
        .fetch_all(
            r#"
            SELECT i.id, i.name, i.user_id, i.created_at, u.username
            FROM items i
            JOIN users u ON i.user_id = u.id
            WHERE i.created_at > ?
            ORDER BY i.created_at DESC
            "#,
            &[since.to_rfc3339().into()],
        )
        .await?;

    self.rows_to_items(rows)
}

fn rows_to_items(&self, rows: Vec<DatabaseRow>) -> Result<Vec<Item>> {
    let mut items = Vec::new();
    for row in rows {
        items.push(Item {
            id: row.get_uuid("id")?,
            name: row.get_string("name")?,
            user_id: row.get_uuid("user_id")?,
            username: row.get_string("username")?,
            created_at: row.get_datetime("created_at")?,
        });
    }
    Ok(items)
}

Pattern 3: Transaction Handling

pub async fn create_user_with_profile(&self, user: &User, profile: &Profile) -> Result<()> {
    match self.database.database_type() {
        DatabaseType::PostgreSQL => {
            self.create_user_with_profile_postgres(user, profile).await
        }
        DatabaseType::SQLite => {
            self.create_user_with_profile_sqlite(user, profile).await
        }
    }
}

async fn create_user_with_profile_postgres(&self, user: &User, profile: &Profile) -> Result<()> {
    let mut tx = self.database.begin().await?;

    // Create user
    tx.execute(
        "INSERT INTO users (id, email, username) VALUES ($1, $2, $3)",
        &[user.id.into(), user.email.clone().into(), user.username.clone().into()],
    ).await?;

    // Create profile
    tx.execute(
        "INSERT INTO profiles (user_id, first_name, last_name) VALUES ($1, $2, $3)",
        &[profile.user_id.into(), profile.first_name.clone().into(), profile.last_name.clone().into()],
    ).await?;

    tx.commit().await?;
    Ok(())
}

async fn create_user_with_profile_sqlite(&self, user: &User, profile: &Profile) -> Result<()> {
    let mut tx = self.database.begin().await?;

    // Create user
    tx.execute(
        "INSERT INTO users (id, email, username) VALUES (?, ?, ?)",
        &[user.id.to_string().into(), user.email.clone().into(), user.username.clone().into()],
    ).await?;

    // Create profile
    tx.execute(
        "INSERT INTO profiles (user_id, first_name, last_name) VALUES (?, ?, ?)",
        &[profile.user_id.to_string().into(), profile.first_name.clone().into(), profile.last_name.clone().into()],
    ).await?;

    tx.commit().await?;
    Ok(())
}

Migration Checklist

Code Migration

• Replace PgPool with DatabaseConnection
• Update repository constructors
• Split database operations into database-specific methods
• Handle parameter binding differences ($1 vs ?)
• Handle data type differences (UUID, timestamps, etc.)
• Update imports to use new database modules
• Add convenience constructors (from_pool)
• Update error handling

Database Migration Files

• Create separate migration files for PostgreSQL (*_postgres.sql)
• Create separate migration files for SQLite (*_sqlite.sql)
• Update migration runner configuration
• Test migrations on both database types
• Add rollback migrations
• Document migration dependencies

Configuration

• Update environment variables
• Test with different database URLs
• Update deployment scripts
• Update documentation
• Configure connection pooling
• Set appropriate timeouts

Testing

• Update unit tests to use new abstractions
• Add integration tests for both database types
• Test migration path from old to new architecture
• Performance testing on both databases
• Test rollback scenarios
• Add database-specific test fixtures

Troubleshooting

Common Issues and Solutions

1. UUID Conversion Errors

Problem: SQLite doesn't support UUID natively Solution: Convert UUIDs to strings for SQLite

match self.database.database_type() {
    DatabaseType::PostgreSQL => params.push(uuid.into()),
    DatabaseType::SQLite => params.push(uuid.to_string().into()),
}

2. Parameter Binding Mismatch

Problem: Using PostgreSQL syntax ($1) with SQLite Solution: Use database-specific query strings

let query = match self.database.database_type() {
    DatabaseType::PostgreSQL => "SELECT * FROM users WHERE id = $1",
    DatabaseType::SQLite => "SELECT * FROM users WHERE id = ?",
};

3. Migration File Not Found

Problem: Missing database-specific migration files Solution: Ensure both *_postgres.sql and *_sqlite.sql files exist

4. Type Conversion Errors

Problem: Data type mismatches between databases Solution: Use the DatabaseRow trait methods consistently

// Always use the trait methods
let user_id = row.get_uuid("user_id")?;  // Handles conversion automatically
let created_at = row.get_datetime("created_at")?;  // Handles conversion automatically

5. Connection Pool Issues

Problem: Connection pool exhaustion or configuration issues Solution: Properly configure connection pool settings

let database_config = DatabaseConfig {
    url: database_url.to_string(),
    max_connections: 10,
    min_connections: 1,
    connect_timeout: Duration::from_secs(30),
    idle_timeout: Duration::from_secs(600),
    max_lifetime: Duration::from_secs(3600),
};

Best Practices

1. Database-Agnostic Design

• Write repositories that work with both databases
• Use the database abstraction layer consistently
• Avoid database-specific features when possible
• Test with both databases regularly

2. Migration Strategy

• Migrate one module at a time
• Test thoroughly with both database types
• Keep the old code until migration is complete
• Document the migration process

3. Performance Considerations

• PostgreSQL: Better for complex queries and large datasets
• SQLite: Better for simple queries and small datasets
• Use appropriate database features for your use case
• Monitor query performance on both databases

4. Testing Strategy

• Test all code paths with both databases
• Use SQLite for fast unit tests
• Use PostgreSQL for integration tests
• Test migration rollbacks
• Use database-specific test fixtures

5. Error Handling

• Handle database-specific errors appropriately
• Use consistent error types across databases
• Log database operations for debugging
• Provide meaningful error messages

Example: Complete Repository Migration

Here's a complete example of migrating a repository from PostgreSQL-only to database-agnostic:

Before (PostgreSQL-Only)

use sqlx::PgPool;
use uuid::Uuid;
use chrono::{DateTime, Utc};

pub struct PostRepository {
    pool: PgPool,
}

impl PostRepository {
    pub fn new(pool: PgPool) -> Self {
        Self { pool }
    }

    pub async fn create_post(&self, post: &Post) -> Result<()> {
        sqlx::query!(
            "INSERT INTO posts (id, title, content, author_id, published) VALUES ($1, $2, $3, $4, $5)",
            post.id,
            post.title,
            post.content,
            post.author_id,
            post.published
        )
        .execute(&self.pool)
        .await?;
        Ok(())
    }

    pub async fn find_post(&self, id: Uuid) -> Result<Option<Post>> {
        let row = sqlx::query_as!(
            Post,
            "SELECT id, title, content, author_id, published, created_at FROM posts WHERE id = $1",
            id
        )
        .fetch_optional(&self.pool)
        .await?;
        Ok(row)
    }

    pub async fn find_posts_by_author(&self, author_id: Uuid) -> Result<Vec<Post>> {
        let rows = sqlx::query_as!(
            Post,
            "SELECT id, title, content, author_id, published, created_at FROM posts WHERE author_id = $1 ORDER BY created_at DESC",
            author_id
        )
        .fetch_all(&self.pool)
        .await?;
        Ok(rows)
    }
}

After (Database-Agnostic)

use crate::database::{DatabaseConnection, DatabaseType, DatabaseRow};
use uuid::Uuid;
use chrono::{DateTime, Utc};

pub struct PostRepository {
    database: DatabaseConnection,
}

impl PostRepository {
    pub fn new(database: DatabaseConnection) -> Self {
        Self { database }
    }

    pub fn from_pool(pool: &crate::database::DatabasePool) -> Self {
        let connection = DatabaseConnection::from_pool(pool);
        Self::new(connection)
    }

    pub async fn create_post(&self, post: &Post) -> Result<()> {
        match self.database.database_type() {
            DatabaseType::PostgreSQL => self.create_post_postgres(post).await,
            DatabaseType::SQLite => self.create_post_sqlite(post).await,
        }
    }

    async fn create_post_postgres(&self, post: &Post) -> Result<()> {
        self.database
            .execute(
                "INSERT INTO posts (id, title, content, author_id, published) VALUES ($1, $2, $3, $4, $5)",
                &[
                    post.id.into(),
                    post.title.clone().into(),
                    post.content.clone().into(),
                    post.author_id.into(),
                    post.published.into(),
                ],
            )
            .await?;
        Ok(())
    }

    async fn create_post_sqlite(&self, post: &Post) -> Result<()> {
        self.database
            .execute(
                "INSERT INTO posts (id, title, content, author_id, published) VALUES (?, ?, ?, ?, ?)",
                &[
                    post.id.to_string().into(),
                    post.title.clone().into(),
                    post.content.clone().into(),
                    post.author_id.to_string().into(),
                    (post.published as i32).into(),
                ],
            )
            .await?;
        Ok(())
    }

    pub async fn find_post(&self, id: Uuid) -> Result<Option<Post>> {
        match self.database.database_type() {
            DatabaseType::PostgreSQL => self.find_post_postgres(id).await,
            DatabaseType::SQLite => self.find_post_sqlite(id).await,
        }
    }

    async fn find_post_postgres(&self, id: Uuid) -> Result<Option<Post>> {
        let row = self
            .database
            .fetch_optional(
                "SELECT id, title, content, author_id, published, created_at FROM posts WHERE id = $1",
                &[id.into()],
            )
            .await?;

        self.row_to_post(row)
    }

    async fn find_post_sqlite(&self, id: Uuid) -> Result<Option<Post>> {
        let row = self
            .database
            .fetch_optional(
                "SELECT id, title, content, author_id, published, created_at FROM posts WHERE id = ?",
                &[id.to_string().into()],
            )
            .await?;

        self.row_to_post(row)
    }

    pub async fn find_posts_by_author(&self, author_id: Uuid) -> Result<Vec<Post>> {
        match self.database.database_type() {
            DatabaseType::PostgreSQL => self.find_posts_by_author_postgres(author_id).await,
            DatabaseType::SQLite => self.find_posts_by_author_sqlite(author_id).await,
        }
    }

    async fn find_posts_by_author_postgres(&self, author_id: Uuid) -> Result<Vec<Post>> {
        let rows = self
            .database
            .fetch_all(
                "SELECT id, title, content, author_id, published, created_at FROM posts WHERE author_id = $1 ORDER BY created_at DESC",
                &[author_id.into()],
            )
            .await?;

        self.rows_to_posts(rows)
    }

    async fn find_posts_by_author_sqlite(&self, author_id: Uuid) -> Result<Vec<Post>> {
        let rows = self
            .database
            .fetch_all(
                "SELECT id, title, content, author_id, published, created_at FROM posts WHERE author_id = ? ORDER BY created_at DESC",
                &[author_id.to_string().into()],
            )
            .await?;

        self.rows_to_posts(rows)
    }

    fn row_to_post(&self, row: Option<DatabaseRow>) -> Result<Option<Post>> {
        if let Some(row) = row {
            Ok(Some(Post {
                id: row.get_uuid("id")?,
                title: row.get_string("title")?,
                content: row.get_string("content")?,
                author_id: row.get_uuid("author_id")?,
                published: row.get_bool("published")?,
                created_at: row.get_datetime("created_at")?,
            }))
        } else {
            Ok(None)
        }
    }

    fn rows_to_posts(&self, rows: Vec<DatabaseRow>) -> Result<Vec<Post>> {
        let mut posts = Vec::new();
        for row in rows {
            posts.push(Post {
                id: row.get_uuid("id")?,
                title: row.get_string("title")?,
                content: row.get_string("content")?,
                author_id: row.get_uuid("author_id")?,
                published: row.get_bool("published")?,
                created_at: row.get_datetime("created_at")?,
            });
        }
        Ok(posts)
    }
}

Next Steps

• Database Configuration
• Performance Optimization
• Testing Strategies
• Deployment Guide

This migration guide provides a comprehensive approach to transitioning from PostgreSQL-only to database-agnostic architecture while maintaining functionality and performance across both database systems.