website-htmx-rustelo-code/scripts/content/review/validate-content-consistency.nu

#!/usr/bin/env nu

# Content Consistency Validation Script
# Nushell version of validate-content-consistency.sh
# Validates content consistency across languages and content types

def main [...args] {
    let config = {
        content_dir: ($env.SITE_CONTENT_PATH? | default "site/content"),
        languages: ["en", "es"]
    }
    
    print $"(ansi blue)🔍 Content Consistency Validation(ansi reset)"
    
    if ($args | length) > 0 and ($args | first) in ["-h", "--help", "help"] {
        show_consistency_usage
        exit 0
    }
    
    # Get content types dynamically
    let content_types = get_content_types $config.content_dir
    
    mut total_errors = 0
    
    $total_errors = $total_errors + (validate_language_parity $config $content_types)
    $total_errors = $total_errors + (validate_metadata_consistency $config $content_types)
    $total_errors = $total_errors + (validate_translation_completeness $config $content_types)
    
    show_consistency_summary $config $content_types $total_errors
    
    if $total_errors > 0 {
        exit 1
    }
}

# Show usage information
def show_consistency_usage [] {
    print "Usage: nu validate-content-consistency.nu [OPTIONS]"
    print ""
    print "This script validates content consistency across:"
    print "  • Language parity (same content IDs in all languages)"
    print "  • Metadata consistency (similar structure across languages)"
    print "  • Translation completeness (all content translated)"
    print ""
    print "Options:"
    print "  -h, --help      Show this help message"
    print ""
    print "Environment Variables:"
    print "  SITE_CONTENT_PATH     Content root directory [default: site/content]"
}

# Get content types from directory structure
def get_content_types [content_dir: string] {
    if not ($content_dir | path exists) {
        return []
    }
    
    ls $content_dir 
    | where type == dir 
    | where name !~ "(locales|themes|menus|footer|tmp|routes)" 
    | get name 
    | path basename
}

# Validate language parity - ensure same content exists across languages
def validate_language_parity [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating language parity...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        print $"(ansi yellow)  Checking ($content_type) language parity...(ansi reset)"
        
        # Collect content IDs for each language
        mut language_ids = {}
        
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            
            if ($content_dir | path exists) {
                let md_files = (ls $"($content_dir)/**/*.md" | where type == file)
                let ids = ($md_files | each { |file| $file.name | path basename | path parse | get stem } | sort)
                $language_ids = ($language_ids | upsert $language $ids)
                print $"(ansi blue)    ($language): ($ids | length) files(ansi reset)"
            } else {
                print $"(ansi yellow)    ($language): directory not found(ansi reset)"
                $language_ids = ($language_ids | upsert $language [])
            }
        }
        
        # Compare languages pairwise
        let languages_with_content = ($language_ids | columns)
        if ($languages_with_content | length) < 2 {
            print $"(ansi yellow)    ⚠️  Only one language found for ($content_type)(ansi reset)"
            continue
        }
        
        for i in 0..(($languages_with_content | length) - 1) {
            let lang1 = ($languages_with_content | get $i)
            let ids1 = ($language_ids | get $lang1)
            
            for j in (($i + 1)..($languages_with_content | length)) {
                let lang2 = ($languages_with_content | get $j)
                let ids2 = ($language_ids | get $lang2)
                
                # Find missing content
                let missing_in_lang2 = ($ids1 | where {|id| $id not-in $ids2})
                let missing_in_lang1 = ($ids2 | where {|id| $id not-in $ids1})
                
                if ($missing_in_lang2 | length) > 0 {
                    print $"(ansi red)    ❌ Missing in ($lang2): ($missing_in_lang2 | str join ', ')(ansi reset)"
                    $errors += ($missing_in_lang2 | length)
                }
                
                if ($missing_in_lang1 | length) > 0 {
                    print $"(ansi red)    ❌ Missing in ($lang1): ($missing_in_lang1 | str join ', ')(ansi reset)"
                    $errors += ($missing_in_lang1 | length)
                }
                
                if ($missing_in_lang1 | length) == 0 and ($missing_in_lang2 | length) == 0 {
                    print $"(ansi green)    ✅ Perfect parity between ($lang1) and ($lang2)(ansi reset)"
                }
            }
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ Language parity validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ Language parity validation failed with ($errors) missing translations(ansi reset)"
    }
    
    $errors
}

# Validate metadata consistency across languages
def validate_metadata_consistency [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating metadata consistency...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        print $"(ansi yellow)  Checking ($content_type) metadata consistency...(ansi reset)"
        
        # Get common content IDs across all languages
        mut common_ids = []
        mut first_lang = true
        
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            
            if ($content_dir | path exists) {
                let md_files = (ls $"($content_dir)/**/*.md" | where type == file)
                let ids = ($md_files | each { |file| $file.name | path basename | path parse | get stem } | sort)
                
                if $first_lang {
                    $common_ids = $ids
                    $first_lang = false
                } else {
                    $common_ids = ($common_ids | where {|id| $id in $ids})
                }
            }
        }
        
        print $"(ansi blue)    Checking metadata for ($common_ids | length) common files...(ansi reset)"
        
        # Check metadata consistency for common files
        for content_id in $common_ids {
            mut file_metadata = {}
            
            # Collect metadata from all languages
            for language in $config.languages {
                let content_dir = $"($config.content_dir)/($content_type)/($language)"
                let md_file = $"($content_dir)/($content_id).md"
                
                if ($md_file | path exists) {
                    try {
                        let content_lines = (open $md_file | lines)
                        let frontmatter = (extract_frontmatter $content_lines)
                        
                        let metadata = {
                            category: (extract_frontmatter_field $frontmatter "category" ""),
                            tags: (extract_frontmatter_field $frontmatter "tags" ""),
                            difficulty: (extract_frontmatter_field $frontmatter "difficulty" ""),
                            published: (extract_frontmatter_field $frontmatter "published" "true")
                        }
                        
                        $file_metadata = ($file_metadata | upsert $language $metadata)
                    } catch {
                        print $"(ansi yellow)      ⚠️  Failed to parse ($content_id) in ($language)(ansi reset)"
                    }
                }
            }
            
            # Compare metadata across languages
            let languages_with_metadata = ($file_metadata | columns)
            if ($languages_with_metadata | length) >= 2 {
                let primary_lang = ($languages_with_metadata | first)
                let primary_metadata = ($file_metadata | get $primary_lang)
                
                for lang in ($languages_with_metadata | skip 1) {
                    let lang_metadata = ($file_metadata | get $lang)
                    
                    # Check important fields for consistency
                    if ($primary_metadata.category != $lang_metadata.category) and not ($primary_metadata.category | is-empty) and not ($lang_metadata.category | is-empty) {
                        print $"(ansi yellow)      ⚠️  ($content_id): Category differs between ($primary_lang) and ($lang)(ansi reset)"
                    }
                    
                    if ($primary_metadata.published != $lang_metadata.published) {
                        print $"(ansi red)      ❌ ($content_id): Published status differs between ($primary_lang) and ($lang)(ansi reset)"
                        $errors += 1
                    }
                    
                    if ($primary_metadata.difficulty != $lang_metadata.difficulty) and not ($primary_metadata.difficulty | is-empty) and not ($lang_metadata.difficulty | is-empty) {
                        print $"(ansi yellow)      ⚠️  ($content_id): Difficulty differs between ($primary_lang) and ($lang)(ansi reset)"
                    }
                }
            }
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ Metadata consistency validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ Metadata consistency validation failed with ($errors) inconsistencies(ansi reset)"
    }
    
    $errors
}

# Validate translation completeness
def validate_translation_completeness [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating translation completeness...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        print $"(ansi yellow)  Checking ($content_type) translation completeness...(ansi reset)"
        
        # Check if index files exist and have proper language field
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            let index_file = $"($content_dir)/index.json"
            
            if ($index_file | path exists) {
                try {
                    let index_content = (open $index_file | from json)
                    
                    # Check language field
                    if "language" in ($index_content | columns) {
                        let declared_lang = ($index_content | get language)
                        if $declared_lang != $language {
                            print $"(ansi red)      ❌ ($content_type) index.json declares wrong language: ($declared_lang) vs ($language)(ansi reset)"
                            $errors += 1
                        } else {
                            print $"(ansi green)      ✅ ($language): correct language declaration(ansi reset)"
                        }
                    } else {
                        print $"(ansi red)      ❌ ($content_type)/($language): index.json missing language field(ansi reset)"
                        $errors += 1
                    }
                    
                    # Check if content array exists and is not empty
                    let array_fields = ($index_content | columns | where $it =~ "_posts|prescriptions")
                    if ($array_fields | length) > 0 {
                        let array_name = ($array_fields | first)
                        let content_array = ($index_content | get $array_name)
                        
                        if ($content_array | length) == 0 {
                            print $"(ansi yellow)      ⚠️  ($language): index.json has empty content array(ansi reset)"
                        } else {
                            print $"(ansi green)      ✅ ($language): ($content_array | length) entries in index(ansi reset)"
                        }
                    } else {
                        print $"(ansi red)      ❌ ($content_type)/($language): index.json missing content array(ansi reset)"
                        $errors += 1
                    }
                    
                } catch {
                    print $"(ansi red)      ❌ ($content_type)/($language): failed to parse index.json(ansi reset)"
                    # Note: Error count handled at validation level
                }
            } else {
                print $"(ansi yellow)      ⚠️  ($content_type)/($language): missing index.json(ansi reset)"
            }
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ Translation completeness validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ Translation completeness validation failed with ($errors) issues(ansi reset)"
    }
    
    $errors
}

# Extract frontmatter from content lines
def extract_frontmatter [content_lines: list<string>] {
    if ($content_lines | length) == 0 or ($content_lines | first) != "---" {
        return []
    }
    
    mut frontmatter_lines = []
    mut in_frontmatter = false
    mut line_count = 0
    
    for line in $content_lines {
        $line_count = $line_count + 1
        if $line_count == 1 and $line == "---" {
            $in_frontmatter = true
            continue
        }
        if $in_frontmatter and $line == "---" {
            break
        }
        if $in_frontmatter {
            $frontmatter_lines = ($frontmatter_lines | append $line)
        }
    }
    
    $frontmatter_lines
}

# Extract field value from frontmatter
def extract_frontmatter_field [frontmatter: list<string>, key: string, default: string] {
    for line in $frontmatter {
        if $line =~ $"($key):\\s*(.+)" {
            let matches = ($line | parse --regex $"($key):\\s*\"?([^\"\\n]+)\"?")
            if ($matches | length) > 0 {
                return ($matches | first | get capture0 | str trim)
            }
        }
    }
    $default
}

# Show consistency validation summary
def show_consistency_summary [config, content_types: list<string>, total_errors: int] {
    print ""
    print $"(ansi blue)📊 Consistency Validation Summary(ansi reset)"
    print $"(ansi blue)Content root: ($config.content_dir)(ansi reset)"
    print $"(ansi blue)Languages validated: ($config.languages | str join ', ')(ansi reset)"
    print $"(ansi blue)Content types validated: ($content_types | str join ', ')(ansi reset)"
    print ""
    
    if $total_errors == 0 {
        print $"(ansi green)🎉 All consistency validations passed!(ansi reset)"
        print ""
        print $"(ansi green)Your content is properly synchronized across languages with:(ansi reset)"
        print $"(ansi green)  ✅ Perfect language parity(ansi reset)"
        print $"(ansi green)  ✅ Consistent metadata(ansi reset)"
        print $"(ansi green)  ✅ Complete translations(ansi reset)"
    } else {
        print $"(ansi red)❌ Consistency validation failed with ($total_errors) total issues(ansi reset)"
        print ""
        print $"(ansi yellow)💡 To fix consistency issues:(ansi reset)"
        print "  1. Create missing content files in all languages"
        print "  2. Ensure frontmatter metadata matches across languages"
        print "  3. Verify index.json files have correct language declarations"
        print "  4. Use the content generator to create missing translations:"
        print $"     nu scripts/content/generate-content.nu blog-post --title \"Title\" --lang es"
        print "  5. Run validation again after fixes"
    }
}