website-htmx-rustelo-code/scripts/content/review/validate-id-consistency.nu

#!/usr/bin/env nu

# ID Consistency Validation Script  
# Nushell version of validate-id-consistency.sh
# Validates ID consistency across languages for content files

def main [...args] {
    let config = {
        content_dir: ($env.SITE_CONTENT_PATH? | default "site/content"),
        languages: ["en", "es"]
    }
    
    print $"(ansi blue)🔍 ID Consistency Validation(ansi reset)"
    
    if ($args | length) > 0 and ($args | first) in ["-h", "--help", "help"] {
        show_id_consistency_usage
        exit 0
    }
    
    # Get content types dynamically  
    let content_types = get_content_types $config.content_dir
    
    mut total_errors = 0
    
    $total_errors = $total_errors + (validate_filename_consistency $config $content_types)
    $total_errors = $total_errors + (validate_id_field_consistency $config $content_types)
    $total_errors = $total_errors + (validate_index_id_consistency $config $content_types)
    
    show_id_consistency_summary $config $content_types $total_errors
    
    if $total_errors > 0 {
        exit 1
    }
}

# Show usage information
def show_id_consistency_usage [] {
    print "Usage: nu validate-id-consistency.nu [OPTIONS]"
    print ""
    print "This script validates ID consistency across languages:"
    print "  • Filename consistency (same files exist in all languages)"
    print "  • Frontmatter ID field consistency"
    print "  • Index.json ID consistency with actual files"
    print ""
    print "Options:"
    print "  -h, --help      Show this help message"
    print ""
    print "Environment Variables:"
    print "  SITE_CONTENT_PATH     Content root directory [default: site/content]"
}

# Get content types from directory structure
def get_content_types [content_dir: string] {
    if not ($content_dir | path exists) {
        return []
    }
    
    ls $content_dir 
    | where type == dir 
    | where name !~ "(locales|themes|menus|footer|tmp|routes)" 
    | get name 
    | path basename
}

# Validate filename consistency across languages
def validate_filename_consistency [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating filename consistency...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        print $"(ansi yellow)  Checking ($content_type) filename consistency...(ansi reset)"
        
        # Collect filenames for each language
        mut language_files = {}
        mut all_unique_filenames = []
        
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            
            if ($content_dir | path exists) {
                let md_files = (ls $"($content_dir)/**/*.md" | where type == file)
                let filenames = ($md_files | each { |file| $file.name | path basename | path parse | get stem } | sort)
                $language_files = ($language_files | upsert $language $filenames)
                $all_unique_filenames = ($all_unique_filenames | append $filenames | uniq | sort)
                
                print $"(ansi blue)    ($language): ($filenames | length) files(ansi reset)"
            } else {
                print $"(ansi yellow)    ($language): directory not found(ansi reset)"
                $language_files = ($language_files | upsert $language [])
            }
        }
        
        if ($all_unique_filenames | length) == 0 {
            print $"(ansi yellow)    No content files found for ($content_type)(ansi reset)"
            continue
        }
        
        # Check each unique filename exists in all languages
        let languages_with_content = ($language_files | columns | where {|lang| ($language_files | get $lang | length) > 0})
        
        if ($languages_with_content | length) < 2 {
            print $"(ansi yellow)    Only one language has content for ($content_type)(ansi reset)"
            continue
        }
        
        print $"(ansi blue)    Checking ($all_unique_filenames | length) unique filenames across ($languages_with_content | length) languages...(ansi reset)"
        
        for filename in $all_unique_filenames {
            mut missing_languages = []
            
            for language in $languages_with_content {
                let language_filenames = ($language_files | get $language)
                if $filename not-in $language_filenames {
                    $missing_languages = ($missing_languages | append $language)
                }
            }
            
            if ($missing_languages | length) > 0 {
                print $"(ansi red)      ❌ ($filename): missing in languages: ($missing_languages | str join ', ')(ansi reset)"
                $errors += ($missing_languages | length)
            } else {
                if ($all_unique_filenames | length) <= 10 {  # Only show details for small sets
                    print $"(ansi green)      ✅ ($filename): present in all languages(ansi reset)"
                }
            }
        }
        
        if $errors == 0 and ($all_unique_filenames | length) > 10 {
            print $"(ansi green)    ✅ All ($all_unique_filenames | length) files present in all languages(ansi reset)"
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ Filename consistency validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ Filename consistency validation failed with ($errors) missing files(ansi reset)"
    }
    
    $errors
}

# Validate ID field consistency in frontmatter
def validate_id_field_consistency [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating frontmatter ID field consistency...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            
            if not ($content_dir | path exists) {
                continue
            }
            
            print $"(ansi yellow)  Checking ($content_type) ID fields (($language))...(ansi reset)"
            
            let md_files = (ls $"($content_dir)/**/*.md" | where type == file)
            
            for file_info in $md_files {
                let md_file = $file_info.name
                let expected_id = ($md_file | path basename | path parse | get stem)
                
                try {
                    let content_lines = (open $md_file | lines)
                    let frontmatter = (extract_frontmatter $content_lines)
                    let actual_id = (extract_frontmatter_field $frontmatter "id" "")
                    
                    if not ($actual_id | is-empty) and $actual_id != $expected_id {
                        print $"(ansi red)      ❌ ($expected_id): frontmatter id '($actual_id)' doesn't match filename(ansi reset)"
                        $errors += 1
                    } else if not ($actual_id | is-empty) {
                        print $"(ansi green)      ✅ ($expected_id): ID field matches filename(ansi reset)"
                    }
                    # Note: Empty ID field is acceptable as filename will be used
                    
                } catch {
                    print $"(ansi red)      ❌ ($expected_id): failed to parse frontmatter(ansi reset)"
                    # Note: Error count handled at validation level
                }
            }
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ ID field consistency validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ ID field consistency validation failed with ($errors) inconsistencies(ansi reset)"
    }
    
    $errors
}

# Validate index.json ID consistency with actual files
def validate_index_id_consistency [config, content_types: list<string>] {
    print $"(ansi blue)🔧 Validating index.json ID consistency...(ansi reset)"
    
    mut errors = 0
    
    for content_type in $content_types {
        for language in $config.languages {
            let content_dir = $"($config.content_dir)/($content_type)/($language)"
            let index_file = $"($content_dir)/index.json"
            
            if not ($content_dir | path exists) or not ($index_file | path exists) {
                continue
            }
            
            print $"(ansi yellow)  Checking ($content_type) index consistency (($language))...(ansi reset)"
            
            try {
                # Read index.json
                let index_content = (open $index_file | from json)
                
                # Get content array
                let array_fields = ($index_content | columns | where $it =~ "_posts|prescriptions")
                if ($array_fields | length) == 0 {
                    print $"(ansi yellow)      ⚠️  No content array found in index.json(ansi reset)"
                    continue
                }
                
                let array_name = ($array_fields | first)
                let content_entries = ($index_content | get $array_name)
                
                # Get actual markdown files
                let md_files = (ls $"($content_dir)/**/*.md" | where type == file)
                let actual_filenames = ($md_files | each { |file| $file.name | path basename | path parse | get stem } | sort)
                
                # Extract IDs from index entries
                let index_ids = ($content_entries 
                    | where {|entry| "id" in ($entry | columns)} 
                    | get id 
                    | sort)
                
                print $"(ansi blue)      Index entries: ($content_entries | length), Actual files: ($actual_filenames | length)(ansi reset)"
                
                # Check index entries have corresponding files
                for index_id in $index_ids {
                    if $index_id not-in $actual_filenames {
                        print $"(ansi red)      ❌ Index entry '($index_id)' has no corresponding file ($index_id).md(ansi reset)"
                        $errors += 1
                    }
                }
                
                # Check files have corresponding index entries
                for filename in $actual_filenames {
                    if $filename not-in $index_ids {
                        print $"(ansi yellow)      ⚠️  File '($filename).md' not found in index.json(ansi reset)"
                        # This is a warning, not an error, as files might be unpublished
                    }
                }
                
                # Check for duplicate IDs in index
                let duplicate_ids = ($index_ids | group-by {|x| $x} | where {|group| ($group | get items | length) > 1} | get group | get 0)
                
                for dup_id in $duplicate_ids {
                    print $"(ansi red)      ❌ Duplicate ID in index: '($dup_id)'(ansi reset)"
                    $errors += 1
                }
                
                if $errors == 0 {
                    print $"(ansi green)      ✅ Index consistency validated: ($index_ids | length) entries match files(ansi reset)"
                }
                
            } catch {
                print $"(ansi red)      ❌ Failed to validate index.json(ansi reset)"
                # Note: Error count handled at validation level
            }
        }
    }
    
    if $errors == 0 {
        print $"(ansi green)✅ Index ID consistency validation passed(ansi reset)"
    } else {
        print $"(ansi red)❌ Index ID consistency validation failed with ($errors) issues(ansi reset)"
    }
    
    $errors
}

# Extract frontmatter from content lines
def extract_frontmatter [content_lines: list<string>] {
    if ($content_lines | length) == 0 or ($content_lines | first) != "---" {
        return []
    }
    
    mut frontmatter_lines = []
    mut in_frontmatter = false
    mut line_count = 0
    
    for line in $content_lines {
        $line_count = $line_count + 1
        if $line_count == 1 and $line == "---" {
            $in_frontmatter = true
            continue
        }
        if $in_frontmatter and $line == "---" {
            break
        }
        if $in_frontmatter {
            $frontmatter_lines = ($frontmatter_lines | append $line)
        }
    }
    
    $frontmatter_lines
}

# Extract field value from frontmatter
def extract_frontmatter_field [frontmatter: list<string>, key: string, default: string] {
    for line in $frontmatter {
        if $line =~ $"($key):\\s*(.+)" {
            let matches = ($line | parse --regex $"($key):\\s*\"?([^\"\\n]+)\"?")
            if ($matches | length) > 0 {
                return ($matches | first | get capture0 | str trim)
            }
        }
    }
    $default
}

# Show ID consistency validation summary
def show_id_consistency_summary [config, content_types: list<string>, total_errors: int] {
    print ""
    print $"(ansi blue)📊 ID Consistency Validation Summary(ansi reset)"
    print $"(ansi blue)Content root: ($config.content_dir)(ansi reset)"
    print $"(ansi blue)Languages: ($config.languages | str join ', ')(ansi reset)"
    print $"(ansi blue)Content types: ($content_types | str join ', ')(ansi reset)"
    print ""
    
    if $total_errors == 0 {
        print $"(ansi green)🎉 All ID consistency validations passed!(ansi reset)"
        print ""
        print $"(ansi green)Your content has perfect ID consistency with:(ansi reset)"
        print $"(ansi green)  ✅ Matching filenames across all languages(ansi reset)"
        print $"(ansi green)  ✅ Consistent frontmatter ID fields(ansi reset)"
        print $"(ansi green)  ✅ Accurate index.json entries(ansi reset)"
    } else {
        print $"(ansi red)❌ ID consistency validation failed with ($total_errors) total issues(ansi reset)"
        print ""
        print $"(ansi yellow)💡 To fix ID consistency issues:(ansi reset)"
        print "  1. Create missing content files in all languages using the same filename"
        print "  2. Ensure frontmatter 'id' fields match the filename (or remove the id field)"
        print "  3. Update index.json files to include all published content"
        print "  4. Remove duplicate entries from index.json files"
        print "  5. Use the content manager to regenerate indices:"
        print $"     nu scripts/content/content-manager.nu generate-indices"
        print "  6. Run validation again after fixes"
    }
}