website-htmx-rustelo-code/scripts/sh/build/build-localized-content.sh
2026-07-10 03:44:13 +01:00

704 lines
26 KiB
Bash
Executable file

#!/bin/bash
# Build script for localized content
# This script converts markdown files to HTML and deploys them to localized public directories
set -e
# Configuration - Use environment variable or fallback to default
CONTENT_DIR="${SITE_CONTENT_PATH:-site/content}"
PUBLIC_DIR="public"
# Supported languages
LANGUAGES=("en" "es")
# Function to read content types from content-kinds.toml
get_enabled_content_types() {
local content_kinds_file="${CONTENT_DIR}/content-kinds.toml"
local enabled_types=()
if [ ! -f "$content_kinds_file" ]; then
echo "⚠️ Warning: content-kinds.toml not found at $content_kinds_file, using default types" >&2
echo "blog recipes"
return
fi
# More robust TOML parsing - process line by line and track state
local in_content_kinds_section=false
local current_name=""
local current_directory=""
local current_enabled=""
while IFS= read -r line; do
# Skip comments and empty lines
[[ "$line" =~ ^[[:space:]]*# ]] && continue
[[ -z "${line// }" ]] && continue
# Check for new content_kinds section
if [[ "$line" =~ ^\[\[content_kinds\]\] ]]; then
# Process previous section if we have one
if [[ "$in_content_kinds_section" == true && "$current_enabled" == "true" && -n "$current_directory" ]]; then
if [ -d "${CONTENT_DIR}/${current_directory}" ]; then
enabled_types+=("$current_directory")
echo "✅ Content type '$current_name' -> directory '${current_directory}' exists" >&2
else
echo "⚠️ WARNING: Content type '$current_name' enabled in content-kinds.toml but directory '${CONTENT_DIR}/${current_directory}' does not exist - SKIPPING" >&2
fi
fi
# Reset for new section
in_content_kinds_section=true
current_name=""
current_directory=""
current_enabled=""
continue
fi
# If we're in a content_kinds section, parse the fields
if [[ "$in_content_kinds_section" == true ]]; then
# Check for another section starting (end current one)
if [[ "$line" =~ ^\[\[ ]]; then
# Process current section before moving to next
if [[ "$current_enabled" == "true" && -n "$current_directory" ]]; then
if [ -d "${CONTENT_DIR}/${current_directory}" ]; then
enabled_types+=("$current_directory")
echo "✅ Content type '$current_name' -> directory '${current_directory}' exists" >&2
else
echo "⚠️ WARNING: Content type '$current_name' enabled in content-kinds.toml but directory '${CONTENT_DIR}/${current_directory}' does not exist - SKIPPING" >&2
fi
fi
in_content_kinds_section=false
continue
fi
# Extract fields
if [[ "$line" =~ ^name[[:space:]]*=[[:space:]]*\"([^\"]+)\" ]]; then
current_name="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ^directory[[:space:]]*=[[:space:]]*\"([^\"]+)\" ]]; then
current_directory="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ^enabled[[:space:]]*=[[:space:]]*true ]]; then
current_enabled="true"
fi
fi
done < "$content_kinds_file"
# Don't forget to process the last section
if [[ "$in_content_kinds_section" == true && "$current_enabled" == "true" && -n "$current_directory" ]]; then
if [ -d "${CONTENT_DIR}/${current_directory}" ]; then
enabled_types+=("$current_directory")
echo "✅ Content type '$current_name' -> directory '${current_directory}' exists" >&2
else
echo "⚠️ WARNING: Content type '$current_name' enabled in content-kinds.toml but directory '${CONTENT_DIR}/${current_directory}' does not exist - SKIPPING" >&2
fi
fi
if [ ${#enabled_types[@]} -eq 0 ]; then
echo "⚠️ Warning: No enabled content types found, using defaults" >&2
echo "blog recipes"
else
echo "${enabled_types[@]}"
fi
}
# Get the script directory and project root
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
echo "🌍 Building localized markdown content to HTML..."
echo "📁 Using content root: $CONTENT_DIR"
# Get enabled content types from content-kinds.toml
CONTENT_TYPES=($(get_enabled_content_types))
echo "📋 Enabled content types: ${CONTENT_TYPES[*]}"
# Build the markdown converter with content-static feature
echo "🔧 Building markdown converter..."
cd "$PROJECT_ROOT"
cargo build --bin markdown_converter --features content-static --quiet
CONVERTER="./target/debug/markdown_converter"
# Check if converter was built successfully
if [ ! -f "$CONVERTER" ]; then
echo "❌ Error: Markdown converter not found. Build may have failed."
exit 1
fi
# Create public directories for all content types
for content_type in "${CONTENT_TYPES[@]}"; do
mkdir -p "${PUBLIC_DIR}/${content_type}"
echo "📁 Created directory: ${PUBLIC_DIR}/${content_type}"
done
# Function to extract frontmatter from markdown file
extract_frontmatter() {
local input_file="$1"
# Extract YAML frontmatter between --- delimiters
if head -1 "$input_file" | grep -q "^---$"; then
awk '/^---$/{flag++; next} flag==1 && /^---$/{flag++} flag==1{print}' "$input_file"
fi
}
# Function to extract tags/categories from frontmatter
extract_metadata() {
local input_file="$1"
local metadata_type="$2" # "tags" or "categories"
local frontmatter=$(extract_frontmatter "$input_file")
if [ -n "$frontmatter" ]; then
# Extract array values (both ["item1", "item2"] and [item1, item2] formats)
echo "$frontmatter" | grep -E "^${metadata_type}:" | sed 's/.*: *\[//' | sed 's/\]//' | sed 's/"//g' | sed "s/'//g" | tr ',' '\n' | sed 's/^ *//' | sed 's/ *$//' | grep -v '^$'
fi
}
# Function to extract single category from frontmatter
extract_single_category() {
local input_file="$1"
local frontmatter=$(extract_frontmatter "$input_file")
if [ -n "$frontmatter" ]; then
# Extract single category value
echo "$frontmatter" | grep -E "^category:" | sed 's/category: *"*//' | sed 's/"*$//'
fi
}
# Function to convert markdown to HTML
convert_markdown() {
local input_file="$1"
local output_file="$2"
local title="$3"
local lang="$4"
local content_type="$5"
echo " Converting: $(basename "$input_file") -> $(basename "$output_file")"
# Extract title from frontmatter if not provided
if [ -z "$title" ]; then
title=$(grep -m1 "^# " "$input_file" | sed 's/^# //' || echo "$(basename "$input_file" .md)")
fi
# Create output directory if it doesn't exist
mkdir -p "$(dirname "$output_file")"
# Convert markdown to HTML
$CONVERTER \
"$input_file" \
-o "$output_file" 2>/dev/null || {
echo " ⚠️ Warning: Failed to convert $input_file"
}
}
# Function to generate category index.json
generate_category_index() {
local content_type="$1"
local lang="$2"
local category="$3"
local category_dir="$4"
local category_target="$5"
local content_source="$6"
local index_file="$category_target/index.json"
echo " 🔍 Generating category index: $category"
# Get emoji from global meta.toml file
local global_meta="${CONTENT_DIR}/${content_type}/meta.toml"
local root_emoji="📝"
if [ -f "$global_meta" ]; then
# Extract emoji from [[categories]] section using new global structure
root_emoji=$(awk -v cat="$category" '
/^\[\[categories\]\]/ { in_category = 1; current_id = ""; current_emoji = ""; next }
/^\[\[/ && !/^\[\[categories\]\]/ { in_category = 0; next }
in_category && /^id = / {
gsub(/^id = "/, "", $0);
gsub(/"$/, "", $0);
current_id = $0;
next
}
in_category && /^emoji = / {
gsub(/^emoji = "/, "", $0);
gsub(/"$/, "", $0);
current_emoji = $0;
next
}
# Check if we found a matching category
in_category && current_id == cat && current_emoji != "" {
print current_emoji;
exit
}
# Start of new section resets current values
/^\[/ { in_category = 0 }
' "$global_meta" || echo "📝")
fi
# Read category metadata from category meta.toml
local category_title="$category"
local category_emoji="$root_emoji" # Use root emoji as default
local category_css_class="category-$category"
local category_description=""
if [ -f "$category_dir/meta.toml" ]; then
category_title=$(grep '^title = ' "$category_dir/meta.toml" 2>/dev/null | sed 's/title = "//' | sed 's/"//' || echo "$category")
# Emoji comes from global meta.toml only (no override from category meta.toml)
category_css_class=$(grep '^css_class = ' "$category_dir/meta.toml" 2>/dev/null | sed 's/css_class = "//' | sed 's/"//' || echo "category-$category")
category_description=$(grep '^description = ' "$category_dir/meta.toml" 2>/dev/null | sed 's/description = "//' | sed 's/"//' || echo "")
fi
# Start JSON structure
cat > "$index_file" << EOF
{
"generated_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
"content_type": "$content_type",
"language": "$lang",
"category": "$category",
"title": "$category_title",
"emoji": "$category_emoji",
"css_class": "$category_css_class",
"description": "$category_description",
"posts": [
EOF
local first_post=true
# Process each markdown file in the category
for md_file in "$category_dir"/*.md; do
if [ -f "$md_file" ]; then
local filename=$(basename "$md_file" .md)
local post_id="$filename"
# Extract frontmatter data
local frontmatter_data=""
if head -1 "$md_file" | grep -q "^---$"; then
frontmatter_data=$(awk '/^---$/{flag++; next} flag==1 && /^---$/{flag++} flag==1{print}' "$md_file")
fi
# Extract basic post info
local post_title=$(echo "$frontmatter_data" | grep '^title:' | sed 's/title:[[:space:]]*//' | tr -d '"' || echo "$filename")
local post_css_class=$(echo "$frontmatter_data" | grep '^css_class:' | sed 's/css_class:[[:space:]]*//' | tr -d '"' || echo "")
local published=$(echo "$frontmatter_data" | grep '^published:' | sed 's/published:[[:space:]]*//' | tr -d '"' || echo "true")
# Only include published posts
if [ "$published" = "true" ]; then
if [ "$first_post" = true ]; then
first_post=false
else
echo "," >> "$index_file"
fi
cat >> "$index_file" << EOF
{
"id": "$post_id",
"title": "$post_title",
"css_class": "$post_css_class",
"html_file": "$filename.html"
}
EOF
fi
fi
done
# Close JSON structure
cat >> "$index_file" << EOF
]
}
EOF
echo " ✅ Generated category index: $index_file"
}
# Function to get emoji from TOML file
get_emoji_from_toml() {
local content_type="$1" # "blog" or "prescriptions"
local lang="$2"
local item_name="$3"
local emoji_file="$CONTENT_DIR/$content_type/$lang/categories.toml"
if [ -f "$emoji_file" ]; then
# Simple grep approach to find the emoji for the given item
local emoji
if [ "$content_type" = "blog" ]; then
# Look for the tag in the categories file
emoji=$(grep "^$item_name *= *" "$emoji_file" 2>/dev/null | sed 's/.*= *"\([^"]*\)".*/\1/' || echo "🏷️")
if [ "$emoji" = "🏷️" ]; then
# Try with quotes around the key
emoji=$(grep "^\"$item_name\" *= *" "$emoji_file" 2>/dev/null | sed 's/.*= *"\([^"]*\)".*/\1/' || echo "🏷️")
fi
else
# Look for the category in the categories file
emoji=$(grep "^\"$item_name\" *= *" "$emoji_file" 2>/dev/null | sed 's/.*= *"\([^"]*\)".*/\1/' || echo "🏷️")
fi
if [ -n "$emoji" ] && [ "$emoji" != "🏷️" ]; then
echo "$emoji"
else
echo "🏷️" # Default fallback emoji
fi
else
echo "🏷️" # Default fallback emoji
fi
}
# Function to generate index.json with categories/tags
generate_filter_index() {
local content_type="$1" # "blog" or "prescriptions"
local lang="$2"
local source_dir="$3"
local target_dir="$4"
echo " - Generating filter index for $content_type ($lang)..."
# Create temporary files for metadata collection
local temp_file="/tmp/filter_metadata_$$"
local count_file="/tmp/filter_counts_$$"
local total_files=0
# Clear temp files
> "$temp_file"
> "$count_file"
# Process each markdown file
for md_file in "$source_dir"/*.md; do
if [ -f "$md_file" ]; then
total_files=$((total_files + 1))
if [ "$content_type" = "blog" ]; then
# Extract tags for blog posts
extract_metadata "$md_file" "tags" >> "$temp_file"
else
# Extract categories for other content types (avoid duplicates from single category + categories array)
local file_categories="/tmp/file_categories_$$"
> "$file_categories"
# Extract single category
local single_category=$(extract_single_category "$md_file")
if [ -n "$single_category" ]; then
echo "$single_category" >> "$file_categories"
fi
# Extract tags array
extract_metadata "$md_file" "tags" >> "$file_categories"
# Add unique categories from this file to temp_file
if [ -s "$file_categories" ]; then
sort "$file_categories" | uniq >> "$temp_file"
fi
rm -f "$file_categories"
fi
fi
done
# Count occurrences and sort
if [ -s "$temp_file" ]; then
sort "$temp_file" | uniq -c | sort -nr > "$count_file"
fi
# Generate the filter index JSON
local filter_index_file="$target_dir/filter-index.json"
if [ "$content_type" = "blog" ]; then
# Generate blog tags index
cat > "$filter_index_file" << EOF
{
"generated_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
"content_type": "blog",
"language": "$lang",
"total_posts": $total_files,
"tags": [
EOF
local first_tag=true
while IFS= read -r line; do
if [ -n "$line" ]; then
local count=$(echo "$line" | awk '{print $1}')
local tag=$(echo "$line" | awk '{$1=""; print $0}' | sed 's/^ *//')
if [ "$first_tag" = true ]; then
first_tag=false
else
echo "," >> "$filter_index_file"
fi
local emoji=$(get_emoji_from_toml "$content_type" "$lang" "$tag")
printf ' {"name": "%s", "count": %d, "emoji": "%s"}' "$tag" "$count" "$emoji" >> "$filter_index_file"
fi
done < "$count_file"
cat >> "$filter_index_file" << EOF
]
}
EOF
else
# Generate categories index for other content types
cat > "$filter_index_file" << EOF
{
"generated_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
"content_type": "$content_type",
"language": "$lang",
"total_${content_type}": $total_files,
"tags": [
EOF
local first_category=true
while IFS= read -r line; do
if [ -n "$line" ]; then
local count=$(echo "$line" | awk '{print $1}')
local category=$(echo "$line" | awk '{$1=""; print $0}' | sed 's/^ *//')
if [ "$first_category" = true ]; then
first_category=false
else
echo "," >> "$filter_index_file"
fi
local emoji=$(get_emoji_from_toml "$content_type" "$lang" "$category")
printf ' {"name": "%s", "count": %d, "emoji": "%s"}' "$category" "$count" "$emoji" >> "$filter_index_file"
fi
done < "$count_file"
cat >> "$filter_index_file" << EOF
]
}
EOF
fi
# Clean up temp files
rm -f "$temp_file" "$count_file"
echo " ✅ Generated filter index: $filter_index_file"
}
# Process content for each type and language
echo "📝 Processing content..."
for content_type in "${CONTENT_TYPES[@]}"; do
echo "🔄 Processing content type: $content_type"
for lang in "${LANGUAGES[@]}"; do
content_source="${CONTENT_DIR}/${content_type}/${lang}"
content_target="${PUBLIC_DIR}/${content_type}/${lang}"
if [ -d "$content_source" ]; then
echo " - Processing $content_type content for language: $lang"
mkdir -p "$content_target"
# Convert all markdown files (including category subdirectories)
# First, handle any top-level markdown files (backward compatibility)
for md_file in "$content_source"/*.md; do
if [ -f "$md_file" ]; then
filename=$(basename "$md_file" .md)
# Use filename as base for HTML file (Rust converter will normalize internally)
html_file="$content_target/$filename.html"
# Extract title from filename (convert dashes to spaces and capitalize)
title=$(echo "$filename" | sed 's/-/ /g' | sed 's/\b\w/\U&/g')
convert_markdown "$md_file" "$html_file" "$title" "$lang" "$content_type"
fi
done
# Process category directories
for category_dir in "$content_source"/*/; do
if [ -d "$category_dir" ]; then
category=$(basename "$category_dir")
category_target="$content_target/$category"
mkdir -p "$category_target"
echo " 📁 Processing category: $category"
# Process category meta.toml if it exists
if [ -f "$category_dir/meta.toml" ]; then
echo " Processing category meta.toml for $category..."
$CONVERTER "$category_dir/meta.toml" -o /dev/null
fi
# Convert all markdown files in the category
for md_file in "$category_dir"/*.md; do
if [ -f "$md_file" ]; then
filename=$(basename "$md_file" .md)
# Use filename as base for HTML file
html_file="$category_target/$filename.html"
# Extract title from filename
title=$(echo "$filename" | sed 's/-/ /g' | sed 's/\b\w/\U&/g')
convert_markdown "$md_file" "$html_file" "$title" "$lang" "$content_type"
fi
done
# Generate category index.json if there are posts
category_posts_count=$(find "$category_dir" -name "*.md" 2>/dev/null | wc -l)
if [ $category_posts_count -gt 0 ]; then
generate_category_index "$content_type" "$lang" "$category" "$category_dir" "$category_target" "$content_source"
fi
fi
done
# Copy index.json if it exists
if [ -f "$content_source/index.json" ]; then
cp "$content_source/index.json" "$content_target/"
echo "${content_type} index copied for $lang"
else
echo " ⚠️ Warning: No ${content_type} index.json found for $lang"
fi
# Process meta.toml to generate meta.json for API access
if [ -f "$content_source/meta.toml" ]; then
echo " Processing meta.toml for $content_type ($lang)..."
$CONVERTER "$content_source/meta.toml" -o /dev/null
else
echo " ⚠️ Warning: No ${content_type} meta.toml found for $lang"
fi
# Generate filter index (use content_type name for filter generation)
generate_filter_index "$content_type" "$lang" "$content_source" "$content_target"
# Count converted files
html_count=$(find "$content_target" -name "*.html" | wc -l)
echo " ✅ Converted $html_count ${content_type} items for $lang"
else
echo " ⚠️ Warning: No ${content_type} content directory found for $lang"
fi
done
done
# Create fallback symlinks for backward compatibility
echo "🔗 Creating backward compatibility links..."
# Create fallback index for each content type (default to English)
for content_type in "${CONTENT_TYPES[@]}"; do
if [ -f "${PUBLIC_DIR}/${content_type}/en/index.json" ]; then
ln -sf "en/index.json" "${PUBLIC_DIR}/${content_type}/index.json" 2>/dev/null || true
echo " ✅ Created fallback ${content_type} index"
fi
done
# Validate content structure
echo "🔍 Validating content structure..."
validate_json() {
local file="$1"
local type="$2"
local lang="$3"
if command -v jq >/dev/null 2>&1; then
if jq empty "$file" 2>/dev/null; then
echo " ✅ Valid JSON: $type ($lang)"
else
echo " ❌ Invalid JSON: $type ($lang)"
return 1
fi
else
echo " ⚠️ jq not available, skipping JSON validation"
fi
}
for lang in "${LANGUAGES[@]}"; do
# Validate index for each content type
for content_type in "${CONTENT_TYPES[@]}"; do
content_index="${PUBLIC_DIR}/${content_type}/${lang}/index.json"
if [ -f "$content_index" ]; then
validate_json "$content_index" "$content_type" "$lang"
fi
done
done
# Generate content statistics
echo "📊 Content Statistics:"
# Initialize totals for each content type (using eval for better bash compatibility)
for content_type in "${CONTENT_TYPES[@]}"; do
eval "total_${content_type//[^a-zA-Z0-9]/_}=0"
done
for lang in "${LANGUAGES[@]}"; do
echo " Language: $lang"
# Count HTML files for each content type
for content_type in "${CONTENT_TYPES[@]}"; do
html_count=$(find "${PUBLIC_DIR}/${content_type}/${lang}" -name "*.html" 2>/dev/null | wc -l)
echo " 📝 ${content_type} (HTML): $html_count"
# Update total using eval for compatibility
var_name="total_${content_type//[^a-zA-Z0-9]/_}"
current_total=$(eval echo \$$var_name)
eval "$var_name=$((current_total + html_count))"
done
# Count source markdown files for comparison
echo -n " 📄 Source files: "
for content_type in "${CONTENT_TYPES[@]}"; do
md_count=$(find "${CONTENT_DIR}/${content_type}/${lang}" -name "*.md" -type f 2>/dev/null | wc -l)
echo -n "$md_count ${content_type}, "
done
echo ""
done
echo ""
echo "📈 Total converted files:"
for content_type in "${CONTENT_TYPES[@]}"; do
var_name="total_${content_type//[^a-zA-Z0-9]/_}"
total=$(eval echo \$$var_name)
echo " 📝 ${content_type} HTML files: $total"
done
# Optional: Generate content manifest
MANIFEST_FILE="${PUBLIC_DIR}/content-manifest.json"
echo "📋 Generating content manifest..."
if command -v jq >/dev/null 2>&1; then
cat > "$MANIFEST_FILE" << EOF
{
"generated_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
"version": "1.0",
"languages": $(printf '%s\n' "${LANGUAGES[@]}" | jq -R . | jq -s .),
"content_types": $(printf '%s\n' "${CONTENT_TYPES[@]}" | jq -R . | jq -s .),
"content_root_path": "$CONTENT_DIR",
"build_info": {
"source_format": "markdown",
"output_format": "html",
"converter": "rustelo_markdown_converter",
"features": ["syntax-highlighting", "css-classes", "localization"]
},
"structure": {
$(for content_type in "${CONTENT_TYPES[@]}"; do
echo " \"$content_type\": {"
for lang in "${LANGUAGES[@]}"; do
html_count=$(find "${PUBLIC_DIR}/${content_type}/${lang}" -name "*.html" 2>/dev/null | wc -l)
md_count=$(find "${CONTENT_DIR}/${content_type}/${lang}" -name "*.md" -type f 2>/dev/null | wc -l)
has_index="false"
[ -f "${PUBLIC_DIR}/${content_type}/${lang}/index.json" ] && has_index="true"
echo " \"$lang\": { \"html_count\": $html_count, \"markdown_count\": $md_count, \"has_index\": $has_index },"
done | sed '$ s/,$//'
echo " },"
done | sed '$ s/,$//')
}
}
EOF
echo " ✅ Content manifest generated: $MANIFEST_FILE"
else
echo " ⚠️ jq not available, skipping manifest generation"
fi
echo ""
echo "🎉 Build complete! Your localized content has been converted to HTML."
echo ""
echo "📁 Generated HTML files:"
for content_type in "${CONTENT_TYPES[@]}"; do
find "${PUBLIC_DIR}/${content_type}" -name "*.html" 2>/dev/null | sort
done
echo ""
echo "Next steps:"
echo " 1. Markdown files have been converted to HTML with localization"
echo " 2. Your Leptos application can now serve the generated HTML content"
echo " 3. Users can switch languages and see content in their preferred language"
echo ""
echo "Content locations:"
for content_type in "${CONTENT_TYPES[@]}"; do
echo " 📝 ${content_type} HTML: ${PUBLIC_DIR}/${content_type}/{lang}/*.html"
done
echo " 📋 Manifest: ${PUBLIC_DIR}/content-manifest.json"
echo ""
echo "🔄 To rebuild content after making changes, run this script again."