191 lines
6.9 KiB
Bash
191 lines
6.9 KiB
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# Normalize Prescription Content Metadata
|
||
|
|
# This script normalizes tags and categories in prescription markdown files
|
||
|
|
|
||
|
|
set -e
|
||
|
|
|
||
|
|
echo "🧪 Normalizing prescription content metadata..."
|
||
|
|
|
||
|
|
# Function to normalize a string (Unicode normalization)
|
||
|
|
normalize_text() {
|
||
|
|
local text="$1"
|
||
|
|
# Convert to lowercase, replace spaces with dashes, remove accents
|
||
|
|
echo "$text" | \
|
||
|
|
tr '[:upper:]' '[:lower:]' | \
|
||
|
|
sed 's/á/a/g' | \
|
||
|
|
sed 's/é/e/g' | \
|
||
|
|
sed 's/í/i/g' | \
|
||
|
|
sed 's/ó/o/g' | \
|
||
|
|
sed 's/ú/u/g' | \
|
||
|
|
sed 's/ü/u/g' | \
|
||
|
|
sed 's/ñ/n/g' | \
|
||
|
|
sed 's/ç/c/g' | \
|
||
|
|
sed 's/[[:space:]]\+/-/g' | \
|
||
|
|
sed 's/[^a-z0-9-]//g' | \
|
||
|
|
sed 's/--\+/-/g' | \
|
||
|
|
sed 's/^-\|-$//g'
|
||
|
|
}
|
||
|
|
|
||
|
|
# Process prescription files
|
||
|
|
for lang_dir in content/prescriptions/*/; do
|
||
|
|
if [ -d "$lang_dir" ]; then
|
||
|
|
lang=$(basename "$lang_dir")
|
||
|
|
echo "📂 Processing prescription language: $lang"
|
||
|
|
|
||
|
|
for md_file in "$lang_dir"*.md; do
|
||
|
|
if [ -f "$md_file" ]; then
|
||
|
|
filename=$(basename "$md_file")
|
||
|
|
echo " 📝 Processing: $filename"
|
||
|
|
|
||
|
|
# Create backup
|
||
|
|
cp "$md_file" "$md_file.backup"
|
||
|
|
|
||
|
|
# Process the file
|
||
|
|
awk '
|
||
|
|
BEGIN { in_frontmatter = 0; frontmatter_ended = 0 }
|
||
|
|
/^---$/ {
|
||
|
|
if (!frontmatter_ended) {
|
||
|
|
if (in_frontmatter) {
|
||
|
|
frontmatter_ended = 1
|
||
|
|
} else {
|
||
|
|
in_frontmatter = 1
|
||
|
|
}
|
||
|
|
}
|
||
|
|
print; next
|
||
|
|
}
|
||
|
|
|
||
|
|
# Process frontmatter lines
|
||
|
|
in_frontmatter && !frontmatter_ended {
|
||
|
|
if (/^category:/) {
|
||
|
|
# Extract and normalize category
|
||
|
|
gsub(/^category:[[:space:]]*"?/, "")
|
||
|
|
gsub(/"?[[:space:]]*$/, "")
|
||
|
|
normalized = $0
|
||
|
|
gsub(/[[:upper:]]/, tolower($0), normalized)
|
||
|
|
gsub(/á/, "a", normalized)
|
||
|
|
gsub(/é/, "e", normalized)
|
||
|
|
gsub(/í/, "i", normalized)
|
||
|
|
gsub(/ó/, "o", normalized)
|
||
|
|
gsub(/ú/, "u", normalized)
|
||
|
|
gsub(/ü/, "u", normalized)
|
||
|
|
gsub(/ñ/, "n", normalized)
|
||
|
|
gsub(/ç/, "c", normalized)
|
||
|
|
gsub(/[[:space:]]+/, "-", normalized)
|
||
|
|
gsub(/[^a-z0-9-]/, "", normalized)
|
||
|
|
gsub(/-+/, "-", normalized)
|
||
|
|
gsub(/^-|-$/, "", normalized)
|
||
|
|
print "category: \"" normalized "\""
|
||
|
|
next
|
||
|
|
}
|
||
|
|
|
||
|
|
if (/^tags:/) {
|
||
|
|
# Process tags array
|
||
|
|
if (/\[.*\]/) {
|
||
|
|
# Single line array
|
||
|
|
line = $0
|
||
|
|
gsub(/^tags:[[:space:]]*\[/, "", line)
|
||
|
|
gsub(/\][[:space:]]*$/, "", line)
|
||
|
|
gsub(/"/, "", line)
|
||
|
|
gsub(/'"'"'/, "", line)
|
||
|
|
|
||
|
|
# Split tags and normalize each
|
||
|
|
n = split(line, tags, /,[[:space:]]*/)
|
||
|
|
printf "tags: ["
|
||
|
|
for (i = 1; i <= n; i++) {
|
||
|
|
if (i > 1) printf ", "
|
||
|
|
tag = tags[i]
|
||
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", tag)
|
||
|
|
gsub(/[[:upper:]]/, tolower(tag), tag)
|
||
|
|
gsub(/á/, "a", tag)
|
||
|
|
gsub(/é/, "e", tag)
|
||
|
|
gsub(/í/, "i", tag)
|
||
|
|
gsub(/ó/, "o", tag)
|
||
|
|
gsub(/ú/, "u", tag)
|
||
|
|
gsub(/ü/, "u", tag)
|
||
|
|
gsub(/ñ/, "n", tag)
|
||
|
|
gsub(/ç/, "c", tag)
|
||
|
|
gsub(/[[:space:]]+/, "-", tag)
|
||
|
|
gsub(/[^a-z0-9-]/, "", tag)
|
||
|
|
gsub(/-+/, "-", tag)
|
||
|
|
gsub(/^-|-$/, "", tag)
|
||
|
|
printf "\"%s\"", tag
|
||
|
|
}
|
||
|
|
print "]"
|
||
|
|
next
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# Print all other lines as-is
|
||
|
|
{ print }
|
||
|
|
' "$md_file" > "$md_file.tmp"
|
||
|
|
|
||
|
|
# Replace original file
|
||
|
|
mv "$md_file.tmp" "$md_file"
|
||
|
|
echo " ✅ Normalized metadata in $filename"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# Also normalize the prescription index.json files
|
||
|
|
echo "📋 Normalizing prescription index files..."
|
||
|
|
|
||
|
|
for index_file in content/prescriptions/*/index.json; do
|
||
|
|
if [ -f "$index_file" ]; then
|
||
|
|
lang_dir=$(dirname "$index_file")
|
||
|
|
lang=$(basename "$lang_dir")
|
||
|
|
echo " 📄 Processing index for: $lang"
|
||
|
|
|
||
|
|
# Create backup
|
||
|
|
cp "$index_file" "$index_file.backup"
|
||
|
|
|
||
|
|
# Use Python to normalize JSON tags and categories
|
||
|
|
python3 -c "
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
import unicodedata
|
||
|
|
|
||
|
|
def normalize_text(text):
|
||
|
|
# Remove accents using Unicode normalization
|
||
|
|
text = unicodedata.normalize('NFD', text)
|
||
|
|
text = ''.join(c for c in text if unicodedata.category(c) != 'Mn')
|
||
|
|
|
||
|
|
# Convert to lowercase and replace spaces with dashes
|
||
|
|
text = text.lower()
|
||
|
|
text = re.sub(r'[^\w\s-]', '', text)
|
||
|
|
text = re.sub(r'[\s_]+', '-', text)
|
||
|
|
text = re.sub(r'-+', '-', text)
|
||
|
|
text = text.strip('-')
|
||
|
|
return text
|
||
|
|
|
||
|
|
try:
|
||
|
|
with open('$index_file', 'r', encoding='utf-8') as f:
|
||
|
|
data = json.load(f)
|
||
|
|
|
||
|
|
if 'prescriptions' in data:
|
||
|
|
for prescription in data['prescriptions']:
|
||
|
|
# Normalize category
|
||
|
|
if 'category' in prescription and prescription['category']:
|
||
|
|
prescription['category'] = normalize_text(prescription['category'])
|
||
|
|
|
||
|
|
# Normalize tags
|
||
|
|
if 'tags' in prescription and prescription['tags']:
|
||
|
|
prescription['tags'] = [normalize_text(tag) for tag in prescription['tags']]
|
||
|
|
|
||
|
|
with open('$index_file', 'w', encoding='utf-8') as f:
|
||
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
|
|
|
||
|
|
print(f' ✅ Normalized index for $lang')
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f' ❌ Error processing $lang index: {e}')
|
||
|
|
sys.exit(1)
|
||
|
|
"
|
||
|
|
done
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "✅ Prescription metadata normalization complete!"
|
||
|
|
echo "💡 Backup files created with .backup extension"
|