website-htmx-rustelo-code/scripts/content/md-to-ncl.nu

611 lines
22 KiB
Text
Raw Permalink Normal View History

2026-07-10 03:44:13 +01:00
#!/usr/bin/env nu
# Markdown → NCL Metadata Generator
#
# Reads YAML frontmatter from existing *.md content files and generates
# a co-located *.ncl metadata file for each one.
#
# Generated files use make_post / make_recipe from the project schema.
# Only fields present in PostMetadata / RecipeMetadata are emitted;
# display-only frontmatter (css_class, category_description, etc.) is dropped.
#
# Usage:
# nu md-to-ncl.nu [--content-dir PATH] [--schema-dir PATH]
# [--type TYPE] [--lang LANG]
# [--dry-run] [--force] [--verbose]
# [--fill-translations] # cross-lang stem matching
#
# Environment:
# SITE_CONTENT_PATH — overrides --content-dir default
# Fields emitted for blog posts (PostMetadata).
const POST_FIELDS = [
"id", "title", "slug", "subtitle", "excerpt", "content_file",
"author", "date", "updated_at", "published", "featured", "draft",
"category", "tags", "read_time", "sort_order", "image_url", "translations",
]
# Extra fields emitted for recipes (RecipeMetadata).
const RECIPE_EXTRA_FIELDS = ["difficulty", "duration", "prerequisites", "tools"]
# Content types that use RecipeMetadata.
const RECIPE_TYPES = ["recipes"]
# Fields to silently drop — frontmatter-only, not in NCL schema.
const DROP_FIELDS = [
"css_class", "category_description", "category_published",
]
# Default author value (matches schema default — omit from NCL if equal).
const DEFAULT_AUTHOR = "Jesús Pérez Lorenzo"
# Fields that live in the .ncl but are NOT sourced from .md frontmatter, so they
# must be preserved across regeneration instead of dropped: graph relationships
# (authored / content-graph) and generated card thumbnails (written by the image
# pipeline via update_ncl_field). image_url stays out — it IS a frontmatter field.
const PRESERVE_FIELDS = ["thumbnail", "thumbnail_dark", "graph"]
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main [
--content-dir: string = "" # Content root. Default: $SITE_CONTENT_PATH or site/content
--schema-dir: string = "" # Schema dir. Default: rustelo/nickel/content/metadata
--type: string = "" # Filter to one content type
--lang: string = "" # Filter to one language
--dry-run # Show what would be written without writing
--force # Overwrite existing *.ncl files
--validate-contracts # After generating, run nickel export on every post *.ncl
--fill-translations # Auto-populate translations field via cross-lang stem matching
--verbose # Print per-field details
] {
let root = resolve_content_dir $content_dir
let schema_abs = resolve_schema_dir $schema_dir
if not ($root | path exists) {
error make { msg: $"Content directory not found: ($root)" }
}
if not ($schema_abs | path exists) {
error make { msg: $"Schema directory not found: ($schema_abs)" }
}
print $"(ansi cyan)MD → NCL Generator(ansi reset) — ($root)"
if $dry_run { print $"(ansi yellow)[dry-run] no files will be written(ansi reset)" }
# Build translation map before generation pass (scans all langs unconditionally).
let trans_map = if $fill_translations {
print $"(ansi cyan)Building translation map...(ansi reset)"
let m = build_translation_map $root
print $" ($m | length) slug entries across all languages"
$m
} else {
[]
}
# --fill-translations implies --force: overwrite to update translations field.
let effective_force = $force or $fill_translations
let types = discover_dirs $root $type
if ($types | is-empty) {
print "No content types found."
return
}
mut total_written = 0
mut total_skipped = 0
mut total_errors = 0
for ct in $types {
let langs = discover_dirs $"($root)/($ct)" $lang
for language in $langs {
let result = process_lang $root $ct $language $schema_abs $dry_run $effective_force $verbose $trans_map
$total_written = $total_written + $result.written
$total_skipped = $total_skipped + $result.skipped
$total_errors = $total_errors + $result.errors
}
}
print ""
print $"(ansi green)Done(ansi reset) — written: ($total_written) skipped: ($total_skipped) errors: ($total_errors)"
if $validate_contracts and not $dry_run {
print ""
validate_contracts $root $type $lang
}
}
# ---------------------------------------------------------------------------
# Translation map construction
# ---------------------------------------------------------------------------
# Scan ALL *.md files (no type/lang filter) and build a flat list of
# { ct, stem, lang, slug } records for use by --fill-translations.
def build_translation_map [root: string] {
let types = discover_dirs $root ""
mut entries: list = []
for ct in $types {
let langs = discover_dirs $"($root)/($ct)" ""
for lang in $langs {
let lang_dir = $"($root)/($ct)/($lang)"
let cats = discover_dirs $lang_dir ""
for cat in $cats {
let cat_dir = $"($lang_dir)/($cat)"
let md_files = discover_md_files $cat_dir
for md_path in $md_files {
let stem = $md_path | path parse | get stem
let raw = try { open --raw $md_path } catch { "" }
if ($raw | is-empty) { continue }
let fm = extract_frontmatter $raw
if ($fm | is-empty) { continue }
let parsed = try { $fm | from yaml } catch { null }
if $parsed == null { continue }
let slug_raw = try { $parsed | get slug } catch { null }
let slug = if $slug_raw != null and ($slug_raw | describe) == "string" {
$slug_raw | str trim
} else {
$stem
}
$entries = ($entries | append { ct: $ct, stem: $stem, lang: $lang, slug: $slug })
}
}
}
}
$entries
}
# ---------------------------------------------------------------------------
# Discovery
# ---------------------------------------------------------------------------
def resolve_content_dir [arg: string] {
if $arg != "" { return $arg }
$env.SITE_CONTENT_PATH? | default "site/content"
}
def resolve_schema_dir [arg: string] {
let d = if $arg != "" { $arg } else { "rustelo/nickel/content/metadata" }
$d | path expand
}
def discover_dirs [parent: string, filter: string] {
if not ($parent | path exists) { return [] }
let all = (ls $parent)
| where type == "dir"
| get name
| each { |p| $p | path basename }
| where { |d| not ($d | str starts-with ".") }
if $filter != "" { $all | where { |d| $d == $filter } } else { $all }
}
def discover_md_files [cat_dir: string] {
if not ($cat_dir | path exists) { return [] }
(ls $cat_dir)
| where type == "file"
| where { |e| ($e.name | path parse | get extension) == "md" }
| get name
| sort
}
# ---------------------------------------------------------------------------
# Per-language processing
# ---------------------------------------------------------------------------
def process_lang [
root: string, ct: string, lang: string,
schema_abs: string, dry_run: bool, force: bool, verbose: bool,
trans_map: list
] {
let lang_dir = $"($root)/($ct)/($lang)"
let cats = discover_dirs $lang_dir ""
if ($cats | is-empty) {
if $verbose { print $" ($ct)/($lang): no categories" }
return { written: 0, skipped: 0, errors: 0 }
}
print $" (ansi blue)($ct)/($lang)(ansi reset)"
mut written = 0
mut skipped = 0
mut errors = 0
for cat in $cats {
let cat_dir = $"($lang_dir)/($cat)"
let md_files = discover_md_files $cat_dir
if ($md_files | is-empty) {
if $verbose { print $" ($cat)/: no *.md files" }
continue
}
for md_path in $md_files {
let stem = $md_path | path parse | get stem
let ncl_path = ($md_path | path parse | update extension "ncl" | path join)
let result = process_md_file $md_path $ncl_path $ct $cat $lang $stem $schema_abs $dry_run $force $verbose $trans_map
if $result == "written" { $written = $written + 1 }
if $result == "skipped" { $skipped = $skipped + 1 }
if $result == "error" { $errors = $errors + 1 }
if $verbose or $result == "written" {
print $" ($cat)/($stem).ncl → ($result)"
}
}
}
{ written: $written, skipped: $skipped, errors: $errors }
}
# ---------------------------------------------------------------------------
# Single file processing
# ---------------------------------------------------------------------------
def process_md_file [
md_path: string, ncl_path: string, ct: string, cat: string,
lang: string, stem: string,
schema_abs: string, dry_run: bool, force: bool, verbose: bool,
trans_map: list
] {
# Skip if NCL already exists and --force not set
if (not $force) and ($ncl_path | path exists) { return "skipped" }
let raw = try { open --raw $md_path } catch { return "error" }
let frontmatter = extract_frontmatter $raw
if ($frontmatter | is-empty) {
print $"(ansi yellow) Warning: no frontmatter in ($md_path)(ansi reset)"
return "skipped"
}
let parsed = try { $frontmatter | from yaml } catch { |e|
print $"(ansi red) Error parsing YAML in ($md_path): ($e.msg)(ansi reset)"
return "error"
}
# Compute relative import path from ncl_path to schema_abs.
# Structure is always {content_root}/{type}/{lang}/{cat}/file.ncl
# = 4 levels up from file dir reaches content root's parent (site/).
let ncl_dir = $ncl_path | path dirname | path expand
let rel_schema = compute_rel_path $ncl_dir $schema_abs
let is_recipe = $RECIPE_TYPES | any { |t| $t == $ct }
let schema_file = if $is_recipe { "recipe_metadata.ncl" } else { "post_metadata.ncl" }
let factory = if $is_recipe { "make_recipe" } else { "make_post" }
let import_path = $"($rel_schema)/($schema_file)"
let fields = if $is_recipe {
$POST_FIELDS ++ $RECIPE_EXTRA_FIELDS
} else {
$POST_FIELDS
}
# Preserve graph + generated thumbnails from the existing .ncl (md-to-ncl
# can't source them from frontmatter) so --force regeneration never drops them.
let preserved = (extract_preserved $ncl_path)
let ncl_content = build_ncl $parsed $fields $import_path $factory $md_path $cat $ct $lang $stem $trans_map $preserved $verbose
if $dry_run { return "written" }
try {
$ncl_content | save --force $ncl_path
"written"
} catch { |e|
print $"(ansi red) Error writing ($ncl_path): ($e.msg)(ansi reset)"
"error"
}
}
# ---------------------------------------------------------------------------
# Relative path computation
# ---------------------------------------------------------------------------
def compute_rel_path [from_abs: string, to_abs: string] {
# Split both paths into segments and find common prefix length.
let from_parts = $from_abs | path split
let to_parts = $to_abs | path split
let min_len = [$from_parts $to_parts] | each { length } | math min
mut common = 0
for i in 0..($min_len - 1) {
let fp = $from_parts | get $i
let tp = $to_parts | get $i
if $fp == $tp { $common = $common + 1 } else { break }
}
let up_count = ($from_parts | length) - $common
let down_parts = $to_parts | skip $common
let ups = 0..($up_count - 1) | each { ".." }
let rel_parts = $ups ++ $down_parts
$rel_parts | str join "/"
}
# ---------------------------------------------------------------------------
# Frontmatter extraction
# ---------------------------------------------------------------------------
def extract_frontmatter [content: string] {
let lines = $content | lines
let delims = $lines | enumerate | where { |e| $e.item | str trim | $in == "---" } | get index
if ($delims | length) < 2 { return "" }
let start = ($delims | first) + 1
let stop = $delims | get 1
$lines | skip $start | first ($stop - $start) | str join "\n"
}
# ---------------------------------------------------------------------------
# Preserve non-frontmatter fields across regeneration (graph, thumbnails)
# ---------------------------------------------------------------------------
def count_char [s: string, c: string] {
$s | split chars | where { |x| $x == $c } | length
}
# Extract a top-level field from an existing .ncl's raw text: a single-line
# `f = "..."` or a brace-balanced multi-line `f = { ... }`. Returns "" if absent.
def extract_ncl_field [lines: list<string>, field: string] {
let idxs = ($lines | enumerate | where { |e|
let t = ($e.item | str trim)
($t | str starts-with $"($field) =") or ($t | str starts-with $"($field)=")
} | get index)
if ($idxs | is-empty) { return "" }
let start = ($idxs | first)
let first_line = ($lines | get $start)
if (count_char $first_line "{") == 0 {
return ($first_line | str trim)
}
mut depth = 0
mut out: list<string> = []
mut i = $start
let n = ($lines | length)
loop {
if $i >= $n { break }
let line = ($lines | get $i)
$out = ($out | append $line)
$depth = $depth + (count_char $line "{") - (count_char $line "}")
if $depth <= 0 { break }
$i = $i + 1
}
$out | str join "\n"
}
# Read the existing .ncl (if any) and return PRESERVE_FIELDS as ready-to-inject
# NCL lines (2-space indent on the field, comma-terminated). "" when none found.
def extract_preserved [ncl_path: string] {
if not ($ncl_path | path exists) { return "" }
let lines = (open --raw $ncl_path | lines)
let blocks = ($PRESERVE_FIELDS | each { |f|
let raw = (extract_ncl_field $lines $f)
if ($raw | str trim | is-empty) { null } else {
let norm = ($raw | str trim | str trim --right --char ",")
$" ($norm),"
}
} | where { |x| $x != null })
$blocks | str join "\n"
}
# ---------------------------------------------------------------------------
# NCL content builder
# ---------------------------------------------------------------------------
def build_ncl [
parsed: record, fields: list<string>, import_path: string,
factory: string, source_md: string, cat: string,
ct: string, cur_lang: string, stem: string, trans_map: list,
preserved: string, verbose: bool
] {
let header = $"# Generated from ($source_md | path basename)
# Schema: ($import_path)
# Regenerate: nu scripts/content/md-to-ncl.nu --force
let schema = import \"($import_path)\" in
schema.($factory) \{"
mut field_lines: list<string> = []
for field in $fields {
# Skip drop-list fields
if ($DROP_FIELDS | any { |d| $d == $field }) { continue }
# translations: computed from cross-lang stem map when available,
# otherwise fall back to frontmatter value.
if $field == "translations" {
if not ($trans_map | is-empty) {
let others = $trans_map
| where { |r| $r.ct == $ct and $r.stem == $stem and $r.lang != $cur_lang }
if not ($others | is-empty) {
let entries = $others
| each { |r| $"($r.lang) = \"($r.slug)\"" }
| str join ", "
$field_lines = ($field_lines | append $" translations = { ($entries) },")
}
} else {
let raw_val = try { $parsed | get translations } catch { null }
if $raw_val != null {
let ncl_val = to_ncl_value $raw_val "translations"
if $ncl_val != null {
$field_lines = ($field_lines | append $" translations = ($ncl_val),")
}
}
}
continue
}
let raw_val = try { $parsed | get $field } catch { null }
# Skip null / missing
if $raw_val == null { continue }
# Skip author if it matches the schema default
if $field == "author" {
let s = $raw_val | into string | str trim
if $s == $DEFAULT_AUTHOR or $s == "Jesús Pérez" { continue }
}
# Skip sort_order == 0 (schema default)
if $field == "sort_order" and ($raw_val | into int) == 0 { continue }
# Skip published == true (schema default)
if $field == "published" and $raw_val == true { continue }
# Skip featured == false (schema default)
if $field == "featured" and $raw_val == false { continue }
# Skip draft == false (schema default)
if $field == "draft" and $raw_val == false { continue }
let ncl_val = to_ncl_value $raw_val $field
if $ncl_val == null { continue }
$field_lines = ($field_lines | append $" ($field) = ($ncl_val),")
}
let all_lines = if ($preserved | str trim | is-not-empty) {
$field_lines | append $preserved
} else {
$field_lines
}
if ($all_lines | is-empty) {
return $"($header)\n}"
}
let body = $all_lines | str join "\n"
$"($header)\n($body)\n}"
}
# ---------------------------------------------------------------------------
# YAML → NCL value conversion
# ---------------------------------------------------------------------------
def to_ncl_value [val: any, field: string] {
# Null / empty
if $val == null { return null }
let type_name = $val | describe
# Boolean
if $type_name == "bool" {
return ($val | into string)
}
# Integer / float
if ($type_name | str starts-with "int") or ($type_name | str starts-with "float") {
return ($val | into string)
}
# List → Nickel array (describe returns "list<string>", "list<any>", etc.)
if ($type_name | str starts-with "list") {
let items = $val | each { |v| $"\"($v)\"" } | str join ", "
return $"[($items)]"
}
# Record → Nickel record (e.g. translations: {es: "slug-es"} in YAML)
if ($type_name | str starts-with "record") {
let entries = $val | transpose key value
| each { |e|
let v_esc = ($e.value | into string | str replace --all '\\' '\\\\' | str replace --all '"' '\\"')
$"($e.key) = \"($v_esc)\""
}
| str join ", "
return $"{ ($entries) }"
}
# String
let s = $val | into string | str trim
if $s == "" { return null }
# tags field: might be empty string
if $field == "tags" and $s == "" { return "[]" }
# Escape backslashes and double-quotes for NCL string literal
let escaped = $s | str replace --all '\\' '\\\\' | str replace --all '"' '\\"'
$"\"($escaped)\""
}
# ---------------------------------------------------------------------------
# Contract validation
# ---------------------------------------------------------------------------
# Run `nickel export` on every post *.ncl file in {type}/{lang}/{cat}/ paths.
# Files at shallower depths (blog.ncl, recipes.ncl, content-kinds.ncl) are
# pre-existing type-level configs — not validated here.
def validate_contracts [root: string, type_filter: string, lang_filter: string] {
print $"(ansi cyan)Validating Nickel contracts...(ansi reset)"
# Collect all *.ncl files exactly 4 levels deep: type/lang/cat/file.ncl
# Exclude _index.ncl (generated aggregators, not post metadata).
let all_ncl = (glob $"($root)/*/*/*/*.ncl")
| where { |p|
let base = $p | path basename
$base != "_index.ncl"
}
| where { |p|
# Apply type/lang filters if provided
let parts = $p | path split
let n = $parts | length
# parts[-4] = type, parts[-3] = lang, parts[-2] = cat, parts[-1] = file
let ok_type = if $type_filter != "" {
($parts | get ($n - 4)) == $type_filter
} else { true }
let ok_lang = if $lang_filter != "" {
($parts | get ($n - 3)) == $lang_filter
} else { true }
$ok_type and $ok_lang
}
| sort
if ($all_ncl | is-empty) {
print " No post *.ncl files found."
return
}
let total = $all_ncl | length
mut passed = 0
mut failed = 0
for ncl_path in $all_ncl {
let label = $ncl_path | path split | last 4 | str join "/"
let result = try {
nickel export $ncl_path --format json | ignore
{ ok: true, msg: "" }
} catch { |e|
{ ok: false, msg: ($e.msg | str trim) }
}
if $result.ok {
$passed = $passed + 1
print $" (ansi green)ok(ansi reset) ($label)"
} else {
$failed = $failed + 1
# Extract first meaningful error line from nickel output
let err_line = nickel export $ncl_path --format json out+err>|
| lines
| where { |l| ($l | str trim | str length) > 0 }
| where { |l| not ($l | str starts-with " ") }
| first 1
| str join ""
print $" (ansi red)FAIL(ansi reset) ($label)"
print $" ($err_line)"
}
}
print ""
if $failed == 0 {
print $"(ansi green)All ($total) files pass contract validation.(ansi reset)"
} else {
print $"(ansi red)($failed) / ($total) files failed contract validation.(ansi reset)"
exit 1
}
}