ontoref/reflection/modules/coder.nu

# .coder/ management — author workspaces, inbox triage, insight graduation
#
# Structure:
#   .coder/<author>/author.ncl            — identity (validated)
#   .coder/<author>/inbox/                — dump zone, zero ceremony
#   .coder/<author>/<category>/           — classified content
#   .coder/<author>/<category>/context.ncl — category metadata (validated)
#   .coder/general/<category>/            — published from all authors
#
# File naming: {description}.{kind}.md  where kind ∈ {done,plan,info,review,audit,commit}
# Legacy files with _ separator are accepted and normalized during triage.

use store.nu [daemon-export-safe]

const VALID_KINDS = ["done", "plan", "info", "review", "audit", "commit"]

def coder-root []: nothing -> string {
  let root = if ($env.ONTOREF_PROJECT_ROOT? | is-not-empty) {
    $env.ONTOREF_PROJECT_ROOT
  } else {
    $env.PWD
  }
  $"($root)/.coder"
}

# Extract kind from filename, handling both . and _ separators
def extract-kind [filename: string]: nothing -> string {
  let base = ($filename | str replace '.md' '')
  for kind in $VALID_KINDS {
    if ($base | str ends-with $".($kind)") or ($base | str ends-with $"_($kind)") {
      return $kind
    }
  }
  ""
}

# Map file kind to triage category
def kind-to-category [kind: string]: nothing -> string {
  match $kind {
    "done" => "features"
    "plan" => "features"
    "info" => "investigations"
    "review" => "reviews"
    "audit" => "reviews"
    "commit" => "features"
    _ => ""
  }
}

# Classify by content keywords (fallback when kind is empty)
def classify-by-content [filepath: string]: nothing -> string {
  let first_lines = (open $filepath | lines | first 5 | str join " " | str downcase)
  if ($first_lines =~ "insight|til |learned|pattern|trap|gotcha") {
    "insights"
  } else if ($first_lines =~ "fix|bug|error|broken|crash|hotfix") {
    "bugfixes"
  } else if ($first_lines =~ "decision|chose|adr|rejected|alternative") {
    "decisions"
  } else if ($first_lines =~ "plan|implement|design|proposal") {
    "features"
  } else {
    ""
  }
}

# Extract title from markdown first heading or filename
def extract-title [filepath: string]: nothing -> string {
  let lines = (open $filepath | lines)
  let heading = ($lines | where { $in =~ '^#+ ' } | first | default "")
  if ($heading | is-not-empty) {
    $heading | str replace -r '^#+\s*' '' | str trim
  } else {
    # Fall back to filename without extension and kind suffix
    mut base = ($filepath | path basename | str replace '.md' '')
    for kind in $VALID_KINDS {
      $base = ($base | str replace $".($kind)" '' | str replace $"_($kind)" '')
    }
    $base | str replace -a '_' ' ' | str replace -a '-' ' '
  }
}

# Extract date from filename (YYYY-MM-DD or YYYYMMDD prefix).
# Nushell str substring ranges are inclusive on both ends.
def extract-date [filename: string]: nothing -> string {
  if ($filename =~ '^\d{4}-\d{2}-\d{2}') {
    $filename | str substring 0..9
  } else if ($filename =~ '^\d{8}') {
    let raw = ($filename | str substring 0..7)
    let y = ($raw | str substring 0..3)
    let m = ($raw | str substring 4..5)
    let d = ($raw | str substring 6..7)
    $"($y)-($m)-($d)"
  } else {
    ""
  }
}

# Extract keyword tags from content
def extract-tags [filepath: string]: nothing -> list<string> {
  let content = (open $filepath | str downcase)
  let tag_map = [
    ["nickel", "nickel"], ["nushell", "nushell"], ["rust", "rust"],
    ["cargo", "cargo"], ["bacon", "bacon"], ["ci", "ci"],
    ["encryption", "encryption"], ["ontology", "ontology"],
    ["surrealdb", "surrealdb"], ["nats", "nats"],
    ["agent", "agent"], ["llm", "llm"], ["docker", "docker"],
  ]
  mut tags = []
  for pair in $tag_map {
    let keyword = ($pair | get 0)
    let tag = ($pair | get 1)
    if ($content =~ $keyword) {
      $tags = ($tags | append $tag)
    }
  }
  $tags
}

# Map category string to NCL enum value
def category-to-ncl [cat: string]: nothing -> string {
  match $cat {
    "insights" => "'Insight"
    "features" => "'Feature"
    "bugfixes" => "'Bugfix"
    "investigations" => "'Investigation"
    "decisions" => "'Decision"
    "reviews" => "'Review"
    "resources" => "'Resource"
    _ => "'Inbox"
  }
}

# Generate companion NCL for a classified file
def generate-companion-ncl [
  filepath: string
  author: string
  kind: string
  category: string
]: nothing -> string {
  let title = (extract-title $filepath)
  let date = (extract-date ($filepath | path basename))
  let tags = (extract-tags $filepath)
  let kind_ncl = if ($kind | is-empty) { "'unknown" } else { $"'($kind)" }
  let cat_ncl = (category-to-ncl $category)
  let tags_ncl = ($tags | each {|t| $"\"($t)\""} | str join ", ")

  let safe_title = ($title | str replace -a '"' '\\"')
  let title_line = $'  title    = "($safe_title)",'
  let author_line = $'  author   = "($author)",'
  let kind_line = $'  kind     = ($kind_ncl),'
  let cat_line = $'  category = ($cat_ncl),'
  let date_line = if ($date | is-not-empty) { $'  date     = "($date)",' } else { "" }
  let tags_line = if ($tags | is-not-empty) { $'  tags     = [($tags_ncl)],' } else { "" }

  ([
    'let d = import "coder-defaults.ncl" in'
    'let c = import "coder-constraints.ncl" in'
    ''
    '(d.make_entry {'
    $title_line
    $date_line
    $author_line
    $kind_line
    $cat_line
    $tags_line
    '}) | c.NonEmptyTitle | c.NonEmptyAuthor'
  ] | where {|line| ($line | str length) > 0 } | str join "\n")
}

# List all authors in .coder/
export def "coder authors" []: nothing -> table {
  let root = (coder-root)
  if not ($root | path exists) { return [] }

  ls $root
  | where type == "dir"
  | where { ($in.name | path basename) != "general" and ($in.name | path basename) != "insights" }
  | each {|d|
    let author_ncl = $"($d.name)/author.ncl"
    let name = ($d.name | path basename)
    let has_meta = ($author_ncl | path exists)
    { name: $name, has_metadata: $has_meta, path: $d.name }
  }
}

# Initialize an author workspace
export def "coder init" [
  author: string
  --actor: string = "Human"
  --model: string = ""
]: nothing -> string {
  let root = (coder-root)
  let author_dir = $"($root)/($author)"
  let inbox_dir = $"($author_dir)/inbox"

  if ($author_dir | path exists) {
    return $"author '($author)' already exists at ($author_dir)"
  }

  mkdir $inbox_dir

  let actor_value = match $actor {
    "Human" => "'Human"
    "AgentClaude" => "'AgentClaude"
    "AgentCustom" => "'AgentCustom"
    "CI" => "'CI"
    _ => "'Human"
  }

  let model_line = if ($model | is-empty) { "" } else { $"  model  = \"($model)\"," }

  let ncl_content = ([
    'let s = import "coder.ncl" in'
    ''
    's.Author & {'
    $"  name   = \"($author)\","
    $"  actor  = ($actor_value),"
    $model_line
    '}'
  ] | where {|line| ($line | str length) > 0 } | str join "\n")

  $ncl_content | save --force $"($author_dir)/author.ncl"
  $"initialized ($author_dir) with inbox/"
}

# Triage: classify files in an author's inbox/ into categories
export def "coder triage" [
  author: string
  --dry-run (-n)
  --interactive (-i)
]: nothing -> table {
  let root = (coder-root)
  let inbox_dir = $"($root)/($author)/inbox"

  if not ($inbox_dir | path exists) {
    print $"no inbox/ at ($inbox_dir)"
    return []
  }

  let files = (ls $inbox_dir | where type == "file")
  if ($files | is-empty) {
    print "inbox/ is empty"
    return []
  }

  mut results = []

  for file in $files {
    let filename = ($file.name | path basename)
    let kind = (extract-kind $filename)
    let by_kind = (kind-to-category $kind)
    let by_content = (classify-by-content $file.name)

    let category = if ($by_kind | is-not-empty) {
      $by_kind
    } else if ($by_content | is-not-empty) {
      $by_content
    } else {
      "unclassified"
    }

    let target_dir = $"($root)/($author)/($category)"

    if $category == "unclassified" {
      $results = ($results | append { file: $filename, kind: $kind, category: $category, action: "skip", ncl: "" })
    } else if $dry_run {
      $results = ($results | append { file: $filename, kind: $kind, category: $category, action: "would move", ncl: "" })
    } else if $interactive {
      let answer = (input $"($filename) → ($category)? [y/n/c] (c=change category) ")
      if ($answer | str starts-with "y") {
        let ncl = (generate-companion-ncl $file.name $author $kind $category)
        let ncl_name = ($filename | str replace '.md' '.ncl')
        mkdir $target_dir
        mv $file.name $"($target_dir)/($filename)"
        $ncl | save --force $"($target_dir)/($ncl_name)"
        $results = ($results | append { file: $filename, kind: $kind, category: $category, action: "moved", ncl: $ncl_name })
      } else if ($answer | str starts-with "c") {
        let new_cat = (input "category: ")
        let ncl = (generate-companion-ncl $file.name $author $kind $new_cat)
        let ncl_name = ($filename | str replace '.md' '.ncl')
        let new_dir = $"($root)/($author)/($new_cat)"
        mkdir $new_dir
        mv $file.name $"($new_dir)/($filename)"
        $ncl | save --force $"($new_dir)/($ncl_name)"
        $results = ($results | append { file: $filename, kind: $kind, category: $new_cat, action: "moved", ncl: $ncl_name })
      } else {
        $results = ($results | append { file: $filename, kind: $kind, category: $category, action: "skipped", ncl: "" })
      }
    } else {
      let ncl = (generate-companion-ncl $file.name $author $kind $category)
      let ncl_name = ($filename | str replace '.md' '.ncl')
      mkdir $target_dir
      mv $file.name $"($target_dir)/($filename)"
      $ncl | save --force $"($target_dir)/($ncl_name)"
      $results = ($results | append { file: $filename, kind: $kind, category: $category, action: "moved", ncl: $ncl_name })
    }
  }

  $results
}

# Publish: promote files from author workspace to .coder/general/
# Renames files to include author: {author}-{original-name}
export def "coder publish" [
  author: string
  category: string
  --dry-run (-n)
  --all (-a)             # publish all files in category (default: interactive pick)
]: nothing -> table {
  let root = (coder-root)
  let source_dir = $"($root)/($author)/($category)"

  if not ($source_dir | path exists) {
    print $"no ($category)/ for author ($author)"
    return []
  }

  let general_dir = $"($root)/general/($category)"
  let files = (ls $source_dir
    | where type == "file"
    | where { ($in.name | path basename) != "context.ncl" })

  if ($files | is-empty) {
    print $"no files in ($category)/"
    return []
  }

  mut results = []

  for file in $files {
    let fname = ($file.name | path basename)
    let target_name = $"($author)-($fname)"

    if (not $all) and (not $dry_run) {
      let answer = (input $"publish ($fname) as ($target_name)? [y/n] ")
      if not ($answer | str starts-with "y") {
        $results = ($results | append { file: $fname, target: $target_name, action: "skipped" })
        continue
      }
    }

    if $dry_run {
      $results = ($results | append { file: $fname, target: $target_name, action: "would publish" })
    } else {
      mkdir $general_dir
      cp $file.name $"($general_dir)/($target_name)"
      $results = ($results | append { file: $fname, target: $target_name, action: "published" })
    }
  }

  $results
}

# List contents of an author workspace or general, grouped by category
export def "coder ls" [
  author: string = ""
  --category (-c): string = ""
]: nothing -> table {
  let root = (coder-root)
  if not ($root | path exists) { return [] }

  let search_dir = if ($author | is-empty) {
    $"($root)/general"
  } else {
    $"($root)/($author)"
  }

  if not ($search_dir | path exists) {
    print $"directory not found: ($search_dir)"
    return []
  }

  let dirs = (ls $search_dir | where type == "dir")

  mut items = []
  for dir in $dirs {
    let cat = ($dir.name | path basename)
    if ($category | is-not-empty) and ($cat != $category) { continue }

    let files = (ls $dir.name
      | where type == "file"
      | where { let b = ($in.name | path basename); $b != "context.ncl" and $b != "author.ncl" })
    for file in $files {
      let fname = ($file.name | path basename)
      let kind = (extract-kind $fname)
      let date = if ($fname =~ '^\d{4}-\d{2}-\d{2}') or ($fname =~ '^\d{8}') {
        $fname | str substring 0..9
      } else {
        ""
      }
      $items = ($items | append {
        category: $cat,
        file: $fname,
        kind: $kind,
        date: $date,
        modified: $file.modified,
      })
    }
  }

  $items | sort-by modified --reverse
}

# Search across all authors by content pattern
export def "coder search" [
  pattern: string
  --author (-a): string = ""
]: nothing -> table {
  let root = (coder-root)
  if not ($root | path exists) { return [] }

  let search_path = if ($author | is-not-empty) {
    $"($root)/($author)"
  } else {
    $root
  }

  let md_files = (glob $"($search_path)/**/*.md")

  mut matches = []
  for file in $md_files {
    let content = (open $file | default "")
    if ($content =~ $pattern) {
      let rel = ($file | str replace $"($root)/" "")
      let parts = ($rel | split row "/")
      let file_author = if ($parts | length) > 1 { $parts | first } else { "root" }
      let fname = ($parts | last)
      $matches = ($matches | append { author: $file_author, file: $fname, path: $rel })
    }
  }

  $matches
}

# ── JSON record commands ──────────────────────────────────────────────────────
# Structured entries stored as JSONL (one JSON object per line).
# Machine-readable alternative to markdown — no companion NCL needed.
# Storage: .coder/<author>/<category>/entries.jsonl

const VALID_CATEGORIES = ["insights", "features", "bugfixes", "investigations", "decisions", "reviews", "resources"]
const VALID_DOMAINS = ["Language", "Runtime", "Architecture", "Tooling", "Pattern", "Debugging", "Security", "Performance"]

# Record a structured JSON entry into a category's JSONL log
export def "coder record" [
  author: string
  --kind (-k): string = "info"
  --category (-c): string = ""
  --title (-t): string
  --tags: list<string> = []
  --relates_to: list<string> = []
  --trigger: string = ""
  --files_touched: list<string> = []
  --domain (-d): string = ""
  --reusable (-r)
  content: string
]: nothing -> record {
  let root = (coder-root)
  let author_dir = $"($root)/($author)"

  if not ($author_dir | path exists) {
    error make { msg: $"author '($author)' not initialized — run: coder init ($author)" }
  }

  if ($title | str trim | is-empty) {
    error make { msg: "title must not be empty" }
  }

  if ($kind | is-not-empty) and (not ($kind in $VALID_KINDS)) {
    error make { msg: $"invalid kind '($kind)' — valid: ($VALID_KINDS | str join ', ')" }
  }

  let resolved_category = if ($category | is-not-empty) {
    $category
  } else {
    let by_kind = (kind-to-category $kind)
    if ($by_kind | is-not-empty) { $by_kind } else { "inbox" }
  }

  if ($resolved_category != "inbox") and (not ($resolved_category in $VALID_CATEGORIES)) {
    error make { msg: $"invalid category '($resolved_category)' — valid: ($VALID_CATEGORIES | str join ', ')" }
  }

  let target_dir = $"($author_dir)/($resolved_category)"
  mkdir $target_dir

  let now = (date now | format date "%Y-%m-%d")

  mut entry = {
    title: $title,
    kind: $kind,
    category: $resolved_category,
    author: $author,
    date: $now,
    tags: $tags,
    relates_to: $relates_to,
    content: $content,
  }

  if ($trigger | is-not-empty) or ($files_touched | is-not-empty) {
    $entry = ($entry | merge {
      context: {
        trigger: $trigger,
        files_touched: $files_touched,
      }
    })
  }

  if ($domain | is-not-empty) {
    if not ($domain in $VALID_DOMAINS) {
      error make { msg: $"invalid domain '($domain)' — valid: ($VALID_DOMAINS | str join ', ')" }
    }
    $entry = ($entry | merge { domain: $domain, reusable: $reusable })
  }

  let jsonl_path = $"($target_dir)/entries.jsonl"
  let json_line = ($entry | to json --raw)

  $"($json_line)\n" | save --raw --append $jsonl_path

  $entry
}

# Query JSONL entries across categories/authors with optional filters
export def "coder log" [
  --author (-a): string = ""
  --category (-c): string = ""
  --tag (-t): string = ""
  --kind (-k): string = ""
  --domain (-d): string = ""
  --limit (-l): int = 0
  --json (-j)
]: nothing -> table {
  let root = (coder-root)
  if not ($root | path exists) { return [] }

  let pattern = if ($author | is-not-empty) and ($category | is-not-empty) {
    $"($root)/($author)/($category)/entries.jsonl"
  } else if ($author | is-not-empty) {
    $"($root)/($author)/**/entries.jsonl"
  } else if ($category | is-not-empty) {
    $"($root)/**/($category)/entries.jsonl"
  } else {
    $"($root)/**/entries.jsonl"
  }

  let files = (glob $pattern)
  if ($files | is-empty) { return [] }

  mut entries = []
  for file in $files {
    let lines = (open $file | lines | where { $in | str trim | is-not-empty })
    for line in $lines {
      # Guard: only attempt JSON parse on lines that look like objects.
      # from json is an internal command — errors cannot be captured with | complete.
      let trimmed = ($line | str trim)
      if ($trimmed | str starts-with "{") and ($trimmed | str ends-with "}") {
        let parsed = ($trimmed | from json)
        if ($parsed | describe | str starts-with "record") {
          $entries = ($entries | append $parsed)
        }
      }
    }
  }

  if ($tag | is-not-empty) {
    $entries = ($entries | where { ($in.tags? | default []) | any {|t| $t == $tag } })
  }

  if ($kind | is-not-empty) {
    $entries = ($entries | where { ($in.kind? | default "") == $kind })
  }

  if ($domain | is-not-empty) {
    $entries = ($entries | where { ($in.domain? | default "") == $domain })
  }

  let sorted = ($entries | sort-by date --reverse)

  if $limit > 0 {
    $sorted | first $limit
  } else {
    $sorted
  }
}

# Export all JSONL entries as a single JSON array (for external tools/databases)
export def "coder export" [
  --author (-a): string = ""
  --category (-c): string = ""
  --format (-f): string = "json"
]: nothing -> string {
  let entries = (coder log --author $author --category $category)

  match $format {
    "json" => { $entries | to json }
    "jsonl" => { $entries | each { to json --raw } | str join "\n" }
    "csv" => {
      $entries
      | select title kind category author date tags
      | update tags { str join ";" }
      | to csv
    }
    _ => { error make { msg: $"unsupported format '($format)' — use json, jsonl, or csv" } }
  }
}

# Graduate: copy items to committed project path (e.g. reflection/knowledge/)
export def "coder graduate" [
  source_category: string    # "general/insights" or "jesus/insights"
  --target (-t): string = "reflection/knowledge"
  --dry-run (-n)
]: nothing -> table {
  let root = (coder-root)
  let source_dir = $"($root)/($source_category)"

  if not ($source_dir | path exists) {
    print $"not found: ($source_dir)"
    return []
  }

  # Check if category is graduable
  let context_path = $"($source_dir)/context.ncl"
  if ($context_path | path exists) {
    let ctx = (daemon-export-safe $context_path)
    if ($ctx != null) {
      if not ($ctx.graduable? | default false) {
        print $"'($source_category)' is not marked graduable in context.ncl"
        return []
      }
    }
  }

  let project_root = if ($env.ONTOREF_PROJECT_ROOT? | is-not-empty) {
    $env.ONTOREF_PROJECT_ROOT
  } else {
    $env.PWD
  }
  let target_dir = $"($project_root)/($target)"

  let files = (ls $source_dir
    | where type == "file"
    | where { ($in.name | path basename) != "context.ncl" })

  mut results = []
  for file in $files {
    let fname = ($file.name | path basename)
    if $dry_run {
      $results = ($results | append { file: $fname, action: "would graduate", target: $target_dir })
    } else {
      mkdir $target_dir
      cp $file.name $"($target_dir)/($fname)"
      $results = ($results | append { file: $fname, action: "graduated", target: $target_dir })
    }
  }

  $results
}

# Check pending entries for novelty against published entries and QA store.
# Flags entries whose content overlaps significantly (Jaccard > threshold)
# with existing knowledge — potential slop that adds no new information.
export def "coder novelty-check" [
  author: string
  --threshold: float = 0.60   # Jaccard overlap above this = flagged as redundant
  --category (-c): string = ""  # Check specific category, default all
]: nothing -> table {
  let root = (coder-root)
  let author_dir = $"($root)/($author)"
  if not ($author_dir | path exists) {
    error make { msg: $"author '($author)' not initialized" }
  }

  # Collect pending entries from author workspace
  let pending_pattern = if ($category | is-not-empty) {
    $"($author_dir)/($category)/entries.jsonl"
  } else {
    $"($author_dir)/**/entries.jsonl"
  }
  let pending_files = (glob $pending_pattern)
  mut pending_entries = []
  for file in $pending_files {
    let lines = (open $file | lines | where { $in | str trim | is-not-empty })
    for line in $lines {
      let trimmed = ($line | str trim)
      if ($trimmed | str starts-with "{") and ($trimmed | str ends-with "}") {
        let parsed = ($trimmed | from json)
        if ($parsed | describe | str starts-with "record") {
          $pending_entries = ($pending_entries | append $parsed)
        }
      }
    }
  }

  if ($pending_entries | is-empty) { return [] }

  # Collect existing published entries
  let published_pattern = $"($root)/general/**/entries.jsonl"
  let pub_files = (glob $published_pattern)
  mut published = []
  for file in $pub_files {
    let lines = (open $file | lines | where { $in | str trim | is-not-empty })
    for line in $lines {
      let trimmed = ($line | str trim)
      if ($trimmed | str starts-with "{") and ($trimmed | str ends-with "}") {
        let parsed = ($trimmed | from json)
        if ($parsed | describe | str starts-with "record") {
          $published = ($published | append $parsed)
        }
      }
    }
  }

  # Collect QA entries
  let project_root = ($env.ONTOREF_PROJECT_ROOT? | default ($env.ONTOREF_ROOT? | default "."))
  let qa_file = $"($project_root)/reflection/qa.ncl"
  mut qa_entries = []
  if ($qa_file | path exists) {
    let qa_data = (do { nickel export --format json $qa_file } | complete)
    if $qa_data.exit_code == 0 {
      let parsed = ($qa_data.stdout | from json)
      $qa_entries = ($parsed.entries? | default [] | each { |e|
        { title: ($e.question? | default ""), content: ($e.answer? | default "") }
      })
    }
  }

  let existing = ($published | append $qa_entries)
  if ($existing | is-empty) { return [] }

  # Build existing content corpus
  let existing_texts = ($existing | each { |e|
    $"($e.title? | default '') ($e.content? | default '')"
  })

  let stop_words = ["that", "this", "with", "from", "have", "will", "been", "when", "they", "each", "does", "also", "into", "than"]

  mut results = []
  for entry in $pending_entries {
    let entry_text = $"($entry.title? | default '') ($entry.content? | default '')"
    let entry_words = ($entry_text | str downcase | split row --regex '\W+'
      | where { |w| ($w | str length) > 3 and not ($w in $stop_words) }
      | sort | uniq)

    if ($entry_words | is-empty) { continue }

    mut max_overlap = 0.0
    mut most_similar = ""

    for existing_text in $existing_texts {
      let ex_words = ($existing_text | str downcase | split row --regex '\W+'
        | where { |w| ($w | str length) > 3 and not ($w in $stop_words) }
        | sort | uniq)

      if ($ex_words | is-empty) { continue }

      let intersection = ($entry_words | where { |w| $w in $ex_words } | length)
      let union_size = ($entry_words | append $ex_words | sort | uniq | length)
      let jaccard = (($intersection | into float) / ($union_size | into float))

      if $jaccard > $max_overlap {
        $max_overlap = $jaccard
        $most_similar = ($existing_text | str substring 0..80)
      }
    }

    let status = if $max_overlap > $threshold { "REDUNDANT" } else if $max_overlap > ($threshold * 0.7) { "SIMILAR" } else { "NOVEL" }

    $results = ($results | append {
      title: ($entry.title? | default "?"),
      status: $status,
      overlap: ($max_overlap * 100.0 | math round),
      similar_to: $most_similar,
    })
  }

  $results
}