715 lines
28 KiB
HTML
715 lines
28 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="light sidebar-visible" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>RAG Integration - VAPORA Platform Documentation</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="Comprehensive documentation for VAPORA, an intelligent development orchestration platform built entirely in Rust.">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
|
|
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
|
|
|
|
<!-- Provide site root and default themes to javascript -->
|
|
<script>
|
|
const path_to_root = "../";
|
|
const default_light_theme = "light";
|
|
const default_dark_theme = "dark";
|
|
</script>
|
|
<!-- Start loading toc.js asap -->
|
|
<script src="../toc.js"></script>
|
|
</head>
|
|
<body>
|
|
<div id="mdbook-help-container">
|
|
<div id="mdbook-help-popup">
|
|
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
|
|
<div>
|
|
<p>Press <kbd>←</kbd> or <kbd>→</kbd> to navigate between chapters</p>
|
|
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
|
|
<p>Press <kbd>?</kbd> to show this help</p>
|
|
<p>Press <kbd>Esc</kbd> to hide this help</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div id="body-container">
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
let theme = localStorage.getItem('mdbook-theme');
|
|
let sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
|
let theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
const html = document.documentElement;
|
|
html.classList.remove('light')
|
|
html.classList.add(theme);
|
|
html.classList.add("js");
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
let sidebar = null;
|
|
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<!-- populated by js -->
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
|
<noscript>
|
|
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
|
|
</noscript>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
|
<div class="sidebar-resize-indicator"></div>
|
|
</div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">VAPORA Platform Documentation</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
<a href="https://github.com/vapora-platform/vapora" title="Git repository" aria-label="Git repository">
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
|
</a>
|
|
<a href="https://github.com/vapora-platform/vapora/edit/main/docs/src/../integrations/rag-integration.md" title="Suggest an edit" aria-label="Suggest an edit">
|
|
<i id="git-edit-button" class="fa fa-edit"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<h1 id="-rag-integration"><a class="header" href="#-rag-integration">🔍 RAG Integration</a></h1>
|
|
<h2 id="retrievable-augmented-generation-for-vapora-context"><a class="header" href="#retrievable-augmented-generation-for-vapora-context">Retrievable Augmented Generation for VAPORA Context</a></h2>
|
|
<p><strong>Version</strong>: 0.1.0
|
|
<strong>Status</strong>: Specification (VAPORA v1.0 Integration)
|
|
<strong>Purpose</strong>: RAG system from provisioning integrated into VAPORA for semantic search</p>
|
|
<hr />
|
|
<h2 id="-objetivo"><a class="header" href="#-objetivo">🎯 Objetivo</a></h2>
|
|
<p><strong>RAG (Retrieval-Augmented Generation)</strong> proporciona contexto a los agentes:</p>
|
|
<ul>
|
|
<li>✅ Agentes buscan documentación semánticamente similar</li>
|
|
<li>✅ ADRs, diseños, y guías como contexto para nuevas tareas</li>
|
|
<li>✅ Query LLM con documentación relevante</li>
|
|
<li>✅ Reducir alucinaciones, mejorar decisiones</li>
|
|
<li>✅ Sistema completo de provisioning (2,140 líneas Rust)</li>
|
|
</ul>
|
|
<hr />
|
|
<h2 id="-rag-architecture"><a class="header" href="#-rag-architecture">🏗️ RAG Architecture</a></h2>
|
|
<h3 id="components-from-provisioning"><a class="header" href="#components-from-provisioning">Components (From Provisioning)</a></h3>
|
|
<pre><code>RAG System (2,140 lines, production-ready from provisioning)
|
|
├─ Chunking Engine
|
|
│ ├─ Markdown chunks (with metadata)
|
|
│ ├─ KCL chunks (for infrastructure docs)
|
|
│ ├─ Nushell chunks (for scripts)
|
|
│ └─ Smart splitting (at headers, code blocks)
|
|
│
|
|
├─ Embeddings
|
|
│ ├─ Primary: OpenAI API (text-embedding-3-small)
|
|
│ ├─ Fallback: Local ONNX (nomic-embed-text)
|
|
│ ├─ Dimension: 1536-dim vectors
|
|
│ └─ Batch processing
|
|
│
|
|
├─ Vector Store
|
|
│ ├─ SurrealDB with HNSW index
|
|
│ ├─ Fast similarity search
|
|
│ ├─ Scalar product distance metric
|
|
│ └─ Replication for redundancy
|
|
│
|
|
├─ Retrieval
|
|
│ ├─ Top-K BM25 + semantic hybrid
|
|
│ ├─ Threshold filtering (relevance > 0.7)
|
|
│ ├─ Context enrichment
|
|
│ └─ Ranking/re-ranking
|
|
│
|
|
└─ Integration
|
|
├─ Claude API with full context
|
|
├─ Agent Search tool
|
|
├─ Workflow context injection
|
|
└─ Decision-making support
|
|
</code></pre>
|
|
<h3 id="data-flow"><a class="header" href="#data-flow">Data Flow</a></h3>
|
|
<pre><code>Document Added to docs/
|
|
↓
|
|
doc-lifecycle-manager classifies
|
|
↓
|
|
RAG Chunking Engine
|
|
├─ Split into semantic chunks
|
|
└─ Extract metadata (title, type, date)
|
|
↓
|
|
Embeddings Generator
|
|
├─ Generate 1536-dim vector per chunk
|
|
└─ Batch process for efficiency
|
|
↓
|
|
Vector Store (SurrealDB HNSW)
|
|
├─ Store chunk + vector + metadata
|
|
└─ Create HNSW index
|
|
↓
|
|
Search Ready
|
|
├─ Agent can query
|
|
├─ Semantic similarity search
|
|
└─ Fast < 100ms latency
|
|
</code></pre>
|
|
<hr />
|
|
<h2 id="-rag-in-vapora"><a class="header" href="#-rag-in-vapora">🔧 RAG in VAPORA</a></h2>
|
|
<h3 id="search-tool-available-to-all-agents"><a class="header" href="#search-tool-available-to-all-agents">Search Tool (Available to All Agents)</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>pub struct SearchTool {
|
|
pub vector_store: SurrealDB,
|
|
pub embeddings: EmbeddingsClient,
|
|
pub retriever: HybridRetriever,
|
|
}
|
|
|
|
impl SearchTool {
|
|
pub async fn search(
|
|
&self,
|
|
query: String,
|
|
top_k: u32,
|
|
threshold: f64,
|
|
) -> anyhow::Result<SearchResults> {
|
|
// 1. Embed query
|
|
let query_vector = self.embeddings.embed(&query).await?;
|
|
|
|
// 2. Search vector store
|
|
let chunk_results = self.vector_store.search_hnsw(
|
|
query_vector,
|
|
top_k,
|
|
threshold,
|
|
).await?;
|
|
|
|
// 3. Enrich with context
|
|
let results = self.enrich_results(chunk_results).await?;
|
|
|
|
Ok(SearchResults {
|
|
query,
|
|
results,
|
|
total_chunks_searched: 1000+,
|
|
search_duration_ms: 45,
|
|
})
|
|
}
|
|
|
|
pub async fn search_with_filters(
|
|
&self,
|
|
query: String,
|
|
filters: SearchFilters,
|
|
) -> anyhow::Result<SearchResults> {
|
|
// Filter by document type, date, tags before search
|
|
let filtered_documents = self.filter_documents(&filters).await?;
|
|
// ... rest of search
|
|
}
|
|
}
|
|
|
|
pub struct SearchFilters {
|
|
pub doc_type: Option<Vec<String>>, // ["adr", "guide"]
|
|
pub date_range: Option<(Date, Date)>,
|
|
pub tags: Option<Vec<String>>, // ["orchestrator", "performance"]
|
|
pub lifecycle_state: Option<String>, // "published", "archived"
|
|
}
|
|
|
|
pub struct SearchResults {
|
|
pub query: String,
|
|
pub results: Vec<SearchResult>,
|
|
pub total_chunks_searched: u32,
|
|
pub search_duration_ms: u32,
|
|
}
|
|
|
|
pub struct SearchResult {
|
|
pub document_id: String,
|
|
pub document_title: String,
|
|
pub chunk_text: String,
|
|
pub relevance_score: f64, // 0.0-1.0
|
|
pub metadata: HashMap<String, String>,
|
|
pub source_url: String,
|
|
pub snippet_context: String, // Surrounding text
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<h3 id="agent-usage-example"><a class="header" href="#agent-usage-example">Agent Usage Example</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>// Agent decides to search for context
|
|
impl DeveloperAgent {
|
|
pub async fn implement_feature(
|
|
&mut self,
|
|
task: Task,
|
|
) -> anyhow::Result<()> {
|
|
// 1. Search for similar features implemented before
|
|
let similar_features = self.search_tool.search(
|
|
format!("implement {} feature like {}", task.domain, task.type_),
|
|
top_k: 5,
|
|
threshold: 0.75,
|
|
).await?;
|
|
|
|
// 2. Extract context from results
|
|
let context_docs = similar_features.results
|
|
.iter()
|
|
.map(|r| r.chunk_text.clone())
|
|
.collect::<Vec<_>>();
|
|
|
|
// 3. Build LLM prompt with context
|
|
let prompt = format!(
|
|
"Implement the following feature:\n{}\n\nSimilar features implemented:\n{}",
|
|
task.description,
|
|
context_docs.join("\n---\n")
|
|
);
|
|
|
|
// 4. Generate code with context
|
|
let code = self.llm_router.complete(prompt).await?;
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<h3 id="documenter-agent-integration"><a class="header" href="#documenter-agent-integration">Documenter Agent Integration</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>impl DocumenterAgent {
|
|
pub async fn update_documentation(
|
|
&mut self,
|
|
task: Task,
|
|
) -> anyhow::Result<()> {
|
|
// 1. Get decisions from task
|
|
let decisions = task.extract_decisions().await?;
|
|
|
|
for decision in decisions {
|
|
// 2. Search existing ADRs to avoid duplicates
|
|
let similar_adrs = self.search_tool.search(
|
|
decision.context.clone(),
|
|
top_k: 3,
|
|
threshold: 0.8,
|
|
).await?;
|
|
|
|
// 3. Check if decision already documented
|
|
if similar_adrs.results.is_empty() {
|
|
// Create new ADR
|
|
let adr_content = format!(
|
|
"# {}\n\n## Context\n{}\n\n## Decision\n{}",
|
|
decision.title,
|
|
decision.context,
|
|
decision.chosen_option,
|
|
);
|
|
|
|
// 4. Save and index for RAG
|
|
self.db.save_adr(&adr_content).await?;
|
|
self.rag_system.index_document(&adr_content).await?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<hr />
|
|
<h2 id="-rag-implementation-from-provisioning"><a class="header" href="#-rag-implementation-from-provisioning">📊 RAG Implementation (From Provisioning)</a></h2>
|
|
<h3 id="schema-surrealdb"><a class="header" href="#schema-surrealdb">Schema (SurrealDB)</a></h3>
|
|
<pre><code class="language-sql">-- RAG chunks table
|
|
CREATE TABLE rag_chunks SCHEMAFULL {
|
|
-- Identifiers
|
|
id: string,
|
|
document_id: string,
|
|
chunk_index: int,
|
|
|
|
-- Content
|
|
text: string,
|
|
title: string,
|
|
doc_type: string,
|
|
|
|
-- Vector
|
|
embedding: vector<1536>,
|
|
|
|
-- Metadata
|
|
created_date: datetime,
|
|
last_updated: datetime,
|
|
source_path: string,
|
|
tags: array<string>,
|
|
lifecycle_state: string,
|
|
|
|
-- Indexing
|
|
INDEX embedding ON HNSW (1536) FIELDS embedding
|
|
DISTANCE SCALAR PRODUCT
|
|
M 16
|
|
EF_CONSTRUCTION 200,
|
|
|
|
PERMISSIONS
|
|
FOR select ALLOW (true)
|
|
FOR create ALLOW (true)
|
|
FOR update ALLOW (false)
|
|
FOR delete ALLOW (false)
|
|
};
|
|
</code></pre>
|
|
<h3 id="chunking-strategy"><a class="header" href="#chunking-strategy">Chunking Strategy</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>pub struct ChunkingEngine;
|
|
|
|
impl ChunkingEngine {
|
|
pub async fn chunk_document(
|
|
&self,
|
|
document: Document,
|
|
) -> anyhow::Result<Vec<Chunk>> {
|
|
let chunks = match document.file_type {
|
|
FileType::Markdown => self.chunk_markdown(&document.content)?,
|
|
FileType::KCL => self.chunk_kcl(&document.content)?,
|
|
FileType::Nushell => self.chunk_nushell(&document.content)?,
|
|
_ => self.chunk_text(&document.content)?,
|
|
};
|
|
|
|
Ok(chunks)
|
|
}
|
|
|
|
fn chunk_markdown(&self, content: &str) -> anyhow::Result<Vec<Chunk>> {
|
|
let mut chunks = Vec::new();
|
|
|
|
// Split by headers
|
|
let sections = content.split(|line: &str| line.starts_with('#'));
|
|
|
|
for section in sections {
|
|
// Max 500 tokens per chunk
|
|
if section.len() > 500 {
|
|
// Split further
|
|
for sub_chunk in section.chunks(400) {
|
|
chunks.push(Chunk {
|
|
text: sub_chunk.to_string(),
|
|
metadata: Default::default(),
|
|
});
|
|
}
|
|
} else {
|
|
chunks.push(Chunk {
|
|
text: section.to_string(),
|
|
metadata: Default::default(),
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(chunks)
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<h3 id="embeddings"><a class="header" href="#embeddings">Embeddings</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>pub enum EmbeddingsProvider {
|
|
OpenAI {
|
|
api_key: String,
|
|
model: "text-embedding-3-small", // 1536 dims, fast
|
|
},
|
|
Local {
|
|
model_path: String, // ONNX model
|
|
model: "nomic-embed-text",
|
|
},
|
|
}
|
|
|
|
pub struct EmbeddingsClient {
|
|
provider: EmbeddingsProvider,
|
|
}
|
|
|
|
impl EmbeddingsClient {
|
|
pub async fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
|
|
match &self.provider {
|
|
EmbeddingsProvider::OpenAI { api_key, .. } => {
|
|
// Call OpenAI API
|
|
let response = reqwest::Client::new()
|
|
.post("https://api.openai.com/v1/embeddings")
|
|
.bearer_auth(api_key)
|
|
.json(&serde_json::json!({
|
|
"model": "text-embedding-3-small",
|
|
"input": text,
|
|
}))
|
|
.send()
|
|
.await?;
|
|
|
|
let result: OpenAIResponse = response.json().await?;
|
|
Ok(result.data[0].embedding.clone())
|
|
},
|
|
EmbeddingsProvider::Local { model_path, .. } => {
|
|
// Use local ONNX model (nomic-embed-text)
|
|
let session = ort::Session::builder()?.commit_from_file(model_path)?;
|
|
|
|
let output = session.run(ort::inputs![text]?)?;
|
|
let embedding = output[0].try_extract_tensor()?.view().to_owned();
|
|
|
|
Ok(embedding.iter().map(|x| *x as f32).collect())
|
|
},
|
|
}
|
|
}
|
|
|
|
pub async fn embed_batch(
|
|
&self,
|
|
texts: Vec<String>,
|
|
) -> anyhow::Result<Vec<Vec<f32>>> {
|
|
// Batch embed for efficiency
|
|
// (Use batching API for OpenAI, etc.)
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<h3 id="retrieval"><a class="header" href="#retrieval">Retrieval</a></h3>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>pub struct HybridRetriever {
|
|
vector_store: SurrealDB,
|
|
bm25_index: BM25Index,
|
|
}
|
|
|
|
impl HybridRetriever {
|
|
pub async fn search(
|
|
&self,
|
|
query: String,
|
|
top_k: u32,
|
|
) -> anyhow::Result<Vec<ChunkWithScore>> {
|
|
// 1. Semantic search (vector similarity)
|
|
let query_vector = self.embed(&query).await?;
|
|
let semantic_results = self.vector_store.search_hnsw(
|
|
query_vector,
|
|
top_k * 2, // Get more for re-ranking
|
|
0.5,
|
|
).await?;
|
|
|
|
// 2. BM25 keyword search
|
|
let bm25_results = self.bm25_index.search(&query, top_k * 2)?;
|
|
|
|
// 3. Merge and re-rank
|
|
let mut merged = HashMap::new();
|
|
|
|
for (i, result) in semantic_results.iter().enumerate() {
|
|
let score = 1.0 / (i as f64 + 1.0); // Rank-based score
|
|
merged.entry(result.id.clone())
|
|
.and_modify(|s: &mut f64| *s += score * 0.7) // 70% weight
|
|
.or_insert(score * 0.7);
|
|
}
|
|
|
|
for (i, result) in bm25_results.iter().enumerate() {
|
|
let score = 1.0 / (i as f64 + 1.0);
|
|
merged.entry(result.id.clone())
|
|
.and_modify(|s: &mut f64| *s += score * 0.3) // 30% weight
|
|
.or_insert(score * 0.3);
|
|
}
|
|
|
|
// 4. Sort and return top-k
|
|
let mut final_results: Vec<_> = merged.into_iter().collect();
|
|
final_results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
|
|
|
Ok(final_results.into_iter()
|
|
.take(top_k as usize)
|
|
.map(|(id, score)| {
|
|
// Fetch full chunk with this score
|
|
ChunkWithScore { id, score }
|
|
})
|
|
.collect())
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<hr />
|
|
<h2 id="-indexing-workflow"><a class="header" href="#-indexing-workflow">📚 Indexing Workflow</a></h2>
|
|
<h3 id="automatic-indexing"><a class="header" href="#automatic-indexing">Automatic Indexing</a></h3>
|
|
<pre><code>File added to docs/
|
|
↓
|
|
Git hook or workflow trigger
|
|
↓
|
|
doc-lifecycle-manager processes
|
|
├─ Classifies document
|
|
└─ Publishes "document_added" event
|
|
↓
|
|
RAG system subscribes
|
|
├─ Chunks document
|
|
├─ Generates embeddings
|
|
├─ Stores in SurrealDB
|
|
└─ Updates HNSW index
|
|
↓
|
|
Agent Search Tool ready
|
|
</code></pre>
|
|
<h3 id="batch-reindexing"><a class="header" href="#batch-reindexing">Batch Reindexing</a></h3>
|
|
<pre><code class="language-bash"># Periodic full reindex (daily or on demand)
|
|
vapora rag reindex --all
|
|
|
|
# Incremental reindex (only changed docs)
|
|
vapora rag reindex --since 1d
|
|
|
|
# Rebuild HNSW index from scratch
|
|
vapora rag rebuild-index --optimize
|
|
</code></pre>
|
|
<hr />
|
|
<h2 id="-implementation-checklist"><a class="header" href="#-implementation-checklist">🎯 Implementation Checklist</a></h2>
|
|
<ul>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Port RAG system from provisioning (2,140 lines)</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Integrate with SurrealDB vector store</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
HNSW index setup + optimization</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Chunking strategies (Markdown, KCL, Nushell)</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Embeddings client (OpenAI + local fallback)</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Hybrid retrieval (semantic + BM25)</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Search tool for agents</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
doc-lifecycle-manager hooks</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Indexing workflows</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Batch reindexing</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
CLI: <code>vapora rag search</code>, <code>vapora rag reindex</code></li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Tests + benchmarks</li>
|
|
</ul>
|
|
<hr />
|
|
<h2 id="-success-metrics"><a class="header" href="#-success-metrics">📊 Success Metrics</a></h2>
|
|
<p>✅ Search latency < 100ms (p99)
|
|
✅ Relevance score > 0.8 for top results
|
|
✅ 1000+ documents indexed
|
|
✅ HNSW index memory efficient
|
|
✅ Agents find relevant context automatically
|
|
✅ No hallucinations from out-of-context queries</p>
|
|
<hr />
|
|
<p><strong>Version</strong>: 0.1.0
|
|
<strong>Status</strong>: ✅ Integration Specification Complete
|
|
<strong>Purpose</strong>: RAG system for semantic document search in VAPORA</p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../../integrations/doc-lifecycle-integration.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../../integrations/provisioning-integration.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../../integrations/doc-lifecycle-integration.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../../integrations/provisioning-integration.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js"></script>
|
|
<script src="../mark.min.js"></script>
|
|
<script src="../searcher.js"></script>
|
|
|
|
<script src="../clipboard.min.js"></script>
|
|
<script src="../highlight.js"></script>
|
|
<script src="../book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|