chore: update crates, names and clippy fixes
This commit is contained in:
parent
069c8785a9
commit
fc1c699795
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,5 +1,7 @@
|
|||||||
.p
|
.p
|
||||||
.claude
|
.claude
|
||||||
|
.opencode
|
||||||
|
AGENTS.md
|
||||||
.vscode
|
.vscode
|
||||||
.shellcheckrc
|
.shellcheckrc
|
||||||
.coder
|
.coder
|
||||||
|
|||||||
9
.gitmodules
vendored
Normal file
9
.gitmodules
vendored
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[submodule "secretumvault"]
|
||||||
|
path = secretumvault
|
||||||
|
url = ssh://git@repo.jesusperez.pro:32225/jesus/secretumvault.git
|
||||||
|
[submodule "stratumiops"]
|
||||||
|
path = stratumiops
|
||||||
|
url = ssh://git@repo.jesusperez.pro:32225/jesus/stratumiops.git
|
||||||
|
[submodule "syntaxis"]
|
||||||
|
path = syntaxis
|
||||||
|
url = ssh://git@repo.jesusperez.pro:32225/jesus/syntaxis.git
|
||||||
177
Cargo.toml
177
Cargo.toml
@ -9,11 +9,26 @@ members = [
|
|||||||
"crates/control-center",
|
"crates/control-center",
|
||||||
"crates/control-center-ui",
|
"crates/control-center-ui",
|
||||||
"crates/vault-service",
|
"crates/vault-service",
|
||||||
"crates/rag",
|
|
||||||
"crates/detector",
|
"crates/detector",
|
||||||
"crates/mcp-server",
|
"crates/mcp-server",
|
||||||
"crates/provisioning-daemon",
|
"crates/daemon",
|
||||||
|
"prov-ecosystem/crates/daemon-cli",
|
||||||
|
"prov-ecosystem/crates/machines",
|
||||||
|
"prov-ecosystem/crates/encrypt",
|
||||||
|
"prov-ecosystem/crates/backup",
|
||||||
|
"prov-ecosystem/crates/observability",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
exclude = [
|
||||||
|
"syntaxis",
|
||||||
|
"syntaxis/core",
|
||||||
|
"prov-ecosystem/crates/syntaxis-integration",
|
||||||
|
"prov-ecosystem/crates/audit",
|
||||||
|
"prov-ecosystem/crates/valida",
|
||||||
|
"prov-ecosystem/crates/runtime",
|
||||||
|
"prov-ecosystem/crates/gitops",
|
||||||
|
]
|
||||||
|
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
@ -39,7 +54,7 @@ resolver = "2"
|
|||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
toml = "0.9"
|
toml = "0.9"
|
||||||
uuid = { version = "1.19", features = ["v4", "serde"] }
|
uuid = { version = "1.20", features = ["v4", "serde"] }
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# ERROR HANDLING
|
# ERROR HANDLING
|
||||||
@ -80,7 +95,7 @@ resolver = "2"
|
|||||||
# DATABASE AND STORAGE
|
# DATABASE AND STORAGE
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite", "chrono", "uuid"] }
|
sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite", "chrono", "uuid"] }
|
||||||
surrealdb = { version = "2.4", features = ["kv-mem", "protocol-ws", "protocol-http"] }
|
surrealdb = { version = "2.6", features = ["kv-mem", "protocol-ws", "protocol-http"] }
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# SECURITY AND CRYPTOGRAPHY
|
# SECURITY AND CRYPTOGRAPHY
|
||||||
@ -89,7 +104,7 @@ resolver = "2"
|
|||||||
argon2 = "0.5"
|
argon2 = "0.5"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
hmac = "0.12"
|
hmac = "0.12"
|
||||||
jsonwebtoken = { version = "10.2", features = ["rust_crypto"] }
|
jsonwebtoken = { version = "10.3", features = ["rust_crypto"] }
|
||||||
rand = { version = "0.9", features = ["std_rng", "os_rng"] }
|
rand = { version = "0.9", features = ["std_rng", "os_rng"] }
|
||||||
ring = "0.17"
|
ring = "0.17"
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
@ -127,7 +142,7 @@ resolver = "2"
|
|||||||
|
|
||||||
# Additional cryptography
|
# Additional cryptography
|
||||||
hkdf = "0.12"
|
hkdf = "0.12"
|
||||||
rsa = "0.9.9"
|
rsa = "0.9.10"
|
||||||
zeroize = { version = "1.8", features = ["derive"] }
|
zeroize = { version = "1.8", features = ["derive"] }
|
||||||
|
|
||||||
# Additional security
|
# Additional security
|
||||||
@ -186,7 +201,7 @@ resolver = "2"
|
|||||||
wasm-bindgen-futures = "0.4"
|
wasm-bindgen-futures = "0.4"
|
||||||
|
|
||||||
# Random number generation
|
# Random number generation
|
||||||
getrandom = { version = "0.3" }
|
getrandom = { version = "0.4" }
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# TUI (Terminal User Interface)
|
# TUI (Terminal User Interface)
|
||||||
@ -216,28 +231,164 @@ resolver = "2"
|
|||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
which = "8"
|
which = "8"
|
||||||
yaml-rust = "0.4"
|
yaml-rust = "0.4"
|
||||||
|
humantime-serde = "1.1"
|
||||||
|
|
||||||
|
# Metrics
|
||||||
|
prometheus = "0.14"
|
||||||
|
|
||||||
|
approx = "0.5"
|
||||||
|
|
||||||
|
# Utilities
|
||||||
|
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# RAG FRAMEWORK DEPENDENCIES (Rig)
|
# RAG FRAMEWORK DEPENDENCIES (Rig)
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
rig-core = "0.27"
|
rig-core = "0.30"
|
||||||
rig-surrealdb = "0.1"
|
rig-surrealdb = "0.1"
|
||||||
tokenizers = "0.22"
|
tokenizers = "0.22"
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# PROV-ECOSYSTEM DAEMON (replaces cli-daemon)
|
# STRATUM ECOSYSTEM DEPENDENCIES (for RAG embeddings & LLM)
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
daemon-cli = { path = "../../submodules/prov-ecosystem/crates/daemon-cli" }
|
moka = { version = "0.12", features = ["future"] }
|
||||||
|
sled = "0.34"
|
||||||
|
fastembed = "5.8"
|
||||||
|
lancedb = "0.23"
|
||||||
|
arrow = "=56"
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# SECRETUMVAULT (Enterprise Secrets Management)
|
# INTERNAL WORKSPACE CRATES (Local path dependencies)
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
secretumvault = { path = "../../submodules/secretumvault" }
|
platform-config = { path = "./crates/platform-config" }
|
||||||
|
service-clients = { path = "./crates/service-clients" }
|
||||||
|
rag = { path = "./crates/rag" }
|
||||||
|
mcp-server = { path = "./crates/mcp-server" }
|
||||||
|
ai-service = { path = "./crates/ai-service" }
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# PROV-ECOSYSTEM (Now members of workspace)
|
||||||
|
# ============================================================================
|
||||||
|
daemon-cli = { path = "./prov-ecosystem/crates/daemon-cli" }
|
||||||
|
machines = { path = "./prov-ecosystem/crates/machines" }
|
||||||
|
encrypt = { path = "./prov-ecosystem/crates/encrypt" }
|
||||||
|
backup = { path = "./prov-ecosystem/crates/backup" }
|
||||||
|
observability = { path = "./prov-ecosystem/crates/observability" }
|
||||||
|
init-servs = { path = "./prov-ecosystem/crates/init-servs" }
|
||||||
|
|
||||||
|
# stratum-embeddings and stratum-llm are built in isolated Docker context for RAG
|
||||||
|
# See: crates/rag/docker/Dockerfile
|
||||||
|
stratum-embeddings = { path = "./stratumiops/crates/stratum-embeddings", features = ["openai-provider", "ollama-provider", "fastembed-provider", "memory-cache"] }
|
||||||
|
stratum-llm = { path = "./stratumiops/crates/stratum-llm", features = ["anthropic", "openai", "ollama"] }
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# SECRETUMVAULT (Enterprise Secrets Management - optional)
|
||||||
|
# ============================================================================
|
||||||
|
secretumvault = { path = "./secretumvault" }
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# WASM/WEB-SPECIFIC DEPENDENCIES
|
||||||
|
# ============================================================================
|
||||||
|
web-sys = { version = "0.3", features = [
|
||||||
|
"console",
|
||||||
|
"Window",
|
||||||
|
"Document",
|
||||||
|
"Element",
|
||||||
|
"HtmlElement",
|
||||||
|
"HtmlCanvasElement",
|
||||||
|
"CanvasRenderingContext2d",
|
||||||
|
"EventTarget",
|
||||||
|
"Event",
|
||||||
|
"DragEvent",
|
||||||
|
"DataTransfer",
|
||||||
|
"HtmlInputElement",
|
||||||
|
"HtmlSelectElement",
|
||||||
|
"HtmlTextAreaElement",
|
||||||
|
"HtmlButtonElement",
|
||||||
|
"HtmlDivElement",
|
||||||
|
"Storage",
|
||||||
|
"Location",
|
||||||
|
"History",
|
||||||
|
"Navigator",
|
||||||
|
"ServiceWorkerRegistration",
|
||||||
|
"ServiceWorker",
|
||||||
|
"NotificationPermission",
|
||||||
|
"Notification",
|
||||||
|
"Headers",
|
||||||
|
"Request",
|
||||||
|
"RequestInit",
|
||||||
|
"RequestMode",
|
||||||
|
"Response",
|
||||||
|
"AbortController",
|
||||||
|
"AbortSignal",
|
||||||
|
"WebSocket",
|
||||||
|
"MessageEvent",
|
||||||
|
"CloseEvent",
|
||||||
|
"ErrorEvent",
|
||||||
|
"Blob",
|
||||||
|
"Url",
|
||||||
|
"FileReader",
|
||||||
|
"File",
|
||||||
|
"HtmlAnchorElement",
|
||||||
|
"MouseEvent",
|
||||||
|
"TouchEvent",
|
||||||
|
"KeyboardEvent",
|
||||||
|
"ResizeObserver",
|
||||||
|
"ResizeObserverEntry",
|
||||||
|
"IntersectionObserver",
|
||||||
|
"IntersectionObserverEntry",
|
||||||
|
"MediaQueryList",
|
||||||
|
"MediaQueryListEvent",
|
||||||
|
"CredentialsContainer",
|
||||||
|
"PublicKeyCredential",
|
||||||
|
"PublicKeyCredentialCreationOptions",
|
||||||
|
"PublicKeyCredentialRequestOptions",
|
||||||
|
"AuthenticatorResponse",
|
||||||
|
"AuthenticatorAttestationResponse",
|
||||||
|
"AuthenticatorAssertionResponse",
|
||||||
|
"Crypto",
|
||||||
|
"SubtleCrypto",
|
||||||
|
"CryptoKey",
|
||||||
|
] }
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# ADDITIONAL MISSING DEPENDENCIES (Not in original workspace)
|
||||||
|
# ============================================================================
|
||||||
|
ed25519-dalek = "2.2"
|
||||||
|
http-body-util = "0.1"
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# BYTES MANIPULATION
|
# BYTES MANIPULATION
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
bytes = "1.5"
|
bytes = "1.11"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# HTTP AND PROTOCOL UTILITIES
|
||||||
|
# ============================================================================
|
||||||
|
http = "1"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# CONTAINER MANAGEMENT AND SSH
|
||||||
|
# ============================================================================
|
||||||
|
bollard = "0.20"
|
||||||
|
russh = "0.57"
|
||||||
|
russh-keys = "0.49"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# SECRETS MANAGEMENT
|
||||||
|
# ============================================================================
|
||||||
|
age = "0.11"
|
||||||
|
rusty_vault = "0.2.1"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# ADDITIONAL DATA FORMAT SERIALIZATION
|
||||||
|
# ============================================================================
|
||||||
|
serde_yaml = "0.9"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# PATH AND SHELL UTILITIES
|
||||||
|
# ============================================================================
|
||||||
|
shellexpand = "3.1"
|
||||||
|
|
||||||
[workspace.metadata]
|
[workspace.metadata]
|
||||||
description = "Provisioning Platform - Rust workspace for cloud infrastructure automation tools"
|
description = "Provisioning Platform - Rust workspace for cloud infrastructure automation tools"
|
||||||
|
|||||||
@ -5,6 +5,10 @@ edition.workspace = true
|
|||||||
name = "ai-service"
|
name = "ai-service"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-ai-service"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Workspace dependencies
|
# Workspace dependencies
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
@ -22,7 +26,7 @@ serde_json = { workspace = true }
|
|||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration
|
# Platform configuration
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# Error handling
|
# Error handling
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
@ -40,14 +44,18 @@ uuid = { workspace = true, features = ["v4", "serde"] }
|
|||||||
clap = { workspace = true, features = ["derive"] }
|
clap = { workspace = true, features = ["derive"] }
|
||||||
|
|
||||||
# RAG crate for AI capabilities
|
# RAG crate for AI capabilities
|
||||||
provisioning-rag = { path = "../rag" }
|
rag = { workspace = true }
|
||||||
|
|
||||||
# MCP server tools for real implementations
|
# MCP server tools for real implementations
|
||||||
provisioning-mcp-server = { path = "../mcp-server" }
|
mcp-server = { workspace = true }
|
||||||
|
|
||||||
# Graph operations for DAG
|
# Graph operations for DAG
|
||||||
petgraph = { workspace = true }
|
petgraph = { workspace = true }
|
||||||
|
|
||||||
|
# Stratum ecosystem - embeddings and LLM abstraction (optional - requires external setup)
|
||||||
|
stratum-embeddings = { workspace = true }
|
||||||
|
stratum-llm = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
tokio-test = { workspace = true }
|
tokio-test = { workspace = true }
|
||||||
@ -56,8 +64,3 @@ tokio-test = { workspace = true }
|
|||||||
[lib]
|
[lib]
|
||||||
name = "ai_service"
|
name = "ai_service"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# Binary target
|
|
||||||
[[bin]]
|
|
||||||
name = "ai-service"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|||||||
@ -13,12 +13,24 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
|||||||
#[command(name = "ai-service")]
|
#[command(name = "ai-service")]
|
||||||
#[command(about = "HTTP service for AI capabilities including RAG, MCP tool invocation, DAG operations, and knowledge graphs", long_about = None)]
|
#[command(about = "HTTP service for AI capabilities including RAG, MCP tool invocation, DAG operations, and knowledge graphs", long_about = None)]
|
||||||
struct Args {
|
struct Args {
|
||||||
|
/// Configuration file path (highest priority)
|
||||||
|
#[arg(short = 'c', long, env = "AI_SERVICE_CONFIG")]
|
||||||
|
config: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for ai-service.ncl|toml|json)
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
config_dir: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
#[arg(short = 'm', long, env = "AI_SERVICE_MODE")]
|
||||||
|
mode: Option<String>,
|
||||||
|
|
||||||
/// Service bind address
|
/// Service bind address
|
||||||
#[arg(short, long, default_value = "127.0.0.1")]
|
#[arg(short = 'H', long, default_value = "127.0.0.1")]
|
||||||
host: String,
|
host: String,
|
||||||
|
|
||||||
/// Service bind port
|
/// Service bind port
|
||||||
#[arg(short, long, default_value_t = DEFAULT_PORT)]
|
#[arg(short = 'p', long, default_value_t = DEFAULT_PORT)]
|
||||||
port: u16,
|
port: u16,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,9 +1,11 @@
|
|||||||
[package]
|
[package]
|
||||||
authors = ["Control Center Team"]
|
authors.workspace = true
|
||||||
autobins = false # Disable auto-detection of binary targets
|
autobins = false
|
||||||
description = "Control Center UI - Leptos CSR App for Cloud Infrastructure Management"
|
description = "Control Center UI - Leptos CSR App for Cloud Infrastructure Management"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
name = "control-center-ui"
|
name = "control-center-ui"
|
||||||
|
repository.workspace = true
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
@ -87,89 +89,21 @@ js-sys = { workspace = true }
|
|||||||
wasm-bindgen-futures = { workspace = true }
|
wasm-bindgen-futures = { workspace = true }
|
||||||
|
|
||||||
# Random number generation (WASM-specific override with js feature)
|
# Random number generation (WASM-specific override with js feature)
|
||||||
getrandom = { version = "0.3.4", features = ["wasm_js"] }
|
getrandom = { workspace = true, features = ["wasm_js"] }
|
||||||
|
|
||||||
# ============================================================================
|
# HTTP client
|
||||||
# PROJECT-SPECIFIC DEPENDENCIES (not in workspace)
|
reqwest = { workspace = true, features = ["json"] }
|
||||||
# ============================================================================
|
|
||||||
|
|
||||||
# Web APIs
|
# Tokio with time features
|
||||||
web-sys = { version = "0.3", features = [
|
tokio = { workspace = true, features = ["time"] }
|
||||||
"console",
|
|
||||||
"Window",
|
|
||||||
"Document",
|
|
||||||
"Element",
|
|
||||||
"HtmlElement",
|
|
||||||
"HtmlCanvasElement",
|
|
||||||
"CanvasRenderingContext2d",
|
|
||||||
"EventTarget",
|
|
||||||
"Event",
|
|
||||||
"DragEvent",
|
|
||||||
"DataTransfer",
|
|
||||||
"HtmlInputElement",
|
|
||||||
"HtmlSelectElement",
|
|
||||||
"HtmlTextAreaElement",
|
|
||||||
"HtmlButtonElement",
|
|
||||||
"HtmlDivElement",
|
|
||||||
"Storage",
|
|
||||||
"Location",
|
|
||||||
"History",
|
|
||||||
"Navigator",
|
|
||||||
"ServiceWorkerRegistration",
|
|
||||||
"ServiceWorker",
|
|
||||||
"NotificationPermission",
|
|
||||||
"Notification",
|
|
||||||
"Headers",
|
|
||||||
"Request",
|
|
||||||
"RequestInit",
|
|
||||||
"RequestMode",
|
|
||||||
"Response",
|
|
||||||
"AbortController",
|
|
||||||
"AbortSignal",
|
|
||||||
"WebSocket",
|
|
||||||
"MessageEvent",
|
|
||||||
"CloseEvent",
|
|
||||||
"ErrorEvent",
|
|
||||||
"Blob",
|
|
||||||
"Url",
|
|
||||||
"FileReader",
|
|
||||||
"File",
|
|
||||||
"HtmlAnchorElement",
|
|
||||||
"MouseEvent",
|
|
||||||
"TouchEvent",
|
|
||||||
"KeyboardEvent",
|
|
||||||
"ResizeObserver",
|
|
||||||
"ResizeObserverEntry",
|
|
||||||
"IntersectionObserver",
|
|
||||||
"IntersectionObserverEntry",
|
|
||||||
# Media Query APIs
|
|
||||||
"MediaQueryList",
|
|
||||||
"MediaQueryListEvent",
|
|
||||||
# WebAuthn APIs
|
|
||||||
"CredentialsContainer",
|
|
||||||
"PublicKeyCredential",
|
|
||||||
"PublicKeyCredentialCreationOptions",
|
|
||||||
"PublicKeyCredentialRequestOptions",
|
|
||||||
"AuthenticatorResponse",
|
|
||||||
"AuthenticatorAttestationResponse",
|
|
||||||
"AuthenticatorAssertionResponse",
|
|
||||||
# Crypto APIs
|
|
||||||
"Crypto",
|
|
||||||
"SubtleCrypto",
|
|
||||||
"CryptoKey",
|
|
||||||
] }
|
|
||||||
|
|
||||||
# HTTP client (project-specific for WASM features)
|
# Web APIs (WASM browser APIs)
|
||||||
reqwest = { version = "0.13", features = ["json"] }
|
web-sys = { workspace = true }
|
||||||
|
|
||||||
# Tokio with time features for WASM (project-specific version)
|
|
||||||
tokio = { version = "1.49", features = ["time"] }
|
|
||||||
|
|
||||||
# Profile configurations moved to workspace root
|
# Profile configurations moved to workspace root
|
||||||
|
|
||||||
# WASM pack settings
|
[package.metadata.wasm-pack.profile.release]
|
||||||
[package.metadata.wasm-pack.profile.release]
|
wasm-opt = ['-Oz', '--enable-mutable-globals']
|
||||||
wasm-opt = ['-Oz', '--enable-mutable-globals']
|
|
||||||
|
|
||||||
[package.metadata.wasm-pack.profile.dev]
|
[package.metadata.wasm-pack.profile.dev]
|
||||||
wasm-opt = false
|
wasm-opt = false
|
||||||
|
|||||||
@ -52,10 +52,10 @@ validator = { workspace = true }
|
|||||||
reqwest = { workspace = true }
|
reqwest = { workspace = true }
|
||||||
|
|
||||||
# HTTP service clients (machines, init, AI) - enables remote service calls
|
# HTTP service clients (machines, init, AI) - enables remote service calls
|
||||||
service-clients = { path = "../service-clients" }
|
service-clients = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration management
|
# Platform configuration management
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# Security and cryptography
|
# Security and cryptography
|
||||||
aes-gcm = { workspace = true }
|
aes-gcm = { workspace = true }
|
||||||
@ -153,9 +153,8 @@ compliance = ["core"]
|
|||||||
# Modules: anomaly (detection)
|
# Modules: anomaly (detection)
|
||||||
experimental = ["core"]
|
experimental = ["core"]
|
||||||
|
|
||||||
# Default: Recommended for standard deployments
|
# Default: All features enabled
|
||||||
# Includes auth, KMS, audit - the essentials
|
default = ["core", "kms", "audit", "mfa", "compliance", "experimental"]
|
||||||
default = ["core", "kms", "audit"]
|
|
||||||
|
|
||||||
# Full: All features enabled (development and testing)
|
# Full: All features enabled (development and testing)
|
||||||
all = ["core", "kms", "audit", "mfa", "compliance", "experimental"]
|
all = ["core", "kms", "audit", "mfa", "compliance", "experimental"]
|
||||||
@ -165,8 +164,7 @@ all = ["core", "kms", "audit", "mfa", "compliance", "experimental"]
|
|||||||
name = "control_center"
|
name = "control_center"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# Binary target (uses all features)
|
# Binary target (uses all features by default)
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "control-center"
|
name = "provisioning-control-center"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
required-features = ["all"]
|
|
||||||
|
|||||||
@ -1,65 +0,0 @@
|
|||||||
# Multi-stage build for Control-Center
|
|
||||||
# Builds from platform workspace root
|
|
||||||
|
|
||||||
# Build stage - Using nightly for edition2024 support (required by async-graphql 7.x)
|
|
||||||
FROM rustlang/rust:nightly-bookworm AS builder
|
|
||||||
|
|
||||||
WORKDIR /workspace
|
|
||||||
|
|
||||||
# Install build dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
pkg-config \
|
|
||||||
libssl-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Copy entire platform workspace (required for workspace dependencies)
|
|
||||||
COPY Cargo.toml Cargo.lock ./
|
|
||||||
COPY orchestrator ./orchestrator
|
|
||||||
COPY control-center ./control-center
|
|
||||||
COPY control-center-ui ./control-center-ui
|
|
||||||
COPY mcp-server ./mcp-server
|
|
||||||
COPY installer ./installer
|
|
||||||
|
|
||||||
# Build control-center (workspace-aware)
|
|
||||||
WORKDIR /workspace
|
|
||||||
RUN cargo build --release --package control-center
|
|
||||||
|
|
||||||
# Runtime stage
|
|
||||||
FROM debian:bookworm-slim
|
|
||||||
|
|
||||||
# Install runtime dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
ca-certificates \
|
|
||||||
curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Create non-root user
|
|
||||||
RUN useradd -m -u 1000 provisioning && \
|
|
||||||
mkdir -p /data /var/log/control-center && \
|
|
||||||
chown -R provisioning:provisioning /data /var/log/control-center
|
|
||||||
|
|
||||||
# Copy binary from builder
|
|
||||||
COPY --from=builder /workspace/target/release/control-center /usr/local/bin/control-center
|
|
||||||
RUN chmod +x /usr/local/bin/control-center
|
|
||||||
|
|
||||||
# Copy default configuration
|
|
||||||
COPY control-center/config.defaults.toml /etc/provisioning/config.defaults.toml
|
|
||||||
|
|
||||||
# Switch to non-root user
|
|
||||||
USER provisioning
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Expose port
|
|
||||||
EXPOSE 8081
|
|
||||||
|
|
||||||
# Set environment variables
|
|
||||||
ENV RUST_LOG=info
|
|
||||||
ENV DATA_DIR=/data
|
|
||||||
ENV CONTROL_CENTER_DATABASE_URL=rocksdb:///data/control-center.db
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
|
||||||
CMD curl -f http://localhost:8081/health || exit 1
|
|
||||||
|
|
||||||
# Run the binary with config path
|
|
||||||
CMD ["control-center", "--config", "/etc/provisioning/config.defaults.toml"]
|
|
||||||
@ -47,13 +47,30 @@ use tracing_subscriber::EnvFilter;
|
|||||||
#[command(name = "control-center")]
|
#[command(name = "control-center")]
|
||||||
#[command(about = "Control Center - JWT Authentication & User Management Service")]
|
#[command(about = "Control Center - JWT Authentication & User Management Service")]
|
||||||
#[command(version = env!("CARGO_PKG_VERSION"))]
|
#[command(version = env!("CARGO_PKG_VERSION"))]
|
||||||
|
#[command(after_help = "CONFIGURATION HIERARCHY (highest to lowest priority):\n 1. CLI: -c/--config <path> (explicit file)\n 2. CLI: --config-dir <dir> --mode <mode> (directory + mode)\n 3. CLI: --config-dir <dir> (searches for control-center.ncl|toml|json)\n 4. CLI: --mode <mode> (searches in provisioning/platform/config/)\n 5. ENV: CONTROL_CENTER_CONFIG (explicit file)\n 6. ENV: PROVISIONING_CONFIG_DIR (searches for control-center.ncl|toml|json)\n 7. ENV: CONTROL_CENTER_MODE (mode-based in default path)\n 8. Built-in defaults")]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Configuration file path
|
/// Configuration file path (highest priority)
|
||||||
#[arg(short, long, default_value = "config.toml")]
|
///
|
||||||
|
/// Accepts absolute or relative path. Supports .ncl, .toml, and .json formats.
|
||||||
|
#[arg(short = 'c', long, env = "CONTROL_CENTER_CONFIG")]
|
||||||
config: Option<PathBuf>,
|
config: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for control-center.ncl|toml|json)
|
||||||
|
///
|
||||||
|
/// Searches for configuration files in order of preference: .ncl > .toml > .json
|
||||||
|
/// Can also search for mode-specific files: control-center.{mode}.{ncl|toml|json}
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
config_dir: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
///
|
||||||
|
/// Determines which configuration profile to use. Searches in:
|
||||||
|
/// provisioning/platform/config/control-center.{mode}.{ncl|toml}
|
||||||
|
#[arg(short = 'm', long, env = "CONTROL_CENTER_MODE")]
|
||||||
|
mode: Option<String>,
|
||||||
|
|
||||||
/// Server port (overrides config file)
|
/// Server port (overrides config file)
|
||||||
#[arg(short, long)]
|
#[arg(short = 'p', long)]
|
||||||
port: Option<u16>,
|
port: Option<u16>,
|
||||||
|
|
||||||
/// Server host (overrides config file)
|
/// Server host (overrides config file)
|
||||||
@ -90,9 +107,15 @@ async fn main() -> Result<()> {
|
|||||||
.with_target(false)
|
.with_target(false)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
|
// Resolve config file path using new resolver
|
||||||
|
let resolver = platform_config::ConfigResolver::new()
|
||||||
|
.with_cli_config(cli.config.clone())
|
||||||
|
.with_cli_config_dir(cli.config_dir.clone())
|
||||||
|
.with_cli_mode(cli.mode.clone());
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
let mut config = if let Some(config_path) = cli.config {
|
let mut config = if let Some(path) = resolver.resolve("control-center") {
|
||||||
Config::load_from_file(config_path)?
|
Config::load_from_file(path)?
|
||||||
} else {
|
} else {
|
||||||
Config::load()?
|
Config::load()?
|
||||||
};
|
};
|
||||||
|
|||||||
@ -2,10 +2,14 @@
|
|||||||
authors.workspace = true
|
authors.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
name = "provisioning-daemon"
|
name = "daemon"
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-daemon"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Core daemon library from prov-ecosystem
|
# Core daemon library from prov-ecosystem
|
||||||
daemon-cli = { workspace = true }
|
daemon-cli = { workspace = true }
|
||||||
@ -22,7 +26,7 @@ serde_json = { workspace = true }
|
|||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration
|
# Platform configuration
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# Error handling
|
# Error handling
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
38
crates/daemon/Dockerfile.runtime
Normal file
38
crates/daemon/Dockerfile.runtime
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
# Install runtime dependencies
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
RUN useradd -m -u 1000 provisioning && \
|
||||||
|
mkdir -p /data /var/log/provisioning-daemon /etc/provisioning && \
|
||||||
|
chown -R provisioning:provisioning /data /var/log/provisioning-daemon /etc/provisioning
|
||||||
|
|
||||||
|
# Copy pre-built binary
|
||||||
|
COPY target/release/provisioning-daemon /usr/local/bin/provisioning-daemon
|
||||||
|
RUN chmod +x /usr/local/bin/provisioning-daemon
|
||||||
|
|
||||||
|
# Copy default configuration files (assumes they're available at build time)
|
||||||
|
COPY provisioning/platform/config/runtime/generated/provisioning-daemon.*.toml /etc/provisioning/
|
||||||
|
|
||||||
|
USER provisioning
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
EXPOSE 8079
|
||||||
|
|
||||||
|
ENV RUST_LOG=info
|
||||||
|
ENV DATA_DIR=/data
|
||||||
|
ENV PROVISIONING_DAEMON_MODE=solo
|
||||||
|
ENV PROVISIONING_CONFIG_DIR=/etc/provisioning
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:8079/api/v1/health || exit 1
|
||||||
|
|
||||||
|
# Configuration precedence:
|
||||||
|
# 1. PROVISIONING_DAEMON_CONFIG (explicit path)
|
||||||
|
# 2. PROVISIONING_DAEMON_MODE (mode-specific file)
|
||||||
|
# 3. Default fallback
|
||||||
|
CMD ["provisioning-daemon"]
|
||||||
@ -30,12 +30,20 @@ use tracing_subscriber::EnvFilter;
|
|||||||
#[command(about = "Provisioning platform daemon with Nushell execution and config rendering")]
|
#[command(about = "Provisioning platform daemon with Nushell execution and config rendering")]
|
||||||
#[command(version = env!("CARGO_PKG_VERSION"))]
|
#[command(version = env!("CARGO_PKG_VERSION"))]
|
||||||
struct Args {
|
struct Args {
|
||||||
/// Configuration file path
|
/// Configuration file path (highest priority)
|
||||||
#[arg(short, long)]
|
#[arg(short = 'c', long, env = "PROVISIONING_DAEMON_CONFIG")]
|
||||||
config: Option<PathBuf>,
|
config: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for provisioning-daemon.ncl|toml|json)
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
config_dir: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
#[arg(short = 'm', long, env = "PROVISIONING_DAEMON_MODE")]
|
||||||
|
mode: Option<String>,
|
||||||
|
|
||||||
/// Enable verbose logging
|
/// Enable verbose logging
|
||||||
#[arg(short, long)]
|
#[arg(short = 'v', long)]
|
||||||
verbose: bool,
|
verbose: bool,
|
||||||
|
|
||||||
/// Validate configuration and exit
|
/// Validate configuration and exit
|
||||||
@ -5,6 +5,10 @@ edition.workspace = true
|
|||||||
name = "extension-registry"
|
name = "extension-registry"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-extension-registry"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Workspace dependencies
|
# Workspace dependencies
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
@ -21,7 +25,7 @@ serde = { workspace = true, features = ["derive"] }
|
|||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration
|
# Platform configuration
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# Error handling
|
# Error handling
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
@ -61,7 +65,7 @@ parking_lot = { workspace = true }
|
|||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
http-body-util = "0.1"
|
http-body-util = { workspace = true }
|
||||||
hyper = { workspace = true }
|
hyper = { workspace = true }
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
tokio-test = { workspace = true }
|
tokio-test = { workspace = true }
|
||||||
@ -70,8 +74,3 @@ tokio-test = { workspace = true }
|
|||||||
[lib]
|
[lib]
|
||||||
name = "extension_registry"
|
name = "extension_registry"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# Binary target
|
|
||||||
[[bin]]
|
|
||||||
name = "extension-registry"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|||||||
@ -1,7 +1,34 @@
|
|||||||
# Build stage
|
# Multi-stage build for extension-registry
|
||||||
FROM rust:1.75-slim as builder
|
# Generated from Nickel template - DO NOT EDIT DIRECTLY
|
||||||
|
# Source: provisioning/schemas/platform/templates/docker/Dockerfile.chef.ncl
|
||||||
|
|
||||||
WORKDIR /app
|
# ============================================================================
|
||||||
|
# Stage 1: PLANNER - Generate dependency recipe
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS planner
|
||||||
|
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
# Install cargo-chef
|
||||||
|
RUN cargo install cargo-chef --version 0.1.67
|
||||||
|
|
||||||
|
# Copy workspace manifests
|
||||||
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
COPY crates ./crates
|
||||||
|
COPY daemon-cli ./daemon-cli
|
||||||
|
COPY secretumvault ./secretumvault
|
||||||
|
COPY prov-ecosystem ./prov-ecosystem
|
||||||
|
COPY stratumiops ./stratumiops
|
||||||
|
|
||||||
|
# Generate recipe.json (dependency graph)
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json --bin extension-registry
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Stage 2: CACHER - Build dependencies only
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS cacher
|
||||||
|
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
# Install build dependencies
|
# Install build dependencies
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
@ -9,40 +36,84 @@ RUN apt-get update && apt-get install -y \
|
|||||||
libssl-dev \
|
libssl-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Copy manifests
|
# Install cargo-chef
|
||||||
COPY Cargo.toml Cargo.lock ./
|
RUN cargo install cargo-chef --version 0.1.67
|
||||||
|
|
||||||
|
# sccache disabled
|
||||||
|
|
||||||
|
# Copy recipe from planner
|
||||||
|
COPY --from=planner /workspace/recipe.json recipe.json
|
||||||
|
|
||||||
|
# Build dependencies - This layer will be cached
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Stage 3: BUILDER - Build source code
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS builder
|
||||||
|
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
# Install build dependencies
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# sccache disabled
|
||||||
|
|
||||||
|
# Copy cached dependencies from cacher stage
|
||||||
|
COPY --from=cacher /workspace/target target
|
||||||
|
COPY --from=cacher /usr/local/cargo /usr/local/cargo
|
||||||
|
|
||||||
# Copy source code
|
# Copy source code
|
||||||
COPY src ./src
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
COPY crates ./crates
|
||||||
|
COPY daemon-cli ./daemon-cli
|
||||||
|
COPY secretumvault ./secretumvault
|
||||||
|
COPY prov-ecosystem ./prov-ecosystem
|
||||||
|
COPY stratumiops ./stratumiops
|
||||||
|
|
||||||
# Build release binary
|
# Build release binary with parallelism
|
||||||
RUN cargo build --release
|
ENV CARGO_BUILD_JOBS=4
|
||||||
|
RUN cargo build --release --package extension-registry
|
||||||
|
|
||||||
# Runtime stage
|
# ============================================================================
|
||||||
FROM debian:bookworm-slim
|
# Stage 4: RUNTIME - Minimal runtime image
|
||||||
|
# ============================================================================
|
||||||
|
FROM debian:trixie-slim
|
||||||
|
|
||||||
# Install runtime dependencies
|
# Install runtime dependencies
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
|
curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN useradd -m -u 1000 registry && \
|
RUN useradd -m -u 1000 provisioning && \
|
||||||
mkdir -p /app/data && \
|
mkdir -p /data /var/log/extension-registry && \
|
||||||
chown -R registry:registry /app
|
chown -R provisioning:provisioning /data /var/log/extension-registry
|
||||||
|
|
||||||
USER registry
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /app/target/release/extension-registry /usr/local/bin/
|
COPY --from=builder /workspace/target/release/extension-registry /usr/local/bin/extension-registry
|
||||||
|
RUN chmod +x /usr/local/bin/extension-registry
|
||||||
|
|
||||||
# Expose port
|
# No config file to copy
|
||||||
EXPOSE 8082
|
|
||||||
|
# Switch to non-root user
|
||||||
|
USER provisioning
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Expose service port
|
||||||
|
EXPOSE 9093
|
||||||
|
|
||||||
|
# Environment variables
|
||||||
|
ENV RUST_LOG=info
|
||||||
|
ENV DATA_DIR=/data
|
||||||
|
|
||||||
# Health check
|
# Health check
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||||
CMD curl -f http://localhost:8082/api/v1/health || exit 1
|
CMD curl -f http://localhost:9093/health || exit 1
|
||||||
|
|
||||||
# Run service
|
# Run the binary
|
||||||
CMD ["extension-registry"]
|
CMD ["extension-registry"]
|
||||||
|
|||||||
@ -287,5 +287,6 @@ pub fn routes(state: AppState) -> Router {
|
|||||||
.route("/extensions/:name", get(get_extension))
|
.route("/extensions/:name", get(get_extension))
|
||||||
// Health
|
// Health
|
||||||
.route("/health", get(health))
|
.route("/health", get(health))
|
||||||
|
.route("/api/v1/health", get(health))
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -13,6 +13,18 @@ use handlers::{routes, AppState};
|
|||||||
#[command(name = "extension-registry")]
|
#[command(name = "extension-registry")]
|
||||||
#[command(about = "OCI-compliant extension registry proxy", long_about = None)]
|
#[command(about = "OCI-compliant extension registry proxy", long_about = None)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
|
/// Configuration file path (highest priority)
|
||||||
|
#[arg(short = 'c', long, env = "EXTENSION_REGISTRY_CONFIG")]
|
||||||
|
config: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for extension-registry.ncl|toml|json)
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
config_dir: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
#[arg(short = 'm', long, env = "EXTENSION_REGISTRY_MODE")]
|
||||||
|
mode: Option<String>,
|
||||||
|
|
||||||
/// Host to bind to
|
/// Host to bind to
|
||||||
#[arg(long, default_value = "127.0.0.1")]
|
#[arg(long, default_value = "127.0.0.1")]
|
||||||
host: String,
|
host: String,
|
||||||
|
|||||||
@ -1,14 +1,18 @@
|
|||||||
[package]
|
[package]
|
||||||
authors = ["Jesús Pérez Lorenzo <jpl@jesusperez.pro>"]
|
authors.workspace = true
|
||||||
categories = ["command-line-utilities", "development-tools"]
|
categories = ["command-line-utilities", "development-tools"]
|
||||||
description = "Rust-native MCP server for Infrastructure Automation system"
|
description = "Rust-native MCP server for Infrastructure Automation system"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
keywords = ["mcp", "rust", "infrastructure", "provisioning", "ai"]
|
keywords = ["mcp", "rust", "infrastructure", "provisioning", "ai"]
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
name = "provisioning-mcp-server"
|
name = "mcp-server"
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-mcp-server"
|
||||||
|
path = "src/simple_main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# WORKSPACE DEPENDENCIES
|
# WORKSPACE DEPENDENCIES
|
||||||
@ -23,7 +27,7 @@ serde_json = { workspace = true }
|
|||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration
|
# Platform configuration
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# Error handling
|
# Error handling
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
@ -63,13 +67,13 @@ walkdir = { workspace = true }
|
|||||||
# rust-mcp-sdk = "0.7.0"
|
# rust-mcp-sdk = "0.7.0"
|
||||||
|
|
||||||
# RAG System (from provisioning-rag crate)
|
# RAG System (from provisioning-rag crate)
|
||||||
provisioning-rag = { path = "../rag", features = [] }
|
rag = { path = "../rag", features = [] }
|
||||||
|
|
||||||
# Date/time utilities
|
# Date/time utilities
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
|
|
||||||
# YAML parsing
|
# YAML parsing
|
||||||
serde_yaml = "0.9"
|
serde_yaml = { workspace = true }
|
||||||
|
|
||||||
# Directory utilities
|
# Directory utilities
|
||||||
dirs = { workspace = true }
|
dirs = { workspace = true }
|
||||||
@ -83,14 +87,8 @@ tokio-test = { workspace = true }
|
|||||||
debug = ["tracing-subscriber/json"]
|
debug = ["tracing-subscriber/json"]
|
||||||
default = []
|
default = []
|
||||||
|
|
||||||
[[bin]]
|
# Note: simple_main.rs is the active entry point
|
||||||
name = "provisioning-mcp-server"
|
# main.rs uses incompatible rust_mcp_sdk v0.7.0 API and is disabled
|
||||||
path = "src/simple_main.rs"
|
|
||||||
|
|
||||||
# Disabled: main.rs uses incompatible rust_mcp_sdk v0.7.0 API
|
|
||||||
# [[bin]]
|
|
||||||
# name = "provisioning-mcp-server-full"
|
|
||||||
# path = "src/main.rs"
|
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
name = "provisioning_mcp_server"
|
name = "provisioning_mcp_server"
|
||||||
|
|||||||
@ -1,63 +0,0 @@
|
|||||||
# Build stage
|
|
||||||
FROM rust:1.75-slim as builder
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Install build dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
pkg-config \
|
|
||||||
libssl-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Copy manifests
|
|
||||||
COPY Cargo.toml Cargo.lock ./
|
|
||||||
|
|
||||||
# Create dummy source to cache dependencies
|
|
||||||
RUN mkdir -p src && \
|
|
||||||
echo "fn main() {}" > src/main.rs && \
|
|
||||||
cargo build --release && \
|
|
||||||
rm -rf src
|
|
||||||
|
|
||||||
# Copy actual source code
|
|
||||||
COPY src ./src
|
|
||||||
|
|
||||||
# Build release binary
|
|
||||||
RUN cargo build --release --bin mcp-server
|
|
||||||
|
|
||||||
# Runtime stage
|
|
||||||
FROM debian:bookworm-slim
|
|
||||||
|
|
||||||
# Install runtime dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
ca-certificates \
|
|
||||||
curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Create non-root user
|
|
||||||
RUN useradd -m -u 1000 provisioning && \
|
|
||||||
mkdir -p /data /var/log/mcp-server && \
|
|
||||||
chown -R provisioning:provisioning /data /var/log/mcp-server
|
|
||||||
|
|
||||||
# Copy binary from builder
|
|
||||||
COPY --from=builder /app/target/release/mcp-server /usr/local/bin/
|
|
||||||
|
|
||||||
# Copy default configuration
|
|
||||||
COPY config.defaults.toml /etc/provisioning/mcp-config.defaults.toml
|
|
||||||
|
|
||||||
# Switch to non-root user
|
|
||||||
USER provisioning
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Expose port
|
|
||||||
EXPOSE 8084
|
|
||||||
|
|
||||||
# Set environment variables
|
|
||||||
ENV RUST_LOG=info
|
|
||||||
ENV MCP_PROTOCOL=http
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
|
||||||
CMD curl -f http://localhost:8084/health || exit 1
|
|
||||||
|
|
||||||
# Run the binary
|
|
||||||
CMD ["mcp-server"]
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -2,7 +2,7 @@
|
|||||||
authors.workspace = true
|
authors.workspace = true
|
||||||
description = "Cloud-native infrastructure orchestrator with Nushell integration"
|
description = "Cloud-native infrastructure orchestrator with Nushell integration"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
name = "provisioning-orchestrator"
|
name = "orchestrator"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@ -45,28 +45,28 @@ clap = { workspace = true }
|
|||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
|
|
||||||
# Docker/Container management
|
# Docker/Container management
|
||||||
bollard = "0.17"
|
bollard = { workspace = true }
|
||||||
|
|
||||||
# HTTP client for DNS/OCI/services
|
# HTTP client for DNS/OCI/services
|
||||||
reqwest = { workspace = true }
|
reqwest = { workspace = true }
|
||||||
|
|
||||||
# HTTP service clients (machines, init, AI) - enables remote service calls
|
# HTTP service clients (machines, init, AI) - enables remote service calls
|
||||||
service-clients = { path = "../service-clients" }
|
service-clients = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration management
|
# Platform configuration management
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
# LRU cache for OCI manifests
|
# LRU cache for OCI manifests
|
||||||
lru = "0.12"
|
lru = { workspace = true }
|
||||||
|
|
||||||
# Authorization policy engine
|
# Authorization policy engine
|
||||||
cedar-policy = "4.2"
|
cedar-policy = { workspace = true }
|
||||||
|
|
||||||
# File system watcher for hot reload
|
# File system watcher for hot reload
|
||||||
notify = "6.1"
|
notify = { workspace = true }
|
||||||
|
|
||||||
# Base64 encoding/decoding
|
# Base64 encoding/decoding
|
||||||
base64 = "0.22"
|
base64 = { workspace = true }
|
||||||
|
|
||||||
# JWT token validation
|
# JWT token validation
|
||||||
jsonwebtoken = { workspace = true }
|
jsonwebtoken = { workspace = true }
|
||||||
@ -78,14 +78,14 @@ rsa = { workspace = true }
|
|||||||
sha2 = { workspace = true }
|
sha2 = { workspace = true }
|
||||||
|
|
||||||
# SSH key management
|
# SSH key management
|
||||||
ed25519-dalek = "2.1"
|
ed25519-dalek = { workspace = true }
|
||||||
|
|
||||||
# SSH client library (pure Rust, async-first)
|
# SSH client library (pure Rust, async-first)
|
||||||
russh = "0.44"
|
russh = { workspace = true }
|
||||||
russh-keys = "0.44"
|
russh-keys = { workspace = true }
|
||||||
|
|
||||||
# Path expansion for tilde (~) handling
|
# Path expansion for tilde (~) handling
|
||||||
shellexpand = "3.1"
|
shellexpand = { workspace = true }
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# FEATURE-GATED OPTIONAL DEPENDENCIES
|
# FEATURE-GATED OPTIONAL DEPENDENCIES
|
||||||
@ -141,9 +141,18 @@ http-api = ["core"]
|
|||||||
# SurrealDB: Optional storage backend
|
# SurrealDB: Optional storage backend
|
||||||
surrealdb = ["dep:surrealdb"]
|
surrealdb = ["dep:surrealdb"]
|
||||||
|
|
||||||
# Default: Recommended for standard deployments
|
# Default: All features enabled
|
||||||
# Includes core, audit, compliance, platform, ssh, workflow
|
default = [
|
||||||
default = ["core", "audit", "compliance", "platform", "ssh", "workflow", "http-api"]
|
"core",
|
||||||
|
"audit",
|
||||||
|
"compliance",
|
||||||
|
"platform",
|
||||||
|
"ssh",
|
||||||
|
"workflow",
|
||||||
|
"testing",
|
||||||
|
"http-api",
|
||||||
|
"surrealdb",
|
||||||
|
]
|
||||||
|
|
||||||
# Full: All features enabled (development and testing)
|
# Full: All features enabled (development and testing)
|
||||||
all = [
|
all = [
|
||||||
@ -170,11 +179,10 @@ tower = { workspace = true, features = ["util"] }
|
|||||||
name = "provisioning_orchestrator"
|
name = "provisioning_orchestrator"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# Binary target (requires testing feature for test environment API)
|
# Binary target (uses all features by default)
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "provisioning-orchestrator"
|
name = "provisioning-orchestrator"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
required-features = ["all"]
|
|
||||||
|
|
||||||
[[bench]]
|
[[bench]]
|
||||||
harness = false
|
harness = false
|
||||||
|
|||||||
@ -1,8 +1,32 @@
|
|||||||
# Multi-stage build for Orchestrator
|
# Multi-stage build for provisioning-orchestrator
|
||||||
# Builds from platform workspace root
|
# Generated from Nickel template - DO NOT EDIT DIRECTLY
|
||||||
|
# Source: provisioning/schemas/platform/templates/docker/Dockerfile.chef.ncl
|
||||||
|
|
||||||
# Build stage - Using nightly for consistency with control-center
|
# ============================================================================
|
||||||
FROM rustlang/rust:nightly-bookworm AS builder
|
# Stage 1: PLANNER - Generate dependency recipe
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS planner
|
||||||
|
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
# Install cargo-chef
|
||||||
|
RUN cargo install cargo-chef --version 0.1.67
|
||||||
|
|
||||||
|
# Copy workspace manifests
|
||||||
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
COPY crates ./crates
|
||||||
|
COPY daemon-cli ./daemon-cli
|
||||||
|
COPY secretumvault ./secretumvault
|
||||||
|
COPY prov-ecosystem ./prov-ecosystem
|
||||||
|
COPY stratumiops ./stratumiops
|
||||||
|
|
||||||
|
# Generate recipe.json (dependency graph)
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json --bin provisioning-orchestrator
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Stage 2: CACHER - Build dependencies only
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS cacher
|
||||||
|
|
||||||
WORKDIR /workspace
|
WORKDIR /workspace
|
||||||
|
|
||||||
@ -12,20 +36,52 @@ RUN apt-get update && apt-get install -y \
|
|||||||
libssl-dev \
|
libssl-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Copy entire platform workspace (required for workspace dependencies)
|
# Install cargo-chef
|
||||||
COPY Cargo.toml Cargo.lock ./
|
RUN cargo install cargo-chef --version 0.1.67
|
||||||
COPY orchestrator ./orchestrator
|
|
||||||
COPY control-center ./control-center
|
# sccache disabled
|
||||||
COPY control-center-ui ./control-center-ui
|
|
||||||
COPY mcp-server ./mcp-server
|
# Copy recipe from planner
|
||||||
COPY installer ./installer
|
COPY --from=planner /workspace/recipe.json recipe.json
|
||||||
|
|
||||||
|
# Build dependencies - This layer will be cached
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Stage 3: BUILDER - Build source code
|
||||||
|
# ============================================================================
|
||||||
|
FROM rust:1.82-trixie AS builder
|
||||||
|
|
||||||
# Build orchestrator (workspace-aware)
|
|
||||||
WORKDIR /workspace
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
# Install build dependencies
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# sccache disabled
|
||||||
|
|
||||||
|
# Copy cached dependencies from cacher stage
|
||||||
|
COPY --from=cacher /workspace/target target
|
||||||
|
COPY --from=cacher /usr/local/cargo /usr/local/cargo
|
||||||
|
|
||||||
|
# Copy source code
|
||||||
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
COPY crates ./crates
|
||||||
|
COPY daemon-cli ./daemon-cli
|
||||||
|
COPY secretumvault ./secretumvault
|
||||||
|
COPY prov-ecosystem ./prov-ecosystem
|
||||||
|
COPY stratumiops ./stratumiops
|
||||||
|
|
||||||
|
# Build release binary with parallelism
|
||||||
|
ENV CARGO_BUILD_JOBS=4
|
||||||
RUN cargo build --release --package provisioning-orchestrator
|
RUN cargo build --release --package provisioning-orchestrator
|
||||||
|
|
||||||
# Runtime stage
|
# ============================================================================
|
||||||
FROM debian:bookworm-slim
|
# Stage 4: RUNTIME - Minimal runtime image
|
||||||
|
# ============================================================================
|
||||||
|
FROM debian:trixie-slim
|
||||||
|
|
||||||
# Install runtime dependencies
|
# Install runtime dependencies
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
@ -35,30 +91,29 @@ RUN apt-get update && apt-get install -y \
|
|||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN useradd -m -u 1000 provisioning && \
|
RUN useradd -m -u 1000 provisioning && \
|
||||||
mkdir -p /data /var/log/orchestrator && \
|
mkdir -p /data /var/log/provisioning-orchestrator && \
|
||||||
chown -R provisioning:provisioning /data /var/log/orchestrator
|
chown -R provisioning:provisioning /data /var/log/provisioning-orchestrator
|
||||||
|
|
||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /workspace/target/release/provisioning-orchestrator /usr/local/bin/provisioning-orchestrator
|
COPY --from=builder /workspace/target/release/provisioning-orchestrator /usr/local/bin/provisioning-orchestrator
|
||||||
RUN chmod +x /usr/local/bin/provisioning-orchestrator
|
RUN chmod +x /usr/local/bin/provisioning-orchestrator
|
||||||
|
|
||||||
# Copy default configuration
|
COPY crates/provisioning-orchestrator/config.defaults.toml /etc/provisioning/config.defaults.toml
|
||||||
COPY orchestrator/config.defaults.toml /etc/provisioning/config.defaults.toml
|
|
||||||
|
|
||||||
# Switch to non-root user
|
# Switch to non-root user
|
||||||
USER provisioning
|
USER provisioning
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Expose port
|
# Expose service port
|
||||||
EXPOSE 8080
|
EXPOSE 9090
|
||||||
|
|
||||||
# Set environment variables
|
# Environment variables
|
||||||
ENV RUST_LOG=info
|
ENV RUST_LOG=info
|
||||||
ENV DATA_DIR=/data
|
ENV DATA_DIR=/data
|
||||||
|
|
||||||
# Health check
|
# Health check
|
||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||||
CMD curl -f http://localhost:8080/health || exit 1
|
CMD curl -f http://localhost:9090/health || exit 1
|
||||||
|
|
||||||
# Run the binary
|
# Run the binary
|
||||||
CMD ["provisioning-orchestrator"]
|
CMD ["provisioning-orchestrator"]
|
||||||
|
|||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 14043518-e459-4316-aadd-6ee6d221e644
|
|
||||||
Timestamp: 2026-01-06 12:47:28 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmpdtjZLt/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 1e9b4914-f290-4bec-80f2-35128250f9fd
|
|
||||||
Timestamp: 2026-01-06 12:50:41 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmp9wM3YA/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 21c8a4af-2562-4304-b5ec-90fb1b5fd0ab
|
|
||||||
Timestamp: 2026-01-06 13:10:16 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmpnoxnXR/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 317e31fa-b549-49c9-a212-1f13445d913f
|
|
||||||
Timestamp: 2026-01-06 12:49:14 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmpe5B6TH/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 5da5d888-527e-4aac-ab53-93e9a30014cc
|
|
||||||
Timestamp: 2026-01-06 12:53:52 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmpOI6ga7/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: 7c16746f-24b0-4bcc-8a49-b5dc6bc1f0c7
|
|
||||||
Timestamp: 2026-01-06 12:59:15 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmp8JuU01/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
Manual Rollback Instructions for Migration migration-123
|
|
||||||
=============================================
|
|
||||||
|
|
||||||
Migration ID: migration-123
|
|
||||||
Rollback ID: cb3ced5a-ab49-4754-ba90-c815ab0948ba
|
|
||||||
Timestamp: 2026-01-06 12:53:37 UTC
|
|
||||||
|
|
||||||
Steps to rollback:
|
|
||||||
1. Stop the orchestrator service
|
|
||||||
2. If backup exists at '/var/folders/my/jvl5hd5x6rgbhks6yszk213c0000gn/T/.tmpI1vYTj/backup.json', restore it to target storage
|
|
||||||
3. Verify target storage contains original data
|
|
||||||
4. Remove any partially migrated data
|
|
||||||
5. Restart orchestrator service
|
|
||||||
|
|
||||||
Storage-specific cleanup commands:
|
|
||||||
- For filesystem: Remove or rename target directory
|
|
||||||
- For SurrealDB embedded: Delete database files in target directory
|
|
||||||
- For SurrealDB server: Connect and drop/recreate database
|
|
||||||
|
|
||||||
IMPORTANT: Test data integrity before resuming operations!
|
|
||||||
@ -8,16 +8,14 @@ use std::default::Default;
|
|||||||
|
|
||||||
use anyhow::{anyhow, Context, Result};
|
use anyhow::{anyhow, Context, Result};
|
||||||
use bollard::{
|
use bollard::{
|
||||||
container::{
|
|
||||||
Config, CreateContainerOptions, ListContainersOptions, LogsOptions, NetworkingConfig,
|
|
||||||
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
|
|
||||||
},
|
|
||||||
exec::{CreateExecOptions, StartExecResults},
|
exec::{CreateExecOptions, StartExecResults},
|
||||||
image::CreateImageOptions,
|
models::{
|
||||||
network::CreateNetworkOptions,
|
ContainerCreateBody, ContainerSummary, EndpointSettings, HostConfig, Ipam, IpamConfig,
|
||||||
service::{
|
NetworkingConfig, RestartPolicy, RestartPolicyNameEnum,
|
||||||
ContainerSummary, EndpointSettings, HostConfig, Ipam, IpamConfig, RestartPolicy,
|
},
|
||||||
RestartPolicyNameEnum,
|
query_parameters::{
|
||||||
|
CreateContainerOptions, CreateImageOptions, ListContainersOptions, LogsOptions,
|
||||||
|
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
|
||||||
},
|
},
|
||||||
Docker,
|
Docker,
|
||||||
};
|
};
|
||||||
@ -58,6 +56,8 @@ impl ContainerManager {
|
|||||||
|
|
||||||
/// Create network for test environment
|
/// Create network for test environment
|
||||||
pub async fn create_network(&self, config: &NetworkConfig) -> Result<String> {
|
pub async fn create_network(&self, config: &NetworkConfig) -> Result<String> {
|
||||||
|
use bollard::models::NetworkCreateRequest;
|
||||||
|
|
||||||
let ipam_config = IpamConfig {
|
let ipam_config = IpamConfig {
|
||||||
subnet: Some(config.subnet.clone()),
|
subnet: Some(config.subnet.clone()),
|
||||||
gateway: None,
|
gateway: None,
|
||||||
@ -71,36 +71,32 @@ impl ContainerManager {
|
|||||||
options: None,
|
options: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let create_options = CreateNetworkOptions {
|
let mut labels = HashMap::new();
|
||||||
|
labels.insert(
|
||||||
|
"managed_by".to_string(),
|
||||||
|
"provisioning_orchestrator".to_string(),
|
||||||
|
);
|
||||||
|
labels.insert("type".to_string(), "test_environment".to_string());
|
||||||
|
|
||||||
|
let create_request = NetworkCreateRequest {
|
||||||
name: config.name.clone(),
|
name: config.name.clone(),
|
||||||
check_duplicate: true,
|
driver: Some("bridge".to_string()),
|
||||||
driver: "bridge".to_string(),
|
internal: Some(false),
|
||||||
internal: false,
|
attachable: Some(true),
|
||||||
attachable: true,
|
ipam: Some(ipam),
|
||||||
ingress: false,
|
enable_ipv4: Some(true),
|
||||||
ipam,
|
options: Some(HashMap::new()),
|
||||||
enable_ipv6: false,
|
labels: Some(labels),
|
||||||
options: HashMap::new(),
|
..Default::default()
|
||||||
labels: {
|
|
||||||
let mut labels = HashMap::new();
|
|
||||||
labels.insert(
|
|
||||||
"managed_by".to_string(),
|
|
||||||
"provisioning_orchestrator".to_string(),
|
|
||||||
);
|
|
||||||
labels.insert("type".to_string(), "test_environment".to_string());
|
|
||||||
labels
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let response = self
|
let response = self
|
||||||
.docker
|
.docker
|
||||||
.create_network(create_options)
|
.create_network(create_request)
|
||||||
.await
|
.await
|
||||||
.context("Failed to create Docker network")?;
|
.context("Failed to create Docker network")?;
|
||||||
|
|
||||||
let network_id = response
|
let network_id = response.id;
|
||||||
.id
|
|
||||||
.ok_or_else(|| anyhow!("Network ID not returned"))?;
|
|
||||||
|
|
||||||
info!("Created network {} ({})", config.name, network_id);
|
info!("Created network {} ({})", config.name, network_id);
|
||||||
Ok(network_id)
|
Ok(network_id)
|
||||||
@ -119,12 +115,12 @@ impl ContainerManager {
|
|||||||
|
|
||||||
/// Pull image if not exists
|
/// Pull image if not exists
|
||||||
pub async fn ensure_image(&self, image: &str) -> Result<()> {
|
pub async fn ensure_image(&self, image: &str) -> Result<()> {
|
||||||
let options = Some(CreateImageOptions {
|
let options = CreateImageOptions {
|
||||||
from_image: image,
|
from_image: Some(image.to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
};
|
||||||
|
|
||||||
let mut stream = self.docker.create_image(options, None, None);
|
let mut stream = self.docker.create_image(Some(options), None, None);
|
||||||
|
|
||||||
while let Some(result) = stream.next().await {
|
while let Some(result) = stream.next().await {
|
||||||
match result {
|
match result {
|
||||||
@ -176,7 +172,6 @@ impl ContainerManager {
|
|||||||
endpoint_config.insert(
|
endpoint_config.insert(
|
||||||
net_id.to_string(),
|
net_id.to_string(),
|
||||||
EndpointSettings {
|
EndpointSettings {
|
||||||
network_id: Some(net_id.to_string()),
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -190,31 +185,30 @@ impl ContainerManager {
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Container configuration
|
// Container configuration
|
||||||
let config = Config {
|
let mut labels = HashMap::new();
|
||||||
|
labels.insert(
|
||||||
|
"managed_by".to_string(),
|
||||||
|
"provisioning_orchestrator".to_string(),
|
||||||
|
);
|
||||||
|
labels.insert("type".to_string(), "test_container".to_string());
|
||||||
|
labels.insert("test_name".to_string(), name.to_string());
|
||||||
|
|
||||||
|
let config = ContainerCreateBody {
|
||||||
image: Some(image.to_string()),
|
image: Some(image.to_string()),
|
||||||
hostname: Some(name.to_string()),
|
hostname: Some(name.to_string()),
|
||||||
env: Some(env),
|
env: if env.is_empty() { None } else { Some(env) },
|
||||||
cmd: command,
|
cmd: command,
|
||||||
host_config: Some(host_config),
|
host_config: Some(host_config),
|
||||||
networking_config: Some(NetworkingConfig {
|
networking_config: Some(NetworkingConfig {
|
||||||
endpoints_config: endpoint_config,
|
endpoints_config: Some(endpoint_config),
|
||||||
}),
|
|
||||||
labels: Some({
|
|
||||||
let mut labels = HashMap::new();
|
|
||||||
labels.insert(
|
|
||||||
"managed_by".to_string(),
|
|
||||||
"provisioning_orchestrator".to_string(),
|
|
||||||
);
|
|
||||||
labels.insert("type".to_string(), "test_container".to_string());
|
|
||||||
labels.insert("test_name".to_string(), name.to_string());
|
|
||||||
labels
|
|
||||||
}),
|
}),
|
||||||
|
labels: Some(labels),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let options = CreateContainerOptions {
|
let options = CreateContainerOptions {
|
||||||
name: name.to_string(),
|
name: Some(name.to_string()),
|
||||||
platform: None,
|
platform: String::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let response = self
|
let response = self
|
||||||
@ -240,7 +234,7 @@ impl ContainerManager {
|
|||||||
/// Start container
|
/// Start container
|
||||||
pub async fn start_container(&self, container_id: &str) -> Result<()> {
|
pub async fn start_container(&self, container_id: &str) -> Result<()> {
|
||||||
self.docker
|
self.docker
|
||||||
.start_container(container_id, None::<StartContainerOptions<String>>)
|
.start_container(container_id, None::<StartContainerOptions>)
|
||||||
.await
|
.await
|
||||||
.context(format!("Failed to start container {}", container_id))?;
|
.context(format!("Failed to start container {}", container_id))?;
|
||||||
|
|
||||||
@ -251,7 +245,8 @@ impl ContainerManager {
|
|||||||
/// Stop container
|
/// Stop container
|
||||||
pub async fn stop_container(&self, container_id: &str, timeout: Option<i64>) -> Result<()> {
|
pub async fn stop_container(&self, container_id: &str, timeout: Option<i64>) -> Result<()> {
|
||||||
let options = StopContainerOptions {
|
let options = StopContainerOptions {
|
||||||
t: timeout.unwrap_or(10),
|
t: Some(timeout.unwrap_or(10) as i32),
|
||||||
|
signal: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
self.docker
|
self.docker
|
||||||
@ -329,7 +324,7 @@ impl ContainerManager {
|
|||||||
container_id: &str,
|
container_id: &str,
|
||||||
tail: Option<&str>,
|
tail: Option<&str>,
|
||||||
) -> Result<String> {
|
) -> Result<String> {
|
||||||
let options = LogsOptions::<String> {
|
let options = LogsOptions {
|
||||||
stdout: true,
|
stdout: true,
|
||||||
stderr: true,
|
stderr: true,
|
||||||
tail: tail.unwrap_or("100").to_string(),
|
tail: tail.unwrap_or("100").to_string(),
|
||||||
@ -386,15 +381,15 @@ impl ContainerManager {
|
|||||||
let mut filters = HashMap::new();
|
let mut filters = HashMap::new();
|
||||||
filters.insert("label".to_string(), vec![format!("{}={}", label, value)]);
|
filters.insert("label".to_string(), vec![format!("{}={}", label, value)]);
|
||||||
|
|
||||||
let options = Some(ListContainersOptions {
|
let options = ListContainersOptions {
|
||||||
all: true,
|
all: true,
|
||||||
filters,
|
filters: Some(filters),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
};
|
||||||
|
|
||||||
let containers = self
|
let containers = self
|
||||||
.docker
|
.docker
|
||||||
.list_containers(options)
|
.list_containers(Some(options))
|
||||||
.await
|
.await
|
||||||
.context("Failed to list containers")?;
|
.context("Failed to list containers")?;
|
||||||
|
|
||||||
|
|||||||
@ -85,14 +85,36 @@ pub fn validate_storage_type(s: &str) -> Result<String, String> {
|
|||||||
|
|
||||||
// CLI arguments structure
|
// CLI arguments structure
|
||||||
#[derive(clap::Parser, Clone)]
|
#[derive(clap::Parser, Clone)]
|
||||||
#[command(author, version, about, long_about = None)]
|
#[command(author, version, about = "Multi-service task orchestration and batch workflow engine")]
|
||||||
|
#[command(long_about = "Orchestrator - Manages distributed task execution, batch workflows, and cluster provisioning with state management and rollback recovery")]
|
||||||
|
#[command(after_help = "CONFIGURATION HIERARCHY (highest to lowest priority):\n 1. CLI: -c/--config <path> (explicit file)\n 2. CLI: --config-dir <dir> --mode <mode> (directory + mode)\n 3. CLI: --config-dir <dir> (searches for orchestrator.ncl|toml|json)\n 4. CLI: --mode <mode> (searches in provisioning/platform/config/)\n 5. ENV: ORCHESTRATOR_CONFIG (explicit file)\n 6. ENV: PROVISIONING_CONFIG_DIR (searches for orchestrator.ncl|toml|json)\n 7. ENV: ORCHESTRATOR_MODE (mode-based in default path)\n 8. Built-in defaults\n\nEXAMPLES:\n # Explicit config file\n orchestrator -c ~/my-config.toml\n\n # Config directory with mode\n orchestrator --config-dir ~/configs --mode enterprise\n\n # Config directory (auto-discover file)\n orchestrator --config-dir ~/.config/provisioning\n\n # Via environment variables\n export ORCHESTRATOR_CONFIG=~/.config/orchestrator.toml\n orchestrator\n\n # Mode-based configuration\n orchestrator --mode solo")]
|
||||||
pub struct Args {
|
pub struct Args {
|
||||||
|
/// Configuration file path (highest priority)
|
||||||
|
///
|
||||||
|
/// Accepts absolute or relative path. Supports .ncl, .toml, and .json formats.
|
||||||
|
#[arg(short = 'c', long, env = "ORCHESTRATOR_CONFIG")]
|
||||||
|
pub config: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for orchestrator.ncl|toml|json)
|
||||||
|
///
|
||||||
|
/// Searches for configuration files in order of preference: .ncl > .toml > .json
|
||||||
|
/// Can also search for mode-specific files: orchestrator.{mode}.{ncl|toml|json}
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
pub config_dir: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
///
|
||||||
|
/// Determines which configuration profile to use. Searches in:
|
||||||
|
/// provisioning/platform/config/orchestrator.{mode}.{ncl|toml}
|
||||||
|
#[arg(short = 'm', long, env = "ORCHESTRATOR_MODE")]
|
||||||
|
pub mode: Option<String>,
|
||||||
|
|
||||||
/// Port to listen on
|
/// Port to listen on
|
||||||
#[arg(short, long, default_value = "9090")]
|
#[arg(short = 'p', long, default_value = "9090")]
|
||||||
pub port: u16,
|
pub port: u16,
|
||||||
|
|
||||||
/// Data directory for storage
|
/// Data directory for storage
|
||||||
#[arg(short, long, default_value = "./data")]
|
#[arg(short = 'd', long, default_value = "./data")]
|
||||||
pub data_dir: String,
|
pub data_dir: String,
|
||||||
|
|
||||||
/// Storage backend type
|
/// Storage backend type
|
||||||
|
|||||||
@ -1007,6 +1007,7 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
let app = Router::new()
|
let app = Router::new()
|
||||||
.route("/health", get(health_check))
|
.route("/health", get(health_check))
|
||||||
|
.route("/api/v1/health", get(health_check))
|
||||||
.route("/tasks", get(list_tasks))
|
.route("/tasks", get(list_tasks))
|
||||||
.route("/tasks/{id}", get(get_task_status))
|
.route("/tasks/{id}", get(get_task_status))
|
||||||
.route("/workflows/servers/create", post(create_server_workflow))
|
.route("/workflows/servers/create", post(create_server_workflow))
|
||||||
|
|||||||
@ -8,7 +8,7 @@ const CONFIG_BASE_PATH: &str = "provisioning/platform/config";
|
|||||||
/// 2. Variable de entorno {SERVICE}_MODE + búsqueda de archivo
|
/// 2. Variable de entorno {SERVICE}_MODE + búsqueda de archivo
|
||||||
/// 3. Fallback a defaults
|
/// 3. Fallback a defaults
|
||||||
pub fn resolve_config_path(service_name: &str) -> Option<PathBuf> {
|
pub fn resolve_config_path(service_name: &str) -> Option<PathBuf> {
|
||||||
// Paso 1: Check {SERVICE}_CONFIG env var (explicit path)
|
// Priority 1: Check {SERVICE}_CONFIG env var (explicit path)
|
||||||
let env_var = format!("{}_CONFIG", service_name.to_uppercase().replace('-', "_"));
|
let env_var = format!("{}_CONFIG", service_name.to_uppercase().replace('-', "_"));
|
||||||
if let Ok(path) = env::var(&env_var) {
|
if let Ok(path) = env::var(&env_var) {
|
||||||
let config_path = PathBuf::from(path);
|
let config_path = PathBuf::from(path);
|
||||||
@ -22,7 +22,18 @@ pub fn resolve_config_path(service_name: &str) -> Option<PathBuf> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Paso 2: Check {SERVICE}_MODE env var + find config file
|
// Priority 2: Check PROVISIONING_CONFIG_DIR env var
|
||||||
|
if let Ok(dir) = env::var("PROVISIONING_CONFIG_DIR") {
|
||||||
|
if let Some(config) = super::resolver::find_config_in_dir(std::path::Path::new(&dir), service_name) {
|
||||||
|
tracing::debug!(
|
||||||
|
"Using config from PROVISIONING_CONFIG_DIR: {:?}",
|
||||||
|
config
|
||||||
|
);
|
||||||
|
return Some(config);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Check {SERVICE}_MODE env var + find config file
|
||||||
let mode_var = format!("{}_MODE", service_name.to_uppercase().replace('-', "_"));
|
let mode_var = format!("{}_MODE", service_name.to_uppercase().replace('-', "_"));
|
||||||
let mode = env::var(&mode_var).unwrap_or_else(|_| "solo".to_string());
|
let mode = env::var(&mode_var).unwrap_or_else(|_| "solo".to_string());
|
||||||
|
|
||||||
@ -36,7 +47,7 @@ pub fn resolve_config_path(service_name: &str) -> Option<PathBuf> {
|
|||||||
return Some(path);
|
return Some(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Paso 3: Fallback - no config file found
|
// Fallback - no config file found
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
"No config file found for {}.{} - using defaults",
|
"No config file found for {}.{} - using defaults",
|
||||||
service_name,
|
service_name,
|
||||||
|
|||||||
@ -53,6 +53,7 @@ pub mod format;
|
|||||||
pub mod hierarchy;
|
pub mod hierarchy;
|
||||||
pub mod loader;
|
pub mod loader;
|
||||||
pub mod nickel;
|
pub mod nickel;
|
||||||
|
pub mod resolver;
|
||||||
|
|
||||||
// Re-export main types
|
// Re-export main types
|
||||||
pub use error::{ConfigError, Result};
|
pub use error::{ConfigError, Result};
|
||||||
@ -60,3 +61,4 @@ pub use format::ConfigLoader;
|
|||||||
pub use hierarchy::{config_base_path, find_config_file, resolve_config_path};
|
pub use hierarchy::{config_base_path, find_config_file, resolve_config_path};
|
||||||
pub use loader::{ConfigLoaderExt, ConfigValidator};
|
pub use loader::{ConfigLoaderExt, ConfigValidator};
|
||||||
pub use nickel::is_nickel_available;
|
pub use nickel::is_nickel_available;
|
||||||
|
pub use resolver::{ConfigResolver, find_config_in_dir, find_config_in_dir_with_mode};
|
||||||
|
|||||||
212
crates/platform-config/src/resolver.rs
Normal file
212
crates/platform-config/src/resolver.rs
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
/// Resolves configuration file paths with CLI flags priority
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct ConfigResolver {
|
||||||
|
cli_config: Option<PathBuf>,
|
||||||
|
cli_config_dir: Option<PathBuf>,
|
||||||
|
cli_mode: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConfigResolver {
|
||||||
|
/// Create a new ConfigResolver with no CLI overrides
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
cli_config: None,
|
||||||
|
cli_config_dir: None,
|
||||||
|
cli_mode: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set explicit config file path (highest priority)
|
||||||
|
pub fn with_cli_config(mut self, path: Option<PathBuf>) -> Self {
|
||||||
|
self.cli_config = path;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set config directory (searches for {service}.{ncl|toml|json})
|
||||||
|
pub fn with_cli_config_dir(mut self, dir: Option<PathBuf>) -> Self {
|
||||||
|
self.cli_config_dir = dir;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set deployment mode for config resolution
|
||||||
|
pub fn with_cli_mode(mut self, mode: Option<String>) -> Self {
|
||||||
|
self.cli_mode = mode;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve config file path with priority order:
|
||||||
|
/// 1. CLI flag: -config <path> (explicit file)
|
||||||
|
/// 2. CLI flag: --config-dir <dir> --mode <mode> (directory + mode)
|
||||||
|
/// 3. CLI flag: --config-dir <dir> (directory, default naming)
|
||||||
|
/// 4. CLI flag: --mode <mode> (mode in default path)
|
||||||
|
/// 5. ENV var: {SERVICE}_CONFIG (explicit file)
|
||||||
|
/// 6. ENV var: PROVISIONING_CONFIG_DIR (directory, default naming)
|
||||||
|
/// 7. ENV var: {SERVICE}_MODE (mode in default path)
|
||||||
|
/// 8. None (fallback to defaults)
|
||||||
|
pub fn resolve(&self, service_name: &str) -> Option<PathBuf> {
|
||||||
|
// Priority 1: CLI flag explicit path
|
||||||
|
if let Some(ref path) = self.cli_config {
|
||||||
|
tracing::debug!("Using CLI-provided config file: {:?}", path);
|
||||||
|
return Some(path.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: CLI config-dir + mode
|
||||||
|
if let Some(ref dir) = self.cli_config_dir {
|
||||||
|
if let Some(ref mode) = self.cli_mode {
|
||||||
|
if let Some(config) = find_config_in_dir_with_mode(dir, service_name, mode) {
|
||||||
|
tracing::debug!(
|
||||||
|
"Using config file from CLI config-dir with mode: {:?}",
|
||||||
|
config
|
||||||
|
);
|
||||||
|
return Some(config);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: CLI config-dir only
|
||||||
|
if let Some(ref dir) = self.cli_config_dir {
|
||||||
|
if let Some(config) = find_config_in_dir(dir, service_name) {
|
||||||
|
tracing::debug!("Using config file from CLI config-dir: {:?}", config);
|
||||||
|
return Some(config);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 4: CLI mode only (searches in default path)
|
||||||
|
if let Some(ref mode) = self.cli_mode {
|
||||||
|
if let Some(config) = super::hierarchy::find_config_file(service_name, mode) {
|
||||||
|
tracing::debug!("Using config file with CLI mode: {:?}", config);
|
||||||
|
return Some(config);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 5-7: Fall back to environment variable resolution
|
||||||
|
super::hierarchy::resolve_config_path(service_name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Search for config file in directory with specific mode
|
||||||
|
/// Searches in order: {service}.{mode}.ncl, {service}.{mode}.toml, {service}.{mode}.json
|
||||||
|
pub fn find_config_in_dir_with_mode(dir: &Path, service_name: &str, mode: &str) -> Option<PathBuf> {
|
||||||
|
for ext in &["ncl", "toml", "json"] {
|
||||||
|
let path = dir.join(format!("{}.{}.{}", service_name, mode, ext));
|
||||||
|
if path.exists() {
|
||||||
|
tracing::trace!("Found config with mode: {:?}", path);
|
||||||
|
return Some(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Search for config file in directory with default naming
|
||||||
|
/// Searches in order: {service}.ncl, {service}.toml, {service}.json
|
||||||
|
pub fn find_config_in_dir(dir: &Path, service_name: &str) -> Option<PathBuf> {
|
||||||
|
for ext in &["ncl", "toml", "json"] {
|
||||||
|
let path = dir.join(format!("{}.{}", service_name, ext));
|
||||||
|
if path.exists() {
|
||||||
|
tracing::trace!("Found config in dir: {:?}", path);
|
||||||
|
return Some(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::fs;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cli_config_highest_priority() {
|
||||||
|
let resolver = ConfigResolver::new()
|
||||||
|
.with_cli_config(Some(PathBuf::from("/explicit/path.toml")));
|
||||||
|
|
||||||
|
let resolved = resolver.resolve("orchestrator");
|
||||||
|
assert_eq!(resolved, Some(PathBuf::from("/explicit/path.toml")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_dir_searches_extensions_in_order() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
let ncl_path = temp_dir.path().join("orchestrator.ncl");
|
||||||
|
let toml_path = temp_dir.path().join("orchestrator.toml");
|
||||||
|
|
||||||
|
// Create both files
|
||||||
|
fs::write(&ncl_path, "{}").unwrap();
|
||||||
|
fs::write(&toml_path, "[orchestrator]").unwrap();
|
||||||
|
|
||||||
|
let resolver = ConfigResolver::new()
|
||||||
|
.with_cli_config_dir(Some(temp_dir.path().to_path_buf()));
|
||||||
|
|
||||||
|
let resolved = resolver.resolve("orchestrator").unwrap();
|
||||||
|
// Should prefer .ncl over .toml
|
||||||
|
assert_eq!(resolved, ncl_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_dir_with_mode() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
let enterprise_path = temp_dir.path().join("orchestrator.enterprise.toml");
|
||||||
|
fs::write(&enterprise_path, "[orchestrator]").unwrap();
|
||||||
|
|
||||||
|
let resolver = ConfigResolver::new()
|
||||||
|
.with_cli_config_dir(Some(temp_dir.path().to_path_buf()))
|
||||||
|
.with_cli_mode(Some("enterprise".to_string()));
|
||||||
|
|
||||||
|
let resolved = resolver.resolve("orchestrator").unwrap();
|
||||||
|
assert_eq!(resolved, enterprise_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_find_config_in_dir_prefers_ncl() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
let ncl_path = temp_dir.path().join("test-service.ncl");
|
||||||
|
let toml_path = temp_dir.path().join("test-service.toml");
|
||||||
|
let json_path = temp_dir.path().join("test-service.json");
|
||||||
|
|
||||||
|
fs::write(&ncl_path, "{}").unwrap();
|
||||||
|
fs::write(&toml_path, "[test]").unwrap();
|
||||||
|
fs::write(&json_path, "{}").unwrap();
|
||||||
|
|
||||||
|
let result = find_config_in_dir(temp_dir.path(), "test-service").unwrap();
|
||||||
|
assert_eq!(result, ncl_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_find_config_in_dir_with_mode_json_fallback() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
let json_path = temp_dir.path().join("test-service.solo.json");
|
||||||
|
fs::write(&json_path, "{}").unwrap();
|
||||||
|
|
||||||
|
let result = find_config_in_dir_with_mode(temp_dir.path(), "test-service", "solo").unwrap();
|
||||||
|
assert_eq!(result, json_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_no_config_found_returns_none() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
|
||||||
|
let resolver = ConfigResolver::new()
|
||||||
|
.with_cli_config_dir(Some(temp_dir.path().to_path_buf()));
|
||||||
|
|
||||||
|
let resolved = resolver.resolve("nonexistent-service");
|
||||||
|
assert!(resolved.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cli_config_dir_overrides_env_var() {
|
||||||
|
let temp_dir = TempDir::new().unwrap();
|
||||||
|
let config_path = temp_dir.path().join("orchestrator.toml");
|
||||||
|
fs::write(&config_path, "[orchestrator]").unwrap();
|
||||||
|
|
||||||
|
// This test just verifies the resolver logic works correctly
|
||||||
|
// (actual env var override would need temp_env crate)
|
||||||
|
let resolver = ConfigResolver::new()
|
||||||
|
.with_cli_config_dir(Some(temp_dir.path().to_path_buf()));
|
||||||
|
|
||||||
|
let resolved = resolver.resolve("orchestrator").unwrap();
|
||||||
|
assert_eq!(resolved, config_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -2,9 +2,13 @@
|
|||||||
authors.workspace = true
|
authors.workspace = true
|
||||||
description = "RAG system for provisioning platform with Rig framework and SurrealDB"
|
description = "RAG system for provisioning platform with Rig framework and SurrealDB"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
name = "provisioning-rag"
|
name = "rag"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-rag"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# WORKSPACE DEPENDENCIES - Core async runtime and traits
|
# WORKSPACE DEPENDENCIES - Core async runtime and traits
|
||||||
@ -41,7 +45,7 @@ reqwest = { workspace = true }
|
|||||||
# REST API Framework (Phase 8)
|
# REST API Framework (Phase 8)
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
axum = { workspace = true }
|
axum = { workspace = true }
|
||||||
http = "1"
|
http = { workspace = true }
|
||||||
hyper = { workspace = true, features = ["full"] }
|
hyper = { workspace = true, features = ["full"] }
|
||||||
tower = { workspace = true }
|
tower = { workspace = true }
|
||||||
tower-http = { workspace = true, features = ["cors", "trace"] }
|
tower-http = { workspace = true, features = ["cors", "trace"] }
|
||||||
@ -61,7 +65,11 @@ walkdir = { workspace = true }
|
|||||||
config = { workspace = true }
|
config = { workspace = true }
|
||||||
|
|
||||||
# Platform configuration management
|
# Platform configuration management
|
||||||
platform-config = { path = "../platform-config" }
|
platform-config = { workspace = true }
|
||||||
|
|
||||||
|
# Stratum ecosystem - embeddings and LLM abstraction
|
||||||
|
stratum-embeddings = { workspace = true, features = ["openai-provider", "ollama-provider", "fastembed-provider"] }
|
||||||
|
stratum-llm = { workspace = true, features = ["anthropic", "openai", "ollama"] }
|
||||||
|
|
||||||
# Regex for document parsing
|
# Regex for document parsing
|
||||||
regex = { workspace = true }
|
regex = { workspace = true }
|
||||||
@ -97,13 +105,7 @@ name = "phase8_benchmarks"
|
|||||||
name = "provisioning_rag"
|
name = "provisioning_rag"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# Binary target (optional CLI tool)
|
|
||||||
[[bin]]
|
|
||||||
name = "provisioning-rag"
|
|
||||||
path = "src/main.rs"
|
|
||||||
required-features = ["cli"]
|
|
||||||
|
|
||||||
# Features
|
# Features
|
||||||
[features]
|
[features]
|
||||||
cli = []
|
cli = []
|
||||||
default = []
|
default = ["cli"]
|
||||||
|
|||||||
@ -1,59 +0,0 @@
|
|||||||
# Multi-stage build for Provisioning RAG Service
|
|
||||||
# Stage 1: Builder
|
|
||||||
FROM rust:1.80.1 as builder
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Install dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
pkg-config \
|
|
||||||
libssl-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Copy workspace and source
|
|
||||||
COPY Cargo.toml Cargo.lock ./
|
|
||||||
COPY provisioning/platform/rag ./rag
|
|
||||||
COPY provisioning/platform/rag/src ./src
|
|
||||||
COPY provisioning/platform/rag/benches ./benches
|
|
||||||
|
|
||||||
# Build the orchestrator binary in release mode
|
|
||||||
RUN cd rag && cargo build --release \
|
|
||||||
&& cp target/release/provisioning-rag /app/provisioning-rag
|
|
||||||
|
|
||||||
# Stage 2: Runtime
|
|
||||||
FROM debian:bookworm-slim
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Install runtime dependencies
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
ca-certificates \
|
|
||||||
curl \
|
|
||||||
openssl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Copy binary from builder
|
|
||||||
COPY --from=builder /app/provisioning-rag /app/
|
|
||||||
|
|
||||||
# Create non-root user for security
|
|
||||||
RUN useradd -m -u 1000 provisioning && \
|
|
||||||
chown -R provisioning:provisioning /app
|
|
||||||
|
|
||||||
USER provisioning
|
|
||||||
|
|
||||||
# Environment variables
|
|
||||||
ENV PROVISIONING_LOG_LEVEL=info
|
|
||||||
ENV PROVISIONING_API_HOST=0.0.0.0
|
|
||||||
ENV PROVISIONING_API_PORT=9090
|
|
||||||
ENV PROVISIONING_CACHE_SIZE=1000
|
|
||||||
ENV PROVISIONING_CACHE_TTL_SECS=3600
|
|
||||||
|
|
||||||
# Expose API port
|
|
||||||
EXPOSE 9090
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
|
||||||
CMD curl -f http://localhost:9090/health || exit 1
|
|
||||||
|
|
||||||
# Start the service
|
|
||||||
CMD ["/app/provisioning-rag"]
|
|
||||||
@ -1,197 +1,157 @@
|
|||||||
//! Embeddings module using Rig framework
|
//! Embeddings module using stratum-embeddings
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::sync::Mutex;
|
use stratum_embeddings::{
|
||||||
|
EmbeddingOptions, EmbeddingService, FastEmbedProvider, MemoryCache, OllamaModel,
|
||||||
|
OllamaProvider, OpenAiModel, OpenAiProvider,
|
||||||
|
};
|
||||||
|
|
||||||
use crate::config::EmbeddingConfig;
|
use crate::config::EmbeddingConfig;
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
|
|
||||||
/// Document chunk to be embedded
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct DocumentChunk {
|
pub struct DocumentChunk {
|
||||||
/// Unique chunk ID
|
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
|
||||||
/// Source document path
|
|
||||||
pub source_path: String,
|
pub source_path: String,
|
||||||
|
|
||||||
/// Document type (markdown, kcl, nushell, rust)
|
|
||||||
pub doc_type: String,
|
pub doc_type: String,
|
||||||
|
|
||||||
/// Chunk content (text to embed)
|
|
||||||
pub content: String,
|
pub content: String,
|
||||||
|
|
||||||
/// Document category for filtering
|
|
||||||
pub category: Option<String>,
|
pub category: Option<String>,
|
||||||
|
pub metadata: HashMap<String, String>,
|
||||||
/// Metadata (headings, function names, etc.)
|
|
||||||
pub metadata: std::collections::HashMap<String, String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Embedded document with vector
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct EmbeddedDocument {
|
pub struct EmbeddedDocument {
|
||||||
/// Chunk ID
|
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
|
||||||
/// Source document path
|
|
||||||
pub source_path: String,
|
pub source_path: String,
|
||||||
|
|
||||||
/// Document type
|
|
||||||
pub doc_type: String,
|
pub doc_type: String,
|
||||||
|
|
||||||
/// Original content
|
|
||||||
pub content: String,
|
pub content: String,
|
||||||
|
|
||||||
/// Embedding vector
|
|
||||||
pub embedding: Vec<f32>,
|
pub embedding: Vec<f32>,
|
||||||
|
pub metadata: HashMap<String, String>,
|
||||||
/// Metadata
|
|
||||||
pub metadata: std::collections::HashMap<String, String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// OpenAI HTTP client wrapper for embeddings
|
enum Service {
|
||||||
#[derive(Clone)]
|
OpenAi(EmbeddingService<OpenAiProvider, MemoryCache>),
|
||||||
struct OpenAiClient {
|
Ollama(EmbeddingService<OllamaProvider, MemoryCache>),
|
||||||
api_key: String,
|
FastEmbed(EmbeddingService<FastEmbedProvider, MemoryCache>),
|
||||||
model: String,
|
|
||||||
dimension: usize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl OpenAiClient {
|
|
||||||
/// Create new OpenAI client
|
|
||||||
fn new(api_key: String, model: String, dimension: usize) -> Self {
|
|
||||||
Self {
|
|
||||||
api_key,
|
|
||||||
model,
|
|
||||||
dimension,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate embedding via OpenAI API
|
|
||||||
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
|
|
||||||
let client = reqwest::Client::new();
|
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
|
||||||
struct EmbeddingRequest {
|
|
||||||
input: String,
|
|
||||||
model: String,
|
|
||||||
dimensions: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
|
||||||
struct EmbeddingResponse {
|
|
||||||
data: Vec<EmbeddingData>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
|
||||||
struct EmbeddingData {
|
|
||||||
embedding: Vec<f32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
let request = EmbeddingRequest {
|
|
||||||
input: text.to_string(),
|
|
||||||
model: self.model.clone(),
|
|
||||||
dimensions: if self.dimension == 1536 {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(self.dimension)
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
let response = client
|
|
||||||
.post("https://api.openai.com/v1/embeddings")
|
|
||||||
.header("Authorization", format!("Bearer {}", self.api_key))
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.json(&request)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
crate::error::RagError::embedding(format!("Failed to call OpenAI API: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if !response.status().is_success() {
|
|
||||||
let error_text = response
|
|
||||||
.text()
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|_| "Unknown error".to_string());
|
|
||||||
return Err(crate::error::RagError::embedding(format!(
|
|
||||||
"OpenAI API error: {}",
|
|
||||||
error_text
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
let embedding_response: EmbeddingResponse = response.json().await.map_err(|e| {
|
|
||||||
crate::error::RagError::embedding(format!("Failed to parse OpenAI response: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
embedding_response
|
|
||||||
.data
|
|
||||||
.first()
|
|
||||||
.map(|d| d.embedding.clone())
|
|
||||||
.ok_or_else(|| crate::error::RagError::embedding("No embedding data in response"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Embedding engine using OpenAI via HTTP
|
|
||||||
pub struct EmbeddingEngine {
|
pub struct EmbeddingEngine {
|
||||||
config: EmbeddingConfig,
|
config: EmbeddingConfig,
|
||||||
client: Arc<Mutex<Option<OpenAiClient>>>,
|
service: Service,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EmbeddingEngine {
|
impl EmbeddingEngine {
|
||||||
/// Create a new embedding engine
|
|
||||||
pub fn new(config: EmbeddingConfig) -> Result<Self> {
|
pub fn new(config: EmbeddingConfig) -> Result<Self> {
|
||||||
// Validate configuration
|
let cache = MemoryCache::new(1000, Duration::from_secs(3600));
|
||||||
match config.provider.as_str() {
|
|
||||||
|
let service = match config.provider.as_str() {
|
||||||
"openai" => {
|
"openai" => {
|
||||||
if config.openai_api_key.is_none() && !config.fallback_local {
|
let api_key = config
|
||||||
|
.openai_api_key
|
||||||
|
.clone()
|
||||||
|
.or_else(|| std::env::var("OPENAI_API_KEY").ok());
|
||||||
|
|
||||||
|
let api_key_str = if let Some(key) = api_key {
|
||||||
|
key
|
||||||
|
} else if config.fallback_local {
|
||||||
|
"dummy".to_string() // Will fail, but fallback will take over
|
||||||
|
} else {
|
||||||
return Err(crate::error::RagError::config(
|
return Err(crate::error::RagError::config(
|
||||||
"OpenAI API key required for OpenAI provider (or enable fallback_local). \
|
"OpenAI API key required. Set OPENAI_API_KEY or enable fallback_local",
|
||||||
Set OPENAI_API_KEY environment variable.",
|
|
||||||
));
|
));
|
||||||
|
};
|
||||||
|
|
||||||
|
let model = match config.model.as_str() {
|
||||||
|
"text-embedding-3-small" => OpenAiModel::TextEmbedding3Small,
|
||||||
|
"text-embedding-3-large" => OpenAiModel::TextEmbedding3Large,
|
||||||
|
"text-embedding-ada-002" => OpenAiModel::TextEmbeddingAda002,
|
||||||
|
_ => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Unknown model '{}', using text-embedding-3-small",
|
||||||
|
config.model
|
||||||
|
);
|
||||||
|
OpenAiModel::TextEmbedding3Small
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let provider = OpenAiProvider::new(api_key_str, model).map_err(|e| {
|
||||||
|
crate::error::RagError::embedding(format!("OpenAI provider failed: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut svc = EmbeddingService::new(provider).with_cache(cache);
|
||||||
|
|
||||||
|
if config.fallback_local {
|
||||||
|
tracing::info!("Fallback to local embeddings enabled");
|
||||||
|
let fallback = Arc::new(FastEmbedProvider::small().map_err(|e| {
|
||||||
|
crate::error::RagError::embedding(format!("Fallback failed: {}", e))
|
||||||
|
})?);
|
||||||
|
svc = svc.with_fallback(fallback);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Service::OpenAi(svc)
|
||||||
}
|
}
|
||||||
"ollama" => {
|
"ollama" => {
|
||||||
tracing::info!("Using Ollama for local embeddings (no API costs)");
|
use stratum_embeddings::OllamaModel;
|
||||||
|
|
||||||
|
let model = if config.model == "nomic-embed-text" {
|
||||||
|
OllamaModel::NomicEmbed
|
||||||
|
} else if config.model == "mxbai-embed-large" {
|
||||||
|
OllamaModel::MxbaiEmbed
|
||||||
|
} else if config.model == "all-minilm" {
|
||||||
|
OllamaModel::AllMiniLm
|
||||||
|
} else {
|
||||||
|
OllamaModel::Custom(config.model.clone(), config.dimension)
|
||||||
|
};
|
||||||
|
|
||||||
|
let provider = OllamaProvider::new(model).map_err(|e| {
|
||||||
|
crate::error::RagError::embedding(format!("Ollama provider failed: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Service::Ollama(EmbeddingService::new(provider).with_cache(cache))
|
||||||
}
|
}
|
||||||
"local" => {
|
"local" => {
|
||||||
tracing::info!("Using local embedding model (no API costs)");
|
tracing::info!("Using FastEmbed local embeddings (no API costs)");
|
||||||
|
let provider = FastEmbedProvider::small().map_err(|e| {
|
||||||
|
crate::error::RagError::embedding(format!("FastEmbed provider failed: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Service::FastEmbed(EmbeddingService::new(provider).with_cache(cache))
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
return Err(crate::error::RagError::config(format!(
|
return Err(crate::error::RagError::config(format!(
|
||||||
"Unknown embedding provider: {}. Supported: openai, ollama, local",
|
"Unknown provider: {}. Supported: openai, ollama, local",
|
||||||
config.provider
|
config.provider
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
"Initialized embedding engine: {} (provider: {}, dimension: {})",
|
"Initialized stratum-embeddings: {} (provider: {}, dim: {})",
|
||||||
config.model,
|
config.model,
|
||||||
config.provider,
|
config.provider,
|
||||||
config.dimension
|
config.dimension
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self { config, service })
|
||||||
config,
|
|
||||||
client: Arc::new(Mutex::new(None)),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get embedding configuration
|
|
||||||
pub fn config(&self) -> &EmbeddingConfig {
|
pub fn config(&self) -> &EmbeddingConfig {
|
||||||
&self.config
|
&self.config
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Embed a single chunk using Rig framework
|
|
||||||
///
|
|
||||||
/// This is a stub implementation that creates zero vectors.
|
|
||||||
/// In production, this would call the Rig embeddings API.
|
|
||||||
pub async fn embed_chunk(&self, chunk: &DocumentChunk) -> Result<EmbeddedDocument> {
|
pub async fn embed_chunk(&self, chunk: &DocumentChunk) -> Result<EmbeddedDocument> {
|
||||||
let embedding = self.generate_embedding(&chunk.content).await?;
|
let options = EmbeddingOptions::default_with_cache();
|
||||||
|
|
||||||
|
let embedding = match &self.service {
|
||||||
|
Service::OpenAi(svc) => svc.embed(&chunk.content, &options).await,
|
||||||
|
Service::Ollama(svc) => svc.embed(&chunk.content, &options).await,
|
||||||
|
Service::FastEmbed(svc) => svc.embed(&chunk.content, &options).await,
|
||||||
|
}
|
||||||
|
.map_err(|e| crate::error::RagError::embedding(format!("Embedding failed: {}", e)))?;
|
||||||
|
|
||||||
Ok(EmbeddedDocument {
|
Ok(EmbeddedDocument {
|
||||||
id: chunk.id.clone(),
|
id: chunk.id.clone(),
|
||||||
@ -203,133 +163,33 @@ impl EmbeddingEngine {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Embed multiple chunks in batch
|
|
||||||
pub async fn embed_batch(&self, chunks: &[DocumentChunk]) -> Result<Vec<EmbeddedDocument>> {
|
pub async fn embed_batch(&self, chunks: &[DocumentChunk]) -> Result<Vec<EmbeddedDocument>> {
|
||||||
let mut results = Vec::new();
|
let options = EmbeddingOptions::default_with_cache();
|
||||||
|
let texts: Vec<String> = chunks.iter().map(|c| c.content.clone()).collect();
|
||||||
|
|
||||||
for (idx, chunk) in chunks.iter().enumerate() {
|
let result = match &self.service {
|
||||||
let embedded = self.embed_chunk(chunk).await?;
|
Service::OpenAi(svc) => svc.embed_batch(texts, &options).await,
|
||||||
results.push(embedded);
|
Service::Ollama(svc) => svc.embed_batch(texts, &options).await,
|
||||||
|
Service::FastEmbed(svc) => svc.embed_batch(texts, &options).await,
|
||||||
// Log progress
|
|
||||||
if (idx + 1) % 10 == 0 {
|
|
||||||
tracing::debug!("Embedded {}/{} chunks", idx + 1, chunks.len());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
.map_err(|e| crate::error::RagError::embedding(format!("Batch embed failed: {}", e)))?;
|
||||||
|
|
||||||
tracing::info!("Embedded {} chunks total", chunks.len());
|
let results = chunks
|
||||||
|
.iter()
|
||||||
|
.zip(result.embeddings)
|
||||||
|
.map(|(chunk, embedding)| EmbeddedDocument {
|
||||||
|
id: chunk.id.clone(),
|
||||||
|
source_path: chunk.source_path.clone(),
|
||||||
|
doc_type: chunk.doc_type.clone(),
|
||||||
|
content: chunk.content.clone(),
|
||||||
|
embedding,
|
||||||
|
metadata: chunk.metadata.clone(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
tracing::info!("Embedded {} chunks (stratum-embeddings)", chunks.len());
|
||||||
Ok(results)
|
Ok(results)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate embedding for text using configured provider
|
|
||||||
async fn generate_embedding(&self, text: &str) -> Result<Vec<f32>> {
|
|
||||||
if text.is_empty() {
|
|
||||||
return Err(crate::error::RagError::embedding(
|
|
||||||
"Empty text cannot be embedded",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
tracing::trace!("Generating embedding for text: {} chars", text.len());
|
|
||||||
|
|
||||||
match self.config.provider.as_str() {
|
|
||||||
"openai" => self.embed_openai(text).await,
|
|
||||||
"ollama" => self.embed_ollama(text).await,
|
|
||||||
"local" => self.embed_local(text).await,
|
|
||||||
_ => Err(crate::error::RagError::embedding(format!(
|
|
||||||
"Unknown embedding provider: {}",
|
|
||||||
self.config.provider
|
|
||||||
))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate embedding via OpenAI API
|
|
||||||
async fn embed_openai(&self, text: &str) -> Result<Vec<f32>> {
|
|
||||||
let api_key = self
|
|
||||||
.config
|
|
||||||
.openai_api_key
|
|
||||||
.clone()
|
|
||||||
.or_else(|| std::env::var("OPENAI_API_KEY").ok())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
crate::error::RagError::embedding(
|
|
||||||
"OpenAI API key not found. Set OPENAI_API_KEY env var or config.",
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let mut client_lock = self.client.lock().await;
|
|
||||||
if client_lock.is_none() {
|
|
||||||
*client_lock = Some(OpenAiClient::new(
|
|
||||||
api_key.clone(),
|
|
||||||
self.config.model.clone(),
|
|
||||||
self.config.dimension,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let client = client_lock.as_ref().unwrap();
|
|
||||||
client.embed(text).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate embedding via Ollama (local)
|
|
||||||
async fn embed_ollama(&self, text: &str) -> Result<Vec<f32>> {
|
|
||||||
let client = reqwest::Client::new();
|
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
|
||||||
struct OllamaRequest {
|
|
||||||
model: String,
|
|
||||||
prompt: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
|
||||||
struct OllamaResponse {
|
|
||||||
embedding: Vec<f32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
let request = OllamaRequest {
|
|
||||||
model: self.config.model.clone(),
|
|
||||||
prompt: text.to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let response = client
|
|
||||||
.post("http://localhost:11434/api/embeddings")
|
|
||||||
.json(&request)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
crate::error::RagError::embedding(format!(
|
|
||||||
"Failed to call Ollama API (ensure Ollama is running on localhost:11434): {}",
|
|
||||||
e
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if !response.status().is_success() {
|
|
||||||
let error_text = response
|
|
||||||
.text()
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|_| "Unknown error".to_string());
|
|
||||||
return Err(crate::error::RagError::embedding(format!(
|
|
||||||
"Ollama API error: {}",
|
|
||||||
error_text
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
let embedding_response: OllamaResponse = response.json().await.map_err(|e| {
|
|
||||||
crate::error::RagError::embedding(format!("Failed to parse Ollama response: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(embedding_response.embedding)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate embedding using local model (stub for future implementation)
|
|
||||||
async fn embed_local(&self, text: &str) -> Result<Vec<f32>> {
|
|
||||||
tracing::warn!(
|
|
||||||
"Local embeddings not fully implemented. Returning zero vector for now. For \
|
|
||||||
production local embeddings, use Ollama or integrate huggingface transformers."
|
|
||||||
);
|
|
||||||
|
|
||||||
// Return zero vector of correct dimension
|
|
||||||
// Future: integrate sentence-transformers or ONNX models
|
|
||||||
Ok(vec![0.0; self.config.dimension])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@ -337,57 +197,54 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_embedding_engine_creation_openai() {
|
fn test_engine_openai() {
|
||||||
let config = EmbeddingConfig {
|
let config = EmbeddingConfig {
|
||||||
provider: "openai".to_string(),
|
provider: "openai".to_string(),
|
||||||
openai_api_key: Some("test-key".to_string()),
|
openai_api_key: Some("test-key".to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
assert!(EmbeddingEngine::new(config).is_ok());
|
||||||
let result = EmbeddingEngine::new(config);
|
|
||||||
assert!(result.is_ok());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_embedding_engine_creation_openai_no_key() {
|
fn test_engine_ollama() {
|
||||||
let config = EmbeddingConfig {
|
let config = EmbeddingConfig {
|
||||||
provider: "openai".to_string(),
|
provider: "ollama".to_string(),
|
||||||
openai_api_key: None,
|
model: "nomic-embed-text".to_string(),
|
||||||
fallback_local: false,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
assert!(EmbeddingEngine::new(config).is_ok());
|
||||||
let result = EmbeddingEngine::new(config);
|
|
||||||
assert!(result.is_err());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_local_embedding_engine() {
|
fn test_engine_local() {
|
||||||
let config = EmbeddingConfig {
|
let config = EmbeddingConfig {
|
||||||
provider: "local".to_string(),
|
provider: "local".to_string(),
|
||||||
|
dimension: 384,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
assert!(EmbeddingEngine::new(config).is_ok());
|
||||||
let engine = EmbeddingEngine::new(config);
|
|
||||||
assert!(engine.is_ok());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_embed_chunk() {
|
async fn test_embed_chunk() {
|
||||||
let config = EmbeddingConfig {
|
let config = EmbeddingConfig {
|
||||||
provider: "local".to_string(),
|
provider: "local".to_string(),
|
||||||
|
dimension: 384,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let engine = EmbeddingEngine::new(config).unwrap();
|
let engine = EmbeddingEngine::new(config).unwrap();
|
||||||
|
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
metadata.insert("section".to_string(), "test".to_string());
|
||||||
|
|
||||||
let chunk = DocumentChunk {
|
let chunk = DocumentChunk {
|
||||||
id: "test-1".to_string(),
|
id: "test-1".to_string(),
|
||||||
source_path: "test.md".to_string(),
|
source_path: "/test/doc.md".to_string(),
|
||||||
doc_type: "markdown".to_string(),
|
doc_type: "markdown".to_string(),
|
||||||
content: "This is test content".to_string(),
|
content: "Test document".to_string(),
|
||||||
category: None,
|
category: Some("test".to_string()),
|
||||||
metadata: std::collections::HashMap::new(),
|
metadata,
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = engine.embed_chunk(&chunk).await;
|
let result = engine.embed_chunk(&chunk).await;
|
||||||
@ -395,6 +252,6 @@ mod tests {
|
|||||||
|
|
||||||
let embedded = result.unwrap();
|
let embedded = result.unwrap();
|
||||||
assert_eq!(embedded.id, "test-1");
|
assert_eq!(embedded.id, "test-1");
|
||||||
assert_eq!(embedded.embedding.len(), 1536); // Default dimension
|
assert_eq!(embedded.embedding.len(), 384);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,64 +1,52 @@
|
|||||||
//! LLM (Large Language Model) integration module
|
//! LLM integration using stratum-llm
|
||||||
//! Provides Claude API integration for RAG-based answer generation
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use stratum_llm::{
|
||||||
|
AnthropicProvider, ConfiguredProvider, CredentialSource, GenerationOptions, Message,
|
||||||
|
ProviderChain, Role, UnifiedClient,
|
||||||
|
};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
|
|
||||||
/// Claude API request message
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub struct ClaudeMessage {
|
|
||||||
pub role: String,
|
|
||||||
pub content: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Claude API response
|
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
|
||||||
pub struct ClaudeResponse {
|
|
||||||
pub content: Vec<ClaudeContent>,
|
|
||||||
pub stop_reason: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Claude response content
|
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
|
||||||
pub struct ClaudeContent {
|
|
||||||
#[serde(rename = "type")]
|
|
||||||
pub content_type: String,
|
|
||||||
pub text: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// LLM Client for Claude API
|
|
||||||
pub struct LlmClient {
|
pub struct LlmClient {
|
||||||
api_key: String,
|
client: UnifiedClient,
|
||||||
pub model: String,
|
pub model: String,
|
||||||
base_url: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LlmClient {
|
impl LlmClient {
|
||||||
/// Create a new Claude LLM client
|
|
||||||
pub fn new(model: String) -> Result<Self> {
|
pub fn new(model: String) -> Result<Self> {
|
||||||
// Get API key from environment
|
let api_key = std::env::var("ANTHROPIC_API_KEY").ok();
|
||||||
let api_key = std::env::var("ANTHROPIC_API_KEY").unwrap_or_else(|_| {
|
|
||||||
tracing::warn!("ANTHROPIC_API_KEY not set - Claude API calls will fail");
|
|
||||||
String::new()
|
|
||||||
});
|
|
||||||
|
|
||||||
Ok(Self {
|
if api_key.is_none() {
|
||||||
api_key,
|
tracing::warn!("ANTHROPIC_API_KEY not set - LLM calls will fail");
|
||||||
model,
|
|
||||||
base_url: "https://api.anthropic.com/v1".to_string(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate an answer using Claude
|
|
||||||
pub async fn generate_answer(&self, query: &str, context: &str) -> Result<String> {
|
|
||||||
// If no API key, return placeholder
|
|
||||||
if self.api_key.is_empty() {
|
|
||||||
return Ok(self.generate_placeholder(query, context));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build the system prompt
|
let provider =
|
||||||
|
AnthropicProvider::new(api_key.unwrap_or_default(), model.clone());
|
||||||
|
|
||||||
|
let configured = ConfiguredProvider {
|
||||||
|
provider: Box::new(provider),
|
||||||
|
credential_source: CredentialSource::EnvVar {
|
||||||
|
name: "ANTHROPIC_API_KEY".to_string(),
|
||||||
|
},
|
||||||
|
priority: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
let chain = ProviderChain::with_providers(vec![configured]);
|
||||||
|
|
||||||
|
let client = UnifiedClient::builder()
|
||||||
|
.with_chain(chain)
|
||||||
|
.build()
|
||||||
|
.map_err(|e| {
|
||||||
|
crate::error::RagError::LlmError(format!("Failed to build LLM client: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
info!("Initialized stratum-llm client: {}", model);
|
||||||
|
|
||||||
|
Ok(Self { client, model })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn generate_answer(&self, query: &str, context: &str) -> Result<String> {
|
||||||
let system_prompt = format!(
|
let system_prompt = format!(
|
||||||
r#"You are a helpful assistant answering questions about a provisioning platform.
|
r#"You are a helpful assistant answering questions about a provisioning platform.
|
||||||
You have been provided with relevant documentation context below.
|
You have been provided with relevant documentation context below.
|
||||||
@ -71,118 +59,32 @@ Be concise and accurate.
|
|||||||
context
|
context
|
||||||
);
|
);
|
||||||
|
|
||||||
// Build the user message
|
let messages = vec![
|
||||||
let user_message = query.to_string();
|
Message {
|
||||||
|
role: Role::System,
|
||||||
|
content: system_prompt,
|
||||||
|
},
|
||||||
|
Message {
|
||||||
|
role: Role::User,
|
||||||
|
content: query.to_string(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
// Call Claude API
|
let options = GenerationOptions {
|
||||||
self.call_claude_api(&system_prompt, &user_message).await
|
max_tokens: Some(1024),
|
||||||
}
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
/// Call Claude API with messages
|
let response = self
|
||||||
async fn call_claude_api(&self, system: &str, user_message: &str) -> Result<String> {
|
.client
|
||||||
let client = reqwest::Client::new();
|
.generate(&messages, Some(&options))
|
||||||
|
|
||||||
// Build request payload
|
|
||||||
let payload = serde_json::json!({
|
|
||||||
"model": self.model,
|
|
||||||
"max_tokens": 1024,
|
|
||||||
"system": system,
|
|
||||||
"messages": [
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": user_message
|
|
||||||
}
|
|
||||||
]
|
|
||||||
});
|
|
||||||
|
|
||||||
// Make the API request
|
|
||||||
let response = client
|
|
||||||
.post(format!("{}/messages", self.base_url))
|
|
||||||
.header("anthropic-version", "2023-06-01")
|
|
||||||
.header("x-api-key", &self.api_key)
|
|
||||||
.json(&payload)
|
|
||||||
.send()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
crate::error::RagError::LlmError(format!("Claude API request failed: {}", e))
|
crate::error::RagError::LlmError(format!("LLM generation failed: {}", e))
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
// Check status
|
info!("Generated answer: {} characters", response.content.len());
|
||||||
if !response.status().is_success() {
|
Ok(response.content)
|
||||||
let status = response.status();
|
|
||||||
let error_text = response
|
|
||||||
.text()
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|_| "Unknown error".to_string());
|
|
||||||
return Err(crate::error::RagError::LlmError(format!(
|
|
||||||
"Claude API error {}: {}",
|
|
||||||
status, error_text
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse response
|
|
||||||
let claude_response: ClaudeResponse = response.json().await.map_err(|e| {
|
|
||||||
crate::error::RagError::LlmError(format!("Failed to parse Claude response: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// Extract text from response
|
|
||||||
let answer = claude_response
|
|
||||||
.content
|
|
||||||
.first()
|
|
||||||
.and_then(|c| c.text.clone())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
crate::error::RagError::LlmError("No text in Claude response".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"Claude API call successful, generated {} characters",
|
|
||||||
answer.len()
|
|
||||||
);
|
|
||||||
Ok(answer)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generate placeholder answer when API key is missing
|
|
||||||
fn generate_placeholder(&self, query: &str, context: &str) -> String {
|
|
||||||
format!(
|
|
||||||
"Based on the provided context about the provisioning platform:\n\n{}\n\n(Note: This \
|
|
||||||
is a placeholder response. Set ANTHROPIC_API_KEY environment variable for full \
|
|
||||||
Claude integration.)",
|
|
||||||
self.format_context_summary(query, context)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Format a summary of the context
|
|
||||||
fn format_context_summary(&self, query: &str, context: &str) -> String {
|
|
||||||
let context_lines = context.lines().count();
|
|
||||||
let query_lower = query.to_lowercase();
|
|
||||||
|
|
||||||
if query_lower.contains("deploy") || query_lower.contains("create") {
|
|
||||||
format!(
|
|
||||||
"Your question about deployment is addressed in {} lines of documentation. The \
|
|
||||||
system supports multi-cloud deployment across AWS, UpCloud, and local \
|
|
||||||
environments.",
|
|
||||||
context_lines
|
|
||||||
)
|
|
||||||
} else if query_lower.contains("architecture") || query_lower.contains("design") {
|
|
||||||
format!(
|
|
||||||
"The provisioning platform uses a modular architecture as described in {} lines \
|
|
||||||
of documentation. Core components include Orchestrator, Control Center, and MCP \
|
|
||||||
Server integration.",
|
|
||||||
context_lines
|
|
||||||
)
|
|
||||||
} else if query_lower.contains("security") || query_lower.contains("auth") {
|
|
||||||
format!(
|
|
||||||
"Security features are documented in {} lines. The system implements JWT-based \
|
|
||||||
authentication, Cedar-based authorization, and dynamic secrets management.",
|
|
||||||
context_lines
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
"Your question is addressed in the provided {} lines of documentation. Please \
|
|
||||||
review the context above for details.",
|
|
||||||
context_lines
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,38 +94,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_llm_client_creation() {
|
fn test_llm_client_creation() {
|
||||||
let client = LlmClient::new("claude-opus-4-1".to_string());
|
let client = LlmClient::new("claude-opus-4".to_string());
|
||||||
assert!(client.is_ok());
|
assert!(client.is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_placeholder_generation() {
|
fn test_llm_client_model() {
|
||||||
let client = LlmClient {
|
let client = LlmClient::new("claude-sonnet-4".to_string());
|
||||||
api_key: String::new(),
|
assert!(client.is_ok());
|
||||||
model: "claude-opus-4-1".to_string(),
|
assert_eq!(client.unwrap().model, "claude-sonnet-4");
|
||||||
base_url: "https://api.anthropic.com/v1".to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let query = "How do I deploy the platform?";
|
|
||||||
let context = "Deployment is done using provisioning commands";
|
|
||||||
|
|
||||||
let answer = client.generate_placeholder(query, context);
|
|
||||||
assert!(answer.contains("deployment"));
|
|
||||||
assert!(answer.contains("placeholder"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_context_summary_formatting() {
|
|
||||||
let client = LlmClient {
|
|
||||||
api_key: String::new(),
|
|
||||||
model: "claude-opus-4-1".to_string(),
|
|
||||||
base_url: "https://api.anthropic.com/v1".to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let deployment_query = "How do I deploy?";
|
|
||||||
let context = "Line 1\nLine 2\nLine 3";
|
|
||||||
let summary = client.format_context_summary(deployment_query, context);
|
|
||||||
assert!(summary.contains("deployment"));
|
|
||||||
assert!(summary.contains("3"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,12 +1,34 @@
|
|||||||
//! RAG system command-line tool
|
//! RAG system command-line tool
|
||||||
|
|
||||||
|
use clap::Parser;
|
||||||
use provisioning_rag::config::RagConfig;
|
use provisioning_rag::config::RagConfig;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
#[command(name = "rag")]
|
||||||
|
#[command(about = "Retrieval-Augmented Generation system")]
|
||||||
|
struct Args {
|
||||||
|
/// Configuration file path (highest priority)
|
||||||
|
#[arg(short = 'c', long, env = "RAG_CONFIG")]
|
||||||
|
config: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Configuration directory (searches for rag.ncl|toml|json)
|
||||||
|
#[arg(long, env = "PROVISIONING_CONFIG_DIR")]
|
||||||
|
config_dir: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Deployment mode (solo, multiuser, cicd, enterprise)
|
||||||
|
#[arg(short = 'm', long, env = "RAG_MODE")]
|
||||||
|
mode: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
// Initialize logging
|
// Initialize logging
|
||||||
tracing_subscriber::fmt::init();
|
tracing_subscriber::fmt::init();
|
||||||
|
|
||||||
|
// Parse CLI arguments
|
||||||
|
let _args = Args::parse();
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
let _config = RagConfig::default();
|
let _config = RagConfig::default();
|
||||||
|
|
||||||
|
|||||||
@ -1,11 +1,11 @@
|
|||||||
[package]
|
[package]
|
||||||
authors = { workspace = true }
|
authors.workspace = true
|
||||||
description = "HTTP service client wrappers for provisioning platform services"
|
description = "HTTP service client wrappers for provisioning platform services"
|
||||||
edition = { workspace = true }
|
edition.workspace = true
|
||||||
license = { workspace = true }
|
license.workspace = true
|
||||||
name = "service-clients"
|
name = "service-clients"
|
||||||
repository = { workspace = true }
|
repository.workspace = true
|
||||||
version = { workspace = true }
|
version.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
@ -15,9 +15,7 @@ serde = { workspace = true, features = ["derive"] }
|
|||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["full"] }
|
tokio = { workspace = true, features = ["full"] }
|
||||||
|
machines = { workspace = true }
|
||||||
# Service types (optional - only if not using generic types)
|
|
||||||
machines = { path = "../../../../submodules/prov-ecosystem/crates/machines" }
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
|
|||||||
@ -1,9 +1,15 @@
|
|||||||
[package]
|
[package]
|
||||||
authors = ["Provisioning Team"]
|
authors.workspace = true
|
||||||
description = "Vault Service for Provisioning Platform with secrets and key management (Age dev, Cosmian KMS prod, RustyVault self-hosted)"
|
description = "Vault Service for Provisioning Platform with secrets and key management (Age dev, Cosmian KMS prod, RustyVault self-hosted)"
|
||||||
edition = "2021"
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
name = "vault-service"
|
name = "vault-service"
|
||||||
version = "0.2.0"
|
repository.workspace = true
|
||||||
|
version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "provisioning-vault-service"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Async runtime
|
# Async runtime
|
||||||
@ -23,10 +29,10 @@ toml = { workspace = true }
|
|||||||
reqwest = { workspace = true }
|
reqwest = { workspace = true }
|
||||||
|
|
||||||
# Age encryption (development)
|
# Age encryption (development)
|
||||||
age = "0.11"
|
age = { workspace = true }
|
||||||
|
|
||||||
# RustyVault (self-hosted Vault alternative)
|
# RustyVault (self-hosted Vault alternative)
|
||||||
rusty_vault = "0.2.1"
|
rusty_vault = { workspace = true }
|
||||||
|
|
||||||
# Cryptography
|
# Cryptography
|
||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
@ -46,18 +52,15 @@ chrono = { workspace = true, features = ["serde"] }
|
|||||||
# Configuration
|
# Configuration
|
||||||
config = { workspace = true }
|
config = { workspace = true }
|
||||||
|
|
||||||
# SecretumVault (Enterprise secrets management)
|
# SecretumVault (Enterprise secrets management - optional)
|
||||||
secretumvault = { workspace = true }
|
secretumvault = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
http-body-util = { workspace = true }
|
||||||
mockito = { workspace = true }
|
mockito = { workspace = true }
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
tokio-test = { workspace = true }
|
tokio-test = { workspace = true }
|
||||||
|
|
||||||
[[bin]]
|
|
||||||
name = "vault-service"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
name = "vault_service"
|
name = "vault_service"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|||||||
1
secretumvault
Submodule
1
secretumvault
Submodule
@ -0,0 +1 @@
|
|||||||
|
Subproject commit 91eefc86fa03826997401facee620f3b6dfd65e1
|
||||||
1
stratumiops
Submodule
1
stratumiops
Submodule
@ -0,0 +1 @@
|
|||||||
|
Subproject commit 9864f88c14ac030f0fa914fd46c2cf4c1a412fc0
|
||||||
1
syntaxis
Submodule
1
syntaxis
Submodule
@ -0,0 +1 @@
|
|||||||
|
Subproject commit 48d7503b4817e38dd86f494e09022b77c0652159
|
||||||
Loading…
x
Reference in New Issue
Block a user