104 lines
2.6 KiB
Go
104 lines
2.6 KiB
Go
// ingestion/cmd/server/main.go
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/mcp"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/watcher"
|
|
)
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func envInt(key string, fallback int) int {
|
|
if v := os.Getenv(key); v != "" {
|
|
if n, err := strconv.Atoi(v); err == nil {
|
|
return n
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func main() {
|
|
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
|
|
|
brainDir := envOr("INGEST_BRAIN_DIR", "../brain")
|
|
port := envOr("INGEST_PORT", "3300")
|
|
|
|
llmURL := envOr("INGEST_LLM_URL", "http://iguana:4000/v1")
|
|
llmKey := os.Getenv("INGEST_LLM_KEY")
|
|
llmModel := envOr("INGEST_LLM_MODEL", "koala/qwen35-9b-fast")
|
|
llmTimeoutMins := envInt("INGEST_LLM_TIMEOUT", 15)
|
|
chunkSize := envInt("INGEST_CHUNK_SIZE", 6000)
|
|
watchInterval := envInt("INGEST_WATCH_INTERVAL", 30)
|
|
|
|
llmClient := llm.New(llmURL, llmKey, llmModel, time.Duration(llmTimeoutMins)*time.Minute)
|
|
|
|
pipelineCfg := pipeline.Config{
|
|
Complete: llmClient.Complete,
|
|
ChunkSize: chunkSize,
|
|
}
|
|
|
|
h := api.NewHandler(brainDir, logger, pipelineCfg)
|
|
|
|
mcpSrv := mcp.NewServer(brainDir, &pipelineCfg, llmClient.Complete)
|
|
|
|
mcpToken := os.Getenv("BRAIN_MCP_TOKEN")
|
|
if mcpToken == "" {
|
|
logger.Error("BRAIN_MCP_TOKEN not set")
|
|
os.Exit(1)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
if watchInterval > 0 {
|
|
watcher.Start(ctx, watcher.Config{
|
|
BrainDir: brainDir,
|
|
Interval: time.Duration(watchInterval) * time.Second,
|
|
Pipeline: pipelineCfg,
|
|
})
|
|
}
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /query", h.Query)
|
|
mux.HandleFunc("POST /write", h.Write)
|
|
mux.HandleFunc("POST /ingest", h.Ingest)
|
|
mux.HandleFunc("POST /ingest-path", h.IngestPath)
|
|
mux.HandleFunc("POST /ingest-raw", h.IngestRaw)
|
|
mux.HandleFunc("POST /backfill-refs", h.BackfillRefs)
|
|
mux.HandleFunc("GET /pass-rate", h.PassRate)
|
|
mux.Handle("/mcp", mcp.BearerAuth(mcpToken, mcpSrv))
|
|
|
|
addr := ":" + port
|
|
watchIntervalLog := "disabled"
|
|
if watchInterval > 0 {
|
|
watchIntervalLog = fmt.Sprintf("%ds", watchInterval)
|
|
}
|
|
logger.Info("ingestion server starting",
|
|
"addr", addr,
|
|
"brain_dir", brainDir,
|
|
"llm_url", llmURL,
|
|
"llm_model", llmModel,
|
|
"chunk_size", chunkSize,
|
|
"watch_interval", watchIntervalLog,
|
|
"mcp_enabled", true,
|
|
)
|
|
if err := http.ListenAndServe(addr, mux); err != nil {
|
|
logger.Error("server stopped", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|