e74320a8e8
- Start background watcher on startup when INGEST_WATCH_INTERVAL > 0 - Procfile: add INGEST_WATCH_INTERVAL=30 and INGEST_SVC_URL for supervisor Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
90 lines
2.2 KiB
Go
90 lines
2.2 KiB
Go
// ingestion/cmd/server/main.go
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/watcher"
|
|
)
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func envInt(key string, fallback int) int {
|
|
if v := os.Getenv(key); v != "" {
|
|
if n, err := strconv.Atoi(v); err == nil {
|
|
return n
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func main() {
|
|
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
|
|
|
brainDir := envOr("INGEST_BRAIN_DIR", "../brain")
|
|
port := envOr("INGEST_PORT", "3300")
|
|
|
|
llmURL := envOr("INGEST_LLM_URL", "http://iguana:4000/v1")
|
|
llmKey := os.Getenv("INGEST_LLM_KEY")
|
|
llmModel := envOr("INGEST_LLM_MODEL", "koala/qwen35-9b-fast")
|
|
llmTimeoutMins := envInt("INGEST_LLM_TIMEOUT", 15)
|
|
chunkSize := envInt("INGEST_CHUNK_SIZE", 6000)
|
|
watchInterval := envInt("INGEST_WATCH_INTERVAL", 30)
|
|
|
|
llmClient := llm.New(llmURL, llmKey, llmModel, time.Duration(llmTimeoutMins)*time.Minute)
|
|
|
|
pipelineCfg := pipeline.Config{
|
|
Complete: llmClient.Complete,
|
|
ChunkSize: chunkSize,
|
|
}
|
|
|
|
h := api.NewHandler(brainDir, logger, pipelineCfg)
|
|
|
|
ctx := context.Background()
|
|
if watchInterval > 0 {
|
|
watcher.Start(ctx, watcher.Config{
|
|
BrainDir: brainDir,
|
|
Interval: time.Duration(watchInterval) * time.Second,
|
|
Pipeline: pipelineCfg,
|
|
})
|
|
}
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /query", h.Query)
|
|
mux.HandleFunc("POST /write", h.Write)
|
|
mux.HandleFunc("POST /ingest", h.Ingest)
|
|
mux.HandleFunc("POST /ingest-path", h.IngestPath)
|
|
|
|
addr := ":" + port
|
|
watchIntervalLog := "disabled"
|
|
if watchInterval > 0 {
|
|
watchIntervalLog = fmt.Sprintf("%ds", watchInterval)
|
|
}
|
|
logger.Info("ingestion server starting",
|
|
"addr", addr,
|
|
"brain_dir", brainDir,
|
|
"llm_url", llmURL,
|
|
"llm_model", llmModel,
|
|
"chunk_size", chunkSize,
|
|
"watch_interval", watchIntervalLog,
|
|
)
|
|
if err := http.ListenAndServe(addr, mux); err != nil {
|
|
logger.Error("server stopped", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|