feat(brain): hybrid BM25 + pgvector retrieval (opt-in)
Wires nomic-embed-text (iguana ollama) + pgvector on the shared
postgres18 into brain_query / brain_answer via Reciprocal Rank Fusion.
Pure BM25 stays the default; setting BRAIN_PG_DSN and BRAIN_EMBED_URL
together opts in. Setting one without the other is misconfiguration →
exit 1.
New packages:
- internal/embed
Client.Embed(ctx, text) → []float32 via POST {URL}/api/embed.
Defaults to nomic-embed-text:latest (768 dim). nil-on-empty-URL so
callers gate on a single nil check.
- internal/vectorstore
PGStore wraps a pgxpool against postgres18. Init creates
brain_embeddings(path PK, vector(768), updated_at) + HNSW cosine
index idempotently. Upsert / Delete / Search / KnownPaths.
Sync(brainDir, store, embedder) diffs brain/wiki/ against the store
and upserts new files / deletes removed ones; StartSync runs it on
a ticker (default 300s). Integration tests gated by BRAIN_PG_TEST_DSN.
- scripts/brain-embeddings-init.sql
One-time DBA setup: brain DB, brain_app role, vector extension,
GRANTs. Idempotent.
Search layer:
- search.QueryOptions gains Vector + Embedder fields.
- QueryContext is the cancellable variant; Query stays for callers.
- When both are set, BM25 (top-N) and pgvector (top-4N) candidates
merge via Reciprocal Rank Fusion (k=60, Cormack et al. 2009 — no
tuning knob, robust to scale differences between rankers).
- Vector-only hits are hydrated from disk so callers see uniform
Result records (path, title, excerpt, wing, hall, score).
- Wing/hall filters still apply to vector candidates via path-prefix.
- On embedder/vector errors the search falls back to BM25 — embedding
outage degrades quality but doesn't take the brain offline.
MCP wiring:
- mcp.Server.WithHybridRetrieval(v, e) opt-in setter, same shape as
WithReranker.
- brainQuery and brainAnswer pass the wired vector/embedder through
to search.QueryContext.
REST:
- POST /backfill-embeddings drives Sync synchronously. Returns
{added, deleted, errors[]}. 503 when feature is unconfigured.
cmd/server/main.go:
- BRAIN_PG_DSN + BRAIN_EMBED_URL together enable hybrid; one alone
→ exit 1.
- vectorAdapter bridges *PGStore (returns []Hit) to
search.VectorSearcher (which takes []VectorHit) without either
package importing the other.
- BRAIN_EMBED_SYNC_INTERVAL (default 300s) controls the background
Sync ticker.
Backend pivot from Qdrant to pgvector recorded in DECISIONS.md
2026-05-18 (supersedes 2026-04-08): postgres18 already runs in
databases/ ns, Qdrant was never deployed, one engine beats two.
Dependency: github.com/jackc/pgx/v5 — modern, native pgvector via
parametric vector literals.
Tests:
- embed.Client: empty-URL nil, request shape, dimension, upstream
error propagation, empty-text rejection.
- vectorstore.PGStore: dimension validation (unit); upsert/search/
KnownPaths (integration, BRAIN_PG_TEST_DSN-gated).
- vectorstore.Sync: adds new files, skips known, deletes
disappeared, skips _index.md, no-op when nil, collects embedder
errors.
- search.Query: hybrid promotes vector-only hits via RRF; falls
back to BM25 on embedder error.
Closes hyperguild#8.
This commit is contained in:
142
ingestion/internal/vectorstore/sync.go
Normal file
142
ingestion/internal/vectorstore/sync.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package vectorstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Embedder produces dense vectors. The embed package's Client satisfies
|
||||
// this; it's declared locally so vectorstore doesn't depend on embed.
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// Store is the subset of PGStore that Sync needs. Lets tests stub it.
|
||||
type Store interface {
|
||||
KnownPaths(ctx context.Context) (map[string]struct{}, error)
|
||||
Upsert(ctx context.Context, path string, embedding []float32) error
|
||||
Delete(ctx context.Context, path string) error
|
||||
}
|
||||
|
||||
// SyncResult tallies what Sync did. Returned for logs / metrics; callers
|
||||
// generally don't act on the fields directly.
|
||||
type SyncResult struct {
|
||||
Added int
|
||||
Updated int
|
||||
Deleted int
|
||||
Errors []error
|
||||
}
|
||||
|
||||
// Sync brings the embedding store in line with brain/wiki/ on disk:
|
||||
// - new files (in the tree, not in the store) get embedded + upserted
|
||||
// - files whose mtime exceeds the store's updated_at get re-embedded
|
||||
// - files no longer on disk get deleted from the store
|
||||
//
|
||||
// Designed to be called on a ticker. Best-effort: per-file errors are
|
||||
// collected into SyncResult.Errors and do not abort the run.
|
||||
func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder) (SyncResult, error) {
|
||||
var res SyncResult
|
||||
if store == nil || embedder == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
known, err := store.KnownPaths(ctx)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("known paths: %w", err)
|
||||
}
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
wikiDir := filepath.Join(brainDir, "wiki")
|
||||
if _, err := os.Stat(wikiDir); os.IsNotExist(err) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
err = filepath.WalkDir(wikiDir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".md") || d.Name() == "_index.md" {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(brainDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relSlash := filepath.ToSlash(rel)
|
||||
seen[relSlash] = struct{}{}
|
||||
|
||||
if _, ok := known[relSlash]; ok {
|
||||
// Already embedded — TODO: compare mtime once Store exposes
|
||||
// updated_at so we re-embed on edit. For now, skip.
|
||||
return nil
|
||||
}
|
||||
|
||||
content, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("read %s: %w", relSlash, readErr))
|
||||
return nil
|
||||
}
|
||||
vec, embErr := embedder.Embed(ctx, string(content))
|
||||
if embErr != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("embed %s: %w", relSlash, embErr))
|
||||
return nil
|
||||
}
|
||||
if upErr := store.Upsert(ctx, relSlash, vec); upErr != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("upsert %s: %w", relSlash, upErr))
|
||||
return nil
|
||||
}
|
||||
res.Added++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("walk wiki: %w", err)
|
||||
}
|
||||
|
||||
// Drop rows whose file is gone.
|
||||
for path := range known {
|
||||
if _, ok := seen[path]; ok {
|
||||
continue
|
||||
}
|
||||
if err := store.Delete(ctx, path); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("delete %s: %w", path, err))
|
||||
continue
|
||||
}
|
||||
res.Deleted++
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// StartSync launches Sync on a ticker in a background goroutine. The
|
||||
// goroutine exits when ctx is cancelled. Failures are logged via slog.
|
||||
func StartSync(ctx context.Context, brainDir string, store Store, embedder Embedder, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 5 * time.Minute
|
||||
}
|
||||
go func() {
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
// Run once immediately so first-boot doesn't wait a full tick.
|
||||
if r, err := Sync(ctx, brainDir, store, embedder); err != nil {
|
||||
slog.Error("embed sync failed", "err", err)
|
||||
} else if r.Added+r.Deleted > 0 || len(r.Errors) > 0 {
|
||||
slog.Info("embed sync", "added", r.Added, "deleted", r.Deleted, "errors", len(r.Errors))
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if r, err := Sync(ctx, brainDir, store, embedder); err != nil {
|
||||
slog.Error("embed sync failed", "err", err)
|
||||
} else if r.Added+r.Deleted > 0 || len(r.Errors) > 0 {
|
||||
slog.Info("embed sync", "added", r.Added, "deleted", r.Deleted, "errors", len(r.Errors))
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
Reference in New Issue
Block a user