feat(brain): hybrid BM25 + pgvector retrieval (opt-in)
Wires nomic-embed-text (iguana ollama) + pgvector on the shared
postgres18 into brain_query / brain_answer via Reciprocal Rank Fusion.
Pure BM25 stays the default; setting BRAIN_PG_DSN and BRAIN_EMBED_URL
together opts in. Setting one without the other is misconfiguration →
exit 1.
New packages:
- internal/embed
Client.Embed(ctx, text) → []float32 via POST {URL}/api/embed.
Defaults to nomic-embed-text:latest (768 dim). nil-on-empty-URL so
callers gate on a single nil check.
- internal/vectorstore
PGStore wraps a pgxpool against postgres18. Init creates
brain_embeddings(path PK, vector(768), updated_at) + HNSW cosine
index idempotently. Upsert / Delete / Search / KnownPaths.
Sync(brainDir, store, embedder) diffs brain/wiki/ against the store
and upserts new files / deletes removed ones; StartSync runs it on
a ticker (default 300s). Integration tests gated by BRAIN_PG_TEST_DSN.
- scripts/brain-embeddings-init.sql
One-time DBA setup: brain DB, brain_app role, vector extension,
GRANTs. Idempotent.
Search layer:
- search.QueryOptions gains Vector + Embedder fields.
- QueryContext is the cancellable variant; Query stays for callers.
- When both are set, BM25 (top-N) and pgvector (top-4N) candidates
merge via Reciprocal Rank Fusion (k=60, Cormack et al. 2009 — no
tuning knob, robust to scale differences between rankers).
- Vector-only hits are hydrated from disk so callers see uniform
Result records (path, title, excerpt, wing, hall, score).
- Wing/hall filters still apply to vector candidates via path-prefix.
- On embedder/vector errors the search falls back to BM25 — embedding
outage degrades quality but doesn't take the brain offline.
MCP wiring:
- mcp.Server.WithHybridRetrieval(v, e) opt-in setter, same shape as
WithReranker.
- brainQuery and brainAnswer pass the wired vector/embedder through
to search.QueryContext.
REST:
- POST /backfill-embeddings drives Sync synchronously. Returns
{added, deleted, errors[]}. 503 when feature is unconfigured.
cmd/server/main.go:
- BRAIN_PG_DSN + BRAIN_EMBED_URL together enable hybrid; one alone
→ exit 1.
- vectorAdapter bridges *PGStore (returns []Hit) to
search.VectorSearcher (which takes []VectorHit) without either
package importing the other.
- BRAIN_EMBED_SYNC_INTERVAL (default 300s) controls the background
Sync ticker.
Backend pivot from Qdrant to pgvector recorded in DECISIONS.md
2026-05-18 (supersedes 2026-04-08): postgres18 already runs in
databases/ ns, Qdrant was never deployed, one engine beats two.
Dependency: github.com/jackc/pgx/v5 — modern, native pgvector via
parametric vector literals.
Tests:
- embed.Client: empty-URL nil, request shape, dimension, upstream
error propagation, empty-text rejection.
- vectorstore.PGStore: dimension validation (unit); upsert/search/
KnownPaths (integration, BRAIN_PG_TEST_DSN-gated).
- vectorstore.Sync: adds new files, skips known, deletes
disappeared, skips _index.md, no-op when nil, collects embedder
errors.
- search.Query: hybrid promotes vector-only hits via RRF; falls
back to BM25 on embedder error.
Closes hyperguild#8.
This commit is contained in:
137
ingestion/internal/vectorstore/sync_test.go
Normal file
137
ingestion/internal/vectorstore/sync_test.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package vectorstore_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type stubStore struct {
|
||||
known map[string]struct{}
|
||||
upserts map[string][]float32
|
||||
deletes []string
|
||||
failNext error
|
||||
}
|
||||
|
||||
func (s *stubStore) KnownPaths(_ context.Context) (map[string]struct{}, error) {
|
||||
out := make(map[string]struct{}, len(s.known))
|
||||
for k := range s.known {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *stubStore) Upsert(_ context.Context, path string, v []float32) error {
|
||||
if s.failNext != nil {
|
||||
err := s.failNext
|
||||
s.failNext = nil
|
||||
return err
|
||||
}
|
||||
if s.upserts == nil {
|
||||
s.upserts = make(map[string][]float32)
|
||||
}
|
||||
s.upserts[path] = v
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubStore) Delete(_ context.Context, path string) error {
|
||||
s.deletes = append(s.deletes, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
type stubEmbedder struct {
|
||||
vec []float32
|
||||
err error
|
||||
}
|
||||
|
||||
func (e stubEmbedder) Embed(_ context.Context, _ string) ([]float32, error) {
|
||||
return e.vec, e.err
|
||||
}
|
||||
|
||||
func writeNote(t *testing.T, dir, rel, body string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, rel)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
|
||||
}
|
||||
|
||||
func TestSync_AddsNewFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/jepa-fx/facts/x.md", "body of x")
|
||||
writeNote(t, dir, "wiki/jepa-fx/facts/y.md", "body of y")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, res.Added)
|
||||
assert.Empty(t, res.Deleted)
|
||||
assert.Contains(t, store.upserts, "wiki/jepa-fx/facts/x.md")
|
||||
assert.Contains(t, store.upserts, "wiki/jepa-fx/facts/y.md")
|
||||
}
|
||||
|
||||
func TestSync_SkipsAlreadyKnown(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{"wiki/a/facts/x.md": {}}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, res.Added)
|
||||
assert.Empty(t, store.upserts)
|
||||
}
|
||||
|
||||
func TestSync_DeletesDisappearedFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki"), 0o755))
|
||||
// store has a path that doesn't exist on disk anymore
|
||||
store := &stubStore{known: map[string]struct{}{"wiki/old/facts/ghost.md": {}}}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, &stubStoreWithDelete{stubStore: store}, stubEmbedder{vec: make([]float32, 768)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Deleted)
|
||||
}
|
||||
|
||||
// stubStoreWithDelete is a thin wrapper to capture Delete calls;
|
||||
// stubStore already implements Delete but we need the wrapper to mix
|
||||
// store interfaces with sync-specific expectations.
|
||||
type stubStoreWithDelete struct {
|
||||
*stubStore
|
||||
}
|
||||
|
||||
func TestSync_SkipsIndexFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/_index.md", "moc")
|
||||
writeNote(t, dir, "wiki/a/facts/real.md", "body")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, stubEmbedder{vec: make([]float32, 768)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Added)
|
||||
assert.NotContains(t, store.upserts, "wiki/a/_index.md")
|
||||
}
|
||||
|
||||
func TestSync_NoOpWhenComponentsNil(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
res, err := vectorstore.Sync(context.Background(), dir, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, res.Added)
|
||||
}
|
||||
|
||||
func TestSync_CollectsEmbedderErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
emb := stubEmbedder{err: errors.New("upstream down")}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, res.Added)
|
||||
assert.Len(t, res.Errors, 1)
|
||||
}
|
||||
Reference in New Issue
Block a user