feat(brain): hybrid BM25 + pgvector retrieval (opt-in)
Wires nomic-embed-text (iguana ollama) + pgvector on the shared
postgres18 into brain_query / brain_answer via Reciprocal Rank Fusion.
Pure BM25 stays the default; setting BRAIN_PG_DSN and BRAIN_EMBED_URL
together opts in. Setting one without the other is misconfiguration →
exit 1.
New packages:
- internal/embed
Client.Embed(ctx, text) → []float32 via POST {URL}/api/embed.
Defaults to nomic-embed-text:latest (768 dim). nil-on-empty-URL so
callers gate on a single nil check.
- internal/vectorstore
PGStore wraps a pgxpool against postgres18. Init creates
brain_embeddings(path PK, vector(768), updated_at) + HNSW cosine
index idempotently. Upsert / Delete / Search / KnownPaths.
Sync(brainDir, store, embedder) diffs brain/wiki/ against the store
and upserts new files / deletes removed ones; StartSync runs it on
a ticker (default 300s). Integration tests gated by BRAIN_PG_TEST_DSN.
- scripts/brain-embeddings-init.sql
One-time DBA setup: brain DB, brain_app role, vector extension,
GRANTs. Idempotent.
Search layer:
- search.QueryOptions gains Vector + Embedder fields.
- QueryContext is the cancellable variant; Query stays for callers.
- When both are set, BM25 (top-N) and pgvector (top-4N) candidates
merge via Reciprocal Rank Fusion (k=60, Cormack et al. 2009 — no
tuning knob, robust to scale differences between rankers).
- Vector-only hits are hydrated from disk so callers see uniform
Result records (path, title, excerpt, wing, hall, score).
- Wing/hall filters still apply to vector candidates via path-prefix.
- On embedder/vector errors the search falls back to BM25 — embedding
outage degrades quality but doesn't take the brain offline.
MCP wiring:
- mcp.Server.WithHybridRetrieval(v, e) opt-in setter, same shape as
WithReranker.
- brainQuery and brainAnswer pass the wired vector/embedder through
to search.QueryContext.
REST:
- POST /backfill-embeddings drives Sync synchronously. Returns
{added, deleted, errors[]}. 503 when feature is unconfigured.
cmd/server/main.go:
- BRAIN_PG_DSN + BRAIN_EMBED_URL together enable hybrid; one alone
→ exit 1.
- vectorAdapter bridges *PGStore (returns []Hit) to
search.VectorSearcher (which takes []VectorHit) without either
package importing the other.
- BRAIN_EMBED_SYNC_INTERVAL (default 300s) controls the background
Sync ticker.
Backend pivot from Qdrant to pgvector recorded in DECISIONS.md
2026-05-18 (supersedes 2026-04-08): postgres18 already runs in
databases/ ns, Qdrant was never deployed, one engine beats two.
Dependency: github.com/jackc/pgx/v5 — modern, native pgvector via
parametric vector literals.
Tests:
- embed.Client: empty-URL nil, request shape, dimension, upstream
error propagation, empty-text rejection.
- vectorstore.PGStore: dimension validation (unit); upsert/search/
KnownPaths (integration, BRAIN_PG_TEST_DSN-gated).
- vectorstore.Sync: adds new files, skips known, deletes
disappeared, skips _index.md, no-op when nil, collects embedder
errors.
- search.Query: hybrid promotes vector-only hits via RRF; falls
back to BM25 on embedder error.
Closes hyperguild#8.
This commit is contained in:
76
ingestion/internal/embed/embed.go
Normal file
76
ingestion/internal/embed/embed.go
Normal file
@@ -0,0 +1,76 @@
|
||||
// Package embed produces dense vector embeddings for brain content.
|
||||
//
|
||||
// Wire format is Ollama's `/api/embed`, with the canonical request shape
|
||||
// `{"model": "...", "input": "..."}` and a 2-D `embeddings` response.
|
||||
// Default deployment runs `nomic-embed-text` on iguana, which returns
|
||||
// 768-dim vectors compatible with the brain_embeddings table schema.
|
||||
package embed
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client posts embedding requests to an Ollama-compatible endpoint.
|
||||
type Client struct {
|
||||
URL string
|
||||
Model string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// New constructs a Client. Returns nil when url is empty so callers can
|
||||
// treat a missing BRAIN_EMBED_URL as "feature disabled" via a single nil
|
||||
// check.
|
||||
func New(url, model string) *Client {
|
||||
if url == "" {
|
||||
return nil
|
||||
}
|
||||
return &Client{
|
||||
URL: strings.TrimRight(url, "/"),
|
||||
Model: model,
|
||||
HTTP: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// Embed returns the embedding vector for text. Empty text is rejected
|
||||
// up-front to keep upstream errors from masking caller mistakes.
|
||||
func (c *Client) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return nil, fmt.Errorf("embed: empty text")
|
||||
}
|
||||
reqBody, _ := json.Marshal(map[string]any{
|
||||
"model": c.Model,
|
||||
"input": text,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
c.URL+"/api/embed", bytes.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("embed: status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
var out struct {
|
||||
Embeddings [][]float32 `json:"embeddings"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("embed: decode: %w", err)
|
||||
}
|
||||
if len(out.Embeddings) == 0 || len(out.Embeddings[0]) == 0 {
|
||||
return nil, fmt.Errorf("embed: empty embeddings in response")
|
||||
}
|
||||
return out.Embeddings[0], nil
|
||||
}
|
||||
74
ingestion/internal/embed/embed_test.go
Normal file
74
ingestion/internal/embed/embed_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package embed_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/embed"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNew_EmptyURLReturnsNil(t *testing.T) {
|
||||
assert.Nil(t, embed.New("", "model"))
|
||||
}
|
||||
|
||||
func TestEmbed_ReturnsVector(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/embed", r.URL.Path)
|
||||
var req map[string]any
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
|
||||
assert.Equal(t, "nomic", req["model"])
|
||||
assert.Equal(t, "hello", req["input"])
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"embeddings": [][]float32{{0.1, 0.2, 0.3}},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := embed.New(srv.URL, "nomic")
|
||||
require.NotNil(t, c)
|
||||
v, err := c.Embed(context.Background(), "hello")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []float32{0.1, 0.2, 0.3}, v)
|
||||
}
|
||||
|
||||
func TestEmbed_StripsTrailingSlashFromURL(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/embed", r.URL.Path)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{{1.0}}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := embed.New(srv.URL+"/", "nomic")
|
||||
_, err := c.Embed(context.Background(), "x")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestEmbed_PropagatesUpstreamError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := embed.New(srv.URL, "m")
|
||||
_, err := c.Embed(context.Background(), "x")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestEmbed_RejectsEmptyEmbeddingsArray(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := embed.New(srv.URL, "m")
|
||||
_, err := c.Embed(context.Background(), "x")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestEmbed_RejectsEmptyText(t *testing.T) {
|
||||
c := embed.New("http://127.0.0.1:1", "m")
|
||||
_, err := c.Embed(context.Background(), "")
|
||||
require.Error(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user