feat(brain_answer): Qwen3-Reranker cross-encoder filter (opt-in)
Adds an opt-in cross-encoder rerank step between BM25 retrieval and LLM synthesis. With BRAIN_RERANKER_URL set, brain_answer retrieves BM25 top-20, scores each excerpt against the query via Qwen3-Reranker on Ollama, drops the "no" answers, and forwards up to 5 surviving sources to the LLM. Unset, behaviour is unchanged (BM25 top-10 → LLM). The reranker is a *filter*, not a re-ranker: Qwen3-Reranker emits a binary yes/no token under its native chat template, and ties within the "yes" set are broken by BM25 rank — what got retrieved first stays ahead. New package ingestion/internal/reranker: - Client with URL, Model, HTTP fields. - New(url, model) returns nil on empty url so callers can treat "feature disabled" as a single nil check. - Score(ctx, query, docs) issues one /api/generate call per doc using the Qwen3-Reranker yes/no chat template (verbatim, because the model was trained on this exact wording). Parses the first non-think token. Wiring: - mcp.Server gains a WithReranker fluent setter to keep NewServer signature stable. - brain_answer's BM25 limit jumps to 20 only when a reranker is wired, to give the filter something to do. - cmd/server/main.go reads BRAIN_RERANKER_URL (+ optional BRAIN_RERANKER_MODEL, default dengcao/Qwen3-Reranker-0.6B:F16). Tests cover: nil-on-empty-url, ordered yes/no scoring, request shape (model, prompt contents, yes/no template), ambiguous response → 0, empty doc slice, upstream-error propagation, plus an end-to-end brain_answer integration that proves only the relevant note reaches the LLM when noise.md is rejected. Closes hyperguild#7.
This commit is contained in:
@@ -3,14 +3,17 @@ package mcp_test
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/mcp"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -46,6 +49,55 @@ func callTool(t *testing.T, ts *httptest.Server, name string, arguments map[stri
|
||||
return out
|
||||
}
|
||||
|
||||
func TestBrainAnswer_RerankerFiltersBeforeLLM(t *testing.T) {
|
||||
brainDir := t.TempDir()
|
||||
wikiDir := filepath.Join(brainDir, "wiki")
|
||||
require.NoError(t, os.MkdirAll(wikiDir, 0o755))
|
||||
// Two notes — both BM25-match the query, but only one is truly relevant.
|
||||
require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "good.md"), []byte(
|
||||
"---\ntitle: Pass-rate Logging\n---\nPass-rate logging tracks skill invocations.",
|
||||
), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "noise.md"), []byte(
|
||||
"---\ntitle: Pass-rate Tangent\n---\nPass-rate appears here too but as a tangent.",
|
||||
), 0o644))
|
||||
|
||||
// Fake Ollama reranker: yes only when prompt contains "tracks skill invocations".
|
||||
rrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
yes := strings.Contains(string(raw), "tracks skill invocations")
|
||||
ans := "no"
|
||||
if yes {
|
||||
ans = "yes"
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"response": ans, "done": true})
|
||||
}))
|
||||
defer rrSrv.Close()
|
||||
|
||||
// LLM mock captures the rendered sources so we can assert what reached it.
|
||||
var sawSources string
|
||||
llm := func(_ context.Context, _, user string) (string, error) {
|
||||
sawSources = user
|
||||
return "answer text", nil
|
||||
}
|
||||
|
||||
srv := mcp.NewServer(brainDir, nil, nil, llm).
|
||||
WithReranker(reranker.New(rrSrv.URL, "qwen3"))
|
||||
ts := httptest.NewServer(srv)
|
||||
defer ts.Close()
|
||||
|
||||
rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
|
||||
require.Nil(t, rpc["error"])
|
||||
|
||||
content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
|
||||
var result map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(content), &result))
|
||||
sources := result["sources"].([]any)
|
||||
require.Len(t, sources, 1, "reranker should drop noise.md")
|
||||
assert.Equal(t, "wiki/good.md", sources[0])
|
||||
assert.Contains(t, sawSources, "good.md")
|
||||
assert.NotContains(t, sawSources, "noise.md")
|
||||
}
|
||||
|
||||
func TestBrainAnswer_NoLLM(t *testing.T) {
|
||||
srv := mcp.NewServer(t.TempDir(), nil, nil, nil)
|
||||
ts := httptest.NewServer(srv)
|
||||
|
||||
Reference in New Issue
Block a user