feat(vectorstore): re-embed on file mtime > store updated_at (#23)
Removes the TODO in Sync that left files static after their first embed. Edits to brain/wiki/ and brain/knowledge/ now surface in subsequent syncs without manual /backfill-embeddings calls. Approach - Store interface: KnownPaths → KnownPathsWithTime returning path → updated_at. Callers compare against file mtime to detect edits. - PGStore: SELECT path, updated_at FROM brain_embeddings. - Sync groups known chunks by parent path and tracks the EARLIEST updated_at per parent. A file is stale when its mtime is after that oldest chunk's timestamp — any chunk older than the file means at least one chunk hasn't been refreshed since the last edit. - Stale-path rewrite: delete every old chunk for the parent (handles "file shrunk → fewer chunks → orphan rows at higher #NNNN" cleanly), then re-chunk + re-embed + re-upsert. Tests - New: TestSync_ReembedsFileWhenMtimeNewer — file mtime forced into the future vs store updated_at; Sync deletes old chunk + upserts fresh one. - New: TestSync_SkipsFileWhenMtimeOlder — file mtime backdated; Sync is a no-op (no upserts, no deletes). - Updated: stubStore.known is now map[string]time.Time. A zero value resolves to a far-future sentinel so existing "skip if already known" tests keep passing without per-test setup. - pg_test renamed KnownPaths integration → KnownPathsWithTime; asserts updated_at is non-zero and within 5s of insert wall-clock. Backward compat - brain_embeddings rows pre-dating this change carry valid updated_at values (column was always populated via `DEFAULT now()` + ON CONFLICT `updated_at = now()`). No migration needed. Live pod will start re-embedding any file whose source has been edited since its chunks were originally written. Closes gitea/mathias/hyperguild#23.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -14,16 +15,27 @@ import (
|
||||
)
|
||||
|
||||
type stubStore struct {
|
||||
known map[string]struct{}
|
||||
// known maps chunk-path → updated_at. Tests that don't care about
|
||||
// re-embed-on-mtime use a far-future time so the Sync skip path
|
||||
// always wins. Tests that do exercise the mtime path set the
|
||||
// updated_at explicitly.
|
||||
known map[string]time.Time
|
||||
upserts map[string][]float32
|
||||
deletes []string
|
||||
failNext error
|
||||
}
|
||||
|
||||
func (s *stubStore) KnownPaths(_ context.Context) (map[string]struct{}, error) {
|
||||
out := make(map[string]struct{}, len(s.known))
|
||||
for k := range s.known {
|
||||
out[k] = struct{}{}
|
||||
// farFuture is "newer than any file mtime", used as the default
|
||||
// updated_at in stubs that don't care about re-embed behavior.
|
||||
var farFuture = time.Now().Add(24 * time.Hour)
|
||||
|
||||
func (s *stubStore) KnownPathsWithTime(_ context.Context) (map[string]time.Time, error) {
|
||||
out := make(map[string]time.Time, len(s.known))
|
||||
for k, t := range s.known {
|
||||
if t.IsZero() {
|
||||
t = farFuture
|
||||
}
|
||||
out[k] = t
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -67,7 +79,7 @@ func TestSync_AddsNewFiles(t *testing.T) {
|
||||
writeNote(t, dir, "wiki/jepa-fx/facts/x.md", "body of x")
|
||||
writeNote(t, dir, "wiki/jepa-fx/facts/y.md", "body of y")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
@@ -81,7 +93,7 @@ func TestSync_SkipsAlreadyKnown(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{"wiki/a/facts/x.md#0001": {}}}
|
||||
store := &stubStore{known: map[string]time.Time{"wiki/a/facts/x.md#0001": {}}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
@@ -93,7 +105,7 @@ func TestSync_DeletesDisappearedFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki"), 0o755))
|
||||
// store has a path that doesn't exist on disk anymore
|
||||
store := &stubStore{known: map[string]struct{}{"wiki/old/facts/ghost.md#0001": {}}}
|
||||
store := &stubStore{known: map[string]time.Time{"wiki/old/facts/ghost.md#0001": {}}}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, &stubStoreWithDelete{stubStore: store}, stubEmbedder{vec: make([]float32, 768)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Deleted)
|
||||
@@ -111,7 +123,7 @@ func TestSync_SkipsIndexFiles(t *testing.T) {
|
||||
writeNote(t, dir, "wiki/a/_index.md", "moc")
|
||||
writeNote(t, dir, "wiki/a/facts/real.md", "body")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, stubEmbedder{vec: make([]float32, 768)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Added)
|
||||
@@ -123,7 +135,7 @@ func TestSync_ScansKnowledgeDir(t *testing.T) {
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
writeNote(t, dir, "knowledge/2026-05-19-koala-gpu-setup.md", "knowledge body")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
@@ -143,7 +155,7 @@ func TestSync_ChunksLongFiles(t *testing.T) {
|
||||
}
|
||||
writeNote(t, dir, "knowledge/long.md", body)
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
@@ -164,7 +176,7 @@ func TestSync_ShortFileGetsSingleChunkRow(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/short.md", "tiny body\n")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
@@ -176,7 +188,7 @@ func TestSync_SkipsFileIfAnyChunkAlreadyKnown(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/foo.md", "body\n")
|
||||
|
||||
store := &stubStore{known: map[string]struct{}{
|
||||
store := &stubStore{known: map[string]time.Time{
|
||||
"wiki/foo.md#0001": {},
|
||||
}}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
@@ -189,7 +201,7 @@ func TestSync_SkipsFileIfAnyChunkAlreadyKnown(t *testing.T) {
|
||||
func TestSync_DeletesAllChunksOfDisappearedFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki"), 0o755))
|
||||
store := &stubStore{known: map[string]struct{}{
|
||||
store := &stubStore{known: map[string]time.Time{
|
||||
"wiki/ghost.md#0001": {},
|
||||
"wiki/ghost.md#0002": {},
|
||||
"wiki/ghost.md#0003": {},
|
||||
@@ -199,6 +211,49 @@ func TestSync_DeletesAllChunksOfDisappearedFile(t *testing.T) {
|
||||
assert.Equal(t, 3, res.Deleted)
|
||||
}
|
||||
|
||||
func TestSync_ReembedsFileWhenMtimeNewer(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/edited.md", "original body\n")
|
||||
// Force the file's mtime ahead of any plausible store updated_at.
|
||||
future := time.Now().Add(1 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(filepath.Join(dir, "wiki/edited.md"), future, future))
|
||||
|
||||
store := &stubStore{
|
||||
known: map[string]time.Time{
|
||||
// Existing chunk row pre-dates the file's mtime.
|
||||
"wiki/edited.md#0001": time.Now().Add(-1 * time.Hour),
|
||||
},
|
||||
}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Added, "file with newer mtime should be re-embedded")
|
||||
assert.Contains(t, store.upserts, "wiki/edited.md#0001")
|
||||
// Old chunks of the same parent must be deleted before re-embed so
|
||||
// shrunk files don't leave orphan rows at higher #NNNN indexes.
|
||||
assert.Contains(t, store.deletes, "wiki/edited.md#0001")
|
||||
}
|
||||
|
||||
func TestSync_SkipsFileWhenMtimeOlder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/stable.md", "body\n")
|
||||
// Backdate mtime to before the store's recorded updated_at.
|
||||
past := time.Now().Add(-2 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(filepath.Join(dir, "wiki/stable.md"), past, past))
|
||||
|
||||
store := &stubStore{
|
||||
known: map[string]time.Time{
|
||||
"wiki/stable.md#0001": time.Now(),
|
||||
},
|
||||
}
|
||||
emb := stubEmbedder{vec: make([]float32, 768)}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, res.Added)
|
||||
assert.Empty(t, store.upserts)
|
||||
assert.Empty(t, store.deletes)
|
||||
}
|
||||
|
||||
func TestSync_NoOpWhenComponentsNil(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
@@ -210,7 +265,7 @@ func TestSync_NoOpWhenComponentsNil(t *testing.T) {
|
||||
func TestSync_CollectsEmbedderErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeNote(t, dir, "wiki/a/facts/x.md", "x")
|
||||
store := &stubStore{known: map[string]struct{}{}}
|
||||
store := &stubStore{known: map[string]time.Time{}}
|
||||
emb := stubEmbedder{err: errors.New("upstream down")}
|
||||
res, err := vectorstore.Sync(context.Background(), dir, store, emb)
|
||||
require.NoError(t, err)
|
||||
|
||||
Reference in New Issue
Block a user