feat(vectorstore): re-embed on file mtime > store updated_at (#23)
Removes the TODO in Sync that left files static after their first embed. Edits to brain/wiki/ and brain/knowledge/ now surface in subsequent syncs without manual /backfill-embeddings calls. Approach - Store interface: KnownPaths → KnownPathsWithTime returning path → updated_at. Callers compare against file mtime to detect edits. - PGStore: SELECT path, updated_at FROM brain_embeddings. - Sync groups known chunks by parent path and tracks the EARLIEST updated_at per parent. A file is stale when its mtime is after that oldest chunk's timestamp — any chunk older than the file means at least one chunk hasn't been refreshed since the last edit. - Stale-path rewrite: delete every old chunk for the parent (handles "file shrunk → fewer chunks → orphan rows at higher #NNNN" cleanly), then re-chunk + re-embed + re-upsert. Tests - New: TestSync_ReembedsFileWhenMtimeNewer — file mtime forced into the future vs store updated_at; Sync deletes old chunk + upserts fresh one. - New: TestSync_SkipsFileWhenMtimeOlder — file mtime backdated; Sync is a no-op (no upserts, no deletes). - Updated: stubStore.known is now map[string]time.Time. A zero value resolves to a far-future sentinel so existing "skip if already known" tests keep passing without per-test setup. - pg_test renamed KnownPaths integration → KnownPathsWithTime; asserts updated_at is non-zero and within 5s of insert wall-clock. Backward compat - brain_embeddings rows pre-dating this change carry valid updated_at values (column was always populated via `DEFAULT now()` + ON CONFLICT `updated_at = now()`). No migration needed. Live pod will start re-embedding any file whose source has been edited since its chunks were originally written. Closes gitea/mathias/hyperguild#23.
This commit is contained in:
@@ -18,7 +18,11 @@ type Embedder interface {
|
||||
|
||||
// Store is the subset of PGStore that Sync needs. Lets tests stub it.
|
||||
type Store interface {
|
||||
KnownPaths(ctx context.Context) (map[string]struct{}, error)
|
||||
// KnownPathsWithTime returns every embedded chunk path paired with the
|
||||
// row's updated_at. Sync uses the timestamp to detect edits — a file
|
||||
// whose mtime is newer than ANY of its chunks' updated_at is re-embedded
|
||||
// from scratch (old chunks deleted, fresh chunks upserted).
|
||||
KnownPathsWithTime(ctx context.Context) (map[string]time.Time, error)
|
||||
Upsert(ctx context.Context, path string, embedding []float32) error
|
||||
Delete(ctx context.Context, path string) error
|
||||
}
|
||||
@@ -58,15 +62,31 @@ func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
known, err := store.KnownPaths(ctx)
|
||||
known, err := store.KnownPathsWithTime(ctx)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("known paths: %w", err)
|
||||
}
|
||||
// Build a parent → "any chunk known?" set so we can skip files that
|
||||
// already have at least one chunk row in the store.
|
||||
knownParents := make(map[string]struct{}, len(known))
|
||||
for p := range known {
|
||||
knownParents[ParentPath(p)] = struct{}{}
|
||||
// Group known chunks by parent path and remember the EARLIEST
|
||||
// updated_at per parent. A file is considered stale if its mtime is
|
||||
// after the oldest of its chunk rows — i.e. at least one chunk hasn't
|
||||
// been refreshed since the last edit. Also keep the full chunk-path
|
||||
// list per parent so we can delete every old chunk before re-embedding
|
||||
// (handles "file shrunk → fewer chunks → orphan rows" cleanly).
|
||||
type parentState struct {
|
||||
minUpdatedAt time.Time
|
||||
chunkPaths []string
|
||||
}
|
||||
parents := make(map[string]*parentState, len(known))
|
||||
for p, t := range known {
|
||||
parent := ParentPath(p)
|
||||
ps, ok := parents[parent]
|
||||
if !ok {
|
||||
ps = &parentState{minUpdatedAt: t}
|
||||
parents[parent] = ps
|
||||
} else if t.Before(ps.minUpdatedAt) {
|
||||
ps.minUpdatedAt = t
|
||||
}
|
||||
ps.chunkPaths = append(ps.chunkPaths, p)
|
||||
}
|
||||
seenParents := make(map[string]struct{})
|
||||
|
||||
@@ -90,11 +110,26 @@ func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder)
|
||||
relSlash := filepath.ToSlash(rel)
|
||||
seenParents[relSlash] = struct{}{}
|
||||
|
||||
if _, ok := knownParents[relSlash]; ok {
|
||||
// File has at least one chunk in the store already.
|
||||
// TODO: compare mtime once Store exposes updated_at so we
|
||||
// re-embed on edit. For now, skip.
|
||||
return nil
|
||||
if ps, ok := parents[relSlash]; ok {
|
||||
// File already has chunks in the store. Re-embed only when
|
||||
// the file has been edited since the oldest chunk was
|
||||
// written. Tolerate clock skew with a sub-second grace.
|
||||
info, statErr := d.Info()
|
||||
if statErr != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("stat %s: %w", relSlash, statErr))
|
||||
return nil
|
||||
}
|
||||
if !info.ModTime().After(ps.minUpdatedAt) {
|
||||
return nil
|
||||
}
|
||||
// Stale: delete old chunks before re-embedding so a shrunk
|
||||
// file doesn't leave orphan rows at higher #NNNN indexes.
|
||||
for _, oldPath := range ps.chunkPaths {
|
||||
if delErr := store.Delete(ctx, oldPath); delErr != nil {
|
||||
res.Errors = append(res.Errors, fmt.Errorf("delete %s for re-embed: %w", oldPath, delErr))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
content, readErr := os.ReadFile(path)
|
||||
|
||||
Reference in New Issue
Block a user