feat(ingestion): simplify brain to knowledge/ — write and search use same dir
This commit is contained in:
@@ -79,7 +79,7 @@ func (h *Handler) Write(w http.ResponseWriter, r *http.Request) {
|
|||||||
filename = fmt.Sprintf("%s-auto.md", time.Now().UTC().Format("2006-01-02-150405"))
|
filename = fmt.Sprintf("%s-auto.md", time.Now().UTC().Format("2006-01-02-150405"))
|
||||||
}
|
}
|
||||||
|
|
||||||
rawDir := filepath.Join(h.brainDir, "raw")
|
rawDir := filepath.Join(h.brainDir, "knowledge")
|
||||||
if err := os.MkdirAll(rawDir, 0o755); err != nil {
|
if err := os.MkdirAll(rawDir, 0o755); err != nil {
|
||||||
http.Error(w, "failed to create raw dir", http.StatusInternalServerError)
|
http.Error(w, "failed to create raw dir", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -20,10 +20,9 @@ import (
|
|||||||
func setup(t *testing.T) (string, *api.Handler) {
|
func setup(t *testing.T) (string, *api.Handler) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki", "concepts"), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, "knowledge"), 0o755))
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "raw"), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
filepath.Join(dir, "wiki", "concepts", "tdd.md"),
|
filepath.Join(dir, "knowledge", "tdd.md"),
|
||||||
[]byte("---\ntitle: TDD\ndomain: software\n---\n\nTest-driven development is a discipline.\n"),
|
[]byte("---\ntitle: TDD\ndomain: software\n---\n\nTest-driven development is a discipline.\n"),
|
||||||
0o644,
|
0o644,
|
||||||
))
|
))
|
||||||
@@ -46,7 +45,7 @@ func TestQuery_ReturnsResults(t *testing.T) {
|
|||||||
assert.NotEmpty(t, results)
|
assert.NotEmpty(t, results)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWrite_CreatesRawFile(t *testing.T) {
|
func TestWrite_CreatesKnowledgeFile(t *testing.T) {
|
||||||
dir, h := setup(t)
|
dir, h := setup(t)
|
||||||
body, _ := json.Marshal(map[string]any{
|
body, _ := json.Marshal(map[string]any{
|
||||||
"content": "# Test note\n\nSome content.",
|
"content": "# Test note\n\nSome content.",
|
||||||
@@ -62,8 +61,7 @@ func TestWrite_CreatesRawFile(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
||||||
assert.NotEmpty(t, resp["path"])
|
assert.NotEmpty(t, resp["path"])
|
||||||
|
|
||||||
written := filepath.Join(dir, "raw", "test-note.md")
|
content, err := os.ReadFile(filepath.Join(dir, "knowledge", "test-note.md"))
|
||||||
content, err := os.ReadFile(written)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Contains(t, string(content), "Some content.")
|
assert.Contains(t, string(content), "Some content.")
|
||||||
}
|
}
|
||||||
@@ -93,7 +91,7 @@ func TestWrite_IncludesFrontmatterWhenTypeProvided(t *testing.T) {
|
|||||||
h.Write(rec, req)
|
h.Write(rec, req)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusOK, rec.Code)
|
assert.Equal(t, http.StatusOK, rec.Code)
|
||||||
content, err := os.ReadFile(filepath.Join(dir, "raw", "typed-note.md"))
|
content, err := os.ReadFile(filepath.Join(dir, "knowledge", "typed-note.md"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Contains(t, string(content), "type: concept")
|
assert.Contains(t, string(content), "type: concept")
|
||||||
assert.Contains(t, string(content), "domain: software")
|
assert.Contains(t, string(content), "domain: software")
|
||||||
@@ -109,7 +107,8 @@ func TestWrite_GeneratesFilenameIfAbsent(t *testing.T) {
|
|||||||
h.Write(rec, req)
|
h.Write(rec, req)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusOK, rec.Code)
|
assert.Equal(t, http.StatusOK, rec.Code)
|
||||||
entries, _ := os.ReadDir(filepath.Join(dir, "raw"))
|
entries, _ := os.ReadDir(filepath.Join(dir, "knowledge"))
|
||||||
assert.Len(t, entries, 1)
|
// +1 because setup already wrote tdd.md
|
||||||
assert.True(t, strings.HasSuffix(entries[0].Name(), ".md"))
|
assert.Len(t, entries, 2)
|
||||||
|
assert.True(t, strings.HasSuffix(entries[1].Name(), ".md"))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ func Query(brainDir, query string, limit int) ([]Result, error) {
|
|||||||
|
|
||||||
var results []Result
|
var results []Result
|
||||||
|
|
||||||
err := filepath.WalkDir(filepath.Join(brainDir, "wiki"), func(path string, d os.DirEntry, err error) error {
|
err := filepath.WalkDir(filepath.Join(brainDir, "knowledge"), func(path string, d os.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Warn("search: skipping path", "path", path, "err", err)
|
slog.Warn("search: skipping path", "path", path, "err", err)
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -14,17 +14,15 @@ import (
|
|||||||
|
|
||||||
func TestSearch_ReturnsMatchingPages(t *testing.T) {
|
func TestSearch_ReturnsMatchingPages(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki", "concepts"), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, "knowledge"), 0o755))
|
||||||
|
|
||||||
// Write a concept page mentioning "retry"
|
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
filepath.Join(dir, "wiki", "concepts", "retry-logic.md"),
|
filepath.Join(dir, "knowledge", "retry-logic.md"),
|
||||||
[]byte("---\ntitle: Retry Logic\ndomain: software\n---\n\nRetry logic handles transient failures by re-attempting operations.\n"),
|
[]byte("---\ntitle: Retry Logic\ndomain: software\n---\n\nRetry logic handles transient failures by re-attempting operations.\n"),
|
||||||
0o644,
|
0o644,
|
||||||
))
|
))
|
||||||
// Write an unrelated page
|
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
filepath.Join(dir, "wiki", "concepts", "database.md"),
|
filepath.Join(dir, "knowledge", "database.md"),
|
||||||
[]byte("---\ntitle: Database\ndomain: software\n---\n\nA database stores structured data.\n"),
|
[]byte("---\ntitle: Database\ndomain: software\n---\n\nA database stores structured data.\n"),
|
||||||
0o644,
|
0o644,
|
||||||
))
|
))
|
||||||
@@ -32,7 +30,7 @@ func TestSearch_ReturnsMatchingPages(t *testing.T) {
|
|||||||
results, err := search.Query(dir, "retry transient", 5)
|
results, err := search.Query(dir, "retry transient", 5)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, results, 1)
|
require.Len(t, results, 1)
|
||||||
assert.Equal(t, "wiki/concepts/retry-logic.md", results[0].Path)
|
assert.Equal(t, "knowledge/retry-logic.md", results[0].Path)
|
||||||
assert.Equal(t, "Retry Logic", results[0].Title)
|
assert.Equal(t, "Retry Logic", results[0].Title)
|
||||||
assert.Greater(t, results[0].Score, 0)
|
assert.Greater(t, results[0].Score, 0)
|
||||||
assert.Contains(t, results[0].Excerpt, "Retry")
|
assert.Contains(t, results[0].Excerpt, "Retry")
|
||||||
@@ -40,10 +38,10 @@ func TestSearch_ReturnsMatchingPages(t *testing.T) {
|
|||||||
|
|
||||||
func TestSearch_RespectsLimit(t *testing.T) {
|
func TestSearch_RespectsLimit(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki", "concepts"), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, "knowledge"), 0o755))
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
filepath.Join(dir, "wiki", "concepts", fmt.Sprintf("page-%d.md", i)),
|
filepath.Join(dir, "knowledge", fmt.Sprintf("page-%d.md", i)),
|
||||||
[]byte(fmt.Sprintf("---\ntitle: Page %d\n---\n\nThis page mentions retry.\n", i)),
|
[]byte(fmt.Sprintf("---\ntitle: Page %d\n---\n\nThis page mentions retry.\n", i)),
|
||||||
0o644,
|
0o644,
|
||||||
))
|
))
|
||||||
|
|||||||
Reference in New Issue
Block a user