feat(pipeline): add POST /ingest-raw for direct batch ingestion without LLM

Allows callers to provide pre-structured RawPage data directly, bypassing the
LLM extraction step. The pipeline still handles slug computation, frontmatter,
link canonicalization, source back-references, and dedup — only the extraction
is skipped. Useful when a more capable model or manual curation produces the
structured data.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

ingestion/internal/api/handler.go

@@ -272,6 +272,48 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, ingestResponse{Pages: allPages, Warnings: allWarnings})
 }
 
+type ingestRawRequest struct {
+	Source string             `json:"source"`
+	Pages  []pipeline.RawPage `json:"pages"`
+	DryRun bool               `json:"dry_run"`
+}
+
+// IngestRaw handles POST /ingest-raw — run the pipeline on pre-parsed RawPages,
+// skipping the LLM extraction step. Use when the caller has already produced
+// structured page data (e.g. from a more capable model or manual curation).
+func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) {
+	var req ingestRawRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	if strings.TrimSpace(req.Source) == "" {
+		writeError(w, http.StatusBadRequest, "source is required")
+		return
+	}
+	if len(req.Pages) == 0 {
+		writeError(w, http.StatusBadRequest, "pages is required and must be non-empty")
+		return
+	}
+
+	result, err := pipeline.RunRaw(h.brainDir, req.Source, req.Pages, req.DryRun)
+	if err != nil {
+		h.logger.Error("ingest-raw failed", "source", req.Source, "err", err)
+		writeError(w, http.StatusInternalServerError, "ingest error")
+		return
+	}
+
+	pages := result.Pages
+	if pages == nil {
+		pages = []string{}
+	}
+	warnings := result.Warnings
+	if warnings == nil {
+		warnings = []string{}
+	}
+	writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings})
+}
+
 // BackfillRefs handles POST /backfill-refs — injects source back-references
 // into all concept and entity pages based on existing wiki/sources/ pages.
 func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) {

ingestion/internal/api/handler_test.go

@@ -226,6 +226,85 @@ func TestIngestPath_File(t *testing.T) {
 	assert.NotEmpty(t, pagesSlice)
 }
 
+// ---------------------------------------------------------------------------
+// POST /ingest-raw
+// ---------------------------------------------------------------------------
+
+func TestIngestRaw_Validation(t *testing.T) {
+	cases := []struct {
+		name string
+		body map[string]any
+	}{
+		{"missing source", map[string]any{"pages": []any{map[string]any{"title": "X", "type": "concept", "content": "x"}}}},
+		{"missing pages", map[string]any{"source": "test-source"}},
+		{"empty pages", map[string]any{"source": "test-source", "pages": []any{}}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, h := setup(t)
+			body, _ := json.Marshal(tc.body)
+			req := httptest.NewRequest(http.MethodPost, "/ingest-raw", bytes.NewReader(body))
+			rec := httptest.NewRecorder()
+
+			h.IngestRaw(rec, req)
+
+			assert.Equal(t, http.StatusBadRequest, rec.Code)
+		})
+	}
+}
+
+func TestIngestRaw_Success(t *testing.T) {
+	dir, h := setup(t)
+	body, _ := json.Marshal(map[string]any{
+		"source": "test-article",
+		"pages": []any{
+			map[string]any{"title": "Test Article", "type": "source", "subtype": "article", "domain": "Testing", "content": "## Summary\n\nThis is a test article about [[Test Concept]].\n"},
+			map[string]any{"title": "Test Concept", "type": "concept", "domain": "Testing", "content": "A concept for testing.\n"},
+		},
+	})
+	req := httptest.NewRequest(http.MethodPost, "/ingest-raw", bytes.NewReader(body))
+	rec := httptest.NewRecorder()
+
+	h.IngestRaw(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	var resp map[string]any
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
+	pages := resp["pages"].([]any)
+	assert.Len(t, pages, 2)
+
+	// Verify files were written
+	sourcePath := filepath.Join(dir, "wiki", "sources", "test-article.md")
+	assert.FileExists(t, sourcePath)
+	conceptPath := filepath.Join(dir, "wiki", "concepts", "test-concept.md")
+	assert.FileExists(t, conceptPath)
+}
+
+func TestIngestRaw_DryRun(t *testing.T) {
+	dir, h := setup(t)
+	body, _ := json.Marshal(map[string]any{
+		"source": "dry-run-test",
+		"pages": []any{
+			map[string]any{"title": "Dry Run Source", "type": "source", "subtype": "article", "content": "Content."},
+		},
+		"dry_run": true,
+	})
+	req := httptest.NewRequest(http.MethodPost, "/ingest-raw", bytes.NewReader(body))
+	rec := httptest.NewRecorder()
+
+	h.IngestRaw(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	var resp map[string]any
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
+	pages := resp["pages"].([]any)
+	assert.NotEmpty(t, pages)
+
+	// Verify no files were written
+	sourcePath := filepath.Join(dir, "wiki", "sources", "dry-run-test.md")
+	assert.NoFileExists(t, sourcePath)
+}
+
 func TestIngestPath_Directory(t *testing.T) {
 	_, h := setup(t)