feat(pipeline): add POST /ingest-raw for direct batch ingestion without LLM
Allows callers to provide pre-structured RawPage data directly, bypassing the LLM extraction step. The pipeline still handles slug computation, frontmatter, link canonicalization, source back-references, and dedup — only the extraction is skipped. Useful when a more capable model or manual curation produces the structured data. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -272,6 +272,48 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, ingestResponse{Pages: allPages, Warnings: allWarnings})
|
||||
}
|
||||
|
||||
type ingestRawRequest struct {
|
||||
Source string `json:"source"`
|
||||
Pages []pipeline.RawPage `json:"pages"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
// IngestRaw handles POST /ingest-raw — run the pipeline on pre-parsed RawPages,
|
||||
// skipping the LLM extraction step. Use when the caller has already produced
|
||||
// structured page data (e.g. from a more capable model or manual curation).
|
||||
func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) {
|
||||
var req ingestRawRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON")
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Source) == "" {
|
||||
writeError(w, http.StatusBadRequest, "source is required")
|
||||
return
|
||||
}
|
||||
if len(req.Pages) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "pages is required and must be non-empty")
|
||||
return
|
||||
}
|
||||
|
||||
result, err := pipeline.RunRaw(h.brainDir, req.Source, req.Pages, req.DryRun)
|
||||
if err != nil {
|
||||
h.logger.Error("ingest-raw failed", "source", req.Source, "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "ingest error")
|
||||
return
|
||||
}
|
||||
|
||||
pages := result.Pages
|
||||
if pages == nil {
|
||||
pages = []string{}
|
||||
}
|
||||
warnings := result.Warnings
|
||||
if warnings == nil {
|
||||
warnings = []string{}
|
||||
}
|
||||
writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings})
|
||||
}
|
||||
|
||||
// BackfillRefs handles POST /backfill-refs — injects source back-references
|
||||
// into all concept and entity pages based on existing wiki/sources/ pages.
|
||||
func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
Reference in New Issue
Block a user