Compare commits
5 Commits
cbf5cab5e7
...
400025715a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
400025715a | ||
|
|
986e3e1d12 | ||
|
|
593d1a4c6d | ||
|
|
417bf224eb | ||
|
|
37dbd22eff |
@@ -232,6 +232,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -61,6 +61,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -237,6 +237,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -235,6 +235,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -232,6 +232,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -61,6 +61,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
|||||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||||
shell scripts and non-MCP clients.
|
shell scripts and non-MCP clients.
|
||||||
|
|
||||||
|
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||||
|
endpoint that aggregates `final_status` counts from session logs and returns
|
||||||
|
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||||
|
reads this to decide whether to route skill calls to local models. Pass rate
|
||||||
|
is `null` when no logged invocations are in the window.
|
||||||
|
|
||||||
## Agent instructions
|
## Agent instructions
|
||||||
|
|
||||||
When acting as a coding agent on this project:
|
When acting as a coding agent on this project:
|
||||||
|
|||||||
@@ -69,6 +69,35 @@ EOF
|
|||||||
knowledge/example-lesson.md
|
knowledge/example-lesson.md
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `hyperguild brain pass-rate <skill>`
|
||||||
|
|
||||||
|
Returns the pass rate for a skill over a lookback window. Computed
|
||||||
|
on-demand from `brain/sessions/*.jsonl`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ hyperguild brain pass-rate tdd
|
||||||
|
tdd: 47 / 50 = 94% (window: 7d)
|
||||||
|
|
||||||
|
$ hyperguild brain pass-rate tdd --window 30d --json
|
||||||
|
{
|
||||||
|
"skill": "tdd",
|
||||||
|
"window": "30d",
|
||||||
|
"pass": 142,
|
||||||
|
"fail": 8,
|
||||||
|
"skip": 5,
|
||||||
|
"total": 155,
|
||||||
|
"pass_rate": 0.9467
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
|
||||||
|
- `--window` — lookback window (default `7d`; accepts `Nh`, `Nd`)
|
||||||
|
- `--json` — emit the raw response envelope
|
||||||
|
|
||||||
|
Skills with no logged invocations return zero counts and `pass_rate: null`
|
||||||
|
(indicating "no data", distinct from "always passes").
|
||||||
|
|
||||||
### `hyperguild mode <cloud|client-local|sovereign>`
|
### `hyperguild mode <cloud|client-local|sovereign>`
|
||||||
|
|
||||||
Writes a `.mcp.json` template for the chosen operating mode.
|
Writes a `.mcp.json` template for the chosen operating mode.
|
||||||
|
|||||||
@@ -11,15 +11,17 @@ import (
|
|||||||
|
|
||||||
func runBrain(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
func runBrain(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("subcommand required (query|write)")
|
return errors.New("subcommand required (query|write|pass-rate)")
|
||||||
}
|
}
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "query":
|
case "query":
|
||||||
return runBrainQuery(ctx, args[1:], stdin, stdout, stderr)
|
return runBrainQuery(ctx, args[1:], stdin, stdout, stderr)
|
||||||
case "write":
|
case "write":
|
||||||
return runBrainWrite(ctx, args[1:], stdin, stdout, stderr)
|
return runBrainWrite(ctx, args[1:], stdin, stdout, stderr)
|
||||||
|
case "pass-rate":
|
||||||
|
return runBrainPassRate(ctx, args[1:], stdin, stdout, stderr)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown subcommand: %s (expected query|write)", args[0])
|
return fmt.Errorf("unknown subcommand: %s (expected query|write|pass-rate)", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,3 +73,34 @@ func runBrainWrite(ctx context.Context, args []string, stdin io.Reader, stdout,
|
|||||||
fmt.Fprintln(stdout, res.Path) //nolint:errcheck
|
fmt.Fprintln(stdout, res.Path) //nolint:errcheck
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runBrainPassRate(ctx context.Context, args []string, _ io.Reader, stdout, stderr io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("brain pass-rate", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
asJSON := fs.Bool("json", false, "output JSON instead of human-readable")
|
||||||
|
window := fs.String("window", "7d", "lookback window (e.g. 1h, 24h, 7d, 30d)")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("parse flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() < 1 {
|
||||||
|
return errors.New("skill required")
|
||||||
|
}
|
||||||
|
skill := fs.Arg(0)
|
||||||
|
|
||||||
|
res, err := newBrainClient().PassRate(ctx, skill, *window)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *asJSON {
|
||||||
|
enc := json.NewEncoder(stdout)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(res)
|
||||||
|
}
|
||||||
|
if res.PassRate == nil {
|
||||||
|
fmt.Fprintf(stdout, "%s: no data (window: %s)\n", res.Skill, res.Window) //nolint:errcheck
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "%s: %d / %d = %.0f%% (window: %s)\n", res.Skill, res.Pass, res.Total, *res.PassRate*100, res.Window) //nolint:errcheck
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -154,3 +154,67 @@ func TestRunBrainWrite_EmptyStdin(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 0, gotLen, "empty stdin should produce empty content payload")
|
assert.Equal(t, 0, gotLen, "empty stdin should produce empty content payload")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunBrainPassRate_Human(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("BRAIN_URL", srv.URL)
|
||||||
|
|
||||||
|
var out, errBuf bytes.Buffer
|
||||||
|
err := runBrain(context.Background(), []string{"pass-rate", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||||
|
require.NoError(t, err)
|
||||||
|
got := out.String()
|
||||||
|
assert.Contains(t, got, "tdd")
|
||||||
|
assert.Contains(t, got, "47 / 50")
|
||||||
|
assert.Contains(t, got, "94%")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBrainPassRate_NoData(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("BRAIN_URL", srv.URL)
|
||||||
|
|
||||||
|
var out, errBuf bytes.Buffer
|
||||||
|
err := runBrain(context.Background(), []string{"pass-rate", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, out.String(), "no data")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBrainPassRate_JSON(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("BRAIN_URL", srv.URL)
|
||||||
|
|
||||||
|
var out, errBuf bytes.Buffer
|
||||||
|
err := runBrain(context.Background(), []string{"pass-rate", "--json", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, out.String(), `"pass_rate": 0.94`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBrainPassRate_MissingSkill(t *testing.T) {
|
||||||
|
var out, errBuf bytes.Buffer
|
||||||
|
err := runBrain(context.Background(), []string{"pass-rate"}, strings.NewReader(""), &out, &errBuf)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "skill required")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBrainPassRate_WindowFlag(t *testing.T) {
|
||||||
|
gotWindow := ""
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotWindow = r.URL.Query().Get("window")
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"30d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("BRAIN_URL", srv.URL)
|
||||||
|
|
||||||
|
var out, errBuf bytes.Buffer
|
||||||
|
err := runBrain(context.Background(), []string{"pass-rate", "--window", "30d", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "30d", gotWindow)
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -119,3 +120,40 @@ func (c *brainClient) Write(ctx context.Context, kind, slug string, content io.R
|
|||||||
}
|
}
|
||||||
return &out, nil
|
return &out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PassRateResult mirrors the /pass-rate response envelope.
|
||||||
|
type PassRateResult struct {
|
||||||
|
Skill string `json:"skill"`
|
||||||
|
Window string `json:"window"`
|
||||||
|
Pass int `json:"pass"`
|
||||||
|
Fail int `json:"fail"`
|
||||||
|
Skip int `json:"skip"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
PassRate *float64 `json:"pass_rate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *brainClient) PassRate(ctx context.Context, skill, window string) (*PassRateResult, error) {
|
||||||
|
q := url.Values{}
|
||||||
|
q.Set("skill", skill)
|
||||||
|
q.Set("window", window)
|
||||||
|
u := c.baseURL + "/pass-rate?" + q.Encode()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build request: %w", err)
|
||||||
|
}
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("brain GET /pass-rate: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("brain GET /pass-rate: status %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
var out PassRateResult
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode /pass-rate response: %w", err)
|
||||||
|
}
|
||||||
|
return &out, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -84,6 +84,40 @@ func TestBrainClient_Write_Success(t *testing.T) {
|
|||||||
assert.Equal(t, "knowledge/find-h.md", res.Path)
|
assert.Equal(t, "knowledge/find-h.md", res.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBrainClient_PassRate_Success(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
assert.Equal(t, http.MethodGet, r.Method)
|
||||||
|
assert.Equal(t, "/pass-rate", r.URL.Path)
|
||||||
|
assert.Equal(t, "tdd", r.URL.Query().Get("skill"))
|
||||||
|
assert.Equal(t, "7d", r.URL.Query().Get("window"))
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := &brainClient{baseURL: srv.URL, http: srv.Client()}
|
||||||
|
res, err := c.PassRate(context.Background(), "tdd", "7d")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "tdd", res.Skill)
|
||||||
|
assert.Equal(t, 47, res.Pass)
|
||||||
|
assert.Equal(t, 3, res.Fail)
|
||||||
|
require.NotNil(t, res.PassRate)
|
||||||
|
assert.InDelta(t, 0.94, *res.PassRate, 0.001)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBrainClient_PassRate_NullRate(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := &brainClient{baseURL: srv.URL, http: srv.Client()}
|
||||||
|
res, err := c.PassRate(context.Background(), "tdd", "7d")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, res.PassRate)
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewBrainClient_DefaultURL(t *testing.T) {
|
func TestNewBrainClient_DefaultURL(t *testing.T) {
|
||||||
t.Setenv("BRAIN_URL", "")
|
t.Setenv("BRAIN_URL", "")
|
||||||
c := newBrainClient()
|
c := newBrainClient()
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ func main() {
|
|||||||
mux.HandleFunc("POST /ingest-path", h.IngestPath)
|
mux.HandleFunc("POST /ingest-path", h.IngestPath)
|
||||||
mux.HandleFunc("POST /ingest-raw", h.IngestRaw)
|
mux.HandleFunc("POST /ingest-raw", h.IngestRaw)
|
||||||
mux.HandleFunc("POST /backfill-refs", h.BackfillRefs)
|
mux.HandleFunc("POST /backfill-refs", h.BackfillRefs)
|
||||||
|
mux.HandleFunc("GET /pass-rate", h.PassRate)
|
||||||
mux.Handle("POST /mcp", mcpSrv)
|
mux.Handle("POST /mcp", mcpSrv)
|
||||||
|
|
||||||
addr := ":" + port
|
addr := ":" + port
|
||||||
|
|||||||
140
ingestion/internal/api/passrate.go
Normal file
140
ingestion/internal/api/passrate.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type passRateResponse struct {
|
||||||
|
Skill string `json:"skill"`
|
||||||
|
Window string `json:"window"`
|
||||||
|
Pass int `json:"pass"`
|
||||||
|
Fail int `json:"fail"`
|
||||||
|
Skip int `json:"skip"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
PassRate *float64 `json:"pass_rate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PassRate handles GET /pass-rate?skill=X&window=Y.
|
||||||
|
// Walks brainDir/sessions/*.jsonl, filters by skill name and timestamp,
|
||||||
|
// returns aggregated counts and pass rate.
|
||||||
|
func (h *Handler) PassRate(w http.ResponseWriter, r *http.Request) {
|
||||||
|
skill := r.URL.Query().Get("skill")
|
||||||
|
if skill == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "skill is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
windowStr := r.URL.Query().Get("window")
|
||||||
|
if windowStr == "" {
|
||||||
|
windowStr = "7d"
|
||||||
|
}
|
||||||
|
window, err := parseWindow(windowStr)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid window: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cutoff := time.Now().UTC().Add(-window)
|
||||||
|
pass, fail, skip := 0, 0, 0
|
||||||
|
|
||||||
|
sessionsDir := filepath.Join(h.brainDir, "sessions")
|
||||||
|
entries, err := os.ReadDir(sessionsDir)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
writeError(w, http.StatusInternalServerError, "read sessions dir: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jsonl") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(filepath.Join(sessionsDir, entry.Name()))
|
||||||
|
if err != nil {
|
||||||
|
continue // skip unreadable files
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(string(body), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
Skill string `json:"skill"`
|
||||||
|
FinalStatus string `json:"final_status"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
||||||
|
continue // malformed — skip
|
||||||
|
}
|
||||||
|
if rec.Skill != skill {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ts, err := time.Parse(time.RFC3339, rec.Timestamp)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ts.Before(cutoff) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch normalizeStatus(rec.FinalStatus) {
|
||||||
|
case "pass":
|
||||||
|
pass++
|
||||||
|
case "fail":
|
||||||
|
fail++
|
||||||
|
case "skip":
|
||||||
|
skip++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
total := pass + fail + skip
|
||||||
|
resp := passRateResponse{
|
||||||
|
Skill: skill,
|
||||||
|
Window: windowStr,
|
||||||
|
Pass: pass,
|
||||||
|
Fail: fail,
|
||||||
|
Skip: skip,
|
||||||
|
Total: total,
|
||||||
|
}
|
||||||
|
if pass+fail > 0 {
|
||||||
|
rate := float64(pass) / float64(pass+fail)
|
||||||
|
resp.PassRate = &rate
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeStatus maps both new (pass/fail/skip) and legacy (ok/error/skipped)
|
||||||
|
// vocabularies to the canonical pass/fail/skip set. Unknown values are treated
|
||||||
|
// as skip for safety.
|
||||||
|
func normalizeStatus(s string) string {
|
||||||
|
switch s {
|
||||||
|
case "pass", "ok":
|
||||||
|
return "pass"
|
||||||
|
case "fail", "error":
|
||||||
|
return "fail"
|
||||||
|
case "skip", "skipped":
|
||||||
|
return "skip"
|
||||||
|
default:
|
||||||
|
return "skip"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseWindow accepts Go-style durations plus "Nd" for days.
|
||||||
|
func parseWindow(s string) (time.Duration, error) {
|
||||||
|
if strings.HasSuffix(s, "d") {
|
||||||
|
// Replace "d" with "h" * 24
|
||||||
|
days := strings.TrimSuffix(s, "d")
|
||||||
|
d, err := time.ParseDuration(days + "h")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return d * 24, nil
|
||||||
|
}
|
||||||
|
return time.ParseDuration(s)
|
||||||
|
}
|
||||||
172
ingestion/internal/api/passrate_test.go
Normal file
172
ingestion/internal/api/passrate_test.go
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// writeSession writes one or more JSONL entries to <dir>/sessions/<sessionID>.jsonl.
|
||||||
|
// The handler scans <brainDir>/sessions/, so test fixtures must mirror that layout.
|
||||||
|
func writeSession(t *testing.T, dir, sessionID string, entries ...string) {
|
||||||
|
t.Helper()
|
||||||
|
sessionsDir := filepath.Join(dir, "sessions")
|
||||||
|
require.NoError(t, os.MkdirAll(sessionsDir, 0o755))
|
||||||
|
path := filepath.Join(sessionsDir, sessionID+".jsonl")
|
||||||
|
body := ""
|
||||||
|
for _, e := range entries {
|
||||||
|
body += e + "\n"
|
||||||
|
}
|
||||||
|
require.NoError(t, os.WriteFile(path, []byte(body), 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_HappyPath(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||||
|
|
||||||
|
writeSession(t, dir, "s1",
|
||||||
|
`{"timestamp":"`+recent+`","skill":"tdd","phase":"red","final_status":"pass"}`,
|
||||||
|
`{"timestamp":"`+recent+`","skill":"tdd","phase":"green","final_status":"pass"}`,
|
||||||
|
`{"timestamp":"`+recent+`","skill":"tdd","phase":"refactor","final_status":"fail"}`,
|
||||||
|
)
|
||||||
|
writeSession(t, dir, "s2",
|
||||||
|
`{"timestamp":"`+recent+`","skill":"code-review","phase":"review","final_status":"pass"}`,
|
||||||
|
)
|
||||||
|
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
resp := w.Result()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||||
|
assert.Equal(t, "tdd", got.Skill)
|
||||||
|
assert.Equal(t, "24h", got.Window)
|
||||||
|
assert.Equal(t, 2, got.Pass)
|
||||||
|
assert.Equal(t, 1, got.Fail)
|
||||||
|
assert.Equal(t, 0, got.Skip)
|
||||||
|
assert.Equal(t, 3, got.Total)
|
||||||
|
require.NotNil(t, got.PassRate)
|
||||||
|
assert.InDelta(t, 0.6667, *got.PassRate, 0.001)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_LegacyVocabulary(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
now := time.Now().UTC().Format(time.RFC3339)
|
||||||
|
writeSession(t, dir, "s1",
|
||||||
|
`{"timestamp":"`+now+`","skill":"tdd","final_status":"ok"}`,
|
||||||
|
`{"timestamp":"`+now+`","skill":"tdd","final_status":"error"}`,
|
||||||
|
`{"timestamp":"`+now+`","skill":"tdd","final_status":"skipped"}`,
|
||||||
|
)
|
||||||
|
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||||
|
assert.Equal(t, 1, got.Pass, "ok→pass")
|
||||||
|
assert.Equal(t, 1, got.Fail, "error→fail")
|
||||||
|
assert.Equal(t, 1, got.Skip, "skipped→skip")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_OutsideWindow_Excluded(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
old := time.Now().UTC().Add(-30 * 24 * time.Hour).Format(time.RFC3339)
|
||||||
|
recent := time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339)
|
||||||
|
writeSession(t, dir, "s1",
|
||||||
|
`{"timestamp":"`+old+`","skill":"tdd","final_status":"pass"}`,
|
||||||
|
`{"timestamp":"`+recent+`","skill":"tdd","final_status":"pass"}`,
|
||||||
|
)
|
||||||
|
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||||
|
assert.Equal(t, 1, got.Pass)
|
||||||
|
assert.Equal(t, 1, got.Total)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_NoData_ReturnsZerosAndNullRate(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||||
|
assert.Equal(t, 0, got.Pass)
|
||||||
|
assert.Equal(t, 0, got.Fail)
|
||||||
|
assert.Equal(t, 0, got.Skip)
|
||||||
|
assert.Equal(t, 0, got.Total)
|
||||||
|
assert.Nil(t, got.PassRate, "pass_rate must be null when pass+fail == 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_DefaultsTo7d(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
now := time.Now().UTC().Format(time.RFC3339)
|
||||||
|
writeSession(t, dir, "s1", `{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`)
|
||||||
|
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd", nil) // no window
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||||
|
assert.Equal(t, "7d", got.Window)
|
||||||
|
assert.Equal(t, 1, got.Pass)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_MissingSkill_ReturnsBadRequest(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_BadWindow_ReturnsBadRequest(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=foo", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassRate_MalformedLine_Skipped(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
now := time.Now().UTC().Format(time.RFC3339)
|
||||||
|
writeSession(t, dir, "s1",
|
||||||
|
`{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`,
|
||||||
|
`not valid json`,
|
||||||
|
`{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`,
|
||||||
|
)
|
||||||
|
|
||||||
|
h := &Handler{brainDir: dir}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.PassRate(w, req)
|
||||||
|
|
||||||
|
var got passRateResponse
|
||||||
|
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||||
|
assert.Equal(t, 2, got.Pass, "the malformed line is silently skipped")
|
||||||
|
}
|
||||||
@@ -77,7 +77,7 @@ func (s *Server) tools() []map[string]any {
|
|||||||
"skill": str("skill name"),
|
"skill": str("skill name"),
|
||||||
"phase": str("phase within the skill"),
|
"phase": str("phase within the skill"),
|
||||||
"project_root": str("absolute project root"),
|
"project_root": str("absolute project root"),
|
||||||
"final_status": str("ok | error | skipped"),
|
"final_status": str("pass | fail | skip (legacy: ok | error | skipped also accepted)"),
|
||||||
"file_path": str("optional file produced"),
|
"file_path": str("optional file produced"),
|
||||||
"model_used": str("optional model identifier"),
|
"model_used": str("optional model identifier"),
|
||||||
"duration_ms": int_("optional duration in ms"),
|
"duration_ms": int_("optional duration in ms"),
|
||||||
|
|||||||
Reference in New Issue
Block a user