feat: pass-rate /pass-rate endpoint and CLI — Plan 5 of hyperguild migration
Adds the read side of pass-rate logging that consumes the SKILL.md
instrumentation landed in the local-dev companion merge:
- GET /pass-rate?skill=X&window=Y on the brain pod (ingestion module)
walks brain/sessions/*.jsonl, normalizes legacy {ok,error,skipped}
to canonical {pass,fail,skip}, returns aggregate counts plus
pass_rate (null when pass+fail == 0).
- hyperguild brain pass-rate <skill> [--window 7d] [--json] CLI
subcommand (third nested verb under brain).
- session_log MCP tool docstring updated to lead with the new
pass|fail|skip vocabulary; legacy values still accepted by both
the tool's loose validator and the aggregator's normalization.
This is the brain HTTP REST API's first GET endpoint — pure reads
follow REST semantics; legacy POST routes (query, write, ingest, etc.)
all take JSON bodies. Future read endpoints SHOULD use GET.
Plan 6 routing pod is the consumer; one week of usage data between
this merge and Plan 6 deploy will mean Plan 6 lands on real numbers.
Plan: docs/superpowers/plans/2026-05-03-pass-rate-logging.md
Spec: docs/superpowers/specs/2026-05-03-pass-rate-logging-design.md
This commit is contained in:
@@ -232,6 +232,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -61,6 +61,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -237,6 +237,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -235,6 +235,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -232,6 +232,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -61,6 +61,12 @@ The brain HTTP REST API (`/query`, `/write`, `/ingest`, `/ingest-raw`,
|
||||
`/ingest-path`, `/backfill-refs`) remains available on the same port (3300) for
|
||||
shell scripts and non-MCP clients.
|
||||
|
||||
The brain HTTP REST API also serves a read-only `GET /pass-rate?skill=X&window=Y`
|
||||
endpoint that aggregates `final_status` counts from session logs and returns
|
||||
`{skill, window, pass, fail, skip, total, pass_rate}`. Plan 6 (routing pod)
|
||||
reads this to decide whether to route skill calls to local models. Pass rate
|
||||
is `null` when no logged invocations are in the window.
|
||||
|
||||
## Agent instructions
|
||||
|
||||
When acting as a coding agent on this project:
|
||||
|
||||
@@ -69,6 +69,35 @@ EOF
|
||||
knowledge/example-lesson.md
|
||||
```
|
||||
|
||||
### `hyperguild brain pass-rate <skill>`
|
||||
|
||||
Returns the pass rate for a skill over a lookback window. Computed
|
||||
on-demand from `brain/sessions/*.jsonl`.
|
||||
|
||||
```bash
|
||||
$ hyperguild brain pass-rate tdd
|
||||
tdd: 47 / 50 = 94% (window: 7d)
|
||||
|
||||
$ hyperguild brain pass-rate tdd --window 30d --json
|
||||
{
|
||||
"skill": "tdd",
|
||||
"window": "30d",
|
||||
"pass": 142,
|
||||
"fail": 8,
|
||||
"skip": 5,
|
||||
"total": 155,
|
||||
"pass_rate": 0.9467
|
||||
}
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
- `--window` — lookback window (default `7d`; accepts `Nh`, `Nd`)
|
||||
- `--json` — emit the raw response envelope
|
||||
|
||||
Skills with no logged invocations return zero counts and `pass_rate: null`
|
||||
(indicating "no data", distinct from "always passes").
|
||||
|
||||
### `hyperguild mode <cloud|client-local|sovereign>`
|
||||
|
||||
Writes a `.mcp.json` template for the chosen operating mode.
|
||||
|
||||
@@ -11,15 +11,17 @@ import (
|
||||
|
||||
func runBrain(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("subcommand required (query|write)")
|
||||
return errors.New("subcommand required (query|write|pass-rate)")
|
||||
}
|
||||
switch args[0] {
|
||||
case "query":
|
||||
return runBrainQuery(ctx, args[1:], stdin, stdout, stderr)
|
||||
case "write":
|
||||
return runBrainWrite(ctx, args[1:], stdin, stdout, stderr)
|
||||
case "pass-rate":
|
||||
return runBrainPassRate(ctx, args[1:], stdin, stdout, stderr)
|
||||
default:
|
||||
return fmt.Errorf("unknown subcommand: %s (expected query|write)", args[0])
|
||||
return fmt.Errorf("unknown subcommand: %s (expected query|write|pass-rate)", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,3 +73,34 @@ func runBrainWrite(ctx context.Context, args []string, stdin io.Reader, stdout,
|
||||
fmt.Fprintln(stdout, res.Path) //nolint:errcheck
|
||||
return nil
|
||||
}
|
||||
|
||||
func runBrainPassRate(ctx context.Context, args []string, _ io.Reader, stdout, stderr io.Writer) error {
|
||||
fs := flag.NewFlagSet("brain pass-rate", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
asJSON := fs.Bool("json", false, "output JSON instead of human-readable")
|
||||
window := fs.String("window", "7d", "lookback window (e.g. 1h, 24h, 7d, 30d)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("parse flags: %w", err)
|
||||
}
|
||||
if fs.NArg() < 1 {
|
||||
return errors.New("skill required")
|
||||
}
|
||||
skill := fs.Arg(0)
|
||||
|
||||
res, err := newBrainClient().PassRate(ctx, skill, *window)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *asJSON {
|
||||
enc := json.NewEncoder(stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(res)
|
||||
}
|
||||
if res.PassRate == nil {
|
||||
fmt.Fprintf(stdout, "%s: no data (window: %s)\n", res.Skill, res.Window) //nolint:errcheck
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(stdout, "%s: %d / %d = %.0f%% (window: %s)\n", res.Skill, res.Pass, res.Total, *res.PassRate*100, res.Window) //nolint:errcheck
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -154,3 +154,67 @@ func TestRunBrainWrite_EmptyStdin(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, gotLen, "empty stdin should produce empty content payload")
|
||||
}
|
||||
|
||||
func TestRunBrainPassRate_Human(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("BRAIN_URL", srv.URL)
|
||||
|
||||
var out, errBuf bytes.Buffer
|
||||
err := runBrain(context.Background(), []string{"pass-rate", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||
require.NoError(t, err)
|
||||
got := out.String()
|
||||
assert.Contains(t, got, "tdd")
|
||||
assert.Contains(t, got, "47 / 50")
|
||||
assert.Contains(t, got, "94%")
|
||||
}
|
||||
|
||||
func TestRunBrainPassRate_NoData(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("BRAIN_URL", srv.URL)
|
||||
|
||||
var out, errBuf bytes.Buffer
|
||||
err := runBrain(context.Background(), []string{"pass-rate", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, out.String(), "no data")
|
||||
}
|
||||
|
||||
func TestRunBrainPassRate_JSON(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("BRAIN_URL", srv.URL)
|
||||
|
||||
var out, errBuf bytes.Buffer
|
||||
err := runBrain(context.Background(), []string{"pass-rate", "--json", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, out.String(), `"pass_rate": 0.94`)
|
||||
}
|
||||
|
||||
func TestRunBrainPassRate_MissingSkill(t *testing.T) {
|
||||
var out, errBuf bytes.Buffer
|
||||
err := runBrain(context.Background(), []string{"pass-rate"}, strings.NewReader(""), &out, &errBuf)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "skill required")
|
||||
}
|
||||
|
||||
func TestRunBrainPassRate_WindowFlag(t *testing.T) {
|
||||
gotWindow := ""
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotWindow = r.URL.Query().Get("window")
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"30d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("BRAIN_URL", srv.URL)
|
||||
|
||||
var out, errBuf bytes.Buffer
|
||||
err := runBrain(context.Background(), []string{"pass-rate", "--window", "30d", "tdd"}, strings.NewReader(""), &out, &errBuf)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "30d", gotWindow)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
@@ -119,3 +120,40 @@ func (c *brainClient) Write(ctx context.Context, kind, slug string, content io.R
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// PassRateResult mirrors the /pass-rate response envelope.
|
||||
type PassRateResult struct {
|
||||
Skill string `json:"skill"`
|
||||
Window string `json:"window"`
|
||||
Pass int `json:"pass"`
|
||||
Fail int `json:"fail"`
|
||||
Skip int `json:"skip"`
|
||||
Total int `json:"total"`
|
||||
PassRate *float64 `json:"pass_rate"`
|
||||
}
|
||||
|
||||
func (c *brainClient) PassRate(ctx context.Context, skill, window string) (*PassRateResult, error) {
|
||||
q := url.Values{}
|
||||
q.Set("skill", skill)
|
||||
q.Set("window", window)
|
||||
u := c.baseURL + "/pass-rate?" + q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("brain GET /pass-rate: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("brain GET /pass-rate: status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
var out PassRateResult
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("decode /pass-rate response: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
@@ -84,6 +84,40 @@ func TestBrainClient_Write_Success(t *testing.T) {
|
||||
assert.Equal(t, "knowledge/find-h.md", res.Path)
|
||||
}
|
||||
|
||||
func TestBrainClient_PassRate_Success(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, "/pass-rate", r.URL.Path)
|
||||
assert.Equal(t, "tdd", r.URL.Query().Get("skill"))
|
||||
assert.Equal(t, "7d", r.URL.Query().Get("window"))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":47,"fail":3,"skip":0,"total":50,"pass_rate":0.94}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &brainClient{baseURL: srv.URL, http: srv.Client()}
|
||||
res, err := c.PassRate(context.Background(), "tdd", "7d")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "tdd", res.Skill)
|
||||
assert.Equal(t, 47, res.Pass)
|
||||
assert.Equal(t, 3, res.Fail)
|
||||
require.NotNil(t, res.PassRate)
|
||||
assert.InDelta(t, 0.94, *res.PassRate, 0.001)
|
||||
}
|
||||
|
||||
func TestBrainClient_PassRate_NullRate(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"skill":"tdd","window":"7d","pass":0,"fail":0,"skip":0,"total":0,"pass_rate":null}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &brainClient{baseURL: srv.URL, http: srv.Client()}
|
||||
res, err := c.PassRate(context.Background(), "tdd", "7d")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, res.PassRate)
|
||||
}
|
||||
|
||||
func TestNewBrainClient_DefaultURL(t *testing.T) {
|
||||
t.Setenv("BRAIN_URL", "")
|
||||
c := newBrainClient()
|
||||
|
||||
@@ -73,6 +73,7 @@ func main() {
|
||||
mux.HandleFunc("POST /ingest-path", h.IngestPath)
|
||||
mux.HandleFunc("POST /ingest-raw", h.IngestRaw)
|
||||
mux.HandleFunc("POST /backfill-refs", h.BackfillRefs)
|
||||
mux.HandleFunc("GET /pass-rate", h.PassRate)
|
||||
mux.Handle("POST /mcp", mcpSrv)
|
||||
|
||||
addr := ":" + port
|
||||
|
||||
140
ingestion/internal/api/passrate.go
Normal file
140
ingestion/internal/api/passrate.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type passRateResponse struct {
|
||||
Skill string `json:"skill"`
|
||||
Window string `json:"window"`
|
||||
Pass int `json:"pass"`
|
||||
Fail int `json:"fail"`
|
||||
Skip int `json:"skip"`
|
||||
Total int `json:"total"`
|
||||
PassRate *float64 `json:"pass_rate"`
|
||||
}
|
||||
|
||||
// PassRate handles GET /pass-rate?skill=X&window=Y.
|
||||
// Walks brainDir/sessions/*.jsonl, filters by skill name and timestamp,
|
||||
// returns aggregated counts and pass rate.
|
||||
func (h *Handler) PassRate(w http.ResponseWriter, r *http.Request) {
|
||||
skill := r.URL.Query().Get("skill")
|
||||
if skill == "" {
|
||||
writeError(w, http.StatusBadRequest, "skill is required")
|
||||
return
|
||||
}
|
||||
|
||||
windowStr := r.URL.Query().Get("window")
|
||||
if windowStr == "" {
|
||||
windowStr = "7d"
|
||||
}
|
||||
window, err := parseWindow(windowStr)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid window: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-window)
|
||||
pass, fail, skip := 0, 0, 0
|
||||
|
||||
sessionsDir := filepath.Join(h.brainDir, "sessions")
|
||||
entries, err := os.ReadDir(sessionsDir)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
writeError(w, http.StatusInternalServerError, "read sessions dir: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jsonl") {
|
||||
continue
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(sessionsDir, entry.Name()))
|
||||
if err != nil {
|
||||
continue // skip unreadable files
|
||||
}
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var rec struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
Skill string `json:"skill"`
|
||||
FinalStatus string `json:"final_status"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
||||
continue // malformed — skip
|
||||
}
|
||||
if rec.Skill != skill {
|
||||
continue
|
||||
}
|
||||
ts, err := time.Parse(time.RFC3339, rec.Timestamp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if ts.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
switch normalizeStatus(rec.FinalStatus) {
|
||||
case "pass":
|
||||
pass++
|
||||
case "fail":
|
||||
fail++
|
||||
case "skip":
|
||||
skip++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
total := pass + fail + skip
|
||||
resp := passRateResponse{
|
||||
Skill: skill,
|
||||
Window: windowStr,
|
||||
Pass: pass,
|
||||
Fail: fail,
|
||||
Skip: skip,
|
||||
Total: total,
|
||||
}
|
||||
if pass+fail > 0 {
|
||||
rate := float64(pass) / float64(pass+fail)
|
||||
resp.PassRate = &rate
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// normalizeStatus maps both new (pass/fail/skip) and legacy (ok/error/skipped)
|
||||
// vocabularies to the canonical pass/fail/skip set. Unknown values are treated
|
||||
// as skip for safety.
|
||||
func normalizeStatus(s string) string {
|
||||
switch s {
|
||||
case "pass", "ok":
|
||||
return "pass"
|
||||
case "fail", "error":
|
||||
return "fail"
|
||||
case "skip", "skipped":
|
||||
return "skip"
|
||||
default:
|
||||
return "skip"
|
||||
}
|
||||
}
|
||||
|
||||
// parseWindow accepts Go-style durations plus "Nd" for days.
|
||||
func parseWindow(s string) (time.Duration, error) {
|
||||
if strings.HasSuffix(s, "d") {
|
||||
// Replace "d" with "h" * 24
|
||||
days := strings.TrimSuffix(s, "d")
|
||||
d, err := time.ParseDuration(days + "h")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return d * 24, nil
|
||||
}
|
||||
return time.ParseDuration(s)
|
||||
}
|
||||
172
ingestion/internal/api/passrate_test.go
Normal file
172
ingestion/internal/api/passrate_test.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeSession writes one or more JSONL entries to <dir>/sessions/<sessionID>.jsonl.
|
||||
// The handler scans <brainDir>/sessions/, so test fixtures must mirror that layout.
|
||||
func writeSession(t *testing.T, dir, sessionID string, entries ...string) {
|
||||
t.Helper()
|
||||
sessionsDir := filepath.Join(dir, "sessions")
|
||||
require.NoError(t, os.MkdirAll(sessionsDir, 0o755))
|
||||
path := filepath.Join(sessionsDir, sessionID+".jsonl")
|
||||
body := ""
|
||||
for _, e := range entries {
|
||||
body += e + "\n"
|
||||
}
|
||||
require.NoError(t, os.WriteFile(path, []byte(body), 0o644))
|
||||
}
|
||||
|
||||
func TestPassRate_HappyPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
|
||||
writeSession(t, dir, "s1",
|
||||
`{"timestamp":"`+recent+`","skill":"tdd","phase":"red","final_status":"pass"}`,
|
||||
`{"timestamp":"`+recent+`","skill":"tdd","phase":"green","final_status":"pass"}`,
|
||||
`{"timestamp":"`+recent+`","skill":"tdd","phase":"refactor","final_status":"fail"}`,
|
||||
)
|
||||
writeSession(t, dir, "s2",
|
||||
`{"timestamp":"`+recent+`","skill":"code-review","phase":"review","final_status":"pass"}`,
|
||||
)
|
||||
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
resp := w.Result()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
assert.Equal(t, "tdd", got.Skill)
|
||||
assert.Equal(t, "24h", got.Window)
|
||||
assert.Equal(t, 2, got.Pass)
|
||||
assert.Equal(t, 1, got.Fail)
|
||||
assert.Equal(t, 0, got.Skip)
|
||||
assert.Equal(t, 3, got.Total)
|
||||
require.NotNil(t, got.PassRate)
|
||||
assert.InDelta(t, 0.6667, *got.PassRate, 0.001)
|
||||
}
|
||||
|
||||
func TestPassRate_LegacyVocabulary(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
writeSession(t, dir, "s1",
|
||||
`{"timestamp":"`+now+`","skill":"tdd","final_status":"ok"}`,
|
||||
`{"timestamp":"`+now+`","skill":"tdd","final_status":"error"}`,
|
||||
`{"timestamp":"`+now+`","skill":"tdd","final_status":"skipped"}`,
|
||||
)
|
||||
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||
assert.Equal(t, 1, got.Pass, "ok→pass")
|
||||
assert.Equal(t, 1, got.Fail, "error→fail")
|
||||
assert.Equal(t, 1, got.Skip, "skipped→skip")
|
||||
}
|
||||
|
||||
func TestPassRate_OutsideWindow_Excluded(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
old := time.Now().UTC().Add(-30 * 24 * time.Hour).Format(time.RFC3339)
|
||||
recent := time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
writeSession(t, dir, "s1",
|
||||
`{"timestamp":"`+old+`","skill":"tdd","final_status":"pass"}`,
|
||||
`{"timestamp":"`+recent+`","skill":"tdd","final_status":"pass"}`,
|
||||
)
|
||||
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||
assert.Equal(t, 1, got.Pass)
|
||||
assert.Equal(t, 1, got.Total)
|
||||
}
|
||||
|
||||
func TestPassRate_NoData_ReturnsZerosAndNullRate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||
assert.Equal(t, 0, got.Pass)
|
||||
assert.Equal(t, 0, got.Fail)
|
||||
assert.Equal(t, 0, got.Skip)
|
||||
assert.Equal(t, 0, got.Total)
|
||||
assert.Nil(t, got.PassRate, "pass_rate must be null when pass+fail == 0")
|
||||
}
|
||||
|
||||
func TestPassRate_DefaultsTo7d(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
writeSession(t, dir, "s1", `{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`)
|
||||
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd", nil) // no window
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||
assert.Equal(t, "7d", got.Window)
|
||||
assert.Equal(t, 1, got.Pass)
|
||||
}
|
||||
|
||||
func TestPassRate_MissingSkill_ReturnsBadRequest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode)
|
||||
}
|
||||
|
||||
func TestPassRate_BadWindow_ReturnsBadRequest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=foo", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode)
|
||||
}
|
||||
|
||||
func TestPassRate_MalformedLine_Skipped(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
writeSession(t, dir, "s1",
|
||||
`{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`,
|
||||
`not valid json`,
|
||||
`{"timestamp":"`+now+`","skill":"tdd","final_status":"pass"}`,
|
||||
)
|
||||
|
||||
h := &Handler{brainDir: dir}
|
||||
req := httptest.NewRequest(http.MethodGet, "/pass-rate?skill=tdd&window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.PassRate(w, req)
|
||||
|
||||
var got passRateResponse
|
||||
require.NoError(t, json.NewDecoder(w.Result().Body).Decode(&got))
|
||||
assert.Equal(t, 2, got.Pass, "the malformed line is silently skipped")
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func (s *Server) tools() []map[string]any {
|
||||
"skill": str("skill name"),
|
||||
"phase": str("phase within the skill"),
|
||||
"project_root": str("absolute project root"),
|
||||
"final_status": str("ok | error | skipped"),
|
||||
"final_status": str("pass | fail | skip (legacy: ok | error | skipped also accepted)"),
|
||||
"file_path": str("optional file produced"),
|
||||
"model_used": str("optional model identifier"),
|
||||
"duration_ms": int_("optional duration in ms"),
|
||||
|
||||
Reference in New Issue
Block a user