11 Commits

Author SHA1 Message Date
Mathias Bergqvist
76f195de2a docs: model orchestration design spec for Phase 3
All checks were successful
CI / Lint / Test / Vet (push) Successful in 1m8s
CI / Mirror to GitHub (push) Successful in 4s
2026-04-20 07:45:32 +02:00
Mathias Bergqvist
f901d4e67d fix(ci): use --follow-tags instead of --tags to avoid re-pushing existing tags
All checks were successful
CI / Lint / Test / Vet (push) Successful in 1m9s
CI / Mirror to GitHub (push) Successful in 3s
2026-04-19 19:16:22 +02:00
Mathias Bergqvist
509c04b6e4 fix(session): use fmt.Fprintf with nolint to satisfy both staticcheck and errcheck
Some checks failed
CI / Lint / Test / Vet (push) Successful in 1m7s
CI / Mirror to GitHub (push) Failing after 3s
2026-04-19 18:56:12 +02:00
Mathias Bergqvist
738275252c feat: hyperguild phase 2 — review/debug/spec/trainer skills with session history injection
Some checks failed
CI / Lint / Test / Vet (push) Failing after 3s
CI / Mirror to GitHub (push) Has been skipped
2026-04-19 14:38:05 +02:00
Mathias Bergqvist
38fcac4cba feat(trainer): add trainer MCP skill with reader→writer sub-agent chain
Reader agent scans session logs for SFT/DPO candidates; writer receives
reader output and formats+writes training pairs to brain/training-data/.
Adds trainer-reader.md and trainer-writer.md discipline prompts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 14:06:00 +02:00
Mathias Bergqvist
7697e901d2 feat(spec): add spec writing MCP skill
Adds the spec skill that generates structured implementation specs from
requirements and writes them to a configurable output path in the project.
Follows the same pattern as review/debug skills with session history injection.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 11:59:28 +02:00
Mathias Bergqvist
8cff57009a feat(debug): add debug MCP skill with hypothesis generation
Implements the debug skill following the same pattern as review. The skill
accepts project_root + error (+ optional context/model/session_id), prepends
session history, and calls the executor to produce 3-5 ordered hypotheses —
diagnosis only, no fixes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 11:29:58 +02:00
Mathias Bergqvist
8fb44affef feat(review): add code review MCP skill with session history injection
Implements the review skill following the same pattern as retrospective/tdd.
Validates project_root and files args, prepends session history when a
session_id is provided, and delegates to the executor with Read,Bash tools.
Iron-law discipline prompt enforces CRITICAL/WARNING/SUGGESTION output format.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 11:11:29 +02:00
Mathias Bergqvist
582ca5019b feat(tdd): inject session history into green and refactor worker prompts
Adds SessionsDir to tdd.Config, session_id to tool input schemas, and a
prependHistory method that reads the session JSONL log and prepends a
formatted history block to the task prompt before worker invocation.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 10:18:23 +02:00
Mathias Bergqvist
858a9ba1a1 fix(exec): expand validPhases and remove schema enum constraint for phase 2026-04-19 10:03:21 +02:00
Mathias Bergqvist
cbef2da8de feat(session): add FormatHistory for worker context injection
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-19 09:40:41 +02:00
28 changed files with 1525 additions and 4 deletions

View File

@@ -53,6 +53,6 @@ jobs:
chmod 600 ~/.ssh/id_rsa_gh_mirror
ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa_gh_mirror -o IdentitiesOnly=yes" \
git push git@github.com:mathiasb/hyperguild.git HEAD:main --tags
git push git@github.com:mathiasb/hyperguild.git HEAD:main --follow-tags
rm ~/.ssh/id_rsa_gh_mirror
echo "✓ Mirrored to GitHub"

View File

@@ -13,6 +13,10 @@ import (
"github.com/mathiasbq/supervisor/internal/skills/brain"
"github.com/mathiasbq/supervisor/internal/skills/org"
"github.com/mathiasbq/supervisor/internal/skills/retrospective"
skilldebug "github.com/mathiasbq/supervisor/internal/skills/debug"
"github.com/mathiasbq/supervisor/internal/skills/review"
"github.com/mathiasbq/supervisor/internal/skills/spec"
"github.com/mathiasbq/supervisor/internal/skills/trainer"
"github.com/mathiasbq/supervisor/internal/skills/sessionlog"
"github.com/mathiasbq/supervisor/internal/skills/tdd"
"github.com/mathiasbq/supervisor/internal/tier"
@@ -51,6 +55,35 @@ func main() {
os.Exit(1)
}
reviewPrompt, err := os.ReadFile(cfg.ConfigDir + "/review.md")
if err != nil {
logger.Error("read review.md", "path", cfg.ConfigDir+"/review.md", "err", err)
os.Exit(1)
}
debugPrompt, err := os.ReadFile(cfg.ConfigDir + "/debug.md")
if err != nil {
logger.Error("read debug.md", "path", cfg.ConfigDir+"/debug.md", "err", err)
os.Exit(1)
}
specPrompt, err := os.ReadFile(cfg.ConfigDir + "/spec.md")
if err != nil {
logger.Error("read spec.md", "path", cfg.ConfigDir+"/spec.md", "err", err)
os.Exit(1)
}
trainerReaderPrompt, err := os.ReadFile(cfg.ConfigDir + "/trainer-reader.md")
if err != nil {
logger.Error("read trainer-reader.md", "path", cfg.ConfigDir+"/trainer-reader.md", "err", err)
os.Exit(1)
}
trainerWriterPrompt, err := os.ReadFile(cfg.ConfigDir + "/trainer-writer.md")
if err != nil {
logger.Error("read trainer-writer.md", "path", cfg.ConfigDir+"/trainer-writer.md", "err", err)
os.Exit(1)
}
executor := iexec.New(iexec.Config{
SystemPrompt: string(systemPrompt),
LiteLLMBaseURL: cfg.LiteLLMBaseURL,
@@ -67,6 +100,7 @@ func main() {
SkillPrompt: string(tddPrompt),
DefaultModel: models.Resolve("tdd", ""),
ExecutorFn: executor.Run,
SessionsDir: cfg.SessionsDir,
}))
reg.Register(brain.New(brain.Config{
IngestBaseURL: cfg.IngestBaseURL,
@@ -83,6 +117,32 @@ func main() {
SessionsDir: cfg.SessionsDir,
ExecutorFn: executor.Run,
}))
reg.Register(review.New(review.Config{
SkillPrompt: string(reviewPrompt),
DefaultModel: models.Resolve("review", ""),
ExecutorFn: executor.Run,
SessionsDir: cfg.SessionsDir,
}))
reg.Register(skilldebug.New(skilldebug.Config{
SkillPrompt: string(debugPrompt),
DefaultModel: models.Resolve("debug", ""),
ExecutorFn: executor.Run,
SessionsDir: cfg.SessionsDir,
}))
reg.Register(spec.New(spec.Config{
SkillPrompt: string(specPrompt),
DefaultModel: models.Resolve("spec", ""),
ExecutorFn: executor.Run,
SessionsDir: cfg.SessionsDir,
}))
reg.Register(trainer.New(trainer.Config{
ReaderPrompt: string(trainerReaderPrompt),
WriterPrompt: string(trainerWriterPrompt),
DefaultModel: models.Resolve("trainer", ""),
ExecutorFn: executor.Run,
SessionsDir: cfg.SessionsDir,
BrainDir: cfg.BrainDir,
}))
srv := mcp.NewServer(reg)
mux := http.NewServeMux()

View File

@@ -9,3 +9,5 @@ skills:
review: ollama/devstral-tuned
debug: ollama/deepseek-r1-tuned
retrospective: ollama/qwen3-coder-30b-tuned
spec: ollama/qwen3-coder-30b-tuned
trainer: ollama/qwen3-coder-30b-tuned

View File

@@ -0,0 +1,31 @@
# Debug Discipline
You are a systematic debugger. Form hypotheses before suggesting fixes.
## Iron laws
1. Never suggest "try X and see what happens" — every hypothesis must have a specific expected outcome if correct
2. Generate exactly 3-5 hypotheses, ordered by likelihood (most likely first)
3. Never fix the bug — diagnose only; the caller decides what to do with the hypotheses
## Output contract
Return JSON result with:
- `status`: "pass" (hypotheses generated) or "error" (error too ambiguous to analyse)
- `phase`: "debug"
- `skill`: "debug"
- `file_path`: the most relevant file to the error (read it)
- `runner_output`: your hypotheses, formatted as:
```
HYPOTHESIS 1 (likelihood: high): <mechanism>
VERIFY: <exact command or file to check> → expected if correct: <specific output>
HYPOTHESIS 2 (likelihood: medium): <mechanism>
VERIFY: <exact command or file to check> → expected if correct: <specific output>
```
- `verified`: false — verification is the caller's job
- `message`: "N hypotheses for: <one-line error summary>"
## Rules
1. Read the error and any context files provided before forming hypotheses
2. Identify the failure mode first — what actually went wrong, not just what the error says
3. For each hypothesis: name the mechanism, explain why it would produce this exact error, give a concrete verification command with expected output
4. If the error is clearly a typo or trivial mistake, still form 3 hypotheses — surface the most likely cause as #1

View File

@@ -0,0 +1,30 @@
# Code Review Discipline
You are a disciplined code reviewer. Read files carefully before commenting.
## Iron laws
1. Never approve security vulnerabilities: command injection, SQL injection, credential exposure, path traversal, unchecked input at system boundaries
2. Never approve silently swallowed errors — `err != nil` without wrapping or handling is always wrong
3. Never approve missing validation at system boundaries (user input, external APIs, file reads)
## Output contract
Return JSON result with:
- `status`: "pass" if no blocking issues; "fail" if any iron law is violated
- `phase`: "review"
- `skill`: "review"
- `file_path`: first file reviewed
- `runner_output`: full review formatted as:
```
CRITICAL: <issue> at <file>:<line>
WARNING: <issue> at <file>:<line>
SUGGESTION: <issue> at <file>:<line>
```
- `verified`: true if you read all specified files; false if any were missing or unreadable
- `message`: "N critical, M warnings, K suggestions" or "clean: <which iron law checks passed and why>"
## Rules
1. Read every file listed before writing feedback
2. Check iron laws first — any violation is CRITICAL and sets status to "fail"
3. Then check: correctness, test coverage for new code, Go style conventions
4. Never rubber-stamp — if nothing is wrong, explain specifically which iron law checks you ran and why they passed
5. Line references are required for every finding — "roughly around the middle" is not acceptable

46
config/supervisor/spec.md Normal file
View File

@@ -0,0 +1,46 @@
# Spec Writing Discipline
You write structured implementation specs. Nothing is left ambiguous.
## Iron laws
1. Success criteria must be measurable — "the system is fast" is banned; "p99 < 200ms under 100 RPS" is valid
2. Always include an explicit "Out of scope" section — if you don't draw the boundary, the developer will guess wrong
3. Every technical decision in the approach must have a rationale
## Output contract
Return JSON result with:
- `status`: "pass" (spec written) or "error" (requirements too ambiguous to spec without more input)
- `phase`: "spec"
- `skill`: "spec"
- `file_path`: the output_path where the spec was written (absolute path)
- `runner_output`: ""
- `verified`: true if the file was written successfully
- `message`: "spec written: <one-line summary of what was specced>"
## Spec structure
Write the spec as markdown to the output_path:
```markdown
# [Feature] Spec
## Problem statement
[What problem does this solve? For whom? Why now?]
## Success criteria
- [ ] [Criterion 1 — measurable and verifiable]
- [ ] [Criterion 2 — measurable and verifiable]
## Constraints
[Non-negotiable requirements the solution must satisfy]
## Out of scope
[What we are explicitly NOT doing in this iteration]
## Technical approach
[Architecture decisions, key components, rationale for each choice]
## Risks
[What could go wrong, and how we'd mitigate it]
```
If the requirements are too vague to produce measurable success criteria, return status "error" with a message listing the specific questions that need answers.

View File

@@ -0,0 +1,31 @@
# Trainer Reader Discipline
You scan session logs and identify candidate learning moments worth converting to training data.
## What to look for
- **SFT candidates**: the worker did exactly the right thing — a clean pattern worth reinforcing
- **DPO candidates**: the worker first produced a wrong or suboptimal response, then corrected — you have both rejected and chosen
## Scoring (15)
- 5: novel pattern, clearly correct, generalises across projects
- 4: good pattern, correct, somewhat project-specific but still useful
- 3: correct but obvious — include only if especially clean
- 2 or below: skip — too ambiguous or too context-specific
## Output contract
Return JSON result with:
- `status`: "pass" or "error"
- `phase`: "trainer"
- `skill`: "trainer"
- `file_path`: ""
- `runner_output`: JSON array of candidates (valid JSON, not markdown):
[{"type":"sft","moment":"<what happened>","prompt":"<what was asked>","completion":"<what was done right>","score":4},
{"type":"dpo","moment":"<what happened>","prompt":"<what was asked>","chosen":"<correct>","rejected":"<incorrect>","score":3}]
- `verified`: true
- `message`: "N sft candidates, M dpo candidates found"
## Rules
1. Read all session entries in the task prompt
2. Score each entry — only include entries scoring >= 3
3. Prompt/completion fields must be phrased to generalise: no project-specific paths or names
4. If no candidates score >= 3, return an empty array `[]` — never force low-quality candidates

View File

@@ -0,0 +1,35 @@
# Trainer Writer Discipline
You receive candidate learning moments from the reader and write clean SFT/DPO training pairs.
## Quality gate (apply before writing)
- SFT: prompt must be phrased so it could come from any project, not just this one
- DPO: chosen and rejected must be clearly distinguishable — skip if a reader can't tell which is better
- Never include project-specific paths, variable names, or identifiers in any pair
## Output contract
Return JSON result with:
- `status`: "pass" (pairs written or skipped due to quality) or "error" (candidates JSON was malformed)
- `phase`: "trainer"
- `skill`: "trainer"
- `file_path`: path of the last file written (empty if nothing passed quality gate)
- `runner_output`: "N SFT pairs written to brain/training-data/sft/, M DPO pairs to brain/training-data/dpo/" or "0 pairs passed quality gate"
- `verified`: true if files were written; false if nothing passed
- `message`: "N sft + M dpo pairs for session <id>" or "no pairs passed quality gate"
## File format
JSONL — one JSON object per line.
SFT: `{"prompt": "...", "completion": "..."}`
DPO: `{"prompt": "...", "chosen": "...", "rejected": "..."}`
Write SFT to: `<brain_dir>/training-data/sft/<session_id>.jsonl`
Write DPO to: `<brain_dir>/training-data/dpo/<session_id>.jsonl`
Append to existing files if they exist (don't overwrite).
## Rules
1. Parse the `reader_candidates` JSON from the task prompt
2. For each candidate: apply quality gate
3. Write passing SFT candidates to sft JSONL, DPO candidates to dpo JSONL
4. If nothing passes, return status "pass" with verified: false and message "no pairs passed quality gate"

View File

@@ -0,0 +1,322 @@
# Model Orchestration Design
**Date:** 2026-04-20
**Status:** Approved for implementation
## Problem statement
The hyperguild supervisor currently spawns a `claude --print` subprocess for every skill call. The model routing config (`models.yaml`) exists but is dead weight — the model name is injected as text into the task prompt and ignored. Every skill call costs Claude tokens regardless of task complexity or data sensitivity.
## Goal
Route skill work to the most appropriate model — weighing cost, latency, and quality — with Claude acting as the real supervisor: verifying outputs and deciding when to escalate. Local models on owned hardware handle the common case; Claude escalates through a chain to frontier models only when local quality is insufficient.
## Success criteria
- [ ] Each skill dispatches generation to its configured local model via LiteLLM by default
- [ ] Claude verifies every local output and either accepts or escalates
- [ ] Escalation walks a per-skill chain (local small → local large → Sonnet → Opus) with one attempt per tier
- [ ] Every attempt (model, tier, duration, warm state, verdict) is logged in the session JSONL
- [ ] Cloud tiers (Sonnet/Opus) self-certify — no separate verifier call
- [ ] Zero changes to skill handlers — they call `ExecutorFn` exactly as today
- [ ] `LiteLTMBaseURL` already in config; no new env vars required beyond `LLAMA_SWAP_URL`
## Constraints
- One attempt per tier before escalating (no retry within a tier)
- Anthropic T&C: Claude is called normally via Anthropic API; local models are called directly via LiteLLM HTTP — no API redirection
- `models.yaml` remains the single routing config file
## Out of scope
- Auto-rerouting based on real-time warm state (logged, not acted on — Phase 4)
- Multi-tenant / public service exposure
- RAG/CAG model boosting
- Managed Agent cloud delegation (chain stub only in Phase 3)
---
## Architecture
```
MCP tool call (Claude Code)
Skill handler — calls ExecutorFn (unchanged)
Orchestrator.Run (implements ExecutorFn)
├─ Resolve chain from models.yaml
├─ For each model in chain:
│ ├─ [ollama/*] → LiteLLM executor → generate
│ │ ↓
│ │ Claude verifier (task + output + discipline)
│ │ ├─ accept → return Result (log attempt)
│ │ └─ escalate → next tier (log attempt)
│ │
│ └─ [claude-*] → Claude executor (current) → generate + self-certify
│ └─ return Result (log attempt)
└─ All tiers exhausted → return best attempt with escalation note
```
Claude is always the verifier for local tiers. At cloud tiers, Claude generates and self-certifies — the verifier call is skipped.
---
## Components
### 1. `internal/exec/litellm.go` — LiteLLM executor
Calls `POST /v1/chat/completions` on the configured LiteLLM server. Implements the same `ExecutorFn` signature as the existing claude executor.
```go
type LiteLLMExecutor struct {
BaseURL string
APIKey string
HTTPClient *http.Client
Timeout time.Duration
}
func NewLiteLLM(baseURL, apiKey string, timeout time.Duration) *LiteLLMExecutor
func (e *LiteLLMExecutor) Run(ctx context.Context, req Request) (Result, error)
```
Request mapping:
- `req.SkillPrompt` → system message
- `req.TaskPrompt` → user message
- `req.Model``model` field in the chat completions request
Response handling: local models are prompted (via the discipline file output contract) to return a JSON object matching the `Result` schema. The executor attempts `json.Unmarshal` into `Result` directly — no envelope unwrapping needed (unlike the `--output-format json` claude envelope). If unmarshalling fails, the executor returns an error that the orchestrator treats as an automatic escalation trigger.
### 2. `internal/exec/verifier.go` — Claude verifier
A focused Claude call that judges local model output. Uses the existing `Executor` (claude subprocess) internally.
```go
type Verdict struct {
Accept bool `json:"accept"`
Feedback string `json:"feedback"` // reason if not accepting; empty if accept
}
type Verifier struct {
executor *Executor // the existing claude executor
}
func NewVerifier(executor *Executor) *Verifier
func (v *Verifier) Verify(ctx context.Context, skillPrompt, taskPrompt string, output Result) (Verdict, error)
```
The verifier prompt gives Claude:
1. The skill discipline file (so it knows the iron laws and output contract)
2. The original task prompt (informed verification — Claude sees what was asked)
3. The generated output
4. A short instruction: "Does this output satisfy the discipline's iron laws and output contract? Reply with JSON: `{\"accept\": true|false, \"feedback\": \"...\"}`"
The verifier uses a lightweight JSON schema for its own output (a `Verdict` schema), keeping the call fast.
### 3. `internal/exec/orchestrator.go` — chain walker
Implements `ExecutorFn`. Walks the escalation chain, delegating generation and verification per tier.
```go
type Chain []ChainEntry
type ChainEntry struct {
Model string // e.g. "ollama/phi4", "claude-sonnet-4-5"
Tier string // "local" | "subagent" | "managed"
IsCloud bool // true for claude-* models; skips verifier
}
type Orchestrator struct {
chain Chain
litellm *LiteLLMExecutor
claude *Executor
verifier *Verifier
llamaSwapURL string // for warm-state probe
}
func NewOrchestrator(chain Chain, litellm *LiteLLMExecutor, claude *Executor, verifier *Verifier, llamaSwapURL string) *Orchestrator
func (o *Orchestrator) Run(ctx context.Context, req Request) (Result, error)
```
Algorithm:
```
for each entry in chain:
warm = probe llama-swap (if local tier)
start = now()
if entry.IsCloud:
result, err = claude.Run(ctx, req with entry.Model)
log attempt(model, tier, duration, warm, verified=true)
if err == nil: return result
else:
result, err = litellm.Run(ctx, req with entry.Model)
duration = now() - start
if err != nil:
log attempt(model, tier, duration, warm, verified=false)
continue // automatic escalation on parse/network error
verdict = verifier.Verify(ctx, req.SkillPrompt, req.TaskPrompt, result)
log attempt(model, tier, duration, warm, verified=verdict.Accept)
if verdict.Accept: return result
// inject verifier feedback into next tier's task prompt
req.TaskPrompt = req.TaskPrompt + "\n\nPrior attempt feedback: " + verdict.Feedback
return error("all tiers exhausted")
```
### 4. `internal/config/models.go` — chain parser
Replaces the current single-model resolution with chain parsing.
Updated `models.yaml` format:
```yaml
verifier: claude-sonnet-4-6 # fixed verifier for all local tiers
llama_swap_url: http://koala:8080 # for warm-state probing
default_chain:
- ollama/qwen3-coder-30b-tuned
- claude-sonnet-4-5
skills:
tdd:
chain:
- ollama/qwen3-coder-30b-tuned
- claude-sonnet-4-5
review:
chain:
- ollama/devstral-tuned
- ollama/gemma4
- claude-sonnet-4-5
debug:
chain:
- ollama/deepseek-r1-tuned
- claude-sonnet-4-5
spec:
chain:
- ollama/phi4
- ollama/gemma4
- claude-sonnet-4-5
- claude-opus-4-6
retrospective:
chain:
- ollama/qwen3-coder-30b-tuned
- claude-sonnet-4-5
trainer:
chain:
- ollama/qwen3-coder-30b-tuned
- claude-sonnet-4-5
```
The parser exposes:
```go
func (m *Models) ChainFor(skill string) Chain
func (m *Models) Verifier() string
func (m *Models) LlamaSwapURL() string
```
Caller override (`model` param in MCP tool call) pins the chain to a single entry — one model, no escalation. This preserves the existing override behaviour for power users.
### 5. `internal/session/session.go` — updated `Attempt` struct
```go
type Attempt struct {
Attempt int `json:"attempt"`
Model string `json:"model"`
Tier string `json:"tier"` // local | subagent | managed
DurationMs int64 `json:"duration_ms"`
WarmStart bool `json:"warm_start"` // model was already loaded in llama-swap
Verified bool `json:"verified"`
Verdict string `json:"verdict,omitempty"` // accept | escalate | error
Feedback string `json:"feedback,omitempty"` // verifier feedback on escalation
OutputSummary string `json:"output_summary,omitempty"`
RunnerOutput string `json:"runner_output,omitempty"`
}
```
### 6. `cmd/supervisor/main.go` — one wiring change
```go
// Before:
reg.Register(review.New(review.Config{ExecutorFn: executor.Run, ...}))
// After:
chain := models.ChainFor("review")
orch := exec.NewOrchestrator(chain, litellmExec, claudeExec, verifier, models.LlamaSwapURL())
reg.Register(review.New(review.Config{ExecutorFn: orch.Run, ...}))
```
One orchestrator per skill, sharing the same `litellmExec`, `claudeExec`, and `verifier` instances.
---
## Data flow example: `review` skill call
1. Claude Code calls `review` tool with `files: ["internal/foo.go"]`
2. Skill handler builds task prompt, calls `orch.Run`
3. Orchestrator resolves chain: `[devstral, gemma4, sonnet]`
4. Probes llama-swap: devstral is warm
5. LiteLLM calls devstral → returns JSON result
6. Verifier asks Claude: "does this review satisfy the iron laws?"
7. Claude: `{"accept": false, "feedback": "missing line references for all findings"}`
8. Orchestrator logs attempt #1 (devstral, local, 4200ms, warm, escalate)
9. Injects feedback into task prompt, calls gemma4
10. Verifier: `{"accept": true}`
11. Orchestrator logs attempt #2 (gemma4, local, 6100ms, cold, accept)
12. Returns result to skill handler → MCP response
Session JSONL records both attempts. You can see: devstral was warm but produced weak output; gemma4 was cold but passed.
---
## Observability
Session JSONL is the primary store. Each `Entry.Attempts` slice records the full escalation trail. To analyse across sessions:
```bash
# Which models are escalating most?
jq -r '.attempts[] | select(.verdict == "escalate") | .model' brain/sessions/*.jsonl | sort | uniq -c
# Average latency per model
jq -r '.attempts[] | [.model, .duration_ms] | @tsv' brain/sessions/*.jsonl | awk '{sum[$1]+=$2; n[$1]++} END {for (m in sum) print m, sum[m]/n[m]}'
# Cold start frequency
jq -r '.attempts[] | select(.warm_start == false) | .model' brain/sessions/*.jsonl | sort | uniq -c
```
No new metrics infrastructure needed for Phase 3. Phase 4 can build a dashboard on top of this data.
---
## Error handling
| Scenario | Behaviour |
|----------|-----------|
| LiteLLM unreachable | Log attempt as error, escalate immediately |
| Local model returns unparseable JSON | Log attempt as error, escalate |
| Verifier call fails | Log, treat as escalate (safe default) |
| All tiers exhausted | Return error to skill handler; skill returns MCP error to caller |
| Caller passes `model` override | Single-entry chain, no escalation, no verifier call |
---
## Testing approach
- `TestLiteLLMExecutor`: mock HTTP server returning valid/invalid JSON; verify parse logic and error escalation
- `TestVerifier`: fake claude executor returning accept/escalate verdicts; verify prompt construction
- `TestOrchestrator`: table-driven — chains of 1/2/3 tiers, various accept/escalate/error combinations; verify attempt log contents and final result
- `TestModelsChainFor`: YAML parsing for all skill overrides and default_chain fallback
- Integration smoke test: start real LiteLLM (or mock), call `review` tool via MCP, verify attempt log written
---
## Risks
| Risk | Mitigation |
|------|------------|
| Local models ignore output contract → bad JSON | Discipline files already specify JSON output contract; parse failure auto-escalates |
| Verifier Claude call adds latency to every local attempt | Verifier prompt is small and fast; acceptable tradeoff for quality gate |
| llama-swap warm probe adds overhead | Probe is a single lightweight HTTP GET; timeout at 200ms, treat failure as `warm_start: false` |
| Chain exhaustion leaves caller with no result | Return structured error via MCP; caller can retry with explicit `model` override |

View File

@@ -10,7 +10,7 @@ import (
// validates its own output before returning.
type Result struct {
Status string `json:"status"` // pass | fail | error
Phase string `json:"phase"` // red | green | refactor
Phase string `json:"phase"` // red | green | refactor | retrospective | review | debug | spec | trainer
Skill string `json:"skill"` // tdd | review | ...
FilePath string `json:"file_path"` // absolute path to generated file
RunnerOutput string `json:"runner_output"` // raw stdout+stderr from test runner
@@ -25,6 +25,10 @@ var validPhases = map[string]bool{
"green": true,
"refactor": true,
"retrospective": true,
"review": true,
"debug": true,
"spec": true,
"trainer": true,
}
func (r Result) Validate() error {
@@ -33,7 +37,7 @@ func (r Result) Validate() error {
errs = append(errs, "status must be pass|fail|error, got: "+r.Status)
}
if !validPhases[r.Phase] {
errs = append(errs, "phase must be red|green|refactor, got: "+r.Phase)
errs = append(errs, "phase must be one of red|green|refactor|retrospective|review|debug|spec|trainer, got: "+r.Phase)
}
if r.Skill == "" {
errs = append(errs, "skill is required")
@@ -50,7 +54,7 @@ const Schema = `{
"required": ["status","phase","skill","file_path","runner_output","verified","model_used","message"],
"properties": {
"status": {"type": "string", "enum": ["pass","fail","error"]},
"phase": {"type": "string", "enum": ["red","green","refactor"]},
"phase": {"type": "string"},
"skill": {"type": "string"},
"file_path": {"type": "string"},
"runner_output": {"type": "string"},

View File

@@ -69,3 +69,11 @@ func TestResultValidation(t *testing.T) {
})
}
}
func TestValidateAcceptsAllPhases(t *testing.T) {
phases := []string{"red", "green", "refactor", "retrospective", "review", "debug", "spec", "trainer"}
for _, phase := range phases {
r := exec.Result{Status: "pass", Phase: phase, Skill: "test", ModelUsed: "self", Message: "ok"}
assert.NoError(t, r.Validate(), "phase %q should be valid", phase)
}
}

View File

@@ -0,0 +1,38 @@
// internal/session/history.go
package session
import (
"fmt"
"strings"
)
// FormatHistory formats prior session entries as a structured block for
// injection into a worker task prompt. Entries matching excludePhase are
// omitted (pass the current phase to avoid circular injection).
func FormatHistory(entries []Entry, excludePhase string) string {
var filtered []Entry
for _, e := range entries {
if e.Phase != excludePhase {
filtered = append(filtered, e)
}
}
if len(filtered) == 0 {
return ""
}
var b strings.Builder
b.WriteString("## Session history\n\n")
for _, e := range filtered {
fmt.Fprintf(&b, "### Phase: %s\n", e.Phase) //nolint:errcheck // strings.Builder never errors
fmt.Fprintf(&b, "- Skill: %s\n", e.Skill) //nolint:errcheck
fmt.Fprintf(&b, "- Status: %s\n", e.FinalStatus) //nolint:errcheck
if e.FilePath != "" {
fmt.Fprintf(&b, "- File: %s\n", e.FilePath) //nolint:errcheck
}
if e.Message != "" {
fmt.Fprintf(&b, "- Summary: %s\n", e.Message) //nolint:errcheck
}
b.WriteString("\n")
}
return b.String()
}

View File

@@ -0,0 +1,41 @@
// internal/session/history_test.go
package session_test
import (
"testing"
"time"
"github.com/mathiasbq/supervisor/internal/session"
"github.com/stretchr/testify/assert"
)
func TestFormatHistoryEmpty(t *testing.T) {
result := session.FormatHistory(nil, "")
assert.Equal(t, "", result)
}
func TestFormatHistoryFormatsEntries(t *testing.T) {
entries := []session.Entry{
{
Skill: "tdd", Phase: "red", FinalStatus: "pass",
FilePath: "internal/foo/foo_test.go",
Message: "wrote failing test for Foo",
Timestamp: time.Now(),
},
}
result := session.FormatHistory(entries, "")
assert.Contains(t, result, "## Session history")
assert.Contains(t, result, "Phase: red")
assert.Contains(t, result, "wrote failing test for Foo")
assert.Contains(t, result, "internal/foo/foo_test.go")
}
func TestFormatHistoryExcludesCurrentPhase(t *testing.T) {
entries := []session.Entry{
{Skill: "tdd", Phase: "red", Message: "red done", FinalStatus: "pass"},
{Skill: "tdd", Phase: "green", Message: "green done", FinalStatus: "pass"},
}
result := session.FormatHistory(entries, "green")
assert.Contains(t, result, "red done")
assert.NotContains(t, result, "green done")
}

View File

@@ -0,0 +1,80 @@
// internal/skills/debug/handlers.go
package debug
import (
"context"
"encoding/json"
"fmt"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
)
type debugArgs struct {
ProjectRoot string `json:"project_root"`
Error string `json:"error"`
Context string `json:"context"`
Model string `json:"model"`
SessionID string `json:"session_id"`
}
// Handle dispatches the MCP tool call to the appropriate handler.
func (s *Skill) Handle(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) {
if tool != "debug" {
return nil, fmt.Errorf("unknown tool: %s", tool)
}
var a debugArgs
if err := json.Unmarshal(args, &a); err != nil {
return nil, fmt.Errorf("parse args: %w", err)
}
if a.ProjectRoot == "" {
return nil, fmt.Errorf("project_root is required")
}
if a.Error == "" {
return nil, fmt.Errorf("error is required")
}
model := a.Model
if model == "" {
model = s.cfg.DefaultModel
}
task := fmt.Sprintf(
"phase: debug\nproject_root: %s\nerror: %s\ncontext: %s\nmodel: %s",
a.ProjectRoot, a.Error, a.Context, model,
)
task = s.prependHistory(a.SessionID, "debug", task)
if s.cfg.ExecutorFn == nil {
return nil, fmt.Errorf("no executor configured")
}
result, err := s.cfg.ExecutorFn(ctx, iexec.Request{
SkillPrompt: s.cfg.SkillPrompt,
TaskPrompt: task,
Model: model,
Tools: "Read,Bash",
})
if err != nil {
return nil, err
}
b, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("marshal result: %w", err)
}
return b, nil
}
func (s *Skill) prependHistory(sessionID, currentPhase, task string) string {
if sessionID == "" || s.cfg.SessionsDir == "" {
return task
}
entries, err := session.Read(s.cfg.SessionsDir, sessionID)
if err != nil || len(entries) == 0 {
return task
}
history := session.FormatHistory(entries, currentPhase)
if history == "" {
return task
}
return history + "\n---\n\n" + task
}

View File

@@ -0,0 +1,61 @@
// internal/skills/debug/handlers_test.go
package debug_test
import (
"context"
"encoding/json"
"testing"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/skills/debug"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDebugToolRegistered(t *testing.T) {
sk := debug.New(debug.Config{SkillPrompt: "debug rules"})
names := make([]string, 0)
for _, tool := range sk.Tools() {
names = append(names, tool.Name)
}
assert.Contains(t, names, "debug")
}
func TestDebugRequiresProjectRoot(t *testing.T) {
sk := debug.New(debug.Config{SkillPrompt: "d"})
_, err := sk.Handle(context.Background(), "debug", json.RawMessage(`{"error":"panic: nil pointer"}`))
assert.ErrorContains(t, err, "project_root")
}
func TestDebugRequiresError(t *testing.T) {
sk := debug.New(debug.Config{SkillPrompt: "d"})
_, err := sk.Handle(context.Background(), "debug", json.RawMessage(`{"project_root":"/tmp"}`))
assert.ErrorContains(t, err, "error")
}
func TestDebugCallsExecutor(t *testing.T) {
called := false
var capturedTask string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
called = true
capturedTask = req.TaskPrompt
return iexec.Result{
Status: "pass", Phase: "debug", Skill: "debug",
RunnerOutput: "HYPOTHESIS 1 (likelihood: high): nil map access\nVERIFY: go test ./... → expected: panic line reference",
Verified: false, ModelUsed: "self", Message: "3 hypotheses for: panic nil pointer at foo.go:42",
}, nil
}
sk := debug.New(debug.Config{SkillPrompt: "debug rules", ExecutorFn: fakeFn, SessionsDir: t.TempDir()})
out, err := sk.Handle(context.Background(), "debug", json.RawMessage(
`{"project_root":"/tmp/proj","error":"panic: nil pointer dereference at foo.go:42","context":"occurs on startup"}`,
))
require.NoError(t, err)
assert.True(t, called)
assert.Contains(t, capturedTask, "panic: nil pointer dereference")
assert.Contains(t, capturedTask, "occurs on startup")
var result iexec.Result
require.NoError(t, json.Unmarshal(out, &result))
assert.Equal(t, "debug", result.Phase)
}

View File

@@ -0,0 +1,55 @@
// internal/skills/debug/skill.go
package debug
import (
"context"
"encoding/json"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/registry"
)
// ExecutorFn is the function signature for running a worker subprocess.
type ExecutorFn func(ctx context.Context, req iexec.Request) (iexec.Result, error)
// Config holds dependencies for the debug skill.
type Config struct {
SkillPrompt string
DefaultModel string
ExecutorFn ExecutorFn
SessionsDir string
}
// Skill implements the debug MCP tool.
type Skill struct{ cfg Config }
// New creates a new debug Skill.
func New(cfg Config) *Skill { return &Skill{cfg: cfg} }
// Name returns the skill identifier.
func (s *Skill) Name() string { return "debug" }
// Tools returns the MCP tool definitions for this skill.
func (s *Skill) Tools() []registry.ToolDef {
schema := func(required []string, props map[string]any) json.RawMessage {
b, _ := json.Marshal(map[string]any{"type": "object", "required": required, "properties": props})
return b
}
str := map[string]any{"type": "string"}
return []registry.ToolDef{
{
Name: "debug",
Description: "Analyse an error and return 3-5 hypotheses ordered by likelihood, each with a concrete verification step.",
InputSchema: schema(
[]string{"project_root", "error"},
map[string]any{
"project_root": str,
"error": str,
"context": str,
"model": str,
"session_id": str,
},
),
},
}
}

View File

@@ -0,0 +1,81 @@
// internal/skills/review/handlers.go
package review
import (
"context"
"encoding/json"
"fmt"
"strings"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
)
type reviewArgs struct {
ProjectRoot string `json:"project_root"`
Files []string `json:"files"`
Context string `json:"context"`
Model string `json:"model"`
SessionID string `json:"session_id"`
}
// Handle dispatches the MCP tool call to the appropriate handler.
func (s *Skill) Handle(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) {
if tool != "review" {
return nil, fmt.Errorf("unknown tool: %s", tool)
}
var a reviewArgs
if err := json.Unmarshal(args, &a); err != nil {
return nil, fmt.Errorf("parse args: %w", err)
}
if a.ProjectRoot == "" {
return nil, fmt.Errorf("project_root is required")
}
if len(a.Files) == 0 {
return nil, fmt.Errorf("files is required")
}
model := a.Model
if model == "" {
model = s.cfg.DefaultModel
}
task := fmt.Sprintf(
"phase: review\nproject_root: %s\nfiles: %s\ncontext: %s\nmodel: %s",
a.ProjectRoot, strings.Join(a.Files, ", "), a.Context, model,
)
task = s.prependHistory(a.SessionID, "review", task)
if s.cfg.ExecutorFn == nil {
return nil, fmt.Errorf("no executor configured")
}
result, err := s.cfg.ExecutorFn(ctx, iexec.Request{
SkillPrompt: s.cfg.SkillPrompt,
TaskPrompt: task,
Model: model,
Tools: "Read,Bash",
})
if err != nil {
return nil, err
}
b, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("marshal result: %w", err)
}
return b, nil
}
func (s *Skill) prependHistory(sessionID, currentPhase, task string) string {
if sessionID == "" || s.cfg.SessionsDir == "" {
return task
}
entries, err := session.Read(s.cfg.SessionsDir, sessionID)
if err != nil || len(entries) == 0 {
return task
}
history := session.FormatHistory(entries, currentPhase)
if history == "" {
return task
}
return history + "\n---\n\n" + task
}

View File

@@ -0,0 +1,61 @@
// internal/skills/review/handlers_test.go
package review_test
import (
"context"
"encoding/json"
"testing"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/skills/review"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReviewToolRegistered(t *testing.T) {
sk := review.New(review.Config{SkillPrompt: "review rules"})
names := make([]string, 0)
for _, tool := range sk.Tools() {
names = append(names, tool.Name)
}
assert.Contains(t, names, "review")
}
func TestReviewRequiresProjectRoot(t *testing.T) {
sk := review.New(review.Config{SkillPrompt: "r"})
_, err := sk.Handle(context.Background(), "review", json.RawMessage(`{"files":["main.go"]}`))
assert.ErrorContains(t, err, "project_root")
}
func TestReviewRequiresFiles(t *testing.T) {
sk := review.New(review.Config{SkillPrompt: "r"})
_, err := sk.Handle(context.Background(), "review", json.RawMessage(`{"project_root":"/tmp"}`))
assert.ErrorContains(t, err, "files")
}
func TestReviewCallsExecutor(t *testing.T) {
called := false
var capturedTask string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
called = true
capturedTask = req.TaskPrompt
return iexec.Result{
Status: "pass", Phase: "review", Skill: "review",
Verified: true, ModelUsed: "self", Message: "2 warnings found",
}, nil
}
sk := review.New(review.Config{SkillPrompt: "review rules", ExecutorFn: fakeFn, SessionsDir: t.TempDir()})
out, err := sk.Handle(context.Background(), "review", json.RawMessage(
`{"project_root":"/tmp/proj","files":["internal/foo/foo.go"],"context":"PR: add Foo helper"}`,
))
require.NoError(t, err)
assert.True(t, called)
assert.Contains(t, capturedTask, "internal/foo/foo.go")
assert.Contains(t, capturedTask, "PR: add Foo helper")
var result iexec.Result
require.NoError(t, json.Unmarshal(out, &result))
assert.Equal(t, "pass", result.Status)
assert.Equal(t, "review", result.Phase)
}

View File

@@ -0,0 +1,55 @@
// internal/skills/review/skill.go
package review
import (
"context"
"encoding/json"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/registry"
)
// ExecutorFn is the function signature for running a worker subprocess.
type ExecutorFn func(ctx context.Context, req iexec.Request) (iexec.Result, error)
// Config holds dependencies for the review skill.
type Config struct {
SkillPrompt string
DefaultModel string
ExecutorFn ExecutorFn
SessionsDir string
}
// Skill implements the review MCP tool.
type Skill struct{ cfg Config }
// New creates a new review Skill.
func New(cfg Config) *Skill { return &Skill{cfg: cfg} }
// Name returns the skill identifier.
func (s *Skill) Name() string { return "review" }
// Tools returns the MCP tool definitions for this skill.
func (s *Skill) Tools() []registry.ToolDef {
schema := func(required []string, props map[string]any) json.RawMessage {
b, _ := json.Marshal(map[string]any{"type": "object", "required": required, "properties": props})
return b
}
str := map[string]any{"type": "string"}
return []registry.ToolDef{
{
Name: "review",
Description: "Perform a structured code review of the specified files. Returns findings with severity levels.",
InputSchema: schema(
[]string{"project_root", "files"},
map[string]any{
"project_root": str,
"files": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
"context": str,
"model": str,
"session_id": str,
},
),
},
}
}

View File

@@ -0,0 +1,85 @@
// internal/skills/spec/handlers.go
package spec
import (
"context"
"encoding/json"
"fmt"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
)
type specArgs struct {
ProjectRoot string `json:"project_root"`
Requirements string `json:"requirements"`
OutputPath string `json:"output_path"`
Context string `json:"context"`
Model string `json:"model"`
SessionID string `json:"session_id"`
}
// Handle dispatches the MCP tool call to the appropriate handler.
func (s *Skill) Handle(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) {
if tool != "spec" {
return nil, fmt.Errorf("unknown tool: %s", tool)
}
var a specArgs
if err := json.Unmarshal(args, &a); err != nil {
return nil, fmt.Errorf("parse args: %w", err)
}
if a.ProjectRoot == "" {
return nil, fmt.Errorf("project_root is required")
}
if a.Requirements == "" {
return nil, fmt.Errorf("requirements is required")
}
outputPath := a.OutputPath
if outputPath == "" {
outputPath = "docs/spec.md"
}
model := a.Model
if model == "" {
model = s.cfg.DefaultModel
}
task := fmt.Sprintf(
"phase: spec\nproject_root: %s\nrequirements: %s\noutput_path: %s\ncontext: %s\nmodel: %s",
a.ProjectRoot, a.Requirements, outputPath, a.Context, model,
)
task = s.prependHistory(a.SessionID, "spec", task)
if s.cfg.ExecutorFn == nil {
return nil, fmt.Errorf("no executor configured")
}
result, err := s.cfg.ExecutorFn(ctx, iexec.Request{
SkillPrompt: s.cfg.SkillPrompt,
TaskPrompt: task,
Model: model,
Tools: "Read,Write",
})
if err != nil {
return nil, err
}
b, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("marshal result: %w", err)
}
return b, nil
}
func (s *Skill) prependHistory(sessionID, currentPhase, task string) string {
if sessionID == "" || s.cfg.SessionsDir == "" {
return task
}
entries, err := session.Read(s.cfg.SessionsDir, sessionID)
if err != nil || len(entries) == 0 {
return task
}
history := session.FormatHistory(entries, currentPhase)
if history == "" {
return task
}
return history + "\n---\n\n" + task
}

View File

@@ -0,0 +1,61 @@
// internal/skills/spec/handlers_test.go
package spec_test
import (
"context"
"encoding/json"
"testing"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/skills/spec"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSpecToolRegistered(t *testing.T) {
sk := spec.New(spec.Config{SkillPrompt: "spec rules"})
names := make([]string, 0)
for _, tool := range sk.Tools() {
names = append(names, tool.Name)
}
assert.Contains(t, names, "spec")
}
func TestSpecRequiresProjectRoot(t *testing.T) {
sk := spec.New(spec.Config{SkillPrompt: "s"})
_, err := sk.Handle(context.Background(), "spec", json.RawMessage(`{"requirements":"add login"}`))
assert.ErrorContains(t, err, "project_root")
}
func TestSpecRequiresRequirements(t *testing.T) {
sk := spec.New(spec.Config{SkillPrompt: "s"})
_, err := sk.Handle(context.Background(), "spec", json.RawMessage(`{"project_root":"/tmp"}`))
assert.ErrorContains(t, err, "requirements")
}
func TestSpecCallsExecutor(t *testing.T) {
called := false
var capturedTask string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
called = true
capturedTask = req.TaskPrompt
return iexec.Result{
Status: "pass", Phase: "spec", Skill: "spec",
FilePath: "/tmp/proj/docs/login-spec.md",
Verified: true, ModelUsed: "self", Message: "spec written: login feature",
}, nil
}
sk := spec.New(spec.Config{SkillPrompt: "spec rules", ExecutorFn: fakeFn, SessionsDir: t.TempDir()})
out, err := sk.Handle(context.Background(), "spec", json.RawMessage(
`{"project_root":"/tmp/proj","requirements":"add OAuth2 login","output_path":"docs/login-spec.md"}`,
))
require.NoError(t, err)
assert.True(t, called)
assert.Contains(t, capturedTask, "OAuth2 login")
assert.Contains(t, capturedTask, "docs/login-spec.md")
var result iexec.Result
require.NoError(t, json.Unmarshal(out, &result))
assert.Equal(t, "spec", result.Phase)
}

View File

@@ -0,0 +1,56 @@
// internal/skills/spec/skill.go
package spec
import (
"context"
"encoding/json"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/registry"
)
// ExecutorFn is the function signature for running a worker subprocess.
type ExecutorFn func(ctx context.Context, req iexec.Request) (iexec.Result, error)
// Config holds dependencies for the spec skill.
type Config struct {
SkillPrompt string
DefaultModel string
ExecutorFn ExecutorFn
SessionsDir string
}
// Skill implements the spec MCP tool.
type Skill struct{ cfg Config }
// New creates a new spec Skill.
func New(cfg Config) *Skill { return &Skill{cfg: cfg} }
// Name returns the skill identifier.
func (s *Skill) Name() string { return "spec" }
// Tools returns the MCP tool definitions for this skill.
func (s *Skill) Tools() []registry.ToolDef {
schema := func(required []string, props map[string]any) json.RawMessage {
b, _ := json.Marshal(map[string]any{"type": "object", "required": required, "properties": props})
return b
}
str := map[string]any{"type": "string"}
return []registry.ToolDef{
{
Name: "spec",
Description: "Generate a structured implementation spec from requirements. Writes the spec to output_path in the project.",
InputSchema: schema(
[]string{"project_root", "requirements"},
map[string]any{
"project_root": str,
"requirements": str,
"output_path": str,
"context": str,
"model": str,
"session_id": str,
},
),
},
}
}

View File

@@ -6,6 +6,7 @@ import (
"fmt"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
)
func (s *Skill) Handle(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) {
@@ -51,6 +52,7 @@ type greenArgs struct {
TestPath string `json:"test_path"`
Model string `json:"model"`
TestCmd string `json:"test_cmd"`
SessionID string `json:"session_id"`
}
func (s *Skill) handleGreen(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) {
@@ -68,6 +70,7 @@ func (s *Skill) handleGreen(ctx context.Context, raw json.RawMessage) (json.RawM
"phase: green\nproject_root: %s\ntest_path: %s\nmodel: %s\ntest_cmd: %s",
args.ProjectRoot, args.TestPath, s.resolveModel(args.Model), args.TestCmd,
)
task = s.prependHistory(args.SessionID, "green", task)
return s.execute(ctx, task)
}
@@ -77,6 +80,7 @@ type refactorArgs struct {
ImplPath string `json:"impl_path"`
Model string `json:"model"`
TestCmd string `json:"test_cmd"`
SessionID string `json:"session_id"`
}
func (s *Skill) handleRefactor(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) {
@@ -97,9 +101,25 @@ func (s *Skill) handleRefactor(ctx context.Context, raw json.RawMessage) (json.R
"phase: refactor\nproject_root: %s\ntest_path: %s\nimpl_path: %s\nmodel: %s\ntest_cmd: %s",
args.ProjectRoot, args.TestPath, args.ImplPath, s.resolveModel(args.Model), args.TestCmd,
)
task = s.prependHistory(args.SessionID, "refactor", task)
return s.execute(ctx, task)
}
func (s *Skill) prependHistory(sessionID, currentPhase, task string) string {
if sessionID == "" || s.cfg.SessionsDir == "" {
return task
}
entries, err := session.Read(s.cfg.SessionsDir, sessionID)
if err != nil || len(entries) == 0 {
return task
}
history := session.FormatHistory(entries, currentPhase)
if history == "" {
return task
}
return history + "\n---\n\n" + task
}
func (s *Skill) resolveModel(override string) string {
if override != "" {
return override

View File

@@ -5,6 +5,8 @@ import (
"encoding/json"
"testing"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
"github.com/mathiasbq/supervisor/internal/skills/tdd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -41,5 +43,43 @@ func TestTDDRedRequiresSpec(t *testing.T) {
assert.ErrorContains(t, err, "spec")
}
func TestTDDGreenInjectsSessionHistory(t *testing.T) {
sessDir := t.TempDir()
require.NoError(t, session.Append(sessDir, "sess-1", session.Entry{
SessionID: "sess-1", Skill: "tdd", Phase: "red", FinalStatus: "pass",
FilePath: "internal/foo/foo_test.go",
Message: "wrote failing test for Foo",
}))
var capturedPrompt string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
capturedPrompt = req.TaskPrompt
return iexec.Result{Status: "pass", Phase: "green", Skill: "tdd", Verified: true, ModelUsed: "self", Message: "ok"}, nil
}
sk := tdd.New(tdd.Config{SkillPrompt: "tdd", ExecutorFn: fakeFn, SessionsDir: sessDir})
_, err := sk.Handle(context.Background(), "tdd_green", json.RawMessage(
`{"project_root":"/tmp","test_path":"internal/foo/foo_test.go","test_cmd":"go test ./...","session_id":"sess-1"}`,
))
require.NoError(t, err)
assert.Contains(t, capturedPrompt, "## Session history")
assert.Contains(t, capturedPrompt, "wrote failing test for Foo")
}
func TestTDDGreenNoHistoryWhenSessionIDEmpty(t *testing.T) {
var capturedPrompt string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
capturedPrompt = req.TaskPrompt
return iexec.Result{Status: "pass", Phase: "green", Skill: "tdd", Verified: true, ModelUsed: "self", Message: "ok"}, nil
}
sk := tdd.New(tdd.Config{SkillPrompt: "tdd", ExecutorFn: fakeFn, SessionsDir: t.TempDir()})
_, err := sk.Handle(context.Background(), "tdd_green", json.RawMessage(
`{"project_root":"/tmp","test_path":"internal/foo/foo_test.go"}`,
))
require.NoError(t, err)
assert.NotContains(t, capturedPrompt, "## Session history")
}
// Ensure require is used (avoids import error).
var _ = require.New

View File

@@ -16,6 +16,7 @@ type Config struct {
SkillPrompt string
ExecutorFn ExecutorFn // nil = no executor (tests that don't reach execute())
DefaultModel string
SessionsDir string // optional: path to brain/sessions/ for history injection
}
type Skill struct {
@@ -63,6 +64,7 @@ func (s *Skill) Tools() []registry.ToolDef {
"test_path": strProp,
"model": strProp,
"test_cmd": strProp,
"session_id": strProp,
},
),
},
@@ -77,6 +79,7 @@ func (s *Skill) Tools() []registry.ToolDef {
"impl_path": strProp,
"model": strProp,
"test_cmd": strProp,
"session_id": strProp,
},
),
},

View File

@@ -0,0 +1,80 @@
// internal/skills/trainer/handlers.go
package trainer
import (
"context"
"encoding/json"
"fmt"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
)
type trainArgs struct {
SessionID string `json:"session_id"`
Model string `json:"model"`
}
// Handle dispatches the MCP tool call to the trainer handler.
func (s *Skill) Handle(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) {
if tool != "trainer" {
return nil, fmt.Errorf("unknown tool: %s", tool)
}
var a trainArgs
if err := json.Unmarshal(args, &a); err != nil {
return nil, fmt.Errorf("parse args: %w", err)
}
if a.SessionID == "" {
return nil, fmt.Errorf("session_id is required")
}
if s.cfg.ExecutorFn == nil {
return nil, fmt.Errorf("no executor configured")
}
model := a.Model
if model == "" {
model = s.cfg.DefaultModel
}
entries, err := session.Read(s.cfg.SessionsDir, a.SessionID)
if err != nil {
return nil, fmt.Errorf("read session log: %w", err)
}
// ── Step 1: Reader agent ─────────────────────────────────────────────────
history := session.FormatHistory(entries, "")
readerTask := fmt.Sprintf(
"role: reader\nsession_id: %s\nbrain_dir: %s\n\n%s",
a.SessionID, s.cfg.BrainDir, history,
)
readerResult, err := s.cfg.ExecutorFn(ctx, iexec.Request{
SkillPrompt: s.cfg.ReaderPrompt,
TaskPrompt: readerTask,
Model: model,
Tools: "Read",
})
if err != nil {
return nil, fmt.Errorf("reader agent: %w", err)
}
// ── Step 2: Writer agent (receives reader candidates) ────────────────────
writerTask := fmt.Sprintf(
"role: writer\nsession_id: %s\nbrain_dir: %s\n\nreader_summary: %s\nreader_candidates:\n%s",
a.SessionID, s.cfg.BrainDir, readerResult.Message, readerResult.RunnerOutput,
)
writerResult, err := s.cfg.ExecutorFn(ctx, iexec.Request{
SkillPrompt: s.cfg.WriterPrompt,
TaskPrompt: writerTask,
Model: model,
Tools: "Read,Write",
})
if err != nil {
return nil, fmt.Errorf("writer agent: %w", err)
}
b, err := json.Marshal(writerResult)
if err != nil {
return nil, fmt.Errorf("marshal result: %w", err)
}
return b, nil
}

View File

@@ -0,0 +1,82 @@
// internal/skills/trainer/handlers_test.go
package trainer_test
import (
"context"
"encoding/json"
"testing"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/session"
"github.com/mathiasbq/supervisor/internal/skills/trainer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTrainerToolRegistered(t *testing.T) {
sk := trainer.New(trainer.Config{ReaderPrompt: "r", WriterPrompt: "w"})
names := make([]string, 0)
for _, tool := range sk.Tools() {
names = append(names, tool.Name)
}
assert.Contains(t, names, "trainer")
}
func TestTrainerRequiresSessionID(t *testing.T) {
sk := trainer.New(trainer.Config{ReaderPrompt: "r", WriterPrompt: "w"})
_, err := sk.Handle(context.Background(), "trainer", json.RawMessage(`{}`))
assert.ErrorContains(t, err, "session_id")
}
func TestTrainerCallsReaderThenWriter(t *testing.T) {
sessDir := t.TempDir()
require.NoError(t, session.Append(sessDir, "sess-1", session.Entry{
SessionID: "sess-1", Skill: "tdd", Phase: "red", FinalStatus: "pass",
Message: "wrote failing test", FilePath: "internal/foo/foo_test.go",
}))
callCount := 0
var readerTask, writerTask string
fakeFn := func(_ context.Context, req iexec.Request) (iexec.Result, error) {
callCount++
if callCount == 1 {
// reader call
readerTask = req.TaskPrompt
return iexec.Result{
Status: "pass", Phase: "trainer", Skill: "trainer",
RunnerOutput: `[{"type":"sft","moment":"first-pass clean TDD","score":4}]`,
Verified: true, ModelUsed: "self", Message: "1 sft candidate found",
}, nil
}
// writer call
writerTask = req.TaskPrompt
return iexec.Result{
Status: "pass", Phase: "trainer", Skill: "trainer",
FilePath: sessDir + "/training-data/sft/sess-1.jsonl",
Verified: true, ModelUsed: "self", Message: "1 sft pair written",
}, nil
}
sk := trainer.New(trainer.Config{
ReaderPrompt: "reader rules",
WriterPrompt: "writer rules",
ExecutorFn: fakeFn,
SessionsDir: sessDir,
BrainDir: t.TempDir(),
})
out, err := sk.Handle(context.Background(), "trainer", json.RawMessage(`{"session_id":"sess-1"}`))
require.NoError(t, err)
assert.Equal(t, 2, callCount, "executor must be called exactly twice: reader then writer")
assert.Contains(t, readerTask, "role: reader")
assert.Contains(t, readerTask, "sess-1")
assert.Contains(t, readerTask, "wrote failing test") // session history in reader prompt
assert.Contains(t, writerTask, "role: writer")
assert.Contains(t, writerTask, "sft candidate") // reader output passed to writer
var result iexec.Result
require.NoError(t, json.Unmarshal(out, &result))
assert.Equal(t, "trainer", result.Phase)
assert.Equal(t, "pass", result.Status)
}

View File

@@ -0,0 +1,53 @@
// internal/skills/trainer/skill.go
package trainer
import (
"context"
"encoding/json"
iexec "github.com/mathiasbq/supervisor/internal/exec"
"github.com/mathiasbq/supervisor/internal/registry"
)
// ExecutorFn is the function signature for running a worker subprocess.
type ExecutorFn func(ctx context.Context, req iexec.Request) (iexec.Result, error)
// Config holds dependencies for the trainer skill.
type Config struct {
ReaderPrompt string
WriterPrompt string
DefaultModel string
ExecutorFn ExecutorFn
SessionsDir string
BrainDir string // root of brain/ directory; writer writes to BrainDir/training-data/
}
// Skill implements the trainer MCP tool.
type Skill struct{ cfg Config }
// New creates a new trainer Skill.
func New(cfg Config) *Skill { return &Skill{cfg: cfg} }
// Name returns the skill identifier.
func (s *Skill) Name() string { return "trainer" }
// Tools returns the MCP tool definitions for this skill.
func (s *Skill) Tools() []registry.ToolDef {
schema := func(required []string, props map[string]any) json.RawMessage {
b, _ := json.Marshal(map[string]any{"type": "object", "required": required, "properties": props})
return b
}
return []registry.ToolDef{
{
Name: "trainer",
Description: "Extract SFT and DPO training pairs from a session log. Runs a reader→writer chain: reader identifies learning moments, writer formats and writes pairs to brain/training-data/.",
InputSchema: schema(
[]string{"session_id"},
map[string]any{
"session_id": map[string]any{"type": "string"},
"model": map[string]any{"type": "string"},
},
),
},
}
}