From e96b39a812569506bf1cb537c81fc9e5f22bb5be Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Mon, 20 Apr 2026 11:02:59 +0200 Subject: [PATCH] feat(exec): add Claude verifier for local model output quality gate --- internal/exec/verifier.go | 99 ++++++++++++++++++++++++++++++++++ internal/exec/verifier_test.go | 74 +++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 internal/exec/verifier.go create mode 100644 internal/exec/verifier_test.go diff --git a/internal/exec/verifier.go b/internal/exec/verifier.go new file mode 100644 index 0000000..c915f80 --- /dev/null +++ b/internal/exec/verifier.go @@ -0,0 +1,99 @@ +package exec + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "time" +) + +// Verdict is the output of a Claude verification call. +type Verdict struct { + Accept bool `json:"accept"` + Feedback string `json:"feedback"` // empty when Accept is true +} + +// Verifier runs a focused Claude call to judge local model output. +type Verifier struct { + claudeBinary string + model string + timeout time.Duration +} + +// NewVerifier creates a Verifier that calls claude with the given binary path and model. +// Empty claudeBinary defaults to "claude". Zero timeout defaults to 30s. +func NewVerifier(claudeBinary, model string, timeout time.Duration) *Verifier { + if claudeBinary == "" { + claudeBinary = "claude" + } + if timeout == 0 { + timeout = 30 * time.Second + } + return &Verifier{ + claudeBinary: claudeBinary, + model: model, + timeout: timeout, + } +} + +// Verify asks Claude whether output satisfies the skill discipline's iron laws. +// Returns Verdict{Accept: true} to accept or Verdict{Accept: false, Feedback: "..."} +// to escalate. Returns an error on subprocess failure or unparseable response. +func (v *Verifier) Verify(ctx context.Context, skillPrompt, taskPrompt string, output Result) (Verdict, error) { + ctx, cancel := context.WithTimeout(ctx, v.timeout) + defer cancel() + + outputJSON, err := json.Marshal(output) + if err != nil { + return Verdict{}, fmt.Errorf("verifier: marshal output: %w", err) + } + + prompt := fmt.Sprintf(`You are a quality verifier for an AI supervisor system. + +Given the skill discipline, the original task, and the generated output, decide whether the output satisfies the discipline's iron laws and output contract. + +Reply with JSON only — no other text: +{"accept": true, "feedback": ""} +or +{"accept": false, "feedback": ""} + +## Skill discipline +%s + +## Original task +%s + +## Generated output +%s`, skillPrompt, taskPrompt, string(outputJSON)) + + args := []string{ + "--print", + "--permission-mode", "bypassPermissions", + } + if v.model != "" { + args = append(args, "--model", v.model) + } + args = append(args, prompt) + + cmd := exec.CommandContext(ctx, v.claudeBinary, args...) + cmd.Env = os.Environ() + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return Verdict{}, fmt.Errorf("verifier: timeout after %s", v.timeout) + } + return Verdict{}, fmt.Errorf("verifier: claude exited with error: %w — stderr: %s", err, stderr.String()) + } + + var verdict Verdict + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &verdict); err != nil { + return Verdict{}, fmt.Errorf("verifier: parse verdict JSON: %w — raw: %s", err, stdout.String()) + } + return verdict, nil +} diff --git a/internal/exec/verifier_test.go b/internal/exec/verifier_test.go new file mode 100644 index 0000000..83c9cb8 --- /dev/null +++ b/internal/exec/verifier_test.go @@ -0,0 +1,74 @@ +package exec_test + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + iexec "github.com/mathiasbq/supervisor/internal/exec" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func fakeVerifierClaude(t *testing.T, verdict iexec.Verdict) string { + t.Helper() + data, err := json.Marshal(verdict) + require.NoError(t, err) + dir := t.TempDir() + script := filepath.Join(dir, "claude") + content := fmt.Sprintf("#!/bin/sh\necho '%s'\n", string(data)) + require.NoError(t, os.WriteFile(script, []byte(content), 0755)) + return script +} + +func TestVerifierAccepts(t *testing.T) { + claude := fakeVerifierClaude(t, iexec.Verdict{Accept: true, Feedback: ""}) + v := iexec.NewVerifier(claude, "claude-sonnet-4-6", 5*time.Second) + + verdict, err := v.Verify(context.Background(), "skill rules", "do the task", iexec.Result{ + Status: "pass", Phase: "review", Skill: "review", Message: "ok", + }) + require.NoError(t, err) + assert.True(t, verdict.Accept) + assert.Empty(t, verdict.Feedback) +} + +func TestVerifierEscalates(t *testing.T) { + claude := fakeVerifierClaude(t, iexec.Verdict{Accept: false, Feedback: "missing line references"}) + v := iexec.NewVerifier(claude, "claude-sonnet-4-6", 5*time.Second) + + verdict, err := v.Verify(context.Background(), "skill rules", "do the task", iexec.Result{ + Status: "pass", Phase: "review", Skill: "review", Message: "incomplete", + }) + require.NoError(t, err) + assert.False(t, verdict.Accept) + assert.Equal(t, "missing line references", verdict.Feedback) +} + +func TestVerifierErrorOnUnparsableOutput(t *testing.T) { + dir := t.TempDir() + script := filepath.Join(dir, "claude") + require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\necho 'not json'\n"), 0755)) + + v := iexec.NewVerifier(script, "claude-sonnet-4-6", 5*time.Second) + _, err := v.Verify(context.Background(), "rules", "task", iexec.Result{ + Status: "pass", Phase: "review", Skill: "review", Message: "ok", + }) + assert.Error(t, err) +} + +func TestVerifierErrorOnNonZeroExit(t *testing.T) { + dir := t.TempDir() + script := filepath.Join(dir, "claude") + require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\nexit 1\n"), 0755)) + + v := iexec.NewVerifier(script, "claude-sonnet-4-6", 5*time.Second) + _, err := v.Verify(context.Background(), "rules", "task", iexec.Result{ + Status: "pass", Phase: "review", Skill: "review", Message: "ok", + }) + assert.Error(t, err) +}