133 lines
4.3 KiB
Go
133 lines
4.3 KiB
Go
package exec_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
iexec "github.com/mathiasbq/supervisor/internal/exec"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// fakeClaudePath writes a shell script that prints fixed output and returns its path.
|
|
func fakeClaudePath(t *testing.T, output string, exitCode int) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
script := filepath.Join(dir, "claude")
|
|
var content string
|
|
if exitCode != 0 {
|
|
content = "#!/bin/sh\necho 'error' >&2\nexit 1\n"
|
|
} else {
|
|
content = "#!/bin/sh\necho '" + output + "'\n"
|
|
}
|
|
require.NoError(t, os.WriteFile(script, []byte(content), 0755))
|
|
return script
|
|
}
|
|
|
|
func TestExecutorParsesValidResult(t *testing.T) {
|
|
// Fake claude emits the --output-format json envelope that the real CLI produces.
|
|
// The executor extracts the result from the "structured_output" field.
|
|
envelope := `{"type":"result","subtype":"success","is_error":false,"structured_output":{"status":"pass","phase":"red","skill":"tdd","file_path":"/tmp/x_test.go","runner_output":"FAIL","verified":true,"model_used":"self","message":"ok"}}`
|
|
claude := fakeClaudePath(t, envelope, 0)
|
|
|
|
ex := iexec.New(iexec.Config{
|
|
ClaudeBinary: claude,
|
|
SystemPrompt: "you are a supervisor",
|
|
Timeout: 5 * time.Second,
|
|
})
|
|
|
|
result, err := ex.Run(context.Background(), iexec.Request{
|
|
SkillPrompt: "tdd rules",
|
|
TaskPrompt: "run red phase",
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "pass", result.Status)
|
|
assert.True(t, result.Verified)
|
|
}
|
|
|
|
func TestExecutorReturnsErrorOnNonZeroExit(t *testing.T) {
|
|
claude := fakeClaudePath(t, "", 1)
|
|
|
|
ex := iexec.New(iexec.Config{
|
|
ClaudeBinary: claude,
|
|
SystemPrompt: "you are a supervisor",
|
|
Timeout: 5 * time.Second,
|
|
})
|
|
|
|
_, err := ex.Run(context.Background(), iexec.Request{TaskPrompt: "fail"})
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestExecutorTimesOut(t *testing.T) {
|
|
dir := t.TempDir()
|
|
script := filepath.Join(dir, "claude")
|
|
require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\nsleep 60\n"), 0755))
|
|
|
|
ex := iexec.New(iexec.Config{
|
|
ClaudeBinary: script,
|
|
SystemPrompt: "you are a supervisor",
|
|
Timeout: 100 * time.Millisecond,
|
|
})
|
|
|
|
_, err := ex.Run(context.Background(), iexec.Request{TaskPrompt: "slow"})
|
|
assert.ErrorContains(t, err, "timeout")
|
|
}
|
|
|
|
func TestExecutorPassesModelFlagForCloudModel(t *testing.T) {
|
|
// The script captures its args to a temp file so we can assert --model was passed.
|
|
argsFile := filepath.Join(t.TempDir(), "args.txt")
|
|
envelope := `{"type":"result","subtype":"success","is_error":false,"structured_output":{"status":"pass","phase":"review","skill":"review","file_path":"","runner_output":"","verified":true,"model_used":"claude-sonnet-4-6","message":"ok"}}`
|
|
|
|
dir := t.TempDir()
|
|
script := filepath.Join(dir, "claude")
|
|
content := "#!/bin/sh\necho \"$@\" > " + argsFile + "\necho '" + envelope + "'\n"
|
|
require.NoError(t, os.WriteFile(script, []byte(content), 0755))
|
|
|
|
ex := iexec.New(iexec.Config{
|
|
ClaudeBinary: script,
|
|
SystemPrompt: "sys",
|
|
Timeout: 5 * time.Second,
|
|
})
|
|
|
|
_, err := ex.Run(context.Background(), iexec.Request{
|
|
SkillPrompt: "review rules",
|
|
TaskPrompt: "do review",
|
|
Model: "claude-sonnet-4-6",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
argsData, err := os.ReadFile(argsFile)
|
|
require.NoError(t, err)
|
|
assert.Contains(t, string(argsData), "--model claude-sonnet-4-6")
|
|
}
|
|
|
|
func TestExecutorSkipsModelFlagForLocalModel(t *testing.T) {
|
|
argsFile := filepath.Join(t.TempDir(), "args.txt")
|
|
envelope := `{"type":"result","subtype":"success","is_error":false,"structured_output":{"status":"pass","phase":"review","skill":"review","file_path":"","runner_output":"","verified":true,"model_used":"ollama/devstral","message":"ok"}}`
|
|
|
|
dir := t.TempDir()
|
|
script := filepath.Join(dir, "claude")
|
|
content := "#!/bin/sh\necho \"$@\" > " + argsFile + "\necho '" + envelope + "'\n"
|
|
require.NoError(t, os.WriteFile(script, []byte(content), 0755))
|
|
|
|
ex := iexec.New(iexec.Config{
|
|
ClaudeBinary: script,
|
|
SystemPrompt: "sys",
|
|
Timeout: 5 * time.Second,
|
|
})
|
|
|
|
_, err := ex.Run(context.Background(), iexec.Request{
|
|
SkillPrompt: "review rules",
|
|
TaskPrompt: "do review",
|
|
Model: "ollama/devstral",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
argsData, err := os.ReadFile(argsFile)
|
|
require.NoError(t, err)
|
|
assert.NotContains(t, string(argsData), "--model")
|
|
}
|