experimental: codex / ai integration
This commit is contained in:
477
ai_test.go
Normal file
477
ai_test.go
Normal file
@@ -0,0 +1,477 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
func TestAIDefaultsAreDisabledAndBounded(t *testing.T) {
|
||||
config := defaultConfig()
|
||||
if config.AI.Enabled {
|
||||
t.Fatal("AI must be disabled by default")
|
||||
}
|
||||
config.AI.Enabled = true
|
||||
if err := validateAIConfig(config.AI); err != nil {
|
||||
t.Fatalf("default AI limits should validate when enabled: %v", err)
|
||||
}
|
||||
if config.AI.MaxRunBytes < config.AI.MaxRequestBytes || config.AI.MaxCalls < 1 {
|
||||
t.Fatalf("unbounded defaults: %#v", config.AI)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareAIDiffFiltersAndRedacts(t *testing.T) {
|
||||
config := defaultAIConfig()
|
||||
raw := `diff --git a/main.go b/main.go
|
||||
--- a/main.go
|
||||
+++ b/main.go
|
||||
@@ -1 +1,2 @@
|
||||
package main
|
||||
+token = "ghp_abcdefghijklmnopqrstuvwxyz123456"
|
||||
diff --git a/go.sum b/go.sum
|
||||
--- a/go.sum
|
||||
+++ b/go.sum
|
||||
@@ -1 +1 @@
|
||||
-old
|
||||
+new
|
||||
diff --git a/.env b/.env
|
||||
--- a/.env
|
||||
+++ b/.env
|
||||
@@ -0,0 +1 @@
|
||||
+PASSWORD=do-not-send
|
||||
`
|
||||
files, excluded, redactions := prepareAIDiff(raw, config)
|
||||
if len(files) != 1 || files[0].Path != "main.go" {
|
||||
t.Fatalf("files = %#v, want only main.go", files)
|
||||
}
|
||||
if len(excluded) != 2 {
|
||||
t.Fatalf("excluded = %#v, want go.sum and .env", excluded)
|
||||
}
|
||||
if redactions != 1 || strings.Contains(files[0].Text, "ghp_") ||
|
||||
!strings.Contains(files[0].Text, "[REDACTED]") {
|
||||
t.Fatalf("redaction failed: count=%d text=%q", redactions, files[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAIDiffTracksDeletedSide(t *testing.T) {
|
||||
file, ok := parseAIDiffFile(`diff --git a/old.go b/old.go
|
||||
deleted file mode 100644
|
||||
--- a/old.go
|
||||
+++ /dev/null
|
||||
@@ -7,2 +0,0 @@
|
||||
-dangerous()
|
||||
-cleanup()
|
||||
`)
|
||||
if !ok {
|
||||
t.Fatal("deletion-only diff was excluded")
|
||||
}
|
||||
if file.Path != "old.go" || !file.DeletedLines[7] || !file.DeletedLines[8] ||
|
||||
len(file.ChangedLines) != 0 {
|
||||
t.Fatalf("parsed deletion = %#v", file)
|
||||
}
|
||||
if hunk := boundedDiffHunk(file.Text, 8, "LEFT"); !strings.Contains(hunk, "cleanup") {
|
||||
t.Fatalf("left-side hunk = %q", hunk)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedDiffHunkKeepsTargetFromLargeHunk(t *testing.T) {
|
||||
var diff strings.Builder
|
||||
diff.WriteString("@@ -1,100 +1,101 @@\n")
|
||||
for line := 1; line <= 100; line++ {
|
||||
if line == 90 {
|
||||
diff.WriteString("+TARGET\n")
|
||||
}
|
||||
diff.WriteString(" " + strings.Repeat("x", 400) + "\n")
|
||||
}
|
||||
hunk := boundedDiffHunk(diff.String(), 90, "RIGHT")
|
||||
if !strings.Contains(hunk, "TARGET") || len(hunk) > 24_000 {
|
||||
t.Fatalf("bounded hunk length=%d contains target=%t", len(hunk), strings.Contains(hunk, "TARGET"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCodexEventsRejectsToolUse(t *testing.T) {
|
||||
data := []byte("{\"type\":\"item.completed\",\"item\":{\"type\":\"command_execution\",\"command\":\"pwd\"}}\n")
|
||||
if _, _, err := parseCodexEvents(data); err == nil {
|
||||
t.Fatal("tool event was accepted")
|
||||
}
|
||||
valid := []byte("{\"type\":\"item.completed\",\"model\":\"gpt-test\",\"item\":{\"type\":\"agent_message\",\"text\":\"{\\\"findings\\\":[],\\\"thread_comments\\\":[]}\"}}\n")
|
||||
content, model, err := parseCodexEvents(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if model != "gpt-test" || !strings.Contains(string(content), `"findings"`) {
|
||||
t.Fatalf("content=%q model=%q", content, model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexFailureDetailReadsJSONErrorEvents(t *testing.T) {
|
||||
stdout := []byte(
|
||||
"{\"type\":\"thread.started\",\"thread_id\":\"x\"}\n" +
|
||||
"{\"type\":\"turn.failed\",\"error\":{\"message\":\"model unavailable\\u001b]8;;bad\"}}\n",
|
||||
)
|
||||
got := codexFailureDetail(stdout, nil)
|
||||
if !strings.Contains(got, "model unavailable") || strings.ContainsRune(got, '\x1b') {
|
||||
t.Fatalf("failure detail = %q", got)
|
||||
}
|
||||
if got := codexFailureDetail(nil, []byte("plain stderr")); got != "plain stderr" {
|
||||
t.Fatalf("stderr detail = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathWithinUsesPathBoundaries(t *testing.T) {
|
||||
parent := filepath.Join(t.TempDir(), "repo")
|
||||
if !pathWithin(parent, filepath.Join(parent, "nested", "file")) {
|
||||
t.Fatal("nested path was not recognized")
|
||||
}
|
||||
if pathWithin(parent, parent+"-other/file") {
|
||||
t.Fatal("sibling prefix was treated as inside the workspace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullRequestDiffUsesAuthenticatedGHESRESTEndpoint(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/api/v3/repos/owner/repo/pulls/12" {
|
||||
t.Fatalf("path = %q", request.URL.Path)
|
||||
}
|
||||
if request.Header.Get("Authorization") != "Bearer token" {
|
||||
t.Fatalf("authorization = %q", request.Header.Get("Authorization"))
|
||||
}
|
||||
if request.Header.Get("Accept") != "application/vnd.github.v3.diff" {
|
||||
t.Fatalf("accept = %q", request.Header.Get("Accept"))
|
||||
}
|
||||
_, _ = writer.Write([]byte("diff --git a/a.go b/a.go\n"))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := NewGitHubClient(server.URL+"/api/graphql", "token")
|
||||
diff, err := client.PullRequestDiff(context.Background(), "owner", "repo", 12)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(diff, "diff --git") {
|
||||
t.Fatalf("diff = %q", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIStoreIsPrivateAndMarksOldHeadOutdated(t *testing.T) {
|
||||
store := NewAIStore(t.TempDir())
|
||||
pr := PRDetails{PullRequest: PullRequest{Owner: "o", Repository: "r", Number: 7}, HeadOID: "head-1"}
|
||||
state, err := store.Load(pr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.Threads = []ReviewThread{{
|
||||
ID: "local-ai-x", Path: "main.go", Line: 3, HeadOID: "head-1",
|
||||
Origin: reviewOriginLocalAI,
|
||||
}}
|
||||
if err := store.Save(pr, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(store.path(pr))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("store mode = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
pr.HeadOID = "head-2"
|
||||
merged := state.Merge(pr)
|
||||
if len(merged.Threads) != 1 || !merged.Threads[0].IsOutdated {
|
||||
t.Fatalf("merged threads = %#v", merged.Threads)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeAIProvider struct {
|
||||
response AIInferenceResponse
|
||||
requests []AIInferenceRequest
|
||||
}
|
||||
|
||||
func (p *fakeAIProvider) Name() string { return "fake" }
|
||||
func (p *fakeAIProvider) Status(context.Context) AIProviderStatus {
|
||||
return AIProviderStatus{Ready: true, Summary: "ready", Model: "gpt-test"}
|
||||
}
|
||||
func (p *fakeAIProvider) Generate(_ context.Context, request AIInferenceRequest) (AIInferenceResponse, error) {
|
||||
p.requests = append(p.requests, request)
|
||||
return p.response, nil
|
||||
}
|
||||
|
||||
type fakeStreamingAIProvider struct {
|
||||
fakeAIProvider
|
||||
progress []AIProviderProgress
|
||||
}
|
||||
|
||||
func (p *fakeStreamingAIProvider) GenerateWithProgress(
|
||||
_ context.Context,
|
||||
request AIInferenceRequest,
|
||||
report func(AIProviderProgress),
|
||||
) (AIInferenceResponse, error) {
|
||||
p.requests = append(p.requests, request)
|
||||
for _, progress := range p.progress {
|
||||
report(progress)
|
||||
}
|
||||
return p.response, nil
|
||||
}
|
||||
|
||||
type fakeAIDiffService struct{ diff string }
|
||||
|
||||
func (s fakeAIDiffService) PullRequestDiff(context.Context, string, string, int) (string, error) {
|
||||
return s.diff, nil
|
||||
}
|
||||
|
||||
func TestAIControllerAcceptsOnlyChangedLinesAndDeduplicates(t *testing.T) {
|
||||
output := aiOutput{}
|
||||
output.Findings = append(output.Findings, aiFinding{
|
||||
Path: "main.go", Side: "RIGHT", StartLine: 2, EndLine: 2,
|
||||
Severity: "high", Title: "Bug", Body: "This is broken.",
|
||||
Suggestion: "var broken = false",
|
||||
})
|
||||
output.Findings = append(output.Findings, aiFinding{
|
||||
Path: "main.go", Side: "RIGHT", StartLine: 99, EndLine: 99,
|
||||
Severity: "high", Title: "Invented", Body: "Not changed.",
|
||||
})
|
||||
content, _ := json.Marshal(output)
|
||||
provider := &fakeAIProvider{response: AIInferenceResponse{Model: "gpt-test", Content: content}}
|
||||
config := defaultAIConfig()
|
||||
config.Enabled = true
|
||||
config.MaxRequestBytes = 16_000
|
||||
config.MaxRunBytes = 32_000
|
||||
store := NewAIStore(filepath.Join(t.TempDir(), "ai"))
|
||||
controller := &AIController{
|
||||
config: config, provider: provider,
|
||||
diffs: fakeAIDiffService{diff: `diff --git a/main.go b/main.go
|
||||
--- a/main.go
|
||||
+++ b/main.go
|
||||
@@ -1 +1,2 @@
|
||||
package main
|
||||
+var broken = true
|
||||
`},
|
||||
store: store,
|
||||
}
|
||||
pr := PRDetails{
|
||||
PullRequest: PullRequest{Owner: "o", Repository: "r", Number: 1, Title: "PR"},
|
||||
HeadOID: "abc",
|
||||
}
|
||||
preview, err := controller.Prepare(context.Background(), pr, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := controller.Run(context.Background(), preview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Findings != 1 {
|
||||
t.Fatalf("findings = %d, want 1", result.Findings)
|
||||
}
|
||||
if len(result.Details.Threads) != 1 ||
|
||||
!strings.Contains(result.Details.Threads[0].Comments[0].DiffHunk, "var broken") {
|
||||
t.Fatalf("local finding did not retain its review-time hunk: %#v", result.Details.Threads)
|
||||
}
|
||||
parsed := parseCommentBody(result.Details.Threads[0].Comments[0].Body)
|
||||
if len(parsed.Suggestions) != 1 || parsed.Suggestions[0] != "var broken = false" {
|
||||
t.Fatalf("AI suggestion = %#v", parsed.Suggestions)
|
||||
}
|
||||
second, err := controller.Run(context.Background(), preview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Findings != 0 {
|
||||
t.Fatalf("duplicate findings = %d, want 0", second.Findings)
|
||||
}
|
||||
if len(provider.requests) != 2 || provider.requests[0].Model != "gpt-test" {
|
||||
t.Fatalf("requests = %#v", provider.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIProviderTestUsesOneMinimalRequestAndForwardsProgress(t *testing.T) {
|
||||
provider := &fakeStreamingAIProvider{
|
||||
fakeAIProvider: fakeAIProvider{response: AIInferenceResponse{
|
||||
Model: "gpt-test", Content: []byte(`{"ok":true}`),
|
||||
}},
|
||||
progress: []AIProviderProgress{{Kind: "reasoning", Text: "Checking response shape"}},
|
||||
}
|
||||
config := defaultAIConfig()
|
||||
config.Enabled = true
|
||||
controller := &AIController{config: config, provider: provider}
|
||||
var progress []AIRunProgress
|
||||
model, err := controller.TestProvider(context.Background(), func(update AIRunProgress) {
|
||||
progress = append(progress, update)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if model != "gpt-test" || len(provider.requests) != 1 {
|
||||
t.Fatalf("model=%q requests=%d", model, len(provider.requests))
|
||||
}
|
||||
request := provider.requests[0]
|
||||
if len(request.Prompt) > 100 || strings.Contains(request.Prompt, "PR ") ||
|
||||
strings.Contains(request.Prompt, "DIFF") {
|
||||
t.Fatalf("provider test sent non-minimal or PR-related input: %q", request.Prompt)
|
||||
}
|
||||
if len(progress) < 3 || progress[1].SummaryKind != "reasoning" ||
|
||||
progress[len(progress)-1].CompletedCalls != 1 {
|
||||
t.Fatalf("progress = %#v", progress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexProgressWriterHandlesFragmentedReasoningEvents(t *testing.T) {
|
||||
var updates []AIProviderProgress
|
||||
output := &limitedBuffer{limit: 4096}
|
||||
writer := &codexProgressWriter{
|
||||
output: output,
|
||||
report: func(progress AIProviderProgress) {
|
||||
updates = append(updates, progress)
|
||||
},
|
||||
}
|
||||
first := `{"type":"item.completed","item":{"type":"reasoning","text":"Check`
|
||||
second := "ing\\u001b[31m result\"}}\n"
|
||||
if _, err := writer.Write([]byte(first)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(updates) != 0 {
|
||||
t.Fatalf("reported incomplete event: %#v", updates)
|
||||
}
|
||||
if _, err := writer.Write([]byte(second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(updates) != 1 || updates[0].Kind != "reasoning" ||
|
||||
updates[0].Text != "Checking[31m result" ||
|
||||
strings.ContainsRune(updates[0].Text, '\x1b') {
|
||||
t.Fatalf("updates = %#v", updates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIProgressBarHasStableWidth(t *testing.T) {
|
||||
first := renderAIProgressBar(20, AIRunProgress{TotalCalls: 3}, 0)
|
||||
second := renderAIProgressBar(20, AIRunProgress{TotalCalls: 3, CompletedCalls: 2}, 7)
|
||||
if ansi.StringWidth(first) != ansi.StringWidth(second) {
|
||||
t.Fatalf("progress bar width changed: %q (%d), %q (%d)",
|
||||
first, ansi.StringWidth(first), second, ansi.StringWidth(second))
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatedAISuggestionRejectsUnsafeOrUncontainedReplacement(t *testing.T) {
|
||||
base := aiFinding{
|
||||
Path: "main.go", Side: "RIGHT", StartLine: 2, EndLine: 2,
|
||||
Suggestion: " replacement()",
|
||||
}
|
||||
if got := validatedAISuggestion(base, map[int]bool{2: true}); got != " replacement()" {
|
||||
t.Fatalf("valid suggestion = %q", got)
|
||||
}
|
||||
left := base
|
||||
left.Side = "LEFT"
|
||||
if got := validatedAISuggestion(left, map[int]bool{2: true}); got != "" {
|
||||
t.Fatalf("LEFT-side suggestion accepted: %q", got)
|
||||
}
|
||||
fenced := base
|
||||
fenced.Suggestion = "```go\nreplacement()\n```"
|
||||
if got := validatedAISuggestion(fenced, map[int]bool{2: true}); got != "" {
|
||||
t.Fatalf("fenced suggestion accepted: %q", got)
|
||||
}
|
||||
uncontained := base
|
||||
uncontained.EndLine = 3
|
||||
if got := validatedAISuggestion(uncontained, map[int]bool{2: true}); got != "" {
|
||||
t.Fatalf("suggestion over unchanged line accepted: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExistingLocalAIFindingCanGainSuggestion(t *testing.T) {
|
||||
finding := aiFinding{
|
||||
Path: "main.go", Side: "RIGHT", StartLine: 2, EndLine: 2,
|
||||
Severity: "high", Title: "Bug", Body: "This is broken.",
|
||||
Suggestion: "fixed()",
|
||||
}
|
||||
fingerprint := aiFingerprint(
|
||||
finding.Path, finding.StartLine, finding.EndLine, finding.Title, finding.Body,
|
||||
)
|
||||
state := &aiStoredState{
|
||||
Version: aiStoreVersion, Annotations: make(map[string][]ReviewComment),
|
||||
Threads: []ReviewThread{{
|
||||
ID: "local-ai-" + fingerprint, Fingerprint: fingerprint,
|
||||
Origin: reviewOriginLocalAI,
|
||||
Comments: []ReviewComment{{Body: "**high: Bug**\n\nThis is broken."}},
|
||||
}},
|
||||
}
|
||||
added, _ := state.Apply(
|
||||
PRDetails{}, aiOutput{Findings: []aiFinding{finding}}, "fake", "model", "",
|
||||
map[string]map[int]bool{"main.go": {2: true}}, nil, nil,
|
||||
)
|
||||
if added != 0 {
|
||||
t.Fatalf("duplicate finding count = %d", added)
|
||||
}
|
||||
suggestions := parseCommentBody(state.Threads[0].Comments[0].Body).Suggestions
|
||||
if len(suggestions) != 1 || suggestions[0] != "fixed()" {
|
||||
t.Fatalf("enriched suggestions = %#v", suggestions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIStoreSkipsUnchangedWrites(t *testing.T) {
|
||||
store := NewAIStore(t.TempDir())
|
||||
pr := PRDetails{PullRequest: PullRequest{Owner: "o", Repository: "r", Number: 2}}
|
||||
state, _ := store.Load(pr)
|
||||
if err := store.Save(pr, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, _ := os.Stat(store.path(pr))
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if err := store.Save(pr, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, _ := os.Stat(store.path(pr))
|
||||
if !first.ModTime().Equal(second.ModTime()) {
|
||||
t.Fatal("unchanged AI state rewrote the file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutLocalAIDoesNotMutateVisibleDetails(t *testing.T) {
|
||||
remote := ReviewThread{
|
||||
ID: "remote", Comments: []ReviewComment{
|
||||
{ID: "github"}, {ID: "local", Origin: reviewOriginLocalAI},
|
||||
},
|
||||
}
|
||||
local := ReviewThread{ID: "local-thread", Origin: reviewOriginLocalAI}
|
||||
pr := PRDetails{Threads: []ReviewThread{remote, local}}
|
||||
clean := withoutLocalAI(pr)
|
||||
if len(clean.Threads) != 1 || len(clean.Threads[0].Comments) != 1 {
|
||||
t.Fatalf("clean details = %#v", clean.Threads)
|
||||
}
|
||||
if len(pr.Threads) != 2 || len(pr.Threads[0].Comments) != 2 {
|
||||
t.Fatal("filter mutated the visible PR details")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplyOnLocalAIThreadStartsInlineDiscussion(t *testing.T) {
|
||||
config := defaultAIConfig()
|
||||
config.Enabled = true
|
||||
store := NewAIStore(t.TempDir())
|
||||
controller := &AIController{config: config, store: store}
|
||||
app := NewAppWithSettings(nil, "", "", false, 10, time.Minute, AppSettings{
|
||||
FoldResolved: true, ThreadListWidthPercent: 33,
|
||||
ThreadStatusOrder: []string{"unresolved", "outdated", "resolved"},
|
||||
ThreadWithinStatus: "file", DashboardMode: "hotkey", EditorMode: "vim",
|
||||
KeyBindings: defaultKeyBindings(), AI: controller, AIStore: store,
|
||||
})
|
||||
app.screen = threadScreen
|
||||
app.details.Threads = []ReviewThread{{ID: "local", Origin: reviewOriginLocalAI}}
|
||||
app.startReply()
|
||||
if app.aiMode != aiDiscussion || app.writeThreadID != "local" {
|
||||
t.Fatalf("AI mode=%v thread=%q", app.aiMode, app.writeThreadID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAITextSimilarityDetectsCloseRestatement(t *testing.T) {
|
||||
left := "This nil pointer can panic when the response body is absent"
|
||||
right := "The absent response body causes a nil pointer panic"
|
||||
if got := aiTextSimilarity(left, right); got < 0.68 {
|
||||
t.Fatalf("similarity = %v, want close restatement", got)
|
||||
}
|
||||
if rangesNear(10, 12, 30, 31) {
|
||||
t.Fatal("distant line ranges were considered overlapping")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user