93 lines
3.2 KiB
Go
93 lines
3.2 KiB
Go
package validators
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
)
|
|
|
|
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
|
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
|
|
for _, tc := range []struct {
|
|
name string
|
|
responses []string
|
|
wantCalls int
|
|
wantApproved bool
|
|
wantFailure outcomes.ReasonCode
|
|
}{
|
|
{"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, ""},
|
|
{"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, outcomes.ReasonInvalidOutputExhausted},
|
|
{"malformed envelope excludes batch", []string{"MALFORMED_ENVELOPE"}, 1, false, outcomes.ReasonProviderResponseInvalid},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
calls := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
output := tc.responses[len(tc.responses)-1]
|
|
if calls <= len(tc.responses) {
|
|
output = tc.responses[calls-1]
|
|
}
|
|
if output == "MALFORMED_ENVELOPE" {
|
|
_, _ = fmt.Fprint(w, `{"choices":[]}`)
|
|
return
|
|
}
|
|
content, _ := json.Marshal(output)
|
|
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
|
|
}))
|
|
defer server.Close()
|
|
dir := t.TempDir()
|
|
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
main := filepath.Join(dir, "config.yml")
|
|
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
|
|
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := runtime.NewRoleClient("validation", llm.CallHooks{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.Config = &loaded.Config
|
|
req.LLMClient = client
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if calls != tc.wantCalls {
|
|
t.Fatalf("calls=%d, want %d", calls, tc.wantCalls)
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || len(result.Operations) != 1 || result.Operations[0].FailureReason != tc.wantFailure {
|
|
t.Fatalf("domain result: %+v", result)
|
|
}
|
|
})
|
|
}
|
|
}
|