Move LLM validators to PromptKit prompt variables

internal/framework/validators/llm_integration_test.go

@@ -0,0 +1,96 @@
+package validators
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
+)
+
+func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
+	valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}`
+	for _, tc := range []struct {
+		name                                   string
+		responses                              []string
+		wantCalls                              int
+		wantApproved, wantWarning, wantRuntime bool
+	}{
+		{"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, false, false},
+		{"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, true, false},
+		{"malformed envelope propagates", []string{"MALFORMED_ENVELOPE"}, 1, false, false, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			calls := 0
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				calls++
+				output := tc.responses[len(tc.responses)-1]
+				if calls <= len(tc.responses) {
+					output = tc.responses[calls-1]
+				}
+				if output == "MALFORMED_ENVELOPE" {
+					_, _ = fmt.Fprint(w, `{"choices":[]}`)
+					return
+				}
+				content, _ := json.Marshal(output)
+				_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
+			}))
+			defer server.Close()
+			dir := t.TempDir()
+			if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
+				t.Fatal(err)
+			}
+			main := filepath.Join(dir, "config.yml")
+			content := fmt.Sprintf("version: 2\nbackends:\n  offline:\n    endpoint: %s/v1\nllm:\n  proposal_profile: base\n  transient_retries: 0\n  repair_attempts: 1\n", server.URL)
+			if err := os.WriteFile(main, []byte(content), 0600); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
+				t.Fatal(err)
+			}
+			loaded, err := config.Load(ck.File(main), "")
+			if err != nil {
+				t.Fatal(err)
+			}
+			runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
+			if err != nil {
+				t.Fatal(err)
+			}
+			client, err := runtime.NewRoleClient("validation", llm.CallHooks{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
+			if err != nil {
+				t.Fatal(err)
+			}
+			req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
+			req.Config = &loaded.Config
+			req.LLMClient = client
+			result, err := validator.Validate(context.Background(), req)
+			if calls != tc.wantCalls {
+				t.Fatalf("calls=%d, want %d", calls, tc.wantCalls)
+			}
+			if tc.wantRuntime {
+				if err == nil || len(result.Warnings) != 0 {
+					t.Fatalf("runtime failure downgraded: %+v, %v", result, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || (len(result.Warnings) > 0) != tc.wantWarning {
+				t.Fatalf("domain result: %+v", result)
+			}
+		})
+	}
+}