415 lines
20 KiB
Go
415 lines
20 KiB
Go
package validators
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
|
)
|
|
|
|
type fakeStructuredLLMClient struct {
|
|
responses []LLMValidationResponse
|
|
err error
|
|
calls []StructuredCompletionRequest
|
|
}
|
|
|
|
type captureValidationDiagnosticsWriter struct {
|
|
err error
|
|
path string
|
|
lastRequestMetadata any
|
|
lastResponsePayload any
|
|
}
|
|
|
|
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
|
_ = stage
|
|
_ = requestPayload
|
|
_ = errorPayload
|
|
w.lastRequestMetadata = requestMetadata
|
|
w.lastResponsePayload = responsePayload
|
|
return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
|
|
}
|
|
|
|
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
|
_ = ctx
|
|
f.calls = append(f.calls, req)
|
|
if f.err != nil {
|
|
return StructuredCompletionResponse{}, f.err
|
|
}
|
|
if len(f.responses) == 0 {
|
|
return StructuredCompletionResponse{}, errors.New("unexpected call")
|
|
}
|
|
resp := f.responses[0]
|
|
f.responses = f.responses[1:]
|
|
target, ok := out.(*LLMValidationResponse)
|
|
if !ok {
|
|
return StructuredCompletionResponse{}, errors.New("unexpected output type")
|
|
}
|
|
*target = resp
|
|
return StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
|
|
cfg := config.Config{}
|
|
cfg.Validation.MaxPromptTokens = 10000
|
|
return Request{
|
|
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
|
|
CandidateProposal: candidates,
|
|
ModuleKey: "homophones",
|
|
ModuleInstance: "homophones",
|
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
|
Config: &cfg,
|
|
}
|
|
}
|
|
|
|
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
|
return proposals.EnrichedCorrectionProposal{
|
|
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
}
|
|
}
|
|
|
|
func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, segment, original, replacement, result string
|
|
}{
|
|
{name: "deletion", segment: "uh hello", original: "uh ", replacement: "", result: "hello"},
|
|
{name: "spacing", segment: "hello world", original: " ", replacement: " ", result: "hello world"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)})
|
|
req.WorkingTranscript.Segments[0].Text = tc.segment
|
|
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
|
if len(immediate) != 0 || len(payload.Items) != 1 {
|
|
t.Fatalf("expected previewable request: %+v, %+v", payload, immediate)
|
|
}
|
|
raw, err := json.Marshal(payload)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var decoded struct {
|
|
ValidatorName string `json:"validator_name"`
|
|
Corrections []map[string]any `json:"corrections"`
|
|
}
|
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 {
|
|
t.Fatalf("wrong validator identity or item count: %s", raw)
|
|
}
|
|
item := decoded.Corrections[0]
|
|
value, present := item["corrected_text"]
|
|
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) {
|
|
t.Fatalf("literal spans or full segment text lost: %s", raw)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkLLMValidationItemsOneSmallBatch(t *testing.T) {
|
|
items := []LLMValidationItem{{CorrectionIndex: 0, OriginalText: "a", CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}}
|
|
batches, err := ChunkLLMValidationItems(items, 1000, chunking.NewSimpleTokenEstimator())
|
|
if err != nil {
|
|
t.Fatalf("Chunk error: %v", err)
|
|
}
|
|
if len(batches) != 1 {
|
|
t.Fatalf("expected 1 batch, got %d", len(batches))
|
|
}
|
|
}
|
|
|
|
func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
|
items := []LLMValidationItem{
|
|
{CorrectionIndex: 0, OriginalText: strings.Repeat("a", 20), CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
{CorrectionIndex: 1, OriginalText: strings.Repeat("c", 20), CorrectedText: "d", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
{CorrectionIndex: 2, OriginalText: strings.Repeat("e", 20), CorrectedText: "f", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
}
|
|
batches, err := ChunkLLMValidationItems(items, 40, chunking.NewSimpleTokenEstimator())
|
|
if err != nil {
|
|
t.Fatalf("Chunk error: %v", err)
|
|
}
|
|
if len(batches) < 2 {
|
|
t.Fatalf("expected multiple batches, got %d", len(batches))
|
|
}
|
|
seen := make([]int, 0)
|
|
for _, b := range batches {
|
|
for _, it := range b.Items {
|
|
seen = append(seen, it.CorrectionIndex)
|
|
}
|
|
}
|
|
if len(seen) != 3 || seen[0] != 0 || seen[1] != 1 || seen[2] != 2 {
|
|
t.Fatalf("unexpected ordering/drops/dups: %v", seen)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
|
|
}}}}
|
|
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
|
if err != nil {
|
|
t.Fatalf("new validator error: %v", err)
|
|
}
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "Lyra")})
|
|
req.LLMClient = client
|
|
res, err := v.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("validate err: %v", err)
|
|
}
|
|
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
|
|
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
|
}
|
|
if len(client.calls) != 1 {
|
|
t.Fatalf("expected one LLM call, got %d", len(client.calls))
|
|
}
|
|
call := client.calls[0]
|
|
if call.Schema != responseschema.ValidatorDecisionSetKey || call.PromptID != prompts.PromptIDValidatorSpokenFormPlausibility || call.Context.Role != "validation" || call.Context.ValidatorName != v.Name() || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") {
|
|
t.Fatalf("unexpected validator call: %+v", call)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
res, err := v.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("expected malformed output downgrade, got %v", err)
|
|
}
|
|
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable || res.Decisions[0].ReasonCode != string(outcomes.ReasonInvalidOutputExhausted) {
|
|
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
|
}
|
|
if len(res.Operations) != 1 || res.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
|
|
t.Fatalf("expected failed operation, got %+v", res.Operations)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
result, err := v.Validate(context.Background(), req)
|
|
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonInternalFailure {
|
|
t.Fatalf("protocol error was downgraded: %+v, %v", result, err)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
|
approve := func(index int) LLMValidationDecision {
|
|
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
|
|
}
|
|
reject := func(index int) LLMValidationDecision {
|
|
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
|
|
}
|
|
type expectedDecision struct {
|
|
index int
|
|
approved bool
|
|
reason string
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
entries []LLMValidationDecision
|
|
want []expectedDecision
|
|
unavailable []int
|
|
ignored []int
|
|
}{
|
|
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
|
|
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
|
|
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
|
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
|
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
|
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
|
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
|
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
|
|
writer := &captureValidationDiagnosticsWriter{path: "validation-response.json"}
|
|
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
|
|
req.LLMClient = client
|
|
req.DiagnosticsWriter = writer
|
|
result, err := v.Validate(context.Background(), req)
|
|
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
|
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
|
|
}
|
|
for i, want := range tc.want {
|
|
got := result.Decisions[i]
|
|
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
|
|
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
|
|
}
|
|
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
|
|
t.Fatalf("negative reason lost: %+v", got)
|
|
}
|
|
}
|
|
op := result.Operations[0]
|
|
if op.Succeeded != (len(tc.unavailable) == 0) || !equalIndices(op.ProposalIndices, tc.unavailable) || (op.FailureReason == outcomes.ReasonValidationDecisionsInvalid) != (len(tc.unavailable) > 0) || len(op.Filtered) != 0 {
|
|
t.Fatalf("operation=%+v", op)
|
|
}
|
|
if len(tc.unavailable) > 0 && len(op.RecoveryReasons) != 0 || len(tc.unavailable) == 0 && (len(op.RecoveryReasons) == 1) != (len(tc.ignored) > 0) {
|
|
t.Fatalf("recovery=%+v", op)
|
|
}
|
|
if len(tc.ignored) > 0 && len(tc.unavailable) == 0 && op.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered {
|
|
t.Fatalf("recovery reason=%+v", op)
|
|
}
|
|
payload := writer.lastResponsePayload.(map[string]any)
|
|
decoded := payload["decoded"].(LLMValidationResponse)
|
|
if len(decoded.Validations) != len(tc.entries) {
|
|
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
|
|
}
|
|
reconciliation := payload["reconciliation"].(validationReconciliation)
|
|
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
|
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func equalIndices(a, b []int) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
}}}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
|
huge := strings.Repeat("gestures ", 200)
|
|
req := Request{
|
|
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
|
|
{ID: 1, Text: huge},
|
|
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
|
|
}},
|
|
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
|
{
|
|
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
},
|
|
{
|
|
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
},
|
|
},
|
|
ModuleKey: "homophones",
|
|
ModuleInstance: "homophones",
|
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
|
}
|
|
req.LLMClient = client
|
|
cfg := config.Config{}
|
|
cfg.Validation.MaxPromptTokens = 200
|
|
req.Config = &cfg
|
|
|
|
res, err := v.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("expected oversize downgrade, got %v", err)
|
|
}
|
|
if len(res.Decisions) != 2 {
|
|
t.Fatalf("expected two decisions, got %+v", res.Decisions)
|
|
}
|
|
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable {
|
|
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
|
|
}
|
|
if !res.Decisions[1].Approved {
|
|
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
|
|
}
|
|
if len(res.Operations) != 2 || res.Operations[0].Reference.Kind != outcomes.OperationValidationInput || res.Operations[0].FailureReason != outcomes.ReasonValidationInputTooLarge || res.Operations[0].Reference.BatchIndex != nil || len(res.Operations[0].ProposalIndices) != 1 || res.Operations[0].ProposalIndices[0] != 0 {
|
|
t.Fatalf("expected consolidated oversized input outcome, got %+v", res.Operations)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
}}}}
|
|
writer := &captureValidationDiagnosticsWriter{}
|
|
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
|
if err != nil {
|
|
t.Fatalf("new validator error: %v", err)
|
|
}
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
req.DiagnosticsWriter = writer
|
|
_, err = v.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("validate error: %v", err)
|
|
}
|
|
metadata, ok := writer.lastRequestMetadata.(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected metadata map, got %T", writer.lastRequestMetadata)
|
|
}
|
|
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected response_schema map, got %T", metadata["response_schema"])
|
|
}
|
|
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
|
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
|
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
|
|
}
|
|
|
|
}
|
|
func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
|
cases := []struct {
|
|
kind LLMValidatorType
|
|
prompt string
|
|
}{
|
|
{LLMValidatorTypeSpokenFormPlausibility, prompts.PromptIDValidatorSpokenFormPlausibility},
|
|
{LLMValidatorTypeMeaningReversal, prompts.PromptIDValidatorMeaningReversalReview},
|
|
{LLMValidatorTypeEditorialReview, prompts.PromptIDValidatorEditorialReview},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(string(tc.kind), func(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
|
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
req.Config.Context.Description = " Fantasy scene "
|
|
if _, err := validator.Validate(context.Background(), req); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(client.calls) != 1 {
|
|
t.Fatalf("calls=%d", len(client.calls))
|
|
}
|
|
call := client.calls[0]
|
|
if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
|
|
t.Fatalf("mapping: %+v", call)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
|
cause := errors.New("diagnostic storage unavailable")
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
|
|
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
|
|
var reported error
|
|
req.ReportDiagnosticError = func(err error) { reported = err }
|
|
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
|
|
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
|
|
}
|
|
}
|