package proposal_generation import ( "context" "encoding/json" "errors" "os" "path/filepath" "reflect" "strings" "testing" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) type fakeStructuredClient struct { responses []StructuredCorrectionSet err error completion contracts.StructuredCompletionResponse calls []contracts.StructuredCompletionRequest } func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx f.calls = append(f.calls, req) if f.err != nil { return f.completion, f.err } target, ok := out.(*StructuredCorrectionSet) if !ok { return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") } if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") } *target = f.responses[0] f.responses = f.responses[1:] var content strings.Builder for _, value := range req.Variables { content.WriteString(value) } response := f.completion response.Prepared = &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}} return response, nil } type captureDiagnosticsWriter struct { err error lastStage string lastRequestMetadata any lastRequestPayload any } func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { w.lastStage = stage w.lastRequestMetadata = requestMetadata w.lastRequestPayload = requestPayload _ = responsePayload _ = errorPayload return InteractionArtifacts{}, w.err } func defaultRequest(t *testing.T) Request { t.Helper() cfg := config.Config{} return Request{ ModuleKey: "test_module", ModuleInstance: "test_module", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &cfg, PromptID: "modules.grammar.proposal", PromptVersion: "v1", Variables: map[string]string{"SectionJSON": "[]"}, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}}, StartIndex: 0, } } func TestGenerateCandidatesSuccess(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, }, }, }, } section := contracts.SectionMetadata{Index: 2} req := defaultRequest(t) req.LLMClient = client req.StartIndex = 10 req.Section = §ion got, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("GenerateCandidates error: %v", err) } if len(got.Corrections) != 1 || len(got.Enriched) != 1 { t.Fatalf("unexpected proposal lengths: %+v", got) } if got.Corrections[0].TargetSegmentID != 7 { t.Fatalf("local reference not resolved: %+v", got.Corrections) } if got.Enriched[0].ProposalIndex != 10 { t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex) } if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 { t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex) } if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal { t.Fatalf("proposal operation identity: %+v", got.Operations) } } func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, }, } req := defaultRequest(t) req.LLMClient = client _, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("GenerateCandidates error: %v", err) } if len(client.calls) != 1 { t.Fatalf("expected 1 LLM call, got %d", len(client.calls)) } call := client.calls[0] if call.Schema != responseschema.CorrectionSetKey || call.PromptID != req.PromptID || call.Context.Role != "proposal" { t.Fatalf("unexpected call contract: %+v", call) } } func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, }, } diag := &captureDiagnosticsWriter{} req := defaultRequest(t) req.LLMClient = client req.DiagnosticsWriter = diag _, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("GenerateCandidates error: %v", err) } metadata, ok := diag.lastRequestMetadata.(map[string]any) if !ok { t.Fatalf("expected request metadata map, got %T", diag.lastRequestMetadata) } schemaMap, ok := metadata["response_schema"].(map[string]any) if !ok { t.Fatalf("expected response_schema metadata map, got %T", metadata["response_schema"]) } want := responseschema.MustLookup(responseschema.CorrectionSetKey) if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 { t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want) } } func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ {SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2}, }, }, }, } req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("expected invalid correction to survive generation, got %v", err) } if len(result.Corrections) != 1 { t.Fatalf("expected one correction, got %+v", result) } if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 { t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0]) } if len(result.Operations) != 1 || !result.Operations[0].Succeeded { t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations) } } func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *testing.T) { client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}} req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("expected recoverable model operation, got %v", err) } if len(result.Corrections) != 0 || len(result.Enriched) != 0 { t.Fatalf("expected no proposals on malformed response, got %+v", result) } if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted { t.Fatalf("expected one failed operation, got %+v", result.Operations) } } func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) { client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}} req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonSetupFailure || !result.Operations[0].Counts.Available { t.Fatalf("fatal setup lost call facts: %+v, %v", result, err) } } func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) { client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}} req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded { t.Fatalf("valid empty response: %+v, %v", result, err) } } func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { baseResponse := StructuredCorrectionSet{ Corrections: []StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9}, {SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9}, }, } clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} reqA := defaultRequest(t) reqA.LLMClient = clientA reqA.StartIndex = 3 first, err := GenerateCandidates(context.Background(), reqA) if err != nil { t.Fatalf("first generation failed: %v", err) } reqB := defaultRequest(t) reqB.LLMClient = clientB reqB.StartIndex = 3 second, err := GenerateCandidates(context.Background(), reqB) if err != nil { t.Fatalf("second generation failed: %v", err) } if !reflect.DeepEqual(first.Enriched, second.Enriched) { t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched) } } func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}}, }, } section0 := contracts.SectionMetadata{Index: 0} section1 := contracts.SectionMetadata{Index: 1} req0 := defaultRequest(t) req0.LLMClient = client req0.Section = §ion0 req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}} req0.StartIndex = 0 part0, err := GenerateCandidates(context.Background(), req0) if err != nil { t.Fatalf("section 0 generation failed: %v", err) } req1 := defaultRequest(t) req1.LLMClient = client req1.Section = §ion1 req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}} req1.StartIndex = len(part0.Enriched) part1, err := GenerateCandidates(context.Background(), req1) if err != nil { t.Fatalf("section 1 generation failed: %v", err) } all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...) if len(all) != 2 { t.Fatalf("expected 2 proposals, got %d", len(all)) } if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 { t.Fatalf("section references did not reset: %+v", all) } if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 { t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex) } if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 { t.Fatalf("unexpected section metadata: %+v", all) } } func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { primarySecret := "proposal-primary-secret" validationSecret := "proposal-validation-secret" client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}}, }, } t.Setenv("PRIMARY_SECRET", primarySecret) t.Setenv("VALIDATION_SECRET", validationSecret) loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "") if loadErr != nil { t.Fatal(loadErr) } cfg := loaded.Config req := defaultRequest(t) req.Config = &cfg req.LLMClient = client req.DiagnosticsDir = t.TempDir() req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret} got, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("GenerateCandidates error: %v", err) } if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" { t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts) } for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} { raw, readErr := os.ReadFile(path) if readErr != nil { t.Fatalf("read artifact %q: %v", path, readErr) } if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw)) } if !strings.Contains(string(raw), "[REDACTED]") { t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw)) } } raw, readErr := os.ReadFile(got.Artifacts.RequestMetadataPath) if readErr != nil { t.Fatalf("read metadata artifact %q: %v", got.Artifacts.RequestMetadataPath, readErr) } var metadata map[string]any if err := json.Unmarshal(raw, &metadata); err != nil { t.Fatalf("unmarshal metadata artifact: %v", err) } schemaMap, ok := metadata["response_schema"].(map[string]any) if !ok { t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"]) } want := responseschema.MustLookup(responseschema.CorrectionSetKey) if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 { t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want) } } func TestGenerateCandidatesClientError(t *testing.T) { client := &fakeStructuredClient{err: errors.New("boom")} req := defaultRequest(t) req.LLMClient = client req.DiagnosticsDir = t.TempDir() got, err := GenerateCandidates(context.Background(), req) if err == nil || !strings.Contains(err.Error(), "completion failed") { t.Fatalf("expected completion failure, got %v", err) } if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 { t.Fatalf("expected fatal call facts without proposals, got %+v", got) } matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json")) if globErr != nil { t.Fatalf("glob error: %v", globErr) } if len(matches) == 0 { t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir) } } func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) { cause := errors.New("diagnostic storage unavailable") req := defaultRequest(t) req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}} req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause} var reported error req.ReportDiagnosticError = func(err error) { reported = err } result, err := GenerateCandidates(context.Background(), req) if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" { t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported) } }