diff --git a/docs/internal/validators.md b/docs/internal/validators.md index d2303ab..982682b 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -51,11 +51,15 @@ Decision cardinality is enforced: each candidate proposal must receive exactly o LLM validators: - build canonical validation request payloads; -- batch using the [configured prompt budget](../config.md#concurrency); -- call structured LLM client using response schema registry. +- batch using `validation.max_prompt_tokens` and the existing domain token estimator; +- select an embedded prompt ID and pass payload/description variables to the + role-bound structured client with the validator-decision schema. Oversized single proposals are rejected with `validator_input_too_large`. -Malformed LLM validator responses are downgraded to warnings and rejected batch decisions. +Exhausted structural repair becomes a warning and rejected batch decisions. +Provider, malformed HTTP envelope, deadline, and admission failures propagate +as runtime errors. Schema-valid decisions still undergo exact index/cardinality +checks and domain acceptance without a semantic retry. ## Decision and Rejection Reporting diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 34f2f51..87109d7 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -318,9 +318,11 @@ key and diagnostic context through the prepared client, retaining ordering, warnings, and later domain validation. Updated proposal and module tests and module internals documentation. The isolated variable-encoding test passed with `go test internal/framework/proposal_generation/variables.go internal/framework/proposal_generation/variables_test.go`; -`git diff --check` passed. Assembled proposal/module tests are deferred because -Stage 7's validator imports still reference removed message-rendering types. -The repaired-proposal/domain integration check remains deferred with those imports. +`git diff --check` passed. After the validator imports were migrated, +`go test ./internal/framework/proposal_generation ./internal/modules/glossary +./internal/modules/homophones ./internal/modules/spoken_word ./internal/modules/grammar` +passed. A controlled HTTP integration test confirmed that repaired proposals +reach low-confidence and missing-original-text domain checks without an extra call. **Depends on:** Stage 5. **Owns:** `internal/modules`, `internal/framework/proposal_generation`, and their tests. @@ -343,7 +345,18 @@ ordering. Run the module and proposal-generation suites as imports become builda ## Stage 7: Migrate LLM Validators Without Changing Domain Decisions -**Status:** Pending. **Depends on:** Stage 6. +**Status:** Complete (2026-09-24). All five LLM validator types now select +embedded prompt IDs and shared payload variables. Kept domain batching, +preview, index/cardinality, and deterministic validation; exhausted repair +rejects/warns while provider/protocol failures propagate. Removed duplicate +message builders and per-validator model selection. Updated internal validator +documentation. `go test ./internal/framework/validators +./internal/framework/proposal_generation ./internal/modules/glossary +./internal/modules/homophones ./internal/modules/spoken_word +./internal/modules/grammar ./internal/validators/... ./internal/core/config` +passed, including controlled HTTP repair and malformed-envelope cases; +`git diff --check` passed. Full runner behavior remains for Stage 8. +**Depends on:** Stage 6. **Owns:** `internal/framework/validators`, validation prompt builders/batching, and validator tests. diff --git a/internal/core/config/config.go b/internal/core/config/config.go index bbf98a2..3fb8758 100644 --- a/internal/core/config/config.go +++ b/internal/core/config/config.go @@ -18,6 +18,7 @@ const ( WorkDirRetentionNever WorkDirRetention = "never" DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar" DefaultTranscriptDescriptionMaxChars = 500 + DefaultValidationMaxPromptTokens = 2048 ) // Config is the version-2 application configuration. Invocation paths are not settings. @@ -89,7 +90,7 @@ func defaults() Config { LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute}, Pipeline: Pipeline{Modules: []string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}}, Output: Output{Schema: "bare-segments"}, - Validation: Validation{MaxPromptTokens: 2048}, + Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens}, Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048}, Normalization: Normalization{MaxSegmentGap: 4, EllipsisGap: 3.5, MaxSegmentDuration: 60, MaxSegmentTokens: 2048}, Thresholds: Thresholds{Glossary: .8, Homophones: .8, SpokenWord: .8, Grammar: .8}, diff --git a/internal/framework/proposal_generation/adapter_integration_test.go b/internal/framework/proposal_generation/adapter_integration_test.go new file mode 100644 index 0000000..844580c --- /dev/null +++ b/internal/framework/proposal_generation/adapter_integration_test.go @@ -0,0 +1,85 @@ +package proposal_generation + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + ck "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + "gitea.maximumdirect.net/eric/audita/internal/prompts" +) + +func TestRepairedProposalStillFacesDomainChecks(t *testing.T) { + calls := 0 + outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + output := outputs[len(outputs)-1] + if calls < len(outputs) { + output = outputs[calls] + } + calls++ + content, _ := json.Marshal(output) + _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) + })) + defer server.Close() + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { + t.Fatal(err) + } + main := filepath.Join(dir, "config.yml") + content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL) + if err := os.WriteFile(main, []byte(content), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.File(main), "") + if err != nil { + t.Fatal(err) + } + runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client()) + if err != nil { + t.Fatal(err) + } + client, err := runtime.NewRoleClient("proposal", llm.CallHooks{}) + if err != nil { + t.Fatal(err) + } + transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}} + vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "") + if err != nil { + t.Fatal(err) + } + generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client}) + if err != nil { + t.Fatal(err) + } + if calls != 2 || len(generated.Enriched) != 1 { + t.Fatalf("repair path: calls=%d, result=%+v", calls, generated) + } + cfg := loaded.Config + cfg.Thresholds.Grammar = .8 + input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript} + low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input) + if err != nil { + t.Fatal(err) + } + missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 { + t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls) + } +} diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 84d5165..d449693 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -116,7 +116,7 @@ func (c *sleepingStructuredClient) CompleteStructured(ctx context.Context, req c func defaultRequest(t *testing.T) Request { t.Helper() - cfg := config.Default() + cfg := config.Config{} return Request{ ModuleKey: "test_module", ModuleInstance: "test_module", diff --git a/internal/framework/proposal_generation/module_proposal_test.go b/internal/framework/proposal_generation/module_proposal_test.go index b998c12..f9c3717 100644 --- a/internal/framework/proposal_generation/module_proposal_test.go +++ b/internal/framework/proposal_generation/module_proposal_test.go @@ -19,7 +19,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing }, } section := contracts.SectionMetadata{Index: 7} - cfg := config.Default() + cfg := config.Config{} cfg.Context.Description = "Hearing transcript with role titles." transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}} glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}} diff --git a/internal/framework/validators/llm_integration_test.go b/internal/framework/validators/llm_integration_test.go new file mode 100644 index 0000000..e90360e --- /dev/null +++ b/internal/framework/validators/llm_integration_test.go @@ -0,0 +1,96 @@ +package validators + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + ck "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { + valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}` + for _, tc := range []struct { + name string + responses []string + wantCalls int + wantApproved, wantWarning, wantRuntime bool + }{ + {"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, false, false}, + {"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, true, false}, + {"malformed envelope propagates", []string{"MALFORMED_ENVELOPE"}, 1, false, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + output := tc.responses[len(tc.responses)-1] + if calls <= len(tc.responses) { + output = tc.responses[calls-1] + } + if output == "MALFORMED_ENVELOPE" { + _, _ = fmt.Fprint(w, `{"choices":[]}`) + return + } + content, _ := json.Marshal(output) + _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) + })) + defer server.Close() + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { + t.Fatal(err) + } + main := filepath.Join(dir, "config.yml") + content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL) + if err := os.WriteFile(main, []byte(content), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.File(main), "") + if err != nil { + t.Fatal(err) + } + runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client()) + if err != nil { + t.Fatal(err) + } + client, err := runtime.NewRoleClient("validation", llm.CallHooks{}) + if err != nil { + t.Fatal(err) + } + validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.Config = &loaded.Config + req.LLMClient = client + result, err := validator.Validate(context.Background(), req) + if calls != tc.wantCalls { + t.Fatalf("calls=%d, want %d", calls, tc.wantCalls) + } + if tc.wantRuntime { + if err == nil || len(result.Warnings) != 0 { + t.Fatalf("runtime failure downgraded: %+v, %v", result, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || (len(result.Warnings) > 0) != tc.wantWarning { + t.Fatalf("domain result: %+v", result) + } + }) + } +} diff --git a/internal/framework/validators/llm_prompt_builders.go b/internal/framework/validators/llm_prompt_builders.go deleted file mode 100644 index 9bfaa0d..0000000 --- a/internal/framework/validators/llm_prompt_builders.go +++ /dev/null @@ -1,92 +0,0 @@ -package validators - -import ( - "encoding/json" - "fmt" - - "gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext" - "gitea.maximumdirect.net/eric/audita/internal/prompts" -) - -func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) { - payloadJSON, err := marshalPromptPayload(validationPayload) - if err != nil { - return nil, err - } - system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorSpokenFormPlausibility, map[string]string{ - "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), - "PayloadJSON": payloadJSON, - }) - if err != nil { - return nil, err - } - return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil -} - -func BuildMeaningReversalMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) { - payloadJSON, err := marshalPromptPayload(validationPayload) - if err != nil { - return nil, err - } - system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorMeaningReversalReview, map[string]string{ - "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), - "PayloadJSON": payloadJSON, - }) - if err != nil { - return nil, err - } - return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil -} - -func BuildEditorialMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) { - payloadJSON, err := marshalPromptPayload(validationPayload) - if err != nil { - return nil, err - } - system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorEditorialReview, map[string]string{ - "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), - "PayloadJSON": payloadJSON, - }) - if err != nil { - return nil, err - } - return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil -} - -func BuildGrammarReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) { - payloadJSON, err := marshalPromptPayload(validationPayload) - if err != nil { - return nil, err - } - system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorGrammarReview, map[string]string{ - "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), - "PayloadJSON": payloadJSON, - }) - if err != nil { - return nil, err - } - return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil -} - -func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) { - payloadJSON, err := marshalPromptPayload(validationPayload) - if err != nil { - return nil, err - } - system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorSpokenWordReview, map[string]string{ - "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), - "PayloadJSON": payloadJSON, - }) - if err != nil { - return nil, err - } - return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil -} - -func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) { - payloadJSON, err := json.MarshalIndent(validationPayload, "", " ") - if err != nil { - return "", fmt.Errorf("marshal validation payload: %w", err) - } - return string(payloadJSON), nil -} diff --git a/internal/framework/validators/llm_prompt_variables.go b/internal/framework/validators/llm_prompt_variables.go new file mode 100644 index 0000000..60f96fb --- /dev/null +++ b/internal/framework/validators/llm_prompt_variables.go @@ -0,0 +1,19 @@ +package validators + +import ( + "encoding/json" + "fmt" + + "gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext" +) + +func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) { + payloadJSON, err := json.MarshalIndent(validationPayload, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal validation payload: %w", err) + } + return map[string]string{ + "TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription), + "PayloadJSON": string(payloadJSON), + }, nil +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index c12d4de..158e178 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -17,13 +17,9 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/prompts" ) -type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) - type LLMBackedValidator struct { name string validatorType LLMValidatorType - promptBuilder LLMPromptBuilder - model string estimator chunking.TokenEstimator } @@ -39,10 +35,9 @@ func (v *LLMBackedValidator) SetTokenEstimator(estimator chunking.TokenEstimator v.estimator = estimator } -func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model string) (*LLMBackedValidator, error) { - builder, err := promptBuilderForType(validatorType) - if err != nil { - return nil, err +func NewLLMBackedValidator(name string, validatorType LLMValidatorType) (*LLMBackedValidator, error) { + if validatorPromptMetadata(validatorType).PromptID == "" { + return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType) } if strings.TrimSpace(name) == "" { return nil, fmt.Errorf("validator name must not be empty") @@ -50,8 +45,6 @@ func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model st return &LLMBackedValidator{ name: name, validatorType: validatorType, - promptBuilder: builder, - model: strings.TrimSpace(model), estimator: chunking.NewSimpleTokenEstimator(), }, nil } @@ -77,8 +70,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } maxTokens := config.DefaultValidationMaxPromptTokens - if req.Config != nil && req.Config.ValidationMaxPromptTokens > 0 { - maxTokens = req.Config.ValidationMaxPromptTokens + if req.Config != nil && req.Config.Validation.MaxPromptTokens > 0 { + maxTokens = req.Config.Validation.MaxPromptTokens } warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...) @@ -103,22 +96,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, for _, batch := range batches { transcriptDescription := "" if req.Config != nil { - transcriptDescription = req.Config.TranscriptDescription + transcriptDescription = req.Config.Context.Description } - messages, err := v.promptBuilder(batch.Items, transcriptDescription) + variables, err := BuildValidationVariables(batch.Items, transcriptDescription) if err != nil { return Result{}, err } var response LLMValidationResponse + var completion StructuredCompletionResponse responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) + promptMetadata := validatorPromptMetadata(v.validatorType) stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex) call := func(callCtx context.Context) error { - _, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{ - StageName: stage, - Messages: messages, - Model: resolvedValidationModel(req.Config, v.model), - ResponseSchema: &responseSchema, + completion, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{ + PromptID: promptMetadata.PromptID, PromptVersion: promptMetadata.PromptVersion, + Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, + Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, BatchIndex: &batch.BatchIndex, ValidatorName: v.name}, }, &response) return err } @@ -129,18 +123,17 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } artifacts := InteractionArtifacts{} if req.DiagnosticsWriter != nil { - promptMetadata := validatorPromptMetadata(v.validatorType) artifacts, _ = req.DiagnosticsWriter.WriteInteraction( stage, map[string]any{ "validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex, - "prompt_metadata": promptMetadata.DiagnosticsMap(), + "prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(), }, - map[string]any{"messages": messages, "items": batch.Items}, - response, + completion.Prepared, + map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, errPayload(err), ) } @@ -199,16 +192,6 @@ func errPayload(err error) any { return map[string]any{"error": err.Error()} } -func resolvedValidationModel(cfg *config.Config, override string) string { - if strings.TrimSpace(override) != "" { - return strings.TrimSpace(override) - } - if cfg == nil { - return "" - } - return cfg.EffectiveValidationLLMConfig().Model -} - func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) { items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) immediate := make([]Decision, 0) @@ -255,23 +238,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT }, immediate } -func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, error) { - switch validatorType { - case LLMValidatorTypeSpokenFormPlausibility: - return BuildSpokenFormPlausibilityMessages, nil - case LLMValidatorTypeMeaningReversal: - return BuildMeaningReversalMessages, nil - case LLMValidatorTypeEditorialReview: - return BuildEditorialMessages, nil - case LLMValidatorTypeGrammarReview: - return BuildGrammarReviewMessages, nil - case LLMValidatorTypeSpokenWordReview: - return BuildSpokenWordReviewMessages, nil - default: - return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType) - } -} - func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) { expected := make(map[int]LLMValidationItem, len(items)) for _, item := range items { diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 3c7b449..50bf112 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -15,6 +15,8 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" + "gitea.maximumdirect.net/eric/audita/internal/prompts" ) type fakeStructuredLLMClient struct { @@ -109,8 +111,8 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req St } func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request { - cfg := config.Default() - cfg.ValidationMaxPromptTokens = 10000 + cfg := config.Config{} + cfg.Validation.MaxPromptTokens = 10000 return Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}}, CandidateProposal: candidates, @@ -163,66 +165,12 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) { } } -func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) { - payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}} - tests := []struct { - name string - build func([]LLMValidationItem, string) ([]LLMMessage, error) - mustHas []string - }{ - {"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}}, - {"meaning_reversal", BuildMeaningReversalMessages, []string{"meaning reversals", "correction_index", "original_segment_text", "corrected_segment_text"}}, - {"editorial", BuildEditorialMessages, []string{"acceptable editorial revision", "correction_index", "original_segment_text", "corrected_segment_text"}}, - {"grammar_review", BuildGrammarReviewMessages, []string{"acceptable editorial revision", "correction_index"}}, - {"spoken_word", BuildSpokenWordReviewMessages, []string{"acceptable editorial revision", "correction_index"}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - msgs, err := tt.build(payload, "Discussion among party members in a dungeon.") - if err != nil { - t.Fatalf("build err: %v", err) - } - if len(msgs) != 2 { - t.Fatalf("expected 2 messages, got %d", len(msgs)) - } - combined := msgs[0].Content + "\n" + msgs[1].Content - for _, needle := range tt.mustHas { - if !strings.Contains(combined, needle) { - t.Fatalf("expected prompt to contain %q", needle) - } - } - for _, needle := range []string{ - "Transcript description (background context only):", - "must not override the transcript content", - "Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.", - } { - if !strings.Contains(combined, needle) { - t.Fatalf("expected prompt to contain %q", needle) - } - } - }) - } -} - -func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) { - payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}} - msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ") - if err != nil { - t.Fatalf("build err: %v", err) - } - combined := msgs[0].Content + "\n" + msgs[1].Content - if strings.Contains(combined, "Transcript description (background context only):") { - t.Fatalf("did not expect empty transcript description section in prompt") - } -} - func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"}, }}}} - v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("new validator error: %v", err) } @@ -238,19 +186,15 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { if len(client.calls) != 1 { t.Fatalf("expected one LLM call, got %d", len(client.calls)) } - if client.calls[0].ResponseSchema == nil { - t.Fatalf("expected response schema on structured validation request") - } - want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) - gotSchema := client.calls[0].ResponseSchema - if gotSchema.ID != want.ID || gotSchema.Version != want.Version || gotSchema.Name != want.Name || gotSchema.SHA256 != want.SHA256 { - t.Fatalf("unexpected validator response schema metadata: got=%+v want=%+v", *gotSchema, want) + call := client.calls[0] + if call.Schema != responseschema.ValidatorDecisionSetKey || call.PromptID != prompts.PromptIDValidatorSpokenFormPlausibility || call.Context.Role != "validation" || call.Context.ValidatorName != v.Name() || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") { + t.Fatalf("unexpected validator call: %+v", call) } } func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client res, err := v.Validate(context.Background(), req) @@ -265,26 +209,20 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { } } -func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") +func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) { + client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - res, err := v.Validate(context.Background(), req) - if err != nil { - t.Fatalf("expected malformed provider envelope downgrade, got %v", err) - } - if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { - t.Fatalf("unexpected decisions: %+v", res.Decisions) - } - if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed { - t.Fatalf("expected malformed warning, got %+v", res.Warnings) + result, err := v.Validate(context.Background(), req) + if err == nil || len(result.Warnings) != 0 { + t.Fatalf("protocol error was downgraded: %+v, %v", result, err) } } func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client res, err := v.Validate(context.Background(), req) @@ -301,7 +239,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, }}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client res, err := v.Validate(context.Background(), req) @@ -315,7 +253,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client res, err := v.Validate(context.Background(), req) @@ -331,7 +269,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, }}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) huge := strings.Repeat("gestures ", 200) req := Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{ @@ -353,8 +291,8 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, } req.LLMClient = client - cfg := config.Default() - cfg.ValidationMaxPromptTokens = 200 + cfg := config.Config{} + cfg.Validation.MaxPromptTokens = 200 req.Config = &cfg res, err := v.Validate(context.Background(), req) @@ -381,7 +319,7 @@ func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) { entered: make(chan struct{}, 16), release: make(chan struct{}), } - v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("new validator error: %v", err) } @@ -416,7 +354,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, }}}} writer := &captureValidationDiagnosticsWriter{} - v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("new validator error: %v", err) } @@ -439,19 +377,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 { t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want) } - promptMap, ok := metadata["prompt_metadata"].(map[string]any) - if !ok { - t.Fatalf("expected prompt_metadata map, got %T", metadata["prompt_metadata"]) - } - if promptMap["prompt_id"] != "validators.spoken_form_plausibility" || promptMap["prompt_version"] != "v1" || promptMap["prompt_source"] != "builtin" { - t.Fatalf("unexpected prompt metadata: %v", promptMap) - } - if _, ok := promptMap["embedded_path"].(string); !ok { - t.Fatalf("expected embedded_path in prompt metadata: %v", promptMap) - } - if _, ok := promptMap["sha256"].(string); !ok { - t.Fatalf("expected sha256 in prompt metadata: %v", promptMap) - } + } func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) { @@ -470,3 +396,38 @@ func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) { t.Fatalf("timed out waiting for %d validator calls to enter (got %d)", want, got) } } + +func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { + cases := []struct { + kind LLMValidatorType + prompt string + }{ + {LLMValidatorTypeSpokenFormPlausibility, prompts.PromptIDValidatorSpokenFormPlausibility}, + {LLMValidatorTypeMeaningReversal, prompts.PromptIDValidatorMeaningReversalReview}, + {LLMValidatorTypeEditorialReview, prompts.PromptIDValidatorEditorialReview}, + {LLMValidatorTypeGrammarReview, prompts.PromptIDValidatorGrammarReview}, + {LLMValidatorTypeSpokenWordReview, prompts.PromptIDValidatorSpokenWordReview}, + } + for _, tc := range cases { + t.Run(string(tc.kind), func(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}} + validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind) + if err != nil { + t.Fatal(err) + } + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + req.Config.Context.Description = " Fantasy scene " + if _, err := validator.Validate(context.Background(), req); err != nil { + t.Fatal(err) + } + if len(client.calls) != 1 { + t.Fatalf("calls=%d", len(client.calls)) + } + call := client.calls[0] + if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") { + t.Fatalf("mapping: %+v", call) + } + }) + } +} diff --git a/internal/framework/validators/validators_test.go b/internal/framework/validators/validators_test.go index 0600cc3..a269caf 100644 --- a/internal/framework/validators/validators_test.go +++ b/internal/framework/validators/validators_test.go @@ -50,7 +50,7 @@ func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) { } func TestConfidenceThresholdValidator(t *testing.T) { - cfg := config.Default() + cfg := config.Config{} cfg.Thresholds.Grammar = 0.8 req := Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: []proposals.EnrichedCorrectionProposal{ mkCandidate(0, 1, "teh", "the", 0.9), diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 362aa8f..883d622 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -186,7 +186,7 @@ func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) { t.Fatal("expected confidence_threshold validator") } - cfg := config.Default() + cfg := config.Config{} cfg.Thresholds.Homophones = 0.95 result, err := thresholdValidator.Validate(context.Background(), validators.Request{ Config: &cfg, diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index 632ca85..a1390c6 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -186,7 +186,7 @@ func TestSpokenWordConfidenceThresholdIsUsed(t *testing.T) { t.Fatal("expected confidence_threshold validator") } - cfg := config.Default() + cfg := config.Config{} cfg.Thresholds.SpokenWord = 0.95 result, err := thresholdValidator.Validate(context.Background(), validators.Request{ Config: &cfg, diff --git a/internal/validators/editorial_review/validator.go b/internal/validators/editorial_review/validator.go index 19deef6..f849893 100644 --- a/internal/validators/editorial_review/validator.go +++ b/internal/validators/editorial_review/validator.go @@ -9,7 +9,7 @@ import ( const Key = "editorial_review" func New() (contracts.Validator, error) { - v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview, "") + v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview) if err != nil { return nil, err } diff --git a/internal/validators/meaning_reversal_review/validator.go b/internal/validators/meaning_reversal_review/validator.go index 4abcd4c..c092f41 100644 --- a/internal/validators/meaning_reversal_review/validator.go +++ b/internal/validators/meaning_reversal_review/validator.go @@ -9,7 +9,7 @@ import ( const Key = "meaning_reversal_review" func New() (contracts.Validator, error) { - v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal, "") + v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal) if err != nil { return nil, err } diff --git a/internal/validators/spoken_form_plausibility/validator.go b/internal/validators/spoken_form_plausibility/validator.go index 1b042bd..f98ad60 100644 --- a/internal/validators/spoken_form_plausibility/validator.go +++ b/internal/validators/spoken_form_plausibility/validator.go @@ -9,7 +9,7 @@ import ( const Key = "spoken_form_plausibility" func New() (contracts.Validator, error) { - v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility, "") + v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { return nil, err }