From 12202508bfd845fbb3e3dec00f2e9c20982a5a67 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 12 May 2026 01:25:52 +0000 Subject: [PATCH] Complete Phase 10 LLM validators --- docs/architecture.md | 46 +++- docs/rewrite-notes.md | 57 ++-- internal/cli/run.go | 26 +- internal/cli/run_test.go | 74 ++++- internal/core/reporting/report.go | 11 +- internal/core/reporting/report_test.go | 4 +- internal/framework/runner/runner.go | 111 +++++++- internal/framework/runner/runner_test.go | 246 +++++++++++++++++ internal/framework/validators/llm_batching.go | 78 ++++++ internal/framework/validators/llm_models.go | 71 +++++ .../validators/llm_prompt_builders.go | 90 ++++++ .../framework/validators/llm_validators.go | 260 ++++++++++++++++++ .../validators/llm_validators_test.go | 193 +++++++++++++ internal/framework/validators/models.go | 28 +- 14 files changed, 1213 insertions(+), 82 deletions(-) create mode 100644 internal/framework/validators/llm_batching.go create mode 100644 internal/framework/validators/llm_models.go create mode 100644 internal/framework/validators/llm_prompt_builders.go create mode 100644 internal/framework/validators/llm_validators.go create mode 100644 internal/framework/validators/llm_validators_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 537a7fa..c25e9be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,18 +26,20 @@ Implemented today: - Bounded LLM scheduler/semaphore infrastructure with context-aware permit handling. - Runtime primary/validation LLM effective-config resolution helpers with validation inheritance. - Generic JSON prompt/response diagnostics writer primitives with secret redaction. +- LLM-backed validator models, prompt builders, batching, and runtime execution. +- Runner wiring for LLM validators via the internal structured LLM abstraction and scheduler hooks. +- LLM validator diagnostics artifacts and report-level decision metadata paths. Not implemented in CLI runtime path today: - Real module execution pipeline (`glossary`, `homophones`, `spoken_word`, `grammar`). - Structured LLM proposal generation. -- LLM-backed validators. -- Runtime module/validator usage of the LLM scheduler infrastructure. +- Shared module proposal-generation framework and module registry for real modules. - End-to-end transcript polishing with real module behavior. Phase sequencing note: - Phase 9 LLM infrastructure is complete (structured client, scheduler, effective config resolution, diagnostics primitives); -- runtime wiring from modules/validators/runner into this LLM infrastructure remains future module/validator phase work; -- LLM-backed validators remain Phase 10 work. +- Phase 10 LLM-backed validator runtime integration is complete; +- shared module proposal-generation and module registry work remain Phase 11. ## Actual Go package layout @@ -93,6 +95,10 @@ internal/framework/runner/ internal/framework/validators/ models.go deterministic.go + llm_models.go + llm_prompt_builders.go + llm_batching.go + llm_validators.go internal/framework/llm/ instructor_client.go @@ -122,14 +128,14 @@ Current runtime flow (`internal/cli/run.go`): 11. Write chunking summary artifact. 12. Optionally execute runner modules sequentially when an injected module registry/factory is available (used by deterministic tests today). 13. Output working transcript to `--output` file or stdout. -14. Build process report (`phase` currently set to `phase8-validators`). +14. Build process report (`phase` currently set to `phase10-llm-validators`). 15. Optionally write `--report-json`; always write run-dir `report.json`. 16. Apply work-dir retention. Important behavior details: - Glossary is validated but not yet used for real correction module logic. - Default production CLI behavior remains deterministic normalization/chunking output because no real module implementations are registered yet. -- No LLM calls occur. +- No real LLM calls occur in the default production runtime path because no real modules are registered yet. - Success path is generally quiet on stderr. - Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`. @@ -193,8 +199,8 @@ Current caveat: - API-key redaction in adapter-returned errors. Current runtime boundary: -- the CLI/runner runtime path does not instantiate this adapter yet; -- no production LLM requests are performed by `audita process`. +- the default CLI runtime path still does not instantiate real production modules, so no default end-to-end LLM polishing occurs. +- LLM calls are exercised only when test/injected modules and validators are provided. `internal/framework/llm` also provides: - a bounded `Scheduler` for controlled concurrent LLM calls with reliable permit release; @@ -256,6 +262,24 @@ These primitives are wired into the production runner and report model. Real mod `internal/framework/runner` executes validator chains in order for each module and applies only validator-approved proposals. Validator rejections are reported distinctly from proposal-application skips. +## Implemented LLM-backed validator infrastructure +`internal/framework/validators` now includes LLM-backed validator support: +- typed request/response models for structured LLM validation; +- prompt builders for: + - spoken-form plausibility + - meaning reversal detection + - editorial review + - grammar review + - spoken-word review +- deterministic batching by `validation_max_prompt_tokens`; +- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail); +- safe failure behavior for malformed/invalid structured responses. + +`internal/framework/runner` wires LLM validators into existing validator chains using: +- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`); +- bounded scheduler hooks for validator call execution; +- diagnostics writer hooks for machine-readable prompt/response artifacts with secret redaction. + ## Reports and diagnostics (implemented) Current per-run artifacts include: - `source-transcript.json` @@ -285,6 +309,7 @@ Current process reports also include: - module-level results (when runner modules execute), including applied/skipped proposal changes; - run-level module summary totals and failed module instance metadata. - module-level validator decisions and validator rejections. +- optional decision-level diagnostic artifact paths for validator LLM interactions when available. Retention modes implemented in `ApplyRetention`: - `always`: keep all run directories. @@ -296,7 +321,7 @@ Current runtime note: - real module execution is not implemented yet, so normal successful runs generally have no skipped corrections and `auto` typically removes clean successful run directories. Intentionally deferred to module/LLM phases: -- module prompt/response diagnostics artifacts are not produced yet because module execution and LLM calls are not in the runtime path. +- module proposal-generation prompt/response diagnostics remain tied to later real-module phases. ## Current tests and quality posture Implemented tests currently cover: @@ -311,8 +336,9 @@ Implemented tests currently cover: - runner sequencing and failure behavior with deterministic fake modules (`internal/framework/runner/*_test.go`) - CLI runner integration through injected fake module factories (`internal/cli/run_test.go`) - validator models, cardinality enforcement, and deterministic validators (`internal/framework/validators/*_test.go`) +- LLM-backed validator batching, prompt builders, structured-response safety, scheduler hooks, and diagnostics redaction (`internal/framework/validators/*_test.go`, `internal/framework/runner/*_test.go`) -Not covered yet (because not implemented): real LLM validator/runtime integration and production module behavior. +Not covered yet (because not implemented): shared module proposal generation, real module implementations, and full transcript-polishing runtime behavior. ## Intended final architecture (not yet implemented) The intended end-state still matches the rewrite plan: diff --git a/docs/rewrite-notes.md b/docs/rewrite-notes.md index 18049e9..b115847 100644 --- a/docs/rewrite-notes.md +++ b/docs/rewrite-notes.md @@ -51,6 +51,10 @@ Implemented: - Validator cardinality enforcement (missing/duplicate/unknown proposal index errors). - Deterministic validator-chain execution in the production runner. - Module reports including validator decisions and validator rejections. +- LLM-backed validator request/response models and prompt builders. +- LLM validator batching by validation prompt-token budget. +- LLM validator runtime integration through structured LLM client abstraction and scheduler hooks. +- LLM validator prompt/response diagnostics artifact wiring with secret redaction. - Broad deterministic and CLI/subprocess test coverage for implemented phases through `go test ./...`. - Internal typed structured LLM contract (`StructuredLLMClient.CompleteStructured(ctx, req, out)`). - `internal/framework/llm` instructor-go-backed adapter with: @@ -67,9 +71,7 @@ Implemented: Not yet implemented in runtime pipeline: - Real correction modules. -- Runtime wiring from production runner/modules into the structured LLM adapter. -- LLM-backed validators. -- Module/validator call-site wiring to emit LLM prompt/response diagnostics artifacts. +- Shared module proposal generation and module registry wiring. - End-to-end transcript polishing behavior. ## Completed phases @@ -208,7 +210,7 @@ Not implemented in Phase 8 (by design): ## Remaining work plan -Next recommended phase: **Phase 10 (LLM-backed validators)**. +Next recommended phase: **Phase 11 (shared LLM proposal generation framework and module registry)**. ## Phase 9: Structured LLM client and scheduler infrastructure @@ -275,45 +277,30 @@ Met: ## Phase 10: LLM-backed validators -### Purpose +Completed. -Implement the LLM-backed validator layer used by the Python implementation, and wire it into the runtime validator framework. - -### Scope - -Implement: -- LLM validator request and response models. -- Shared batching logic for validation prompts using validation token limits. -- Prompt builders for LLM validators. -- LLM-backed validation categories needed for parity, such as: +Implemented: +- LLM-backed validator request/response models in `internal/framework/validators`. +- Prompt builders for: - spoken-form plausibility - meaning reversal detection - editorial review - grammar review - spoken-word review -- Validator prompt/response diagnostics. -- Validation LLM scheduler usage. -- Validator error handling and report integration. -- Fake LLM tests for approval, rejection, malformed output, missing decision, duplicate decision, and retry cases. +- Deterministic batching by `validation_max_prompt_tokens` with stable ordering and no drop/dup behavior. +- LLM validator execution through the internal structured client abstraction (no direct provider calls in validator code). +- Scheduler/concurrency hooks for LLM validator calls. +- Prompt/response diagnostics artifact writing for LLM validator batches using Phase 9 diagnostics primitives. +- Secret redaction in validator LLM diagnostics artifacts. +- Strict structured-response safety and cardinality checks (missing/duplicate/unknown indexes fail closed). +- Runner/report integration so LLM validator decisions and rejections appear in module reports. +- Fake-module and fake-client tests for approval/rejection, malformed output, cardinality errors, batching, scheduler usage, and diagnostics redaction. -Do not implement: -- Real correction modules, except for minimal fake/test modules needed to exercise validators. -- Full default pipeline behavior. +Not implemented in Phase 10 (by design): +- Real correction modules (`glossary`, `homophones`, `spoken_word`, `grammar`). +- Shared module proposal generation and module registry work (Phase 11). - Domain proposal prompts. - -### Expected behavior at end of phase - -The runner can execute a mixed deterministic + LLM validator chain against proposals produced by fake modules. LLM validators use the structured LLM client and write diagnostics. - -### Definition of done - -- LLM-backed validators are implemented. -- Validator batching respects configured token limits. -- Validator cardinality rules are enforced for LLM validator output. -- Prompt/response diagnostics are written for LLM validator calls. -- Validator results appear in module reports. -- Fake LLM tests cover success, rejection, malformed output, and retry behavior. -- `go test ./...` passes. +- Default CLI end-to-end transcript polishing behavior. ## Phase 11: Shared LLM proposal generation framework and module registry diff --git a/internal/cli/run.go b/internal/cli/run.go index 9b59ef4..912e0bc 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -30,6 +30,8 @@ type processInvocation struct { } var processModuleFactory runner.ModuleFactory +var processValidationLLMClient contracts.StructuredLLMClient +var processValidationLLMScheduler runner.ValidationScheduler var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) { runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention)) @@ -130,10 +132,13 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio } runnerResult, runErr := runner.New(processModuleFactory).Run(context.Background(), runner.RunInput{ - Config: &inv.Config, - Transcript: normalizedTranscript, - Glossary: glossary, - ModuleSpecs: moduleSpecs, + Config: &inv.Config, + Transcript: normalizedTranscript, + Glossary: glossary, + ModuleSpecs: moduleSpecs, + ValidationLLMClient: processValidationLLMClient, + ValidationLLMScheduler: processValidationLLMScheduler, + ValidationDiagnosticsDir: runDir.Path(), }) runOutput = &runnerResult if runErr != nil { @@ -395,7 +400,7 @@ func extractErrorPhase(err error) (phase string, message string) { func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport { report := reporting.ProcessReport{ - Phase: "phase8-validators", + Phase: "phase10-llm-validators", Status: status, Operation: "process", TranscriptPath: inv.TranscriptPath, @@ -491,11 +496,12 @@ func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.Vali out := make([]reporting.ValidatorDecisionReport, len(in)) for i, d := range in { out[i] = reporting.ValidatorDecisionReport{ - ValidatorName: d.ValidatorName, - ProposalIndex: d.ProposalIndex, - Approved: d.Approved, - ReasonCode: d.ReasonCode, - Message: d.Message, + ValidatorName: d.ValidatorName, + ProposalIndex: d.ProposalIndex, + Approved: d.Approved, + ReasonCode: d.ReasonCode, + Message: d.Message, + DiagnosticArtifactPath: d.DiagnosticArtifactPath, } } return out diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 1ffc310..1e1f744 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -614,8 +614,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) { if report.Chunking.MaxSectionTokens == 0 { t.Errorf("expected max_section_tokens in report") } - if report.Phase != "phase8-validators" { - t.Errorf("expected phase 'phase8-validators', got %q", report.Phase) + if report.Phase != "phase10-llm-validators" { + t.Errorf("expected phase 'phase10-llm-validators', got %q", report.Phase) } } @@ -659,6 +659,26 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq return v.validateF(req) } +type fakeStructuredLLMClient struct { + responses []validators.LLMValidationResponse + err error +} + +func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = ctx + _ = req + if f.err != nil { + return contracts.StructuredCompletionResponse{}, f.err + } + if len(f.responses) == 0 { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm call") + } + target := out.(*validators.LLMValidationResponse) + *target = f.responses[0] + f.responses = f.responses[1:] + return contracts.StructuredCompletionResponse{}, nil +} + func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) { allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) @@ -744,6 +764,56 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) } } +func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) { + llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + if err != nil { + t.Fatalf("NewLLMBackedValidator: %v", err) + } + processValidationLLMClient = &fakeStructuredLLMClient{ + responses: []validators.LLMValidationResponse{{ + Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject"}}, + }}, + } + processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil + }}, + }} + t.Cleanup(func() { + processModuleFactory = nil + processValidationLLMClient = nil + processValidationLLMScheduler = nil + }) + + var stdout, stderr bytes.Buffer + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + outputPath := filepath.Join(t.TempDir(), "out.json") + transcriptPath := writeFile(t, "transcript.json", `[ + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"} +]`) + exitCode := Run([]string{ + "process", transcriptPath, + "--glossary", fixturePath("tiny_glossary.yaml"), + "--modules", "m", + "--output", outputPath, + "--report-json", reportPath, + "--work-dir", workDir, + "--work-dir-retention", "always", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) + } + report := readProcessReport(t, reportPath) + if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].ValidatorDecisions) == 0 { + t.Fatalf("expected llm validator decisions in external report") + } + runReport := readProcessReport(t, filepath.Join(onlyRunDir(t, workDir), "report.json")) + if len(runReport.ModuleResults) != 1 || len(runReport.ModuleResults[0].ValidatorRejected) != 1 { + t.Fatalf("expected llm validator rejection in run report") + } +} + func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) { processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { diff --git a/internal/core/reporting/report.go b/internal/core/reporting/report.go index e86aea1..562ac47 100644 --- a/internal/core/reporting/report.go +++ b/internal/core/reporting/report.go @@ -48,11 +48,12 @@ type ModuleReport struct { } type ValidatorDecisionReport struct { - ValidatorName string `json:"validator_name"` - ProposalIndex int `json:"proposal_index"` - Approved bool `json:"approved"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` + ValidatorName string `json:"validator_name"` + ProposalIndex int `json:"proposal_index"` + Approved bool `json:"approved"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } type ValidatorRejectedReport struct { diff --git a/internal/core/reporting/report_test.go b/internal/core/reporting/report_test.go index 051c017..5c0f760 100644 --- a/internal/core/reporting/report_test.go +++ b/internal/core/reporting/report_test.go @@ -11,7 +11,7 @@ import ( func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) { now := time.Now().UTC() report := ProcessReport{ - Phase: "phase8-validators", + Phase: "phase10-llm-validators", Status: "success", ModuleResults: []ModuleReport{ { @@ -80,7 +80,7 @@ func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) { func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) { now := time.Now().UTC() report := ProcessReport{ - Phase: "phase8-validators", + Phase: "phase10-llm-validators", Status: "failed", ModuleResults: []ModuleReport{ { diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 59731d9..2e44d98 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -3,11 +3,13 @@ package runner import ( "context" "fmt" + "path/filepath" "time" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) @@ -27,6 +29,10 @@ type Runner struct { factory ModuleFactory } +type ValidationScheduler interface { + Run(ctx context.Context, fn func(context.Context) error) error +} + // ModuleResult captures deterministic per-module execution output. type ModuleResult struct { ModuleKey string `json:"module_key"` @@ -44,11 +50,12 @@ type ModuleResult struct { } type ValidatorDecisionRecord struct { - ValidatorName string `json:"validator_name"` - ProposalIndex int `json:"proposal_index"` - Approved bool `json:"approved"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` + ValidatorName string `json:"validator_name"` + ProposalIndex int `json:"proposal_index"` + Approved bool `json:"approved"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } type ValidatorRejectedChange struct { @@ -65,10 +72,13 @@ type ValidatorRejectedChange struct { // RunInput is the deterministic runner input. type RunInput struct { - Config *config.Config - Transcript *schema.Transcript - Glossary *schema.Glossary - ModuleSpecs []contracts.ModuleRunSpec + Config *config.Config + Transcript *schema.Transcript + Glossary *schema.Glossary + ModuleSpecs []contracts.ModuleRunSpec + ValidationLLMClient contracts.StructuredLLMClient + ValidationLLMScheduler ValidationScheduler + ValidationDiagnosticsDir string } // RunOutput is the deterministic runner output. @@ -148,6 +158,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { validatorRejected := make([]ValidatorRejectedChange, 0) eligible := enriched for _, validator := range module.Validators() { + var diagnosticsWriter validators.InteractionDiagnosticsWriter + if input.ValidationDiagnosticsDir != "" { + diagnosticsWriter = &llmDiagnosticsWriterAdapter{ + writer: llm.NewDiagnosticsWriter( + filepath.Join(input.ValidationDiagnosticsDir, spec.InstanceName), + validatorSecrets(input.Config), + ), + } + } + vResult, vErr := validator.Validate(ctx, contracts.ValidationRequest{ WorkingTranscript: working, CandidateProposal: eligible, @@ -156,6 +176,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { ReplacementPolicy: policy, Glossary: input.Glossary, Config: input.Config, + LLMClient: validationLLMClientAdapter{client: input.ValidationLLMClient}, + Scheduler: input.ValidationLLMScheduler, + DiagnosticsWriter: diagnosticsWriter, }) if vErr != nil { failed := ModuleResult{ @@ -197,11 +220,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } for _, d := range vResult.Decisions { validatorDecisions = append(validatorDecisions, ValidatorDecisionRecord{ - ValidatorName: validator.Name(), - ProposalIndex: d.ProposalIndex, - Approved: d.Approved, - ReasonCode: d.ReasonCode, - Message: d.Message, + ValidatorName: validator.Name(), + ProposalIndex: d.ProposalIndex, + Approved: d.Approved, + ReasonCode: d.ReasonCode, + Message: d.Message, + DiagnosticArtifactPath: d.DiagnosticArtifactPath, }) if d.Approved { nextEligible = append(nextEligible, byIndex[d.ProposalIndex]) @@ -265,3 +289,62 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript { } return &schema.Transcript{Segments: segments} } + +type validationLLMClientAdapter struct { + client contracts.StructuredLLMClient +} + +func (a validationLLMClientAdapter) CompleteStructured(ctx context.Context, req validators.StructuredCompletionRequest, out any) (validators.StructuredCompletionResponse, error) { + if a.client == nil { + return validators.StructuredCompletionResponse{}, fmt.Errorf("validation structured LLM client is not configured") + } + messages := make([]contracts.LLMMessage, len(req.Messages)) + for i, m := range req.Messages { + messages[i] = contracts.LLMMessage{Role: m.Role, Content: m.Content} + } + resp, err := a.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: req.StageName, + Messages: messages, + Model: req.Model, + }, out) + if err != nil { + return validators.StructuredCompletionResponse{}, err + } + return validators.StructuredCompletionResponse{ + PromptTokens: resp.PromptTokens, + CompletionTokens: resp.CompletionTokens, + TotalTokens: resp.TotalTokens, + }, nil +} + +type llmDiagnosticsWriterAdapter struct { + writer *llm.DiagnosticsWriter +} + +func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { + if a == nil || a.writer == nil { + return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured") + } + art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) + if err != nil { + return validators.InteractionArtifacts{}, err + } + return validators.InteractionArtifacts{ + RequestMetadataPath: art.RequestMetadataPath, + RequestPayloadPath: art.RequestPayloadPath, + ResponsePayloadPath: art.ResponsePayloadPath, + ErrorPayloadPath: art.ErrorPayloadPath, + }, nil +} + +func validatorSecrets(cfg *config.Config) []string { + if cfg == nil { + return nil + } + effective := cfg.EffectiveValidationLLMConfig() + return []string{ + cfg.PrimaryLLM.APIKey, + effective.APIKey, + cfg.ValidationLLM.APIKey, + } +} diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 3cc0488..7f289a3 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -3,11 +3,14 @@ package runner import ( "context" "errors" + "os" + "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) @@ -307,3 +310,246 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { } func ptrConfig(c config.Config) *config.Config { return &c } + +type fakeStructuredClient struct { + responses []validators.LLMValidationResponse + err error + calls int +} + +func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = ctx + _ = req + f.calls++ + if f.err != nil { + return contracts.StructuredCompletionResponse{}, f.err + } + if len(f.responses) == 0 { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") + } + resp := f.responses[0] + f.responses = f.responses[1:] + target := out.(*validators.LLMValidationResponse) + *target = resp + return contracts.StructuredCompletionResponse{}, nil +} + +type countingScheduler struct{ runs int } + +func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error { + s.runs++ + return fn(ctx) +} + +type lenEstimator struct{} + +func (lenEstimator) EstimateTokens(text string) int { return len(text) } + +func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}} + llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + if err != nil { + t.Fatalf("NewLLMBackedValidator: %v", err) + } + + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil + }}, + }}) + cfg := config.Default() + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + }) + if err != nil { + t.Fatalf("Run error: %v", err) + } + if out.FinalTranscript.Segments[0].Text != "There were Jesters at the temple." { + t.Fatalf("expected applied LLM-approved proposal, got %q", out.FinalTranscript.Segments[0].Text) + } +} + +func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil + }}, + }}) + cfg := config.Default() + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + }) + if err != nil { + t.Fatalf("Run error: %v", err) + } + if out.FinalTranscript.Segments[0].Text != "There were gestures at the temple." { + t.Fatalf("expected rejected proposal not applied") + } + if len(out.ModuleResults[0].ValidatorRejected) != 1 { + t.Fatalf("expected validator rejection record") + } +} + +func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil + }}, + "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil + }}, + }}) + cfg := config.Default() + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, + ValidationLLMClient: client, + }) + if err == nil { + t.Fatal("expected llm validator failure") + } + if out.FinalTranscript.Segments[0].Text != "the cat" { + t.Fatalf("expected partial progress retained") + } +} + +func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil + }}, + }}) + cfg := config.Default() + _, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + }) + if err == nil { + t.Fatal("expected missing decision failure") + } +} + +func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ + {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, + }}}} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil + }}, + }}) + cfg := config.Default() + _, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + }) + if err == nil { + t.Fatal("expected duplicate decision failure") + } +} + +func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, + }} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + llmValidator.SetTokenEstimator(lenEstimator{}) + scheduler := &countingScheduler{} + cfg := config.Default() + cfg.ValidationMaxPromptTokens = 260 + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1}, + {TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1}, + }, nil + }}, + }}) + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + ValidationLLMScheduler: scheduler, + }) + if err != nil { + t.Fatalf("unexpected run error: %v", err) + } + if scheduler.runs < 2 { + t.Fatalf("expected scheduler to run per batch, got %d", scheduler.runs) + } + if client.calls < 2 { + t.Fatalf("expected multiple llm calls for batching, got %d", client.calls) + } + if len(out.ModuleResults[0].AppliedChanges) != 2 { + t.Fatalf("expected both approved proposals applied") + } +} + +func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { + secret := "super-secret-key" + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}} + llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + cfg := config.Default() + cfg.PrimaryLLM.APIKey = secret + cfg.ValidationLLM.APIKey = secret + diagDir := t.TempDir() + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil + }}, + }}) + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ValidationLLMClient: client, + ValidationDiagnosticsDir: diagDir, + }) + if err != nil { + t.Fatalf("Run error: %v", err) + } + if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" { + t.Fatalf("expected diagnostic artifact path on decision") + } + raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath) + if readErr != nil { + t.Fatalf("read diagnostic: %v", readErr) + } + if strings.Contains(string(raw), secret) { + t.Fatalf("secret leaked in diagnostics: %s", string(raw)) + } + if !strings.Contains(string(raw), "[REDACTED]") { + t.Fatalf("expected redaction marker in diagnostics") + } +} + +func TestRunnerAcceptsLLMSchedulerType(t *testing.T) { + s, err := llm.NewScheduler(1) + if err != nil { + t.Fatalf("NewScheduler: %v", err) + } + if s == nil { + t.Fatal("expected scheduler instance") + } +} diff --git a/internal/framework/validators/llm_batching.go b/internal/framework/validators/llm_batching.go new file mode 100644 index 0000000..f5cfa9b --- /dev/null +++ b/internal/framework/validators/llm_batching.go @@ -0,0 +1,78 @@ +package validators + +import ( + "encoding/json" + "fmt" + + "gitea.maximumdirect.net/eric/audita/internal/core/chunking" +) + +type LLMValidationBatch struct { + BatchIndex int `json:"batch_index"` + Items []LLMValidationItem `json:"items"` + TokenCount int `json:"token_count"` +} + +func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) { + if maxPromptTokens <= 0 { + return nil, fmt.Errorf("validation max prompt tokens must be greater than zero") + } + if len(items) == 0 { + return nil, fmt.Errorf("validation input must contain at least one proposal") + } + if estimator == nil { + estimator = chunking.NewSimpleTokenEstimator() + } + + batches := make([]LLMValidationBatch, 0) + current := make([]LLMValidationItem, 0) + currentTokens := 0 + + for _, item := range items { + singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) + if err != nil { + return nil, err + } + if singleTokens > maxPromptTokens { + return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens") + } + + candidate := append(append([]LLMValidationItem(nil), current...), item) + candidateTokens, err := estimateBatchTokens(estimator, candidate) + if err != nil { + return nil, err + } + + if len(current) > 0 && candidateTokens > maxPromptTokens { + batches = append(batches, LLMValidationBatch{ + BatchIndex: len(batches), + Items: append([]LLMValidationItem(nil), current...), + TokenCount: currentTokens, + }) + current = []LLMValidationItem{item} + currentTokens = singleTokens + continue + } + + current = candidate + currentTokens = candidateTokens + } + + if len(current) > 0 { + batches = append(batches, LLMValidationBatch{ + BatchIndex: len(batches), + Items: append([]LLMValidationItem(nil), current...), + TokenCount: currentTokens, + }) + } + + return batches, nil +} + +func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) { + payload, err := json.Marshal(batch) + if err != nil { + return 0, fmt.Errorf("marshal batch payload: %w", err) + } + return estimator.EstimateTokens(string(payload)), nil +} diff --git a/internal/framework/validators/llm_models.go b/internal/framework/validators/llm_models.go new file mode 100644 index 0000000..29ba230 --- /dev/null +++ b/internal/framework/validators/llm_models.go @@ -0,0 +1,71 @@ +package validators + +import ( + "context" + + "gitea.maximumdirect.net/eric/audita/internal/core/schema" +) + +type LLMValidatorType string + +const ( + LLMValidatorTypeSpokenFormPlausibility LLMValidatorType = "spoken_form_plausibility" + LLMValidatorTypeMeaningReversal LLMValidatorType = "meaning_reversal" + LLMValidatorTypeEditorialReview LLMValidatorType = "editorial_review" + LLMValidatorTypeGrammarReview LLMValidatorType = "grammar_review" + LLMValidatorTypeSpokenWordReview LLMValidatorType = "spoken_word_review" +) + +type LLMMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type StructuredCompletionRequest struct { + StageName string `json:"stage_name"` + Messages []LLMMessage `json:"messages"` + Model string `json:"model,omitempty"` +} + +type StructuredCompletionResponse struct { + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` +} + +type StructuredLLMClient interface { + CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) +} + +// LLMValidationItem is one candidate correction payload passed to LLM validators. +type LLMValidationItem struct { + CorrectionIndex int `json:"correction_index"` + SegmentID int `json:"id"` + OriginalText string `json:"original_text"` + CorrectedText string `json:"corrected_text"` + OriginalSegmentText string `json:"original_segment_text"` + CorrectedSegmentText string `json:"corrected_segment_text"` + Categories []string `json:"categories,omitempty"` +} + +// LLMValidationRequest is the canonical request model for LLM-backed validators. +type LLMValidationRequest struct { + ValidatorName string `json:"validator_name"` + ValidatorType LLMValidatorType `json:"validator_type"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy string `json:"replacement_policy"` + Glossary *schema.Glossary `json:"-"` + Items []LLMValidationItem `json:"corrections"` +} + +type LLMValidationDecision struct { + CorrectionIndex int `json:"correction_index"` + Approved bool `json:"approved"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` +} + +type LLMValidationResponse struct { + Validations []LLMValidationDecision `json:"validations"` +} diff --git a/internal/framework/validators/llm_prompt_builders.go b/internal/framework/validators/llm_prompt_builders.go new file mode 100644 index 0000000..6aa1c1f --- /dev/null +++ b/internal/framework/validators/llm_prompt_builders.go @@ -0,0 +1,90 @@ +package validators + +import ( + "encoding/json" + "fmt" +) + +func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) { + payloadJSON, err := marshalPromptPayload(validationPayload) + if err != nil { + return nil, err + } + system := "You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken." + user := "Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n" + + "Rules:\n" + + "- Return one validation decision for every correction_index in the input.\n" + + "- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n" + + "- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", \"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n" + + "- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" + + "- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n" + + "- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n" + + "- If a correction includes categories, treat them as additional segment context.\n" + + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + + "- confidence must be between 0.0 and 1.0.\n\n" + + fmt.Sprintf("Corrections to validate:\n%s", payloadJSON) + return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil +} + +func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) { + payloadJSON, err := marshalPromptPayload(validationPayload) + if err != nil { + return nil, err + } + system := "You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals." + user := "Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n" + + "Rules:\n" + + "- Return one validation decision for every correction_index in the input.\n" + + "- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n" + + "- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n" + + "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" + + "- Do not reject a correction merely because the literal written word changes.\n" + + "- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n" + + "- If a correction includes categories, treat them as additional segment context.\n" + + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + + "- confidence must be between 0.0 and 1.0.\n\n" + + fmt.Sprintf("Corrections to validate:\n%s", payloadJSON) + return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil +} + +func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) { + payloadJSON, err := marshalPromptPayload(validationPayload) + if err != nil { + return nil, err + } + system := "You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup." + user := "Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n" + + "Rules:\n" + + "- Return one validation decision for every correction_index in the input.\n" + + "- Approve editorial revisions that preserve the underlying meaning of the segment.\n" + + "- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" + + "- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n" + + "- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n" + + "- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n" + + "- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" + + "- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" + + "- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n" + + "- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n" + + "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" + + "- If a correction includes categories, treat them as additional segment context.\n" + + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + + "- confidence must be between 0.0 and 1.0.\n\n" + + fmt.Sprintf("Corrections to validate:\n%s", payloadJSON) + return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil +} + +func BuildGrammarReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) { + return BuildEditorialMessages(validationPayload) +} + +func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) { + return BuildEditorialMessages(validationPayload) +} + +func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) { + payloadJSON, err := json.MarshalIndent(validationPayload, "", " ") + if err != nil { + return "", fmt.Errorf("marshal validation payload: %w", err) + } + return string(payloadJSON), nil +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go new file mode 100644 index 0000000..b983eef --- /dev/null +++ b/internal/framework/validators/llm_validators.go @@ -0,0 +1,260 @@ +package validators + +import ( + "context" + "fmt" + "sort" + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error) + +type LLMBackedValidator struct { + name string + validatorType LLMValidatorType + promptBuilder LLMPromptBuilder + model string + estimator chunking.TokenEstimator +} + +func (v *LLMBackedValidator) Name() string { + return v.name +} + +// SetTokenEstimator allows deterministic test control over batching behavior. +func (v *LLMBackedValidator) SetTokenEstimator(estimator chunking.TokenEstimator) { + if v == nil || estimator == nil { + return + } + v.estimator = estimator +} + +func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model string) (*LLMBackedValidator, error) { + builder, err := promptBuilderForType(validatorType) + if err != nil { + return nil, err + } + if strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("validator name must not be empty") + } + return &LLMBackedValidator{ + name: name, + validatorType: validatorType, + promptBuilder: builder, + model: strings.TrimSpace(model), + estimator: chunking.NewSimpleTokenEstimator(), + }, nil +} + +func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, error) { + if v == nil { + return Result{}, fmt.Errorf("validator is nil") + } + if req.LLMClient == nil { + return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name) + } + if len(req.CandidateProposal) == 0 { + return Result{ValidatorName: v.name, Decisions: nil}, nil + } + validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req) + if len(validationReq.Items) == 0 { + all := append([]Decision(nil), immediate...) + sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) + if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { + return Result{}, err + } + return Result{ValidatorName: v.name, Decisions: all}, nil + } + + maxTokens := config.DefaultValidationMaxPromptTokens + if req.Config != nil && req.Config.ValidationMaxPromptTokens > 0 { + maxTokens = req.Config.ValidationMaxPromptTokens + } + + batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator) + if err != nil { + return Result{}, err + } + + llmDecisions := make([]Decision, 0) + for _, batch := range batches { + messages, err := v.promptBuilder(batch.Items) + if err != nil { + return Result{}, err + } + + var response LLMValidationResponse + call := func(callCtx context.Context) error { + _, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{ + StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex), + Messages: messages, + Model: resolvedValidationModel(req.Config, v.model), + }, &response) + return err + } + if req.Scheduler != nil { + err = req.Scheduler.Run(ctx, call) + } else { + err = call(ctx) + } + artifacts := InteractionArtifacts{} + if req.DiagnosticsWriter != nil { + stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex) + artifacts, _ = req.DiagnosticsWriter.WriteInteraction( + stage, + map[string]any{"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex}, + map[string]any{"messages": messages, "items": batch.Items}, + response, + errPayload(err), + ) + } + if err != nil { + return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) + } + + batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) + if err != nil { + return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err) + } + for i := range batchDecisions { + batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath + } + llmDecisions = append(llmDecisions, batchDecisions...) + } + + all := append([]Decision(nil), immediate...) + all = append(all, llmDecisions...) + if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { + return Result{}, err + } + sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) + return Result{ValidatorName: v.name, Decisions: all}, nil +} + +func errPayload(err error) any { + if err == nil { + return nil + } + return map[string]any{"error": err.Error()} +} + +func resolvedValidationModel(cfg *config.Config, override string) string { + if strings.TrimSpace(override) != "" { + return strings.TrimSpace(override) + } + if cfg == nil { + return "" + } + return cfg.EffectiveValidationLLMConfig().Model +} + +func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) { + items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) + immediate := make([]Decision, 0) + segments := make(map[int]schema.Segment) + if req.WorkingTranscript != nil { + segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments)) + for _, seg := range req.WorkingTranscript.Segments { + segments[seg.ID] = seg + } + } + + for _, p := range req.CandidateProposal { + seg, ok := segments[p.TargetSegmentID] + if !ok { + immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found")) + continue + } + + preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy) + if !preview.Applicable { + immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) + continue + } + + items = append(items, LLMValidationItem{ + CorrectionIndex: p.ProposalIndex, + SegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + OriginalSegmentText: seg.Text, + CorrectedSegmentText: preview.CorrectedSegmentText, + Categories: append([]string(nil), seg.Categories...), + }) + } + + return LLMValidationRequest{ + ValidatorName: validatorName, + ValidatorType: validatorType, + ModuleKey: req.ModuleKey, + ModuleInstance: req.ModuleInstance, + ReplacementPolicy: string(req.ReplacementPolicy), + Glossary: req.Glossary, + Items: items, + }, immediate +} + +func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, error) { + switch validatorType { + case LLMValidatorTypeSpokenFormPlausibility: + return BuildSpokenFormPlausibilityMessages, nil + case LLMValidatorTypeMeaningReversal: + return BuildMeaningReversalMessages, nil + case LLMValidatorTypeEditorialReview: + return BuildEditorialMessages, nil + case LLMValidatorTypeGrammarReview: + return BuildGrammarReviewMessages, nil + case LLMValidatorTypeSpokenWordReview: + return BuildSpokenWordReviewMessages, nil + default: + return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType) + } +} + +func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) { + expected := make(map[int]LLMValidationItem, len(items)) + for _, item := range items { + expected[item.CorrectionIndex] = item + } + if len(response.Validations) == 0 { + return nil, fmt.Errorf("missing validations in structured response") + } + + seen := make(map[int]LLMValidationDecision, len(response.Validations)) + for _, d := range response.Validations { + if d.Confidence < 0.0 || d.Confidence > 1.0 { + return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex) + } + if _, ok := expected[d.CorrectionIndex]; !ok { + return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex) + } + if _, exists := seen[d.CorrectionIndex]; exists { + return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex) + } + seen[d.CorrectionIndex] = d + } + + decisions := make([]Decision, 0, len(items)) + for _, item := range items { + d, ok := seen[item.CorrectionIndex] + if !ok { + return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex) + } + reasonCode := ReasonApproved + if !d.Approved { + reasonCode = "llm_rejected" + } + decisions = append(decisions, Decision{ + ProposalIndex: item.CorrectionIndex, + Approved: d.Approved, + ReasonCode: reasonCode, + Message: strings.TrimSpace(d.Reason), + }) + } + return decisions, nil +} diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go new file mode 100644 index 0000000..1a0c31d --- /dev/null +++ b/internal/framework/validators/llm_validators_test.go @@ -0,0 +1,193 @@ +package validators + +import ( + "context" + "errors" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +type fakeStructuredLLMClient struct { + responses []LLMValidationResponse + err error + calls []StructuredCompletionRequest +} + +func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) { + _ = ctx + f.calls = append(f.calls, req) + if f.err != nil { + return StructuredCompletionResponse{}, f.err + } + if len(f.responses) == 0 { + return StructuredCompletionResponse{}, errors.New("unexpected call") + } + resp := f.responses[0] + f.responses = f.responses[1:] + target, ok := out.(*LLMValidationResponse) + if !ok { + return StructuredCompletionResponse{}, errors.New("unexpected output type") + } + *target = resp + return StructuredCompletionResponse{}, nil +} + +func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request { + cfg := config.Default() + cfg.ValidationMaxPromptTokens = 10000 + return Request{ + WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}}, + CandidateProposal: candidates, + ModuleKey: "homophones", + ModuleInstance: "homophones", + ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, + Config: &cfg, + } +} + +func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal { + return proposals.EnrichedCorrectionProposal{ + CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9}, + ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"}, + } +} + +func TestChunkLLMValidationItemsOneSmallBatch(t *testing.T) { + items := []LLMValidationItem{{CorrectionIndex: 0, OriginalText: "a", CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}} + batches, err := ChunkLLMValidationItems(items, 1000, chunking.NewSimpleTokenEstimator()) + if err != nil { + t.Fatalf("Chunk error: %v", err) + } + if len(batches) != 1 { + t.Fatalf("expected 1 batch, got %d", len(batches)) + } +} + +func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) { + items := []LLMValidationItem{ + {CorrectionIndex: 0, OriginalText: strings.Repeat("a", 20), CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}, + {CorrectionIndex: 1, OriginalText: strings.Repeat("c", 20), CorrectedText: "d", OriginalSegmentText: "x", CorrectedSegmentText: "y"}, + {CorrectionIndex: 2, OriginalText: strings.Repeat("e", 20), CorrectedText: "f", OriginalSegmentText: "x", CorrectedSegmentText: "y"}, + } + batches, err := ChunkLLMValidationItems(items, 40, chunking.NewSimpleTokenEstimator()) + if err != nil { + t.Fatalf("Chunk error: %v", err) + } + if len(batches) < 2 { + t.Fatalf("expected multiple batches, got %d", len(batches)) + } + seen := make([]int, 0) + for _, b := range batches { + for _, it := range b.Items { + seen = append(seen, it.CorrectionIndex) + } + } + if len(seen) != 3 || seen[0] != 0 || seen[1] != 1 || seen[2] != 2 { + t.Fatalf("unexpected ordering/drops/dups: %v", seen) + } +} + +func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) { + payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}} + tests := []struct { + name string + build func([]LLMValidationItem) ([]LLMMessage, error) + mustHas []string + }{ + {"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}}, + {"meaning_reversal", BuildMeaningReversalMessages, []string{"meaning reversals", "correction_index", "original_segment_text", "corrected_segment_text"}}, + {"editorial", BuildEditorialMessages, []string{"acceptable editorial revision", "correction_index", "original_segment_text", "corrected_segment_text"}}, + {"grammar_review", BuildGrammarReviewMessages, []string{"acceptable editorial revision", "correction_index"}}, + {"spoken_word", BuildSpokenWordReviewMessages, []string{"acceptable editorial revision", "correction_index"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msgs, err := tt.build(payload) + if err != nil { + t.Fatalf("build err: %v", err) + } + if len(msgs) != 2 { + t.Fatalf("expected 2 messages, got %d", len(msgs)) + } + combined := msgs[0].Content + "\n" + msgs[1].Content + for _, needle := range tt.mustHas { + if !strings.Contains(combined, needle) { + t.Fatalf("expected prompt to contain %q", needle) + } + } + }) + } +} + +func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ + {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"}, + }}}} + v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + if err != nil { + t.Fatalf("new validator error: %v", err) + } + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "Lyra")}) + req.LLMClient = client + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("validate err: %v", err) + } + if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved { + t.Fatalf("unexpected decisions: %+v", res.Decisions) + } +} + +func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) { + client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + _, err := v.Validate(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "completion failed") { + t.Fatalf("expected malformed output error, got %v", err) + } +} + +func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + _, err := v.Validate(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "response invalid") { + t.Fatalf("expected missing decision error, got %v", err) + } +} + +func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ + {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, + }}}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + _, err := v.Validate(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("expected duplicate decision error, got %v", err) + } +} + +func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + _, err := v.Validate(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("expected unknown index error, got %v", err) + } +} diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 1a5ed46..0aae19f 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -29,14 +29,18 @@ type Request struct { ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` Glossary *schema.Glossary `json:"-"` Config *config.Config `json:"-"` + LLMClient StructuredLLMClient `json:"-"` + Scheduler ValidationScheduler `json:"-"` + DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"` } // Decision is one validator decision for one proposal index. type Decision struct { - ProposalIndex int `json:"proposal_index"` - Approved bool `json:"approved"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` + ProposalIndex int `json:"proposal_index"` + Approved bool `json:"approved"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } // Result is one validator output containing exactly one decision per proposal index. @@ -45,6 +49,22 @@ type Result struct { Decisions []Decision `json:"decisions"` } +// ValidationScheduler provides bounded execution for validator LLM calls. +type ValidationScheduler interface { + Run(ctx context.Context, fn func(context.Context) error) error +} + +type InteractionDiagnosticsWriter interface { + WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) +} + +type InteractionArtifacts struct { + RequestMetadataPath string `json:"request_metadata_path,omitempty"` + RequestPayloadPath string `json:"request_payload_path,omitempty"` + ResponsePayloadPath string `json:"response_payload_path,omitempty"` + ErrorPayloadPath string `json:"error_payload_path,omitempty"` +} + // Validator is the deterministic runtime validator interface. type Validator interface { Name() string