Complete Phase 10 LLM validators
This commit is contained in:
@@ -3,11 +3,14 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
@@ -307,3 +310,246 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
}
|
||||
|
||||
func ptrConfig(c config.Config) *config.Config { return &c }
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []validators.LLMValidationResponse
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
resp := f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
target := out.(*validators.LLMValidationResponse)
|
||||
*target = resp
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
type countingScheduler struct{ runs int }
|
||||
|
||||
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
s.runs++
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
type lenEstimator struct{}
|
||||
|
||||
func (lenEstimator) EstimateTokens(text string) int { return len(text) }
|
||||
|
||||
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
|
||||
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
if err != nil {
|
||||
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := config.Default()
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "There were Jesters at the temple." {
|
||||
t.Fatalf("expected applied LLM-approved proposal, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := config.Default()
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "There were gestures at the temple." {
|
||||
t.Fatalf("expected rejected proposal not applied")
|
||||
}
|
||||
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("expected validator rejection record")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := config.Default()
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected llm validator failure")
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected partial progress retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := config.Default()
|
||||
_, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing decision failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := config.Default()
|
||||
_, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate decision failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
|
||||
}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
llmValidator.SetTokenEstimator(lenEstimator{})
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.ValidationMaxPromptTokens = 260
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
ValidationLLMScheduler: scheduler,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected run error: %v", err)
|
||||
}
|
||||
if scheduler.runs < 2 {
|
||||
t.Fatalf("expected scheduler to run per batch, got %d", scheduler.runs)
|
||||
}
|
||||
if client.calls < 2 {
|
||||
t.Fatalf("expected multiple llm calls for batching, got %d", client.calls)
|
||||
}
|
||||
if len(out.ModuleResults[0].AppliedChanges) != 2 {
|
||||
t.Fatalf("expected both approved proposals applied")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
cfg.ValidationLLM.APIKey = secret
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
ValidationDiagnosticsDir: diagDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
|
||||
t.Fatalf("expected diagnostic artifact path on decision")
|
||||
}
|
||||
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic: %v", readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in diagnostics")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerAcceptsLLMSchedulerType(t *testing.T) {
|
||||
s, err := llm.NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
if s == nil {
|
||||
t.Fatal("expected scheduler instance")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user