eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 10 LLM validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 01:25:52 +00:00
parent 6d9a4bd017
commit 12202508bf
14 changed files with 1213 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
internal/framework/runner/runner.go
View File

@@ -3,11 +3,13 @@ package runner
import (
"context"
"fmt"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -27,6 +29,10 @@ type Runner struct {
factory ModuleFactory
}
type ValidationScheduler interface {
Run(ctx context.Context, fn func(context.Context) error) error
}
// ModuleResult captures deterministic per-module execution output.
type ModuleResult struct {
ModuleKey string `json:"module_key"`
@@ -44,11 +50,12 @@ type ModuleResult struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -65,10 +72,13 @@ type ValidatorRejectedChange struct {
// RunInput is the deterministic runner input.
type RunInput struct {
Config *config.Config
Transcript *schema.Transcript
Glossary *schema.Glossary
ModuleSpecs []contracts.ModuleRunSpec
Config *config.Config
Transcript *schema.Transcript
Glossary *schema.Glossary
ModuleSpecs []contracts.ModuleRunSpec
ValidationLLMClient contracts.StructuredLLMClient
ValidationLLMScheduler ValidationScheduler
ValidationDiagnosticsDir string
}
// RunOutput is the deterministic runner output.
@@ -148,6 +158,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
validatorRejected := make([]ValidatorRejectedChange, 0)
eligible := enriched
for _, validator := range module.Validators() {
var diagnosticsWriter validators.InteractionDiagnosticsWriter
if input.ValidationDiagnosticsDir != "" {
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(input.ValidationDiagnosticsDir, spec.InstanceName),
validatorSecrets(input.Config),
),
}
}
vResult, vErr := validator.Validate(ctx, contracts.ValidationRequest{
WorkingTranscript: working,
CandidateProposal: eligible,
@@ -156,6 +176,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy,
Glossary: input.Glossary,
Config: input.Config,
LLMClient: validationLLMClientAdapter{client: input.ValidationLLMClient},
Scheduler: input.ValidationLLMScheduler,
DiagnosticsWriter: diagnosticsWriter,
})
if vErr != nil {
failed := ModuleResult{
@@ -197,11 +220,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for _, d := range vResult.Decisions {
validatorDecisions = append(validatorDecisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
@@ -265,3 +289,62 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
return &schema.Transcript{Segments: segments}
}
type validationLLMClientAdapter struct {
client contracts.StructuredLLMClient
}
func (a validationLLMClientAdapter) CompleteStructured(ctx context.Context, req validators.StructuredCompletionRequest, out any) (validators.StructuredCompletionResponse, error) {
if a.client == nil {
return validators.StructuredCompletionResponse{}, fmt.Errorf("validation structured LLM client is not configured")
}
messages := make([]contracts.LLMMessage, len(req.Messages))
for i, m := range req.Messages {
messages[i] = contracts.LLMMessage{Role: m.Role, Content: m.Content}
}
resp, err := a.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: messages,
Model: req.Model,
}, out)
if err != nil {
return validators.StructuredCompletionResponse{}, err
}
return validators.StructuredCompletionResponse{
PromptTokens: resp.PromptTokens,
CompletionTokens: resp.CompletionTokens,
TotalTokens: resp.TotalTokens,
}, nil
}
type llmDiagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
if a == nil || a.writer == nil {
return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
}
art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}
return validators.InteractionArtifacts{
RequestMetadataPath: art.RequestMetadataPath,
RequestPayloadPath: art.RequestPayloadPath,
ResponsePayloadPath: art.ResponsePayloadPath,
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}
func validatorSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effective := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
effective.APIKey,
cfg.ValidationLLM.APIKey,
}
}

246
internal/framework/runner/runner_test.go
View File

@@ -3,11 +3,14 @@ package runner
import (
"context"
"errors"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -307,3 +310,246 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
}
func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
err error
calls int
}
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
f.calls++
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
resp := f.responses[0]
f.responses = f.responses[1:]
target := out.(*validators.LLMValidationResponse)
*target = resp
return contracts.StructuredCompletionResponse{}, nil
}
type countingScheduler struct{ runs int }
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
s.runs++
return fn(ctx)
}
type lenEstimator struct{}
func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
}},
}})
cfg := config.Default()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "There were Jesters at the temple." {
t.Fatalf("expected applied LLM-approved proposal, got %q", out.FinalTranscript.Segments[0].Text)
}
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
}},
}})
cfg := config.Default()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "There were gestures at the temple." {
t.Fatalf("expected rejected proposal not applied")
}
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("expected validator rejection record")
}
}
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
}},
}})
cfg := config.Default()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected llm validator failure")
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress retained")
}
}
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
}},
}})
cfg := config.Default()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected missing decision failure")
}
}
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
}},
}})
cfg := config.Default()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected duplicate decision failure")
}
}
func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
llmValidator.SetTokenEstimator(lenEstimator{})
scheduler := &countingScheduler{}
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 260
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1},
}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ValidationLLMScheduler: scheduler,
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
}
if scheduler.runs < 2 {
t.Fatalf("expected scheduler to run per batch, got %d", scheduler.runs)
}
if client.calls < 2 {
t.Fatalf("expected multiple llm calls for batching, got %d", client.calls)
}
if len(out.ModuleResults[0].AppliedChanges) != 2 {
t.Fatalf("expected both approved proposals applied")
}
}
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
secret := "super-secret-key"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
cfg.ValidationLLM.APIKey = secret
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ValidationDiagnosticsDir: diagDir,
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("expected diagnostic artifact path on decision")
}
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
if readErr != nil {
t.Fatalf("read diagnostic: %v", readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in diagnostics")
}
}
func TestRunnerAcceptsLLMSchedulerType(t *testing.T) {
s, err := llm.NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
if s == nil {
t.Fatal("expected scheduler instance")
}
}