eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 10 LLM validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 01:25:52 +00:00
parent 6d9a4bd017
commit 12202508bf
14 changed files with 1213 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
internal/framework/runner/runner.go
View File

@@ -3,11 +3,13 @@ package runner
import (
"context"
"fmt"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -27,6 +29,10 @@ type Runner struct {
factory ModuleFactory
}
type ValidationScheduler interface {
Run(ctx context.Context, fn func(context.Context) error) error
}
// ModuleResult captures deterministic per-module execution output.
type ModuleResult struct {
ModuleKey string `json:"module_key"`
@@ -44,11 +50,12 @@ type ModuleResult struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -65,10 +72,13 @@ type ValidatorRejectedChange struct {
// RunInput is the deterministic runner input.
type RunInput struct {
Config *config.Config
Transcript *schema.Transcript
Glossary *schema.Glossary
ModuleSpecs []contracts.ModuleRunSpec
Config *config.Config
Transcript *schema.Transcript
Glossary *schema.Glossary
ModuleSpecs []contracts.ModuleRunSpec
ValidationLLMClient contracts.StructuredLLMClient
ValidationLLMScheduler ValidationScheduler
ValidationDiagnosticsDir string
}
// RunOutput is the deterministic runner output.
@@ -148,6 +158,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
validatorRejected := make([]ValidatorRejectedChange, 0)
eligible := enriched
for _, validator := range module.Validators() {
var diagnosticsWriter validators.InteractionDiagnosticsWriter
if input.ValidationDiagnosticsDir != "" {
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(input.ValidationDiagnosticsDir, spec.InstanceName),
validatorSecrets(input.Config),
),
}
}
vResult, vErr := validator.Validate(ctx, contracts.ValidationRequest{
WorkingTranscript: working,
CandidateProposal: eligible,
@@ -156,6 +176,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy,
Glossary: input.Glossary,
Config: input.Config,
LLMClient: validationLLMClientAdapter{client: input.ValidationLLMClient},
Scheduler: input.ValidationLLMScheduler,
DiagnosticsWriter: diagnosticsWriter,
})
if vErr != nil {
failed := ModuleResult{
@@ -197,11 +220,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for _, d := range vResult.Decisions {
validatorDecisions = append(validatorDecisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
@@ -265,3 +289,62 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
return &schema.Transcript{Segments: segments}
}
type validationLLMClientAdapter struct {
client contracts.StructuredLLMClient
}
func (a validationLLMClientAdapter) CompleteStructured(ctx context.Context, req validators.StructuredCompletionRequest, out any) (validators.StructuredCompletionResponse, error) {
if a.client == nil {
return validators.StructuredCompletionResponse{}, fmt.Errorf("validation structured LLM client is not configured")
}
messages := make([]contracts.LLMMessage, len(req.Messages))
for i, m := range req.Messages {
messages[i] = contracts.LLMMessage{Role: m.Role, Content: m.Content}
}
resp, err := a.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: messages,
Model: req.Model,
}, out)
if err != nil {
return validators.StructuredCompletionResponse{}, err
}
return validators.StructuredCompletionResponse{
PromptTokens: resp.PromptTokens,
CompletionTokens: resp.CompletionTokens,
TotalTokens: resp.TotalTokens,
}, nil
}
type llmDiagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
if a == nil || a.writer == nil {
return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
}
art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}
return validators.InteractionArtifacts{
RequestMetadataPath: art.RequestMetadataPath,
RequestPayloadPath: art.RequestPayloadPath,
ResponsePayloadPath: art.ResponsePayloadPath,
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}
func validatorSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effective := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
effective.APIKey,
cfg.ValidationLLM.APIKey,
}
}