eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Move LLM validators to PromptKit prompt variables

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:07:17 +00:00
parent f577847ca2
commit a4a72168d5
17 changed files with 310 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
internal/framework/validators/llm_validators.go
View File

@@ -17,13 +17,9 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
type LLMBackedValidator struct {
name string
validatorType LLMValidatorType
promptBuilder LLMPromptBuilder
model string
estimator chunking.TokenEstimator
}
@@ -39,10 +35,9 @@ func (v *LLMBackedValidator) SetTokenEstimator(estimator chunking.TokenEstimator
v.estimator = estimator
}
func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model string) (*LLMBackedValidator, error) {
builder, err := promptBuilderForType(validatorType)
if err != nil {
return nil, err
func NewLLMBackedValidator(name string, validatorType LLMValidatorType) (*LLMBackedValidator, error) {
if validatorPromptMetadata(validatorType).PromptID == "" {
return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType)
}
if strings.TrimSpace(name) == "" {
return nil, fmt.Errorf("validator name must not be empty")
@@ -50,8 +45,6 @@ func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model st
return &LLMBackedValidator{
name: name,
validatorType: validatorType,
promptBuilder: builder,
model: strings.TrimSpace(model),
estimator: chunking.NewSimpleTokenEstimator(),
}, nil
}
@@ -77,8 +70,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
maxTokens := config.DefaultValidationMaxPromptTokens
if req.Config != nil && req.Config.ValidationMaxPromptTokens > 0 {
maxTokens = req.Config.ValidationMaxPromptTokens
if req.Config != nil && req.Config.Validation.MaxPromptTokens > 0 {
maxTokens = req.Config.Validation.MaxPromptTokens
}
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
@@ -103,22 +96,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
for _, batch := range batches {
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
transcriptDescription = req.Config.Context.Description
}
messages, err := v.promptBuilder(batch.Items, transcriptDescription)
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
}
var response LLMValidationResponse
var completion StructuredCompletionResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
promptMetadata := validatorPromptMetadata(v.validatorType)
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: stage,
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema,
completion, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID, PromptVersion: promptMetadata.PromptVersion,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
return err
}
@@ -129,18 +123,17 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
promptMetadata := validatorPromptMetadata(v.validatorType)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt_metadata": promptMetadata.DiagnosticsMap(),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
map[string]any{"messages": messages, "items": batch.Items},
response,
completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(err),
)
}
@@ -199,16 +192,6 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func resolvedValidationModel(cfg *config.Config, override string) string {
if strings.TrimSpace(override) != "" {
return strings.TrimSpace(override)
}
if cfg == nil {
return ""
}
return cfg.EffectiveValidationLLMConfig().Model
}
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
@@ -255,23 +238,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
}, immediate
}
func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, error) {
switch validatorType {
case LLMValidatorTypeSpokenFormPlausibility:
return BuildSpokenFormPlausibilityMessages, nil
case LLMValidatorTypeMeaningReversal:
return BuildMeaningReversalMessages, nil
case LLMValidatorTypeEditorialReview:
return BuildEditorialMessages, nil
case LLMValidatorTypeGrammarReview:
return BuildGrammarReviewMessages, nil
case LLMValidatorTypeSpokenWordReview:
return BuildSpokenWordReviewMessages, nil
default:
return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType)
}
}
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
expected := make(map[int]LLMValidationItem, len(items))
for _, item := range items {