eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Connect processing to shared PromptKit backend capacity

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:18:31 +00:00
parent a4a72168d5
commit 4812032c9e
26 changed files with 406 additions and 1816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/framework/validators/llm_validators.go
View File

@@ -53,12 +53,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if v == nil {
return Result{}, fmt.Errorf("validator is nil")
}
if req.LLMClient == nil {
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
}
if len(req.CandidateProposal) == 0 {
return Result{ValidatorName: v.name, Decisions: nil}, nil
}
if req.LLMClient == nil {
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
}
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
if len(validationReq.Items) == 0 {
all := append([]Decision(nil), immediate...)
@@ -108,19 +108,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
promptMetadata := validatorPromptMetadata(v.validatorType)
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
call := func(callCtx context.Context) error {
completion, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID, PromptVersion: promptMetadata.PromptVersion,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
return err
}
if req.Scheduler != nil {
err = req.Scheduler.Run(ctx, call)
} else {
err = call(ctx)
}
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID, PromptVersion: promptMetadata.PromptVersion,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(

111
internal/framework/validators/llm_validators_test.go
View File

@@ -3,12 +3,8 @@ package validators
import (
"context"
"errors"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
@@ -25,17 +21,6 @@ type fakeStructuredLLMClient struct {
calls []StructuredCompletionRequest
}
type sleepingValidationClient struct {
inFlight int32
maxInFlight int32
entered chan struct{}
release chan struct{}
}
type boundedScheduler struct {
permits chan struct{}
}
type captureValidationDiagnosticsWriter struct {
lastRequestMetadata any
}
@@ -49,48 +34,6 @@ func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requ
return InteractionArtifacts{}, nil
}
func newBoundedScheduler(max int) *boundedScheduler {
return &boundedScheduler{permits: make(chan struct{}, max)}
}
func (s *boundedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
select {
case s.permits <- struct{}{}:
case <-ctx.Done():
return ctx.Err()
}
defer func() { <-s.permits }()
return fn(ctx)
}
func (c *sleepingValidationClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
_ = req
current := atomic.AddInt32(&c.inFlight, 1)
for {
prior := atomic.LoadInt32(&c.maxInFlight)
if current <= prior || atomic.CompareAndSwapInt32(&c.maxInFlight, prior, current) {
break
}
}
if c.entered != nil {
c.entered <- struct{}{}
}
select {
case <-c.release:
case <-ctx.Done():
atomic.AddInt32(&c.inFlight, -1)
return StructuredCompletionResponse{}, ctx.Err()
}
atomic.AddInt32(&c.inFlight, -1)
target := out.(*LLMValidationResponse)
*target = LLMValidationResponse{
Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
},
}
return StructuredCompletionResponse{}, nil
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
@@ -313,42 +256,6 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
}
}
func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
scheduler := newBoundedScheduler(2)
client := &sleepingValidationClient{
entered: make(chan struct{}, 16),
release: make(chan struct{}),
}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("new validator error: %v", err)
}
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.Scheduler = scheduler
if _, runErr := v.Validate(context.Background(), req); runErr != nil {
t.Errorf("Validate error: %v", runErr)
}
}()
}
waitForValidationEntries(t, client.entered, 2)
close(client.release)
wg.Wait()
if got := atomic.LoadInt32(&client.maxInFlight); got > 2 {
t.Fatalf("expected scheduler cap <= 2, got %d", got)
}
if got := atomic.LoadInt32(&client.maxInFlight); got < 2 {
t.Fatalf("expected observed concurrency of at least 2, got %d", got)
}
}
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
@@ -379,24 +286,6 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
}
}
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
t.Helper()
deadline := time.Now().Add(300 * time.Millisecond)
got := 0
for got < want && time.Now().Before(deadline) {
select {
case <-entered:
got++
default:
runtime.Gosched()
}
}
if got < want {
t.Fatalf("timed out waiting for %d validator calls to enter (got %d)", want, got)
}
}
func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
cases := []struct {
kind LLMValidatorType

6
internal/framework/validators/models.go
View File

@@ -37,7 +37,6 @@ type Request struct {
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
Scheduler ValidationScheduler `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
}
@@ -57,11 +56,6 @@ type Result struct {
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
}
// ValidationScheduler provides bounded execution for validator LLM calls.
type ValidationScheduler interface {
Run(ctx context.Context, fn func(context.Context) error) error
}
type InteractionDiagnosticsWriter interface {
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
}