Replace structured LLM dependency with Audita adapter

This commit is contained in:
2026-05-13 02:10:24 +00:00
parent 20f612215f
commit de99467ede
24 changed files with 1611 additions and 689 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMValidatorType string
@@ -22,9 +23,10 @@ type LLMMessage struct {
}
type StructuredCompletionRequest struct {
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchema *responseschema.Schema `json:"response_schema,omitempty"`
}
type StructuredCompletionResponse struct {

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
@@ -89,11 +90,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
var response LLMValidationResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema,
}, &response)
return err
}
@@ -107,7 +110,17 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex},
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
"name": responseSchema.Name,
"sha256": responseSchema.SHA256,
},
},
map[string]any{"messages": messages, "items": batch.Items},
response,
errPayload(err),

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type fakeStructuredLLMClient struct {
@@ -33,6 +34,19 @@ type boundedScheduler struct {
permits chan struct{}
}
type captureValidationDiagnosticsWriter struct {
lastRequestMetadata any
}
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
_ = stage
_ = requestPayload
_ = responsePayload
_ = errorPayload
w.lastRequestMetadata = requestMetadata
return InteractionArtifacts{}, nil
}
func newBoundedScheduler(max int) *boundedScheduler {
return &boundedScheduler{permits: make(chan struct{}, max)}
}
@@ -200,6 +214,17 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(client.calls) != 1 {
t.Fatalf("expected one LLM call, got %d", len(client.calls))
}
if client.calls[0].ResponseSchema == nil {
t.Fatalf("expected response schema on structured validation request")
}
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
gotSchema := client.calls[0].ResponseSchema
if gotSchema.ID != want.ID || gotSchema.Version != want.Version || gotSchema.Name != want.Name || gotSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected validator response schema metadata: got=%+v want=%+v", *gotSchema, want)
}
}
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
@@ -285,6 +310,36 @@ func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
}
}
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
if err != nil {
t.Fatalf("new validator error: %v", err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.DiagnosticsWriter = writer
_, err = v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("validate error: %v", err)
}
metadata, ok := writer.lastRequestMetadata.(map[string]any)
if !ok {
t.Fatalf("expected metadata map, got %T", writer.lastRequestMetadata)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema map, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
}
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
t.Helper()
deadline := time.Now().Add(300 * time.Millisecond)