eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Replace structured LLM dependency with Audita adapter

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 02:10:24 +00:00
parent 20f612215f
commit de99467ede
24 changed files with 1611 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
internal/framework/proposal_generation/generate.go
View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
@@ -112,11 +113,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
callErr error
artifacts InteractionArtifacts
)
responseSchema := responseschema.MustLookup(responseschema.CorrectionSetKey)
call := func(callCtx context.Context) error {
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
StageName: stage,
Messages: messages,
Model: model,
StageName: stage,
Messages: messages,
Model: model,
ResponseSchema: &responseSchema,
}, &response)
return callErr
}
@@ -126,17 +129,20 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
callErr = call(ctx)
}
requestMetadata := map[string]any{
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"model": model,
}
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
if writer != nil {
artifacts, _ = writer.WriteInteraction(
stage,
map[string]any{
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"model": model,
},
requestMetadata,
map[string]any{
"messages": messages,
},
@@ -185,6 +191,15 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}, nil
}
func schemaMetadata(schema responseschema.Schema) map[string]any {
return map[string]any{
"id": schema.ID,
"version": schema.Version,
"name": schema.Name,
"sha256": schema.SHA256,
}
}
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if section == nil {

89
internal/framework/proposal_generation/generate_test.go
View File

@@ -2,6 +2,7 @@ package proposal_generation
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -17,6 +18,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type fakeStructuredClient struct {
@@ -52,6 +54,21 @@ func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) er
return fn(ctx)
}
type captureDiagnosticsWriter struct {
lastStage string
lastRequestMetadata any
lastRequestPayload any
}
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
w.lastStage = stage
w.lastRequestMetadata = requestMetadata
w.lastRequestPayload = requestPayload
_ = responsePayload
_ = errorPayload
return InteractionArtifacts{}, nil
}
type sleepingStructuredClient struct {
inFlight int32
maxInFlight int32
@@ -138,6 +155,61 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
}
}
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
req := defaultRequest(t)
req.LLMClient = client
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(client.calls) != 1 {
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
}
call := client.calls[0]
if call.ResponseSchema == nil {
t.Fatalf("expected response schema on structured request")
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if call.ResponseSchema.ID != want.ID || call.ResponseSchema.Version != want.Version || call.ResponseSchema.Name != want.Name || call.ResponseSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected response schema metadata: got=%+v want=%+v", *call.ResponseSchema, want)
}
}
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
diag := &captureDiagnosticsWriter{}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsWriter = diag
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
metadata, ok := diag.lastRequestMetadata.(map[string]any)
if !ok {
t.Fatalf("expected request metadata map, got %T", diag.lastRequestMetadata)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata map, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
}
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
@@ -266,6 +338,23 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
}
}
raw, readErr := os.ReadFile(got.Artifacts.RequestMetadataPath)
if readErr != nil {
t.Fatalf("read metadata artifact %q: %v", got.Artifacts.RequestMetadataPath, readErr)
}
var metadata map[string]any
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
}
}
func TestGenerateCandidatesSchedulerUsage(t *testing.T) {