eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Route proposal modules through embedded PromptKit definitions

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:03:08 +00:00
parent c8a78bf024
commit f577847ca2
19 changed files with 177 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
internal/framework/proposal_generation/generate_test.go
View File

@@ -14,11 +14,13 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
)
type fakeStructuredClient struct {
@@ -42,7 +44,11 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
var content strings.Builder
for _, value := range req.Variables {
content.WriteString(value)
}
return contracts.StructuredCompletionResponse{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}}, nil
}
type countingScheduler struct {
@@ -116,11 +122,10 @@ func defaultRequest(t *testing.T) Request {
ModuleInstance: "test_module",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Config: &cfg,
Messages: []contracts.LLMMessage{
{Role: "system", Content: "system prompt"},
{Role: "user", Content: "user prompt"},
},
StartIndex: 0,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
StartIndex: 0,
}
}
@@ -172,12 +177,8 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
}
call := client.calls[0]
if call.ResponseSchema == nil {
t.Fatalf("expected response schema on structured request")
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if call.ResponseSchema.ID != want.ID || call.ResponseSchema.Version != want.Version || call.ResponseSchema.Name != want.Name || call.ResponseSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected response schema metadata: got=%+v want=%+v", *call.ResponseSchema, want)
if call.Schema != responseschema.CorrectionSetKey || call.PromptID != req.PromptID || call.Context.Role != "proposal" {
t.Fatalf("unexpected call contract: %+v", call)
}
}
@@ -191,13 +192,6 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsWriter = diag
req.PromptMetadata = map[string]any{
"prompt_id": "modules.grammar.proposal",
"prompt_version": "v1",
"prompt_source": "builtin",
"embedded_path": "assets/modules/grammar",
"sha256": "abc",
}
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -215,13 +209,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
promptMap, ok := metadata["prompt_metadata"].(map[string]any)
if !ok {
t.Fatalf("expected prompt_metadata map, got %T", metadata["prompt_metadata"])
}
if promptMap["prompt_id"] != "modules.grammar.proposal" || promptMap["prompt_version"] != "v1" || promptMap["prompt_source"] != "builtin" || promptMap["embedded_path"] != "assets/modules/grammar" || promptMap["sha256"] != "abc" {
t.Fatalf("unexpected prompt metadata map: %v", promptMap)
}
}
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
@@ -253,7 +241,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
req := defaultRequest(t)
req.LLMClient = client
@@ -272,20 +260,13 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
}
}
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
func TestGenerateCandidatesProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warnings: %+v", result.Warnings)
if err == nil || len(result.Warnings) != 0 {
t.Fatalf("protocol failure was downgraded: %+v, %v", result, err)
}
}
@@ -368,17 +349,18 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = primarySecret
cfg.ValidationLLM.APIKey = validationSecret
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
req := defaultRequest(t)
req.Config = &cfg
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Messages = []contracts.LLMMessage{
{Role: "system", Content: "include secret " + primarySecret},
{Role: "user", Content: "fix it"},
}
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {