eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Preserve proposal outcomes and classify filtered proposals

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:09:35 +00:00
parent 51169ff9e9
commit 42bde59827
8 changed files with 144 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/framework/proposal_generation/generate_test.go
View File

@@ -13,22 +13,24 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
)
type fakeStructuredClient struct {
responses []StructuredCorrectionSet
err error
calls []contracts.StructuredCompletionRequest
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
calls []contracts.StructuredCompletionRequest
}
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
return f.completion, f.err
}
target, ok := out.(*StructuredCorrectionSet)
if !ok {
@@ -43,7 +45,9 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
for _, value := range req.Variables {
content.WriteString(value)
}
return contracts.StructuredCompletionResponse{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}}, nil
response := f.completion
response.Prepared = &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}
return response, nil
}
type captureDiagnosticsWriter struct {
@@ -105,6 +109,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
}
}
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
@@ -182,38 +189,45 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Warnings) != 0 {
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
}
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 {
t.Fatalf("expected one warning, got %+v", result.Warnings)
}
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
t.Fatalf("expected one failed operation, got %+v", result.Operations)
}
}
func TestGenerateCandidatesProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err == nil || len(result.Warnings) != 0 {
t.Fatalf("protocol failure was downgraded: %+v, %v", result, err)
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonSetupFailure || !result.Operations[0].Counts.Available {
t.Fatalf("fatal setup lost call facts: %+v, %v", result, err)
}
}
func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}