eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Centralize validator classification and malformed output handling

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-23 18:02:53 +00:00
parent 84be774b34
commit 99391cd18b
12 changed files with 255 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/framework/proposal_generation/generate.go
View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
@@ -159,7 +160,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if callErr != nil {
if isMalformedStructuredOutputError(callErr) {
if structuredoutput.IsMalformedError(callErr) {
return Result{
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
Artifacts: artifacts,
@@ -230,27 +231,6 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
warning := stagewarnings.StageWarning{
Scope: stagewarnings.ScopeProposalGeneration,

17
internal/framework/proposal_generation/generate_test.go
View File

@@ -272,6 +272,23 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
}
}
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warnings: %+v", result.Warnings)
}
}
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{

28
internal/framework/structuredoutput/malformed.go Normal file
View File

@@ -0,0 +1,28 @@
package structuredoutput
import "strings"
var malformedMarkers = []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
}
// IsMalformedError reports whether err matches provider malformed
// structured-output failure markers that should be downgraded.
func IsMalformedError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range malformedMarkers {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

32
internal/framework/structuredoutput/malformed_test.go Normal file
View File

@@ -0,0 +1,32 @@
package structuredoutput
import (
"errors"
"testing"
)
func TestIsMalformedError(t *testing.T) {
cases := []struct {
name string
err error
want bool
}{
{name: "nil", err: nil, want: false},
{name: "generic", err: errors.New("network timeout"), want: false},
{name: "malformed", err: errors.New("malformed structured output"), want: true},
{name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true},
{name: "missing choices", err: errors.New("provider response missing choices"), want: true},
{name: "missing content", err: errors.New("provider response missing assistant message content"), want: true},
{name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true},
{name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := IsMalformedError(tc.err)
if got != tc.want {
t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want)
}
})
}
}

24
internal/framework/validators/llm_validators.go
View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -144,7 +145,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if isMalformedStructuredOutputError(err) {
if structuredoutput.IsMalformedError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
@@ -387,24 +388,3 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
}
return artifacts.ResponsePayloadPath
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

17
internal/framework/validators/llm_validators_test.go
View File

@@ -265,6 +265,23 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
}
}
func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
}
}
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")