eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Preserve proposal outcomes and classify filtered proposals

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:09:35 +00:00
parent 51169ff9e9
commit 42bde59827
8 changed files with 144 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
internal/framework/proposal_generation/generate.go
View File

@@ -5,6 +5,7 @@ package proposal_generation
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
@@ -13,11 +14,11 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
@@ -70,7 +71,7 @@ type Request struct {
type Result struct {
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
}
@@ -121,6 +122,17 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
}, &response)
sectionIndex := 0
if req.Section != nil {
sectionIndex = req.Section.Index
}
operation := outcomes.Operation{
Reference: outcomes.Reference{
ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance,
SectionIndex: sectionIndex, Kind: outcomes.OperationProposal,
},
Counts: completion.Execution,
}
requestMetadata := map[string]any{
"module_key": req.ModuleKey,
@@ -143,13 +155,25 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if callErr != nil {
if structuredoutput.IsMalformedError(callErr) {
return Result{
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
Artifacts: artifacts,
}, nil
var failure *structuredoutput.Failure
if errors.As(callErr, &failure) {
operation.FailureReason = failure.Reason
}
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
if structuredoutput.IsRecoverable(callErr) && failure != nil && failure.Reason != "" {
return result, nil
}
return result, fmt.Errorf("proposal generation completion failed: %w", callErr)
}
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
@@ -181,7 +205,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
return Result{
Corrections: corrections,
Enriched: enriched,
Warnings: nil,
Operations: []outcomes.Operation{operation},
Artifacts: artifacts,
}, nil
}
@@ -193,27 +217,6 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
warning := stagewarnings.StageWarning{
Scope: stagewarnings.ScopeProposalGeneration,
ReasonCode: "proposal_response_malformed",
Message: strings.TrimSpace(err.Error()),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
if section != nil {
sectionIndex := section.Index
warning.SectionIndex = &sectionIndex
}
return warning
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil

52
internal/framework/proposal_generation/generate_test.go
View File

@@ -13,22 +13,24 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
)
type fakeStructuredClient struct {
responses []StructuredCorrectionSet
err error
calls []contracts.StructuredCompletionRequest
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
calls []contracts.StructuredCompletionRequest
}
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
return f.completion, f.err
}
target, ok := out.(*StructuredCorrectionSet)
if !ok {
@@ -43,7 +45,9 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
for _, value := range req.Variables {
content.WriteString(value)
}
return contracts.StructuredCompletionResponse{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}}, nil
response := f.completion
response.Prepared = &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}
return response, nil
}
type captureDiagnosticsWriter struct {
@@ -105,6 +109,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
}
}
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
@@ -182,38 +189,45 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Warnings) != 0 {
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
}
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 {
t.Fatalf("expected one warning, got %+v", result.Warnings)
}
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
t.Fatalf("expected one failed operation, got %+v", result.Operations)
}
}
func TestGenerateCandidatesProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err == nil || len(result.Warnings) != 0 {
t.Fatalf("protocol failure was downgraded: %+v, %v", result, err)
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonSetupFailure || !result.Operations[0].Counts.Available {
t.Fatalf("fatal setup lost call facts: %+v, %v", result, err)
}
}
func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}

6
internal/framework/proposal_generation/module_proposal.go
View File

@@ -71,11 +71,11 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
return contracts.ProposalResult{Operations: generated.Operations}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
Proposals: generated.Corrections,
Operations: generated.Operations,
}, nil
}

22
internal/framework/proposal_generation/module_proposal_test.go
View File

@@ -8,7 +8,9 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -113,3 +115,23 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
t.Fatalf("expected builder error, got %v", err)
}
}
func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
cause := errors.New("missing embedded asset")
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}
section := contracts.SectionMetadata{Index: 4}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{Section: &section},
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_2", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildVariables: func(*schema.Transcript, *schema.Glossary, int, string) (map[string]string, error) {
return map[string]string{"SectionJSON": "[]"}, nil
},
})
if !errors.Is(err, cause) || len(result.Operations) != 1 || result.Operations[0].Reference.ModuleInstance != "grammar_2" || result.Operations[0].Reference.SectionIndex != 4 {
t.Fatalf("fatal operation lost: %+v, %v", result, err)
}
}

21
internal/framework/validators/filtering.go Normal file
View File

@@ -0,0 +1,21 @@
package validators
import (
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
// DefectiveProposalReason recognizes a safely rejected defect in a proposal
// response. Editorial disagreement and later apply-time conflicts are excluded.
func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) {
switch reason {
case ReasonInvalidTargetSegment, ReasonMissingTargetSegment,
ReasonEmptyOriginalText, ReasonMissingOriginalText,
ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText,
string(proposals.SkipReasonAmbiguousOriginal),
string(proposals.SkipReasonInvalidProposal):
return outcomes.ReasonCode(reason), true
default:
return "", false
}
}

20
internal/framework/validators/filtering_test.go Normal file
View File

@@ -0,0 +1,20 @@
package validators
import "testing"
func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) {
for _, reason := range []string{
ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText,
ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect,
ReasonEmptyResultingText, "ambiguous_original_text",
} {
if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason {
t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok)
}
}
for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
if got, ok := DefectiveProposalReason(reason); ok {
t.Fatalf("ordinary rejection %q classified as %q", reason, got)
}
}
}

4
internal/framework/validators/models.go
View File

@@ -113,11 +113,11 @@ func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal
}
func approval(index int) Decision {
return Decision{ProposalIndex: index, Approved: true, ReasonCode: ReasonApproved, Message: "approved"}
return Decision{ProposalIndex: index, Approved: true, Disposition: DispositionApproved, ReasonCode: ReasonApproved, Message: "approved"}
}
func rejection(index int, reasonCode string, msg string) Decision {
return Decision{ProposalIndex: index, Approved: false, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionRejected, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
}
func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {