diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2f61b7d..bc4bf7d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -463,6 +463,16 @@ without examining error strings; counts are available independently of debug. ## Stage 3 — Convert Proposal Failures and Classify Safe Filtering +**Completed 2026-09-24.** Proposal generation and the shared path used by all +four built-in modules now preserve one identified operation with terminal counts +on success, recoverable failure, and fatal return. Recoverable failures yield no +candidates; a valid empty response succeeds. A focused domain classifier separates +defective proposals from editorial rejection without changing deterministic +validator policy. Updated proposal, wrapper, and filtering tests are present. +`git diff --check` passed. Those tests and the existing four-module routing tests +cannot compile until the validator warning consumers are converted in Stage 4; +the attempted `go test` reports those exact stale-field errors. + **Work:** Update `proposal_generation.GenerateCandidates` and all four module wrappers to preserve operation outcomes. A recoverable proposal call returns zero candidates plus a failed operation, not a warning or fatal Go error. A valid empty diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 588ea07..07e70d2 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -5,6 +5,7 @@ package proposal_generation import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -13,11 +14,11 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" - stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) // InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts. @@ -70,7 +71,7 @@ type Request struct { type Result struct { Corrections []proposals.CorrectionProposal `json:"corrections"` Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` - Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` + Operations []outcomes.Operation `json:"operations,omitempty"` Artifacts InteractionArtifacts `json:"artifacts,omitempty"` } @@ -121,6 +122,17 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { Variables: req.Variables, Schema: responseschema.CorrectionSetKey, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}, }, &response) + sectionIndex := 0 + if req.Section != nil { + sectionIndex = req.Section.Index + } + operation := outcomes.Operation{ + Reference: outcomes.Reference{ + ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, + SectionIndex: sectionIndex, Kind: outcomes.OperationProposal, + }, + Counts: completion.Execution, + } requestMetadata := map[string]any{ "module_key": req.ModuleKey, @@ -143,13 +155,25 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } if callErr != nil { - if structuredoutput.IsMalformedError(callErr) { - return Result{ - Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)}, - Artifacts: artifacts, - }, nil + var failure *structuredoutput.Failure + if errors.As(callErr, &failure) { + operation.FailureReason = failure.Reason } - return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr) + if operation.FailureReason == "" { + operation.FailureReason = outcomes.ReasonInternalFailure + } + result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts} + if structuredoutput.IsRecoverable(callErr) && failure != nil && failure.Reason != "" { + return result, nil + } + return result, fmt.Errorf("proposal generation completion failed: %w", callErr) + } + operation.Succeeded = true + if completion.Execution.StructuralRepairs > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) + } + if completion.Execution.TransientRetries > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) } corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) @@ -181,7 +205,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { return Result{ Corrections: corrections, Enriched: enriched, - Warnings: nil, + Operations: []outcomes.Operation{operation}, Artifacts: artifacts, }, nil } @@ -193,27 +217,6 @@ func errPayload(err error) any { return map[string]any{"error": err.Error()} } -func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning { - warning := stagewarnings.StageWarning{ - Scope: stagewarnings.ScopeProposalGeneration, - ReasonCode: "proposal_response_malformed", - Message: strings.TrimSpace(err.Error()), - DiagnosticArtifactPath: diagnosticArtifactPath(artifacts), - } - if section != nil { - sectionIndex := section.Index - warning.SectionIndex = §ionIndex - } - return warning -} - -func diagnosticArtifactPath(artifacts InteractionArtifacts) string { - if artifacts.ErrorPayloadPath != "" { - return artifacts.ErrorPayloadPath - } - return artifacts.ResponsePayloadPath -} - func sectionIndexPtr(section *contracts.SectionMetadata) *int { if section == nil { return nil diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index dd18ea3..114ea86 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -13,22 +13,24 @@ import ( ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" ) type fakeStructuredClient struct { - responses []StructuredCorrectionSet - err error - calls []contracts.StructuredCompletionRequest + responses []StructuredCorrectionSet + err error + completion contracts.StructuredCompletionResponse + calls []contracts.StructuredCompletionRequest } func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx f.calls = append(f.calls, req) if f.err != nil { - return contracts.StructuredCompletionResponse{}, f.err + return f.completion, f.err } target, ok := out.(*StructuredCorrectionSet) if !ok { @@ -43,7 +45,9 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr for _, value := range req.Variables { content.WriteString(value) } - return contracts.StructuredCompletionResponse{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}}, nil + response := f.completion + response.Prepared = &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}} + return response, nil } type captureDiagnosticsWriter struct { @@ -105,6 +109,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) { if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 { t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex) } + if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal { + t.Fatalf("proposal operation identity: %+v", got.Operations) + } } func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) { @@ -182,38 +189,45 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 { t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0]) } - if len(result.Warnings) != 0 { - t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings) + if len(result.Operations) != 1 || !result.Operations[0].Succeeded { + t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations) } } -func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) { - client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}} +func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *testing.T) { + client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}} req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) if err != nil { - t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err) + t.Fatalf("expected recoverable model operation, got %v", err) } if len(result.Corrections) != 0 || len(result.Enriched) != 0 { t.Fatalf("expected no proposals on malformed response, got %+v", result) } - if len(result.Warnings) != 1 { - t.Fatalf("expected one warning, got %+v", result.Warnings) - } - if result.Warnings[0].ReasonCode != "proposal_response_malformed" { - t.Fatalf("unexpected warning: %+v", result.Warnings[0]) + if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted { + t.Fatalf("expected one failed operation, got %+v", result.Operations) } } -func TestGenerateCandidatesProviderMalformedEnvelopePropagates(t *testing.T) { - client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}} +func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) { + client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}} req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) - if err == nil || len(result.Warnings) != 0 { - t.Fatalf("protocol failure was downgraded: %+v, %v", result, err) + if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonSetupFailure || !result.Operations[0].Counts.Available { + t.Fatalf("fatal setup lost call facts: %+v, %v", result, err) + } +} + +func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) { + client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}} + req := defaultRequest(t) + req.LLMClient = client + result, err := GenerateCandidates(context.Background(), req) + if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded { + t.Fatalf("valid empty response: %+v, %v", result, err) } } diff --git a/internal/framework/proposal_generation/module_proposal.go b/internal/framework/proposal_generation/module_proposal.go index 8afa8c1..f7f77a0 100644 --- a/internal/framework/proposal_generation/module_proposal.go +++ b/internal/framework/proposal_generation/module_proposal.go @@ -71,11 +71,11 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont DiagnosticsDir: req.ProposalRequest.DiagnosticsDir, }) if err != nil { - return contracts.ProposalResult{}, err + return contracts.ProposalResult{Operations: generated.Operations}, err } return contracts.ProposalResult{ - Proposals: generated.Corrections, - Warnings: generated.Warnings, + Proposals: generated.Corrections, + Operations: generated.Operations, }, nil } diff --git a/internal/framework/proposal_generation/module_proposal_test.go b/internal/framework/proposal_generation/module_proposal_test.go index f9c3717..4c2c77a 100644 --- a/internal/framework/proposal_generation/module_proposal_test.go +++ b/internal/framework/proposal_generation/module_proposal_test.go @@ -8,7 +8,9 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) @@ -113,3 +115,23 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) { t.Fatalf("expected builder error, got %v", err) } } + +func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) { + cause := errors.New("missing embedded asset") + client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}} + section := contracts.SectionMetadata{Index: 4} + result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ + ProposalRequest: contracts.ProposalRequest{ + ExecutionContext: contracts.ExecutionContext{Section: §ion}, + RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_2", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique}, + LLMClient: client, + }, + PromptID: prompts.PromptIDModuleGrammarProposal, + BuildVariables: func(*schema.Transcript, *schema.Glossary, int, string) (map[string]string, error) { + return map[string]string{"SectionJSON": "[]"}, nil + }, + }) + if !errors.Is(err, cause) || len(result.Operations) != 1 || result.Operations[0].Reference.ModuleInstance != "grammar_2" || result.Operations[0].Reference.SectionIndex != 4 { + t.Fatalf("fatal operation lost: %+v, %v", result, err) + } +} diff --git a/internal/framework/validators/filtering.go b/internal/framework/validators/filtering.go new file mode 100644 index 0000000..e693fc4 --- /dev/null +++ b/internal/framework/validators/filtering.go @@ -0,0 +1,21 @@ +package validators + +import ( + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +// DefectiveProposalReason recognizes a safely rejected defect in a proposal +// response. Editorial disagreement and later apply-time conflicts are excluded. +func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) { + switch reason { + case ReasonInvalidTargetSegment, ReasonMissingTargetSegment, + ReasonEmptyOriginalText, ReasonMissingOriginalText, + ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText, + string(proposals.SkipReasonAmbiguousOriginal), + string(proposals.SkipReasonInvalidProposal): + return outcomes.ReasonCode(reason), true + default: + return "", false + } +} diff --git a/internal/framework/validators/filtering_test.go b/internal/framework/validators/filtering_test.go new file mode 100644 index 0000000..1670035 --- /dev/null +++ b/internal/framework/validators/filtering_test.go @@ -0,0 +1,20 @@ +package validators + +import "testing" + +func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) { + for _, reason := range []string{ + ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText, + ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect, + ReasonEmptyResultingText, "ambiguous_original_text", + } { + if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason { + t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok) + } + } + for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} { + if got, ok := DefectiveProposalReason(reason); ok { + t.Fatalf("ordinary rejection %q classified as %q", reason, got) + } + } +} diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 8ea7fe9..866c51b 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -113,11 +113,11 @@ func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal } func approval(index int) Decision { - return Decision{ProposalIndex: index, Approved: true, ReasonCode: ReasonApproved, Message: "approved"} + return Decision{ProposalIndex: index, Approved: true, Disposition: DispositionApproved, ReasonCode: ReasonApproved, Message: "approved"} } func rejection(index int, reasonCode string, msg string) Decision { - return Decision{ProposalIndex: index, Approved: false, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)} + return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionRejected, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)} } func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {