diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 9245651..3c9a98e 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -406,6 +406,16 @@ classification. Reporting disabled removes only its file requirements. ## Stage 1 — Define Processing Outcomes and Contracts +**Completed 2026-09-24.** Added processing-owned operation identities, execution +counts with availability, safe outcome/recovery reasons, work states, explicit +decision dispositions, and typed recoverable/fatal/canceled failure classes. The +proposal, validator, and runner result boundaries now carry operation outcomes +instead of warning fields. `go test ./internal/framework/outcomes +./internal/framework/structuredoutput` and `git diff --check` passed. Tests of +contracts, validators, and runner are temporarily unavailable because existing +validator/runner consumers still use the removed warning fields; Stages 3, 4, +and 6 reconnect those consumers. + **Work:** Add the small outcome/reference/count types from B–E. Extend `structuredoutput.Response` and `Failure`, module proposal results, validator results/decisions, and runner output to carry actual processing outcomes. Replace diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 4678539..50789d0 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -7,10 +7,10 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" - stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) type StructuredLLMClient = structuredoutput.Client @@ -74,8 +74,8 @@ type ProposalRequest struct { } type ProposalResult struct { - Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"` - Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` + Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"` + Operations []outcomes.Operation `json:"operations,omitempty"` } // ValidationRequest is the input to validator execution. diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go new file mode 100644 index 0000000..f78af8d --- /dev/null +++ b/internal/framework/outcomes/outcomes.go @@ -0,0 +1,92 @@ +// Package outcomes contains processing facts shared by proposal generation, +// validation, and orchestration. Presentation and filesystem concerns live elsewhere. +package outcomes + +// OperationKind identifies one logical model operation. Validation input failures +// occur before a model call and therefore have no batch index. +type OperationKind string + +const ( + OperationProposal OperationKind = "proposal" + OperationValidation OperationKind = "validation" + OperationValidationInput OperationKind = "validation_input" +) + +// Reference is the stable processing identity of an operation. Proposal indices +// are scoped to a module instance, while batch indices are local to a validator +// invocation within one section. +type Reference struct { + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + SectionIndex int `json:"section_index"` + Kind OperationKind `json:"operation"` + ValidatorName string `json:"validator_name,omitempty"` + ValidatorOrder *int `json:"validator_order,omitempty"` + BatchIndex *int `json:"batch_index,omitempty"` +} + +// ExecutionCounts describe one terminal structured-client call. Available is +// false when an external client provides no execution facts; zero is then unknown. +type ExecutionCounts struct { + Available bool `json:"available"` + ProviderAttempts int `json:"provider_attempts"` + TransientRetries int `json:"transient_retries"` + StructuralRepairs int `json:"structural_repairs"` + RetriesExhausted bool `json:"retries_exhausted"` + HTTPStatus *int `json:"http_status,omitempty"` +} + +// ReasonCode is a fixed, safe explanation category. Raw provider errors and +// prompt content are never stored in an operation outcome. +type ReasonCode string + +const ( + ReasonOperationTimeout ReasonCode = "operation_timeout" + ReasonCapacityExceeded ReasonCode = "capacity_exceeded" + ReasonTransportFailed ReasonCode = "transport_failed" + ReasonProviderResponseInvalid ReasonCode = "provider_response_invalid" + ReasonProviderFailed ReasonCode = "provider_failed" + ReasonInvalidOutputExhausted ReasonCode = "invalid_output_exhausted" + ReasonInvalidOutput ReasonCode = "invalid_output" + ReasonValidationInputTooLarge ReasonCode = "validation_input_too_large" + ReasonValidationDecisionsInvalid ReasonCode = "validation_decisions_invalid" + ReasonSetupFailure ReasonCode = "setup_failure" + ReasonCanceled ReasonCode = "canceled" + ReasonInternalFailure ReasonCode = "internal_failure" +) + +type RecoveryReason string + +const ( + RecoveryOutputRepaired RecoveryReason = "output_repaired" + RecoveryServiceRecovered RecoveryReason = "service_recovered" + RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" +) + +// FilteredProposal attributes a defective response item to its proposal identity. +// Ordinary editorial rejection is a decision, not a filtered proposal. +type FilteredProposal struct { + ProposalIndex int `json:"proposal_index"` + Reason ReasonCode `json:"reason"` + Count int `json:"count"` +} + +// Operation is a compact terminal outcome. A successful operation may have one +// recovery with several reasons. A failed operation never has recovery reasons. +type Operation struct { + Reference Reference `json:"reference"` + Counts ExecutionCounts `json:"counts"` + Succeeded bool `json:"succeeded"` + FailureReason ReasonCode `json:"failure_reason,omitempty"` + RecoveryReasons []RecoveryReason `json:"recovery_reasons,omitempty"` + ProposalIndices []int `json:"proposal_indices,omitempty"` + Filtered []FilteredProposal `json:"filtered,omitempty"` +} + +type WorkState string + +const ( + WorkNotStarted WorkState = "not_started" + WorkCompleted WorkState = "completed" + WorkInterrupted WorkState = "interrupted" +) diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index ed19974..50a2cc4 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -13,6 +13,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" @@ -36,19 +37,31 @@ type Runner struct { // ModuleResult captures deterministic per-module execution output. type ModuleResult struct { - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - Status string `json:"status"` - ProposalCount int `json:"proposal_count"` - Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` - ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` - ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` - AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` - SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + Status string `json:"status"` + ProposalCount int `json:"proposal_count"` + Operations []outcomes.Operation `json:"operations,omitempty"` + ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` + ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` + AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` + SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` +} + +// WorkUnitOutcome keeps initial section identity while text evolves between +// module instances. Detailed candidate and decision records are debug evidence. +type WorkUnitOutcome struct { + ModuleIndex int `json:"module_index"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + Section contracts.SectionMetadata `json:"section"` + State outcomes.WorkState `json:"state"` + CorrectionsApplied int `json:"corrections_applied"` + Operations []outcomes.Operation `json:"operations,omitempty"` } type ValidatorDecisionRecord struct { @@ -87,13 +100,15 @@ type RunInput struct { // RunOutput is the deterministic runner output. type RunOutput struct { - FinalTranscript *schema.Transcript `json:"-"` - ModuleResults []ModuleResult `json:"module_results"` - Observations llm.ObservationSnapshot `json:"-"` - ObservationsAvailable bool `json:"-"` - LLMSummary llm.ExecutionSummary `json:"-"` - RunWallTimeMS int64 `json:"-"` - ValidatorTimings []ValidatorTiming `json:"-"` + FinalTranscript *schema.Transcript `json:"-"` + ModuleResults []ModuleResult `json:"module_results"` + Sections []contracts.SectionMetadata `json:"sections"` + WorkUnits []WorkUnitOutcome `json:"work_units"` + Observations llm.ObservationSnapshot `json:"-"` + ObservationsAvailable bool `json:"-"` + LLMSummary llm.ExecutionSummary `json:"-"` + RunWallTimeMS int64 `json:"-"` + ValidatorTimings []ValidatorTiming `json:"-"` } func New(factory ModuleFactory) *Runner { diff --git a/internal/framework/structuredoutput/call.go b/internal/framework/structuredoutput/call.go index f1ad3ef..398a1c8 100644 --- a/internal/framework/structuredoutput/call.go +++ b/internal/framework/structuredoutput/call.go @@ -5,6 +5,7 @@ import ( "encoding/json" "time" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" ) @@ -40,15 +41,16 @@ type PromptMetadata struct { } type Response struct { - Content json.RawMessage `json:"content,omitempty"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - Prompt PromptMetadata `json:"prompt"` - Prepared *PreparedSnapshot `json:"prepared,omitempty"` - FinalContent string `json:"final_content,omitempty"` + Execution outcomes.ExecutionCounts `json:"execution"` + Content json.RawMessage `json:"content,omitempty"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + Prompt PromptMetadata `json:"prompt"` + Prepared *PreparedSnapshot `json:"prepared,omitempty"` + FinalContent string `json:"final_content,omitempty"` } // PreparedSnapshot is the exact rendered request snapshot. Its messages may diff --git a/internal/framework/structuredoutput/failure.go b/internal/framework/structuredoutput/failure.go index 78b7357..612276c 100644 --- a/internal/framework/structuredoutput/failure.go +++ b/internal/framework/structuredoutput/failure.go @@ -1,6 +1,10 @@ package structuredoutput -import "errors" +import ( + "errors" + + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" +) // FailureKind separates invalid generated content from operational failures. type FailureKind string @@ -13,14 +17,31 @@ const ( // Failure exposes a safe message while retaining the typed cause for callers. // The cause must not be serialized or formatted directly in user artifacts. type Failure struct { - Kind FailureKind + Kind FailureKind // Retained until the adapter switches to reason-based classification. + Class FailureClass + Reason outcomes.ReasonCode Message string Cause error } +// FailureClass determines whether domain processing may safely skip a model +// operation or must stop the run. Parent cancellation always stops the run. +type FailureClass string + +const ( + FailureRecoverable FailureClass = "recoverable" + FailureFatal FailureClass = "fatal" + FailureCanceled FailureClass = "canceled" +) + func (f *Failure) Error() string { return f.Message } func (f *Failure) Unwrap() error { return f.Cause } +func IsRecoverable(err error) bool { + var failure *Failure + return errors.As(err, &failure) && failure.Class == FailureRecoverable +} + func IsMalformedError(err error) bool { var failure *Failure return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput diff --git a/internal/framework/structuredoutput/failure_test.go b/internal/framework/structuredoutput/failure_test.go index 1287f02..c528444 100644 --- a/internal/framework/structuredoutput/failure_test.go +++ b/internal/framework/structuredoutput/failure_test.go @@ -3,6 +3,8 @@ package structuredoutput import ( "errors" "testing" + + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" ) func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) { @@ -16,3 +18,28 @@ func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) { t.Fatalf("runtime classification: %v", runtime) } } + +func TestRecoverableFailureKeepsTypedCauseOutOfSafeMessage(t *testing.T) { + cause := errors.New("provider response contained a secret") + failure := &Failure{ + Class: FailureRecoverable, + Reason: outcomes.ReasonTransportFailed, + Message: "model transport failed", + Cause: cause, + } + wrapped := errors.Join(errors.New("operation failed"), failure) + if !IsRecoverable(wrapped) || !errors.Is(wrapped, cause) { + t.Fatalf("recoverable failure lost classification or cause: %v", wrapped) + } + if failure.Error() != "model transport failed" { + t.Fatalf("unsafe failure text: %q", failure.Error()) + } + for _, class := range []FailureClass{FailureFatal, FailureCanceled} { + if IsRecoverable(&Failure{Class: class, Cause: cause}) { + t.Fatalf("%s classified as recoverable", class) + } + } + if IsRecoverable(cause) { + t.Fatal("bare internal error classified as recoverable") + } +} diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index ef083a4..8ea7fe9 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -8,8 +8,8 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" - stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) const ( @@ -43,18 +43,29 @@ type Request struct { // Decision is one validator decision for one proposal index. type Decision struct { - ProposalIndex int `json:"proposal_index"` - Approved bool `json:"approved"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` - DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` + ProposalIndex int `json:"proposal_index"` + Approved bool `json:"approved"` + Disposition DecisionDisposition `json:"disposition"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } +// DecisionDisposition separates normal editorial rejection from a candidate +// excluded because its model operation could not provide a trustworthy decision. +type DecisionDisposition string + +const ( + DispositionApproved DecisionDisposition = "approved" + DispositionRejected DecisionDisposition = "rejected" + DispositionUnavailable DecisionDisposition = "unavailable" +) + // Result is one validator output containing exactly one decision per proposal index. type Result struct { - ValidatorName string `json:"validator_name"` - Decisions []Decision `json:"decisions"` - Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` + ValidatorName string `json:"validator_name"` + Decisions []Decision `json:"decisions"` + Operations []outcomes.Operation `json:"operations,omitempty"` } type InteractionDiagnosticsWriter interface {