392 lines
17 KiB
Go
392 lines
17 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
// producerAttemptKind records why a producer invocation followed the prior
|
|
// one. It is deliberately independent of any artifact family.
|
|
type producerAttemptKind string
|
|
|
|
const (
|
|
producerAttemptInitial producerAttemptKind = "initial"
|
|
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
|
|
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
|
|
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
|
|
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
|
|
)
|
|
|
|
type producerAttemptOutcome string
|
|
|
|
const (
|
|
producerAttemptAccepted producerAttemptOutcome = "accepted"
|
|
producerAttemptRejected producerAttemptOutcome = "rejected"
|
|
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
|
|
producerAttemptRetried producerAttemptOutcome = "retried"
|
|
producerAttemptFailed producerAttemptOutcome = "failed"
|
|
)
|
|
|
|
type producerTerminalAction string
|
|
|
|
const (
|
|
producerTerminalAccepted producerTerminalAction = "accepted"
|
|
producerTerminalRejected producerTerminalAction = "reject_output"
|
|
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
|
|
producerTerminalFailed producerTerminalAction = "fail_run"
|
|
)
|
|
|
|
// producerAttemptRequest contains only attempt-local control material. The
|
|
// producer reconstructs its ordinary request from its own durable inputs.
|
|
type producerAttemptRequest struct {
|
|
Number int
|
|
Kind producerAttemptKind
|
|
Correction *contracts.SemanticCorrection
|
|
}
|
|
|
|
// producerRetryDirective asks for another producer invocation while retaining
|
|
// the current value as a safe fallback if its shared budget is exhausted.
|
|
// Artifact-specific adapters are responsible for validating and populating it.
|
|
type producerRetryDirective struct {
|
|
FallbackWarnings []contracts.Warning
|
|
}
|
|
|
|
func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
|
if directive == nil {
|
|
return nil
|
|
}
|
|
return &producerRetryDirective{FallbackWarnings: cloneWarnings(directive.FallbackWarnings)}
|
|
}
|
|
|
|
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
|
// opaque to the state machine; Candidate is the attempt-local response that
|
|
// may support semantic correction.
|
|
type producerAttemptOutput struct {
|
|
Value any
|
|
Candidate *contracts.ModelCandidate
|
|
Warnings []contracts.Warning
|
|
Retry *producerRetryDirective
|
|
}
|
|
|
|
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
|
|
candidate, err := contracts.CloneModelCandidate(output.Candidate)
|
|
if err != nil {
|
|
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
|
|
}
|
|
return producerAttemptOutput{
|
|
Value: output.Value,
|
|
Candidate: candidate,
|
|
Warnings: cloneWarnings(output.Warnings),
|
|
Retry: output.Retry.clone(),
|
|
}, nil
|
|
}
|
|
|
|
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
|
|
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
|
|
|
|
type producerAttemptConfig struct {
|
|
Retries int
|
|
Policy ValidationPolicy
|
|
AllowStructuralRetry bool
|
|
}
|
|
|
|
// producerAttemptProvenance is ordered by producer invocation. It retains the
|
|
// settled validation report for later debug and durable-provenance adapters.
|
|
type producerAttemptProvenance struct {
|
|
Number int
|
|
Kind producerAttemptKind
|
|
Outcome producerAttemptOutcome
|
|
Validation validationReport
|
|
}
|
|
|
|
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
|
|
provenance.Validation = cloneValidationReport(provenance.Validation)
|
|
return provenance
|
|
}
|
|
|
|
// producerAttemptTerminal describes the state-machine decision without
|
|
// materializing an artifact or persisting stage-specific diagnostics.
|
|
type producerAttemptTerminal struct {
|
|
Action producerTerminalAction
|
|
Value any
|
|
Warnings []contracts.Warning
|
|
Rejection *contracts.RejectedOutput
|
|
Validation validationReport
|
|
ValidationIncomplete bool
|
|
Provenance []producerAttemptProvenance
|
|
}
|
|
|
|
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
|
|
terminal.Warnings = cloneWarnings(terminal.Warnings)
|
|
if terminal.Rejection != nil {
|
|
rejection := *terminal.Rejection
|
|
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
|
|
terminal.Rejection = &rejection
|
|
}
|
|
terminal.Validation = cloneValidationReport(terminal.Validation)
|
|
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
|
|
return terminal
|
|
}
|
|
|
|
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
|
|
if len(provenance) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make([]producerAttemptProvenance, len(provenance))
|
|
for index, item := range provenance {
|
|
cloned[index] = item.clone()
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneValidationReport(report validationReport) validationReport {
|
|
return validationReport{records: report.Records()}
|
|
}
|
|
|
|
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
|
|
if ctx == nil {
|
|
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
|
|
}
|
|
if config.Retries < 0 {
|
|
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
|
|
}
|
|
if produce == nil {
|
|
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
|
|
}
|
|
if validate == nil {
|
|
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
|
|
}
|
|
|
|
attemptLimit := config.Retries + 1
|
|
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
|
|
kind := producerAttemptInitial
|
|
var correction *contracts.SemanticCorrection
|
|
|
|
for number := 1; number <= attemptLimit; number++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return failedProducerAttempt(provenance), err
|
|
}
|
|
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
|
|
if err != nil {
|
|
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
|
|
}
|
|
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
|
|
if err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
if isImmediateProducerFailure(err) {
|
|
return failedProducerAttempt(provenance), err
|
|
}
|
|
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
|
|
if number < attemptLimit {
|
|
kind, correction = producerAttemptStructuralRetry, nil
|
|
continue
|
|
}
|
|
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
|
|
}
|
|
if number < attemptLimit {
|
|
kind, correction = producerAttemptOperationalRetry, nil
|
|
continue
|
|
}
|
|
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
|
|
}
|
|
|
|
output, err = output.clone()
|
|
if err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
return failedProducerAttempt(provenance), err
|
|
}
|
|
if output.Retry != nil && number < attemptLimit {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
|
kind, correction = producerAttemptModuleRetry, nil
|
|
continue
|
|
}
|
|
if output.Retry != nil {
|
|
output.Warnings = append(output.Warnings, cloneWarnings(output.Retry.FallbackWarnings)...)
|
|
}
|
|
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
|
|
if err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
|
|
}
|
|
|
|
report, err := validate(ctx, output)
|
|
if err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
return failedProducerAttempt(provenance), err
|
|
}
|
|
report = cloneValidationReport(report)
|
|
if err := ctx.Err(); err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
return failedProducerAttempt(provenance), err
|
|
}
|
|
|
|
if rejection := report.FirstRejection(); rejection != nil {
|
|
if correctionCandidate != nil {
|
|
if err := correctionCandidate.Validate(); err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
|
|
}
|
|
}
|
|
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
|
correctionRequest, guidanceErr := report.CorrectionRequest()
|
|
if guidanceErr != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
|
|
}
|
|
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
|
|
if err != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
|
|
}
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
|
|
kind = producerAttemptSemanticRetry
|
|
continue
|
|
}
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
|
|
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
|
|
}
|
|
|
|
if incomplete := firstIncompleteValidation(report); incomplete != nil {
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
|
|
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
|
|
warnings := terminalWarnings(output, report)
|
|
warnings = append(warnings, incompleteValidationWarnings(report)...)
|
|
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Warnings: warnings, Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
}
|
|
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
|
|
}
|
|
|
|
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
|
|
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Warnings: terminalWarnings(output, report), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
}
|
|
|
|
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
|
}
|
|
|
|
func isImmediateProducerFailure(err error) bool {
|
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
return true
|
|
}
|
|
var debugErr *attemptDebugPersistenceError
|
|
return errors.As(err, &debugErr)
|
|
}
|
|
|
|
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
|
|
switch policy.ProducerStructuralFailure {
|
|
case ProducerStructuralFailureRejectOutput:
|
|
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
case ProducerStructuralFailureFailRun:
|
|
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
|
|
default:
|
|
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
|
|
}
|
|
}
|
|
|
|
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
|
|
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
|
|
switch policy.SemanticRejection {
|
|
case SemanticRejectionRejectOutput:
|
|
return producerAttemptTerminal{Action: producerTerminalRejected, Warnings: terminalWarnings(output, report), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
case SemanticRejectionFailRun:
|
|
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
|
|
default:
|
|
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
|
|
}
|
|
}
|
|
|
|
func firstIncompleteValidation(report validationReport) *validationRecord {
|
|
for _, record := range report.records {
|
|
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
|
clone := record.clone()
|
|
return &clone
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func terminalWarnings(output producerAttemptOutput, report validationReport) []contracts.Warning {
|
|
warnings := cloneWarnings(output.Warnings)
|
|
warnings = append(warnings, report.Warnings()...)
|
|
return warnings
|
|
}
|
|
|
|
// incompleteValidationWarnings reports only validators that exhausted their
|
|
// execution budget. It never reports rejected candidates, and it uses fixed
|
|
// text so provider errors and correction content cannot cross this boundary.
|
|
func incompleteValidationWarnings(report validationReport) []contracts.Warning {
|
|
warnings := make([]contracts.Warning, 0)
|
|
for _, record := range report.records {
|
|
if record.outcome != validationFailed {
|
|
continue
|
|
}
|
|
warnings = append(warnings, contracts.Warning{
|
|
Scope: record.validatorName,
|
|
ReasonCode: "validator_execution_incomplete",
|
|
Message: "Validator execution did not complete within its configured budget.",
|
|
})
|
|
}
|
|
return warnings
|
|
}
|
|
|
|
// validationSummary projects a terminal state-machine result into the durable
|
|
// bounded representation used by manifests, rejections, and receipts.
|
|
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
|
|
summary := artifacts.ValidationSummary{
|
|
Stage: string(stage),
|
|
StepID: stepID,
|
|
LaneID: laneID,
|
|
ModuleKey: moduleKey,
|
|
ChunkID: chunkID,
|
|
ChunkIndex: chunkIndex,
|
|
ProducerAttemptCount: len(terminal.Provenance),
|
|
TerminalAction: string(terminal.Action),
|
|
}
|
|
switch {
|
|
case terminal.Action == producerTerminalRejected:
|
|
summary.Status = "rejected"
|
|
case terminal.Action == producerTerminalFailed:
|
|
summary.Status = "incomplete"
|
|
case terminal.ValidationIncomplete:
|
|
summary.Status = "incomplete"
|
|
default:
|
|
summary.Status = "complete"
|
|
}
|
|
seenValidators := make(map[string]struct{})
|
|
seenReasons := make(map[string]struct{})
|
|
seenIncomplete := make(map[string]struct{})
|
|
for _, record := range terminal.Validation.records {
|
|
if record.reasonCode != "" {
|
|
if _, exists := seenReasons[record.reasonCode]; !exists {
|
|
seenReasons[record.reasonCode] = struct{}{}
|
|
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
|
|
}
|
|
}
|
|
if record.outcome == validationRejected {
|
|
if _, exists := seenValidators[record.validatorName]; !exists {
|
|
seenValidators[record.validatorName] = struct{}{}
|
|
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
|
|
}
|
|
}
|
|
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
|
if _, exists := seenIncomplete[record.validatorName]; !exists {
|
|
seenIncomplete[record.validatorName] = struct{}{}
|
|
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
|
|
}
|
|
}
|
|
}
|
|
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
|
|
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
|
|
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
|
|
}
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
|
|
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
|
|
}
|