Strengthen validation retries and checkpoint safety
This commit is contained in:
@@ -126,11 +126,17 @@ func (candidate ModelCandidate) Validate() error {
|
||||
}
|
||||
|
||||
func ValidateValidationResult(result ValidationResult) error {
|
||||
if !result.Approved && result.ReasonCode == "" {
|
||||
return errors.New("validation rejection reason code must not be empty")
|
||||
}
|
||||
if result.ReasonCode != "" {
|
||||
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !result.Approved && result.CorrectionGuidance == "" {
|
||||
return errors.New("validation rejection correction guidance must not be empty")
|
||||
}
|
||||
if result.CorrectionGuidance != "" {
|
||||
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
|
||||
return err
|
||||
|
||||
@@ -59,13 +59,21 @@ func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
|
||||
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
|
||||
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
|
||||
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
|
||||
{"oversized reason code", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason}) }},
|
||||
{"blank reason code", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: " \t"}) }},
|
||||
{"missing rejection reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
|
||||
{"oversized reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"blank reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"invalid correction guidance utf8", func() error {
|
||||
return ValidateValidationResult(ValidationResult{CorrectionGuidance: string([]byte{0xff})})
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
|
||||
}},
|
||||
{"oversized correction guidance", func() error {
|
||||
return ValidateValidationResult(ValidationResult{CorrectionGuidance: tooLongValidationGuidance})
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -89,6 +89,7 @@ const (
|
||||
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
|
||||
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
|
||||
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
|
||||
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
|
||||
)
|
||||
|
||||
type CheckpointDecision struct {
|
||||
@@ -160,6 +161,8 @@ func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
|
||||
return "accepted normalized artifact is reusable"
|
||||
case CheckpointReasonRecomputeStep:
|
||||
return "selected step requires execution"
|
||||
case CheckpointReasonValidationIncompleteLineage:
|
||||
return "checkpoint reuse is disabled by validation-incomplete input lineage"
|
||||
default:
|
||||
return "checkpoint decision"
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
|
||||
prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
prepared.Steps[1].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
|
||||
@@ -41,6 +41,54 @@ func operationReferenceSet(input RunInput, target ResolvedReferenceTarget) contr
|
||||
return CloneReferenceSet(target.ReferenceSet)
|
||||
}
|
||||
|
||||
// referenceTargetReuseEligible reports whether every generated artifact in a
|
||||
// stage's reference set descends exclusively from fully validated work. Static
|
||||
// references and callers that do not supply lineage metadata are reusable.
|
||||
func referenceTargetReuseEligible(input RunInput, target ResolvedReferenceTarget) bool {
|
||||
if input.referenceReuseEligibility == nil {
|
||||
return true
|
||||
}
|
||||
eligible, ok := input.referenceReuseEligibility[keyForReferenceTarget(target)]
|
||||
return !ok || eligible
|
||||
}
|
||||
|
||||
func laneReferencesReuseEligible(input RunInput, lane ResolvedArtifactLane) bool {
|
||||
return referenceTargetReuseEligible(input, lane.ExtractReferences) &&
|
||||
referenceTargetReuseEligible(input, lane.MergeReferences) &&
|
||||
referenceTargetReuseEligible(input, lane.NormalizeReferences)
|
||||
}
|
||||
|
||||
// buildStepReferenceReuseEligibility carries validation completeness alongside
|
||||
// generated references without exposing the internal lineage flag in artifact
|
||||
// payloads. A target becomes ineligible when any generated input is ineligible.
|
||||
func buildStepReferenceReuseEligibility(step PreparedPipelineStep, outputs map[generatedOutputKey]bool) map[referenceTargetKey]bool {
|
||||
eligibility := make(map[referenceTargetKey]bool)
|
||||
for _, prepared := range step.lanes {
|
||||
lane := prepared.resolved
|
||||
for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||
generated := false
|
||||
eligible := true
|
||||
for _, binding := range target.Bindings {
|
||||
if binding.Artifact == nil {
|
||||
continue
|
||||
}
|
||||
generated = true
|
||||
producer := generatedOutputKeyFor(binding.Artifact.Step, binding.Artifact.Lane)
|
||||
if reusable, ok := outputs[producer]; ok && !reusable {
|
||||
eligible = false
|
||||
}
|
||||
}
|
||||
if generated {
|
||||
eligibility[keyForReferenceTarget(target)] = eligible
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(eligibility) == 0 {
|
||||
return nil
|
||||
}
|
||||
return eligibility
|
||||
}
|
||||
|
||||
// buildStepReferenceSets resolves every generated binding for a step before
|
||||
// any lane in that step is allowed to start. Each returned set is a fresh
|
||||
// operation-time view; prepared reference sets are never modified.
|
||||
|
||||
@@ -232,7 +232,12 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
|
||||
}
|
||||
}
|
||||
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
||||
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, report.CorrectionGuidance())
|
||||
correctionRequest, guidanceErr := report.CorrectionRequest()
|
||||
if guidanceErr != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
|
||||
}
|
||||
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
|
||||
|
||||
@@ -61,10 +61,11 @@ type RunInput struct {
|
||||
// single worker so direct framework callers retain deterministic behavior.
|
||||
ExtractWorkers int
|
||||
|
||||
pipeline ResolvedPipeline
|
||||
llmClient contracts.StructuredLLMClient
|
||||
stepID string
|
||||
references map[referenceTargetKey]contracts.ReferenceSet
|
||||
pipeline ResolvedPipeline
|
||||
llmClient contracts.StructuredLLMClient
|
||||
stepID string
|
||||
references map[referenceTargetKey]contracts.ReferenceSet
|
||||
referenceReuseEligibility map[referenceTargetKey]bool
|
||||
}
|
||||
|
||||
type RunOutput struct {
|
||||
@@ -76,6 +77,8 @@ type RunOutput struct {
|
||||
OutputFiles []contracts.OutputFile `json:"-"`
|
||||
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
|
||||
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||
|
||||
normalizeReuseEligibility map[generatedOutputKey]bool
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
|
||||
@@ -425,6 +428,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
|
||||
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
|
||||
}
|
||||
stepInput.references = stepReferences
|
||||
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
|
||||
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
|
||||
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
|
||||
if err := mergeLaneOutput(output, laneOutput); err != nil {
|
||||
|
||||
@@ -237,7 +237,7 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
|
||||
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
validatorCalls++
|
||||
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate"}, nil
|
||||
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}
|
||||
switch stage {
|
||||
@@ -280,11 +280,12 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
|
||||
|
||||
func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*PreparedPipeline)
|
||||
path string
|
||||
wantError string
|
||||
wantBody string
|
||||
name string
|
||||
configure func(*PreparedPipeline)
|
||||
path string
|
||||
wantError string
|
||||
wantBody string
|
||||
attemptError bool
|
||||
}{
|
||||
{
|
||||
name: "merge module error",
|
||||
@@ -293,8 +294,9 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
return erasedTypedResult{}, errors.New("merge exploded")
|
||||
}
|
||||
},
|
||||
path: "merge/notes/attempt-01.json",
|
||||
wantError: "merge exploded",
|
||||
path: "merge/notes/attempt-01.json",
|
||||
wantError: "merge exploded",
|
||||
attemptError: true,
|
||||
},
|
||||
{
|
||||
name: "normalize validator error",
|
||||
@@ -318,7 +320,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
},
|
||||
@@ -332,8 +334,9 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
return erasedTypedResult{Value: "wrong artifact type"}, nil
|
||||
}
|
||||
},
|
||||
path: "normalize/notes/attempt-01.json",
|
||||
wantError: "serialize normalize candidate",
|
||||
path: "normalize/notes/attempt-01.json",
|
||||
wantError: "serialize normalize candidate",
|
||||
attemptError: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -345,9 +348,15 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
envelope := debug.envelope(t, tc.path)
|
||||
if tc.wantError != "" {
|
||||
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
|
||||
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
|
||||
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else if runErr != nil {
|
||||
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
|
||||
validator := preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator"}, nil
|
||||
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}
|
||||
switch target {
|
||||
|
||||
@@ -72,10 +72,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
|
||||
result.setValidation(report.Warnings(), nil, err)
|
||||
return result, err
|
||||
}
|
||||
if report.FirstRejection() == nil {
|
||||
if incomplete := firstIncompleteValidation(report); incomplete != nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
return result, validatorFailureError(*incomplete)
|
||||
}
|
||||
rejection := report.FirstRejection()
|
||||
incomplete := firstIncompleteValidation(report)
|
||||
if rejection == nil && incomplete == nil {
|
||||
if err := result.setCandidate(record, "reused"); err != nil {
|
||||
return result, fmt.Errorf("clone reused chunk plan record: %w", err)
|
||||
}
|
||||
@@ -85,18 +84,21 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
|
||||
result.accepted = true
|
||||
result.setValidation(report.Warnings(), nil, nil)
|
||||
cachedTerminal := producerAttemptTerminal{Action: producerTerminalAccepted, Validation: report}
|
||||
if firstIncompleteValidation(report) != nil {
|
||||
result.summary.ValidationStatus = "incomplete"
|
||||
cachedTerminal.Action = producerTerminalIncompleteAccepted
|
||||
cachedTerminal.ValidationIncomplete = true
|
||||
}
|
||||
summary := validationSummary(cachedTerminal, StageChunk, "", "", chunker.Key(), "", 0)
|
||||
result.validation = &summary
|
||||
return result, nil
|
||||
}
|
||||
if incomplete != nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
failure := validatorFailureError(*incomplete)
|
||||
result.setValidation(report.Warnings(), nil, failure)
|
||||
failedTerminal := producerAttemptTerminal{Action: producerTerminalFailed, Validation: report, ValidationIncomplete: true}
|
||||
summary := validationSummary(failedTerminal, StageChunk, "", "", chunker.Key(), "", 0)
|
||||
result.validation = &summary
|
||||
return result, failure
|
||||
}
|
||||
// A cache hit is not model material. Its rejection is discarded and
|
||||
// generation begins with the ordinary initial request below.
|
||||
result.setValidation(report.Warnings(), chunkRejection(report, 1, chunker.Key()), nil)
|
||||
// generation begins with the ordinary initial request below. Warnings
|
||||
// from this discarded candidate are intentionally not promoted.
|
||||
}
|
||||
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
|
||||
result.summary.LookupStatus = "invalid"
|
||||
@@ -176,11 +178,6 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
|
||||
if validationErr != nil {
|
||||
return report, candidate.terminal.record(payload, validationErr)
|
||||
}
|
||||
if report.FirstRejection() == nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
if failure := firstIncompleteValidation(report); failure != nil {
|
||||
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
|
||||
}
|
||||
}
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
})
|
||||
terminalSummary := validationSummary(terminal, StageChunk, "", "", chunker.Key(), "", 0)
|
||||
|
||||
@@ -293,7 +293,7 @@ func TestRunnerOmitsChunkMapForRejectedPlan(t *testing.T) {
|
||||
prepared.output = encoder
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject/chunk"), Target: ValidatorTargetChunk},
|
||||
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}},
|
||||
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable chunk plan"}},
|
||||
}}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err != nil {
|
||||
@@ -351,6 +351,92 @@ func TestRunnerDoesNotPublishValidationIncompleteChunkPlan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
calls := 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||
validatorCalls := 0
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("cache-then-generated"), Target: ValidatorTargetChunk},
|
||||
chunk: chunkValidationFunc{name: "cache-then-generated", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
validatorCalls++
|
||||
if validatorCalls == 1 {
|
||||
return contracts.ValidationResult{}, errors.New("cached candidate could not be validated")
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}},
|
||||
}}
|
||||
record := chunkPlanRecord(t, prepared, plan)
|
||||
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
|
||||
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != 1 || validatorCalls != 2 || store.saves != 1 {
|
||||
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 1", calls, validatorCalls, store.saves)
|
||||
}
|
||||
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "approved" {
|
||||
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want discarded cache warnings omitted", output.Warnings)
|
||||
}
|
||||
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
|
||||
}
|
||||
|
||||
func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
calls := 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||
record := chunkPlanRecord(t, prepared, plan)
|
||||
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
|
||||
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != 1 || validator.calls != 2 || store.saves != 0 {
|
||||
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 0", calls, validator.calls, store.saves)
|
||||
}
|
||||
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "incomplete" {
|
||||
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "validator_execution_incomplete" {
|
||||
t.Fatalf("warnings = %#v, want only generated incomplete warning", output.Warnings)
|
||||
}
|
||||
if !reflect.DeepEqual(store.record, record) {
|
||||
t.Fatal("discarded incomplete candidates mutated the stored record")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerFailsOnValidationIncompleteChunkPlanHitUnderFailRun(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
calls := 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||
if err == nil || !strings.Contains(err.Error(), "validator unavailable") {
|
||||
t.Fatalf("Run() error = %v, want cached validation failure", err)
|
||||
}
|
||||
if calls != 0 || validator.calls != 1 || store.saves != 0 {
|
||||
t.Fatalf("calls = producer %d validator %d saves %d, want 0 1 0", calls, validator.calls, store.saves)
|
||||
}
|
||||
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "error" {
|
||||
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
|
||||
prepared, _ := preparedTerminalDebugPipeline(t)
|
||||
encoder := &capturingChunkMapOutput{}
|
||||
@@ -358,7 +444,7 @@ func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
|
||||
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject/lane"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
@@ -392,7 +478,7 @@ func TestRunnerChunkMapRequestDoesNotAliasStoredPlan(t *testing.T) {
|
||||
|
||||
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
|
||||
prepared, _ := preparedTerminalDebugPipeline(t)
|
||||
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no"}}
|
||||
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no", CorrectionGuidance: "return a policy-compliant chunk plan"}}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
|
||||
if err != nil {
|
||||
@@ -508,7 +594,7 @@ func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
|
||||
wantReject bool
|
||||
}{
|
||||
{name: "warning", result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "current", ReasonCode: "observed", Message: "current warning"}}}},
|
||||
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit"}, wantReject: true},
|
||||
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit", CorrectionGuidance: "return an acceptable chunk plan"}, wantReject: true},
|
||||
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
@@ -573,7 +659,7 @@ func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
|
||||
wantReject bool
|
||||
}{
|
||||
{name: "module error", moduleErr: errors.New("generation failed"), wantCalls: 2},
|
||||
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan"}, wantCalls: 1, wantReject: true},
|
||||
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan", CorrectionGuidance: "return an acceptable chunk plan"}, wantCalls: 1, wantReject: true},
|
||||
{name: "cancellation", cancel: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
@@ -568,7 +568,7 @@ func TestRunnerKeepsSuccessfulChunksWhenAnotherExtractIsRejected(t *testing.T) {
|
||||
validator := &prepared.Steps[0].lanes[0].extractValidators.validators[0]
|
||||
validator.typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
if target.chunk != nil && target.chunk.Index == 0 {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ type laneExtractState struct {
|
||||
prepared preparedLaneExecutor
|
||||
deps []CheckpointFingerprint
|
||||
decision CheckpointDecision
|
||||
reuseEligible bool
|
||||
values []erasedExtractArtifact
|
||||
serialized []CheckpointArtifact
|
||||
warnings []contracts.Warning
|
||||
@@ -42,6 +43,7 @@ type finalizedExtractResults struct {
|
||||
incomplete []int
|
||||
validationSummaries []artifacts.ValidationSummary
|
||||
decision CheckpointDecision
|
||||
reuseEligible bool
|
||||
}
|
||||
|
||||
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
@@ -123,6 +125,9 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
|
||||
return states, err
|
||||
}
|
||||
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
|
||||
if !laneReferencesReuseEligible(input, prepared.resolved) {
|
||||
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
|
||||
}
|
||||
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
|
||||
states[i] = state
|
||||
if err != nil {
|
||||
@@ -134,7 +139,7 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
|
||||
if err != nil {
|
||||
return states, err
|
||||
}
|
||||
if !state.decision.Reused {
|
||||
if !state.decision.Reused && state.reuseEligible {
|
||||
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
|
||||
return states, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
|
||||
}
|
||||
@@ -303,7 +308,9 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
|
||||
if result.err != nil {
|
||||
state.failed = true
|
||||
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
|
||||
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
|
||||
if state.reuseEligible {
|
||||
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
|
||||
}
|
||||
e.cancel()
|
||||
} else {
|
||||
state.results[result.chunkIndex] = result
|
||||
@@ -341,7 +348,7 @@ func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.So
|
||||
}
|
||||
resolution, err := resolveCheckpointDecision(&local, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
|
||||
decision = resolution.decision
|
||||
state := &laneExtractState{index: index, prepared: prepared, decision: decision, terminal: true, output: local}
|
||||
state := &laneExtractState{index: index, prepared: prepared, decision: decision, reuseEligible: true, terminal: true, output: local}
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
@@ -354,6 +361,7 @@ func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.So
|
||||
SourceID: doc.ID,
|
||||
Artifact: contracts.CloneSerializedArtifact(hydrated.Artifact),
|
||||
})
|
||||
local.normalizeReuseEligibility = map[generatedOutputKey]bool{generatedOutputKeyFor(input.stepID, lane.ID): true}
|
||||
state.output = local
|
||||
return state, nil
|
||||
}
|
||||
@@ -387,7 +395,12 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
|
||||
}
|
||||
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
|
||||
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
|
||||
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
|
||||
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
|
||||
if !state.reuseEligible {
|
||||
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
|
||||
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
|
||||
return state, nil
|
||||
}
|
||||
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
|
||||
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
|
||||
if err != nil {
|
||||
@@ -460,14 +473,6 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
|
||||
if validationErr != nil {
|
||||
return report, candidate.terminal.record(payload, validationErr)
|
||||
}
|
||||
if report.FirstRejection() != nil {
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
}
|
||||
if report.FirstRejection() == nil && lane.ExtractValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
if failure := firstIncompleteValidation(report); failure != nil {
|
||||
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
|
||||
}
|
||||
}
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
})
|
||||
result.err = err
|
||||
@@ -540,7 +545,10 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
|
||||
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
|
||||
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
|
||||
sort.Ints(state.incomplete)
|
||||
if !state.decision.Reused && len(state.incomplete) == 0 {
|
||||
if len(state.incomplete) > 0 {
|
||||
state.reuseEligible = false
|
||||
}
|
||||
if !state.decision.Reused && state.reuseEligible {
|
||||
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected, state.warnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
@@ -559,6 +567,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints C
|
||||
incomplete: state.incomplete,
|
||||
validationSummaries: state.validationSummaries,
|
||||
decision: state.decision,
|
||||
reuseEligible: state.reuseEligible,
|
||||
}
|
||||
local.Warnings = append(local.Warnings, cloneWarnings(results.warnings)...)
|
||||
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
|
||||
@@ -634,6 +643,14 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
|
||||
dst.Warnings = append(dst.Warnings, cloneWarnings(src.Warnings)...)
|
||||
dst.CheckpointEvents = append(dst.CheckpointEvents, src.CheckpointEvents...)
|
||||
dst.ValidationSummaries = append(dst.ValidationSummaries, cloneValidationSummaries(src.ValidationSummaries)...)
|
||||
if len(src.normalizeReuseEligibility) > 0 {
|
||||
if dst.normalizeReuseEligibility == nil {
|
||||
dst.normalizeReuseEligibility = make(map[generatedOutputKey]bool, len(src.normalizeReuseEligibility))
|
||||
}
|
||||
for key, eligible := range src.normalizeReuseEligibility {
|
||||
dst.normalizeReuseEligibility[key] = eligible
|
||||
}
|
||||
}
|
||||
for i := range dst.Manifest.ArtifactLanes {
|
||||
for j := range src.Manifest.ArtifactLanes {
|
||||
if dst.Manifest.ArtifactLanes[i].ID == src.Manifest.ArtifactLanes[j].ID && dst.Manifest.ArtifactLanes[i].StepID == src.Manifest.ArtifactLanes[j].StepID && src.Manifest.ArtifactLanes[j].Metadata != nil {
|
||||
|
||||
@@ -65,8 +65,8 @@ func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
|
||||
}
|
||||
for index := 0; index < 2; index++ {
|
||||
correction := corrections[index]
|
||||
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || correction.UserGuidance != fmt.Sprintf("correct chunk %d", index) {
|
||||
t.Fatalf("chunk %d correction = %#v, want its exact initial response and guidance", index, correction)
|
||||
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || !strings.Contains(correction.UserGuidance, fmt.Sprintf("correct chunk %d", index)) || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "incorrect_extract") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
|
||||
t.Fatalf("chunk %d correction = %#v, want its exact initial response and semantic replacement guidance only", index, correction)
|
||||
}
|
||||
}
|
||||
terminal := string(debug.json["extract/notes/chunk-000001/terminal.json"])
|
||||
|
||||
@@ -230,7 +230,7 @@ func TestRunnerPromotesOnlyAcceptedExtractRetryWarnings(t *testing.T) {
|
||||
validatorCalls := 0
|
||||
prepared.Steps[0].lanes[0].extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
validatorCalls++
|
||||
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract"}, nil
|
||||
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract", CorrectionGuidance: "return an acceptable extract"}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
|
||||
validator: &preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject-final-fallback"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback", CorrectionGuidance: "return an acceptable normalized artifact"}, nil
|
||||
},
|
||||
},
|
||||
wantCalls: 2,
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestRunnerPromotesOnlyTerminalRejectionWarnings(t *testing.T) {
|
||||
return contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: fmt.Sprintf("validator-%d", attempts), ReasonCode: "validator", Message: "validator warning"}}}
|
||||
}
|
||||
reject := func() contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected"}
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected", CorrectionGuidance: "return an acceptable candidate"}
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
|
||||
238
internal/framework/pipeline/runner_reuse_lineage_test.go
Normal file
238
internal/framework/pipeline/runner_reuse_lineage_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type reuseLineageCheckpointSpy struct {
|
||||
CheckpointLoader
|
||||
CheckpointRecorder
|
||||
loads map[string]int
|
||||
writes map[string]int
|
||||
forbid map[string]struct{}
|
||||
}
|
||||
|
||||
func newReuseLineageCheckpointSpy() *reuseLineageCheckpointSpy {
|
||||
return &reuseLineageCheckpointSpy{
|
||||
CheckpointLoader: NoopCheckpointLoader(),
|
||||
CheckpointRecorder: NoopCheckpointRecorder(),
|
||||
loads: make(map[string]int),
|
||||
writes: make(map[string]int),
|
||||
forbid: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Enabled() bool { return true }
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) load(stage, laneID string) {
|
||||
key := stage + "/" + laneID
|
||||
if _, forbidden := s.forbid[key]; forbidden {
|
||||
panic("checkpoint lookup crossed validation-incomplete lineage: " + key)
|
||||
}
|
||||
s.loads[key]++
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) write(stage, action, laneID string) {
|
||||
s.writes[stage+"/"+action+"/"+laneID]++
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
s.load("extract", laneID)
|
||||
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
s.load("merge", laneID)
|
||||
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
s.load("normalize", laneID)
|
||||
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("extract", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ []CheckpointArtifact, _ []contracts.RejectedOutput, _ []contracts.Warning) error {
|
||||
s.write("extract", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("extract", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("merge", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact, _ []contracts.Warning) error {
|
||||
s.write("merge", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
|
||||
s.write("merge", "rejected", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("merge", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("normalize", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact, _ []contracts.Warning) error {
|
||||
s.write("normalize", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
|
||||
s.write("normalize", "rejected", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("normalize", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) writesFor(stage, laneID string) int {
|
||||
total := 0
|
||||
needle := stage + "/"
|
||||
suffix := "/" + laneID
|
||||
for key, count := range s.writes {
|
||||
if strings.HasPrefix(key, needle) && strings.HasSuffix(key, suffix) {
|
||||
total += count
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func unavailableTypedValidator(kind contracts.ArtifactKind) preparedValidator {
|
||||
return preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("typed/check"), Target: ValidatorTargetTyped, ArtifactKind: kind},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteExtractDisablesDownstreamCheckpointIO(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
|
||||
}
|
||||
if spy.loads["merge/notes"] != 0 || spy.loads["normalize/notes"] != 0 {
|
||||
t.Fatalf("downstream loads = %#v, want none", spy.loads)
|
||||
}
|
||||
if spy.writesFor("merge", "notes") != 0 || spy.writesFor("normalize", "notes") != 0 {
|
||||
t.Fatalf("downstream writes = %#v, want none", spy.writes)
|
||||
}
|
||||
if spy.writes["extract/succeeded/notes"] != 0 {
|
||||
t.Fatalf("extract writes = %#v, want no reusable success", spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteMergeDisablesNormalizeCheckpointIO(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.MergeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.mergeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
|
||||
}
|
||||
if spy.loads["merge/notes"] != 1 || spy.loads["normalize/notes"] != 0 {
|
||||
t.Fatalf("loads = %#v, want merge lookup only", spy.loads)
|
||||
}
|
||||
if spy.writes["merge/succeeded/notes"] != 0 || spy.writesFor("normalize", "notes") != 0 {
|
||||
t.Fatalf("writes = %#v, want no reusable merge or normalize state", spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteNormalizeIsNotPublished(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || spy.writes["normalize/succeeded/notes"] != 0 {
|
||||
t.Fatalf("output/writes = %d / %#v, want in-memory output without normalize publication", len(output.NormalizeOutputs), spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedReferenceFromIncompleteValidationDisablesDependentCheckpointIO(t *testing.T) {
|
||||
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"producer"}})
|
||||
prepared := input.Prepared
|
||||
producer := &prepared.Steps[0].lanes[0]
|
||||
consumer := &prepared.Steps[1].lanes[0]
|
||||
producer.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
producer.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(producer.resolved.ArtifactKind)}
|
||||
referenceItems := 0
|
||||
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
referenceItems = len(request.References.Slots["producer-output"].Items)
|
||||
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
|
||||
}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
for _, stage := range []string{"extract", "merge", "normalize"} {
|
||||
spy.forbid[stage+"/score"] = struct{}{}
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if referenceItems != 1 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("current-run handoff = items %d outputs %d, want one generated item and two outputs", referenceItems, len(output.NormalizeOutputs))
|
||||
}
|
||||
for _, stage := range []string{"extract", "merge", "normalize"} {
|
||||
if spy.writesFor(stage, "score") != 0 {
|
||||
t.Fatalf("dependent writes = %#v, want none for %s", spy.writes, stage)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -61,8 +62,8 @@ func TestRunnerCorrectsRejectedMergeAndNormalizeCandidates(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || correction.UserGuidance != "produce the accepted value" {
|
||||
t.Fatalf("%s correction = %#v, want exact rejected response and guidance", target, correction)
|
||||
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || !strings.Contains(correction.UserGuidance, "produce the accepted value") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "invalid_candidate") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
|
||||
t.Fatalf("%s correction = %#v, want exact rejected response and semantic replacement guidance only", target, correction)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted output", output)
|
||||
@@ -84,7 +85,7 @@ func TestRunnerRejectsDeterministicMergeCandidateWithoutCorrection(t *testing.T)
|
||||
lane.mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted"}, nil
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
|
||||
|
||||
@@ -157,10 +157,11 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
||||
validator terminalChunkValidator
|
||||
wantError string
|
||||
wantRejection bool
|
||||
attemptError bool
|
||||
}{
|
||||
{name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}},
|
||||
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed"},
|
||||
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected"}}, wantRejection: true},
|
||||
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed", attemptError: true},
|
||||
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected", CorrectionGuidance: "return an acceptable chunk plan"}}, wantRejection: true},
|
||||
{name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
@@ -174,9 +175,15 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
||||
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
|
||||
envelope := debug.envelope(t, "chunk/attempt-01.json")
|
||||
if tc.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
@@ -300,12 +307,13 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
||||
candidateFail bool
|
||||
finalFail bool
|
||||
wantError string
|
||||
attemptError bool
|
||||
}{
|
||||
{name: "terminal rejection", reject: true},
|
||||
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed"},
|
||||
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed", attemptError: true},
|
||||
{name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"},
|
||||
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate"},
|
||||
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output"},
|
||||
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate", attemptError: true},
|
||||
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output", attemptError: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -333,7 +341,7 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
||||
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected"}, tc.validatorErr
|
||||
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected", CorrectionGuidance: "return an acceptable extract"}, tc.validatorErr
|
||||
},
|
||||
}}
|
||||
debug := newCapturedDebugRecorder()
|
||||
@@ -343,9 +351,15 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
||||
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
|
||||
envelope := debug.envelope(t, attemptPath)
|
||||
if tc.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil rejection", err)
|
||||
|
||||
@@ -193,6 +193,7 @@ type mergeStageResult struct {
|
||||
serialized CheckpointArtifact
|
||||
terminal bool
|
||||
validationIncomplete bool
|
||||
reuseEligible bool
|
||||
}
|
||||
|
||||
type mergeAttemptValue struct {
|
||||
@@ -223,12 +224,16 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
|
||||
if merged.terminal {
|
||||
return nil
|
||||
}
|
||||
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, output)
|
||||
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, merged.reuseEligible, output)
|
||||
if err != nil {
|
||||
return &laneRunError{stage: StageNormalize, err: err}
|
||||
}
|
||||
if normalized.accepted {
|
||||
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized.Artifact)})
|
||||
if output.normalizeReuseEligibility == nil {
|
||||
output.normalizeReuseEligibility = make(map[generatedOutputKey]bool)
|
||||
}
|
||||
output.normalizeReuseEligibility[generatedOutputKeyFor(input.stepID, lane.ID)] = normalized.reuseEligible
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -241,9 +246,14 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
|
||||
}
|
||||
mergeReferences := operationReferenceSet(input, lane.MergeReferences)
|
||||
stageResult.reuseEligible = extracts.reuseEligible && referenceTargetReuseEligible(input, lane.MergeReferences)
|
||||
mergeDeps := append(artifactCheckpointDigests(extracts.serialized), generatedReferenceDependencies(mergeReferences)...)
|
||||
mergeDeps = normalizeCheckpointFingerprints(mergeDeps)
|
||||
mergeCP, mergeDecision := loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, mergeDeps)
|
||||
var mergeCP MergeCheckpoint
|
||||
mergeDecision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
|
||||
if stageResult.reuseEligible {
|
||||
mergeCP, mergeDecision = loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, mergeDeps)
|
||||
}
|
||||
mergeResolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageMerge, input.stepID, lane.ID, lane.Merge.Module, mergeDecision, typed.codec, []CheckpointArtifact{mergeCP.Output})
|
||||
if err != nil {
|
||||
return stageResult, err
|
||||
@@ -261,8 +271,10 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
mergeWarnings = cloneWarnings(mergeCP.Warnings)
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
} else {
|
||||
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps); err != nil {
|
||||
return stageResult, err
|
||||
if stageResult.reuseEligible {
|
||||
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
}
|
||||
terminalResult, runErr := runProducerAttempts(ctx, producerAttemptConfig{Retries: lane.Merge.Retries, Policy: lane.MergeValidationPolicy, AllowStructuralRetry: lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked}, func(attemptCtx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
started := time.Now().UTC()
|
||||
@@ -295,23 +307,19 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
if validationErr != nil {
|
||||
return report, candidate.terminal.record(payload, validationErr)
|
||||
}
|
||||
if report.FirstRejection() != nil {
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
}
|
||||
if lane.MergeValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
if failure := firstIncompleteValidation(report); failure != nil {
|
||||
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
})
|
||||
terminalSummary := validationSummary(terminalResult, StageMerge, input.stepID, lane.ID, lane.Merge.Module, "", 0)
|
||||
if debugErr := writeProducerTerminalDebug(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "terminal.json"), terminalResult, lane.MergeValidationPolicy, terminalSummary); debugErr != nil {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
|
||||
}
|
||||
return stageResult, debugErr
|
||||
}
|
||||
if runErr != nil {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, runErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, runErr)
|
||||
}
|
||||
return stageResult, runErr
|
||||
}
|
||||
if terminalResult.Action == producerTerminalRejected {
|
||||
@@ -324,8 +332,10 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(terminalSummary))
|
||||
output.Warnings = append(output.Warnings, terminalResult.Warnings...)
|
||||
output.Rejected = append(output.Rejected, *rejected)
|
||||
if err := checkpointMergeRejected(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, *rejected); err != nil {
|
||||
return stageResult, err
|
||||
if stageResult.reuseEligible {
|
||||
if err := checkpointMergeRejected(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, *rejected); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
}
|
||||
stageResult.terminal = true
|
||||
return stageResult, nil
|
||||
@@ -338,19 +348,26 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
payload := map[string]any{"output": debugCheckpointArtifact(candidate.candidate), "warnings": debugWarningEnvelopes(terminalResult.Warnings), "rejection": debugRejectedOutputPtr(nil)}
|
||||
if encodeErr != nil {
|
||||
attemptErr := fmt.Errorf("serialize accepted merge output for lane %q: %w", lane.ID, encodeErr)
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, attemptErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, attemptErr)
|
||||
}
|
||||
return stageResult, candidate.terminal.record(payload, attemptErr)
|
||||
}
|
||||
if debugErr := candidate.terminal.record(payload, nil); debugErr != nil {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
|
||||
}
|
||||
return stageResult, debugErr
|
||||
}
|
||||
merged, serializedMerge = candidate.artifact, stored
|
||||
mergeWarnings = cloneWarnings(terminalResult.Warnings)
|
||||
output.ValidationSummaries = append(output.ValidationSummaries, terminalSummary)
|
||||
stageResult.validationIncomplete = terminalResult.ValidationIncomplete
|
||||
if stageResult.validationIncomplete {
|
||||
stageResult.reuseEligible = false
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if !stageResult.validationIncomplete {
|
||||
if stageResult.reuseEligible {
|
||||
if err := recordMerge(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
@@ -365,18 +382,24 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
|
||||
}
|
||||
|
||||
type normalizeStageResult struct {
|
||||
serialized CheckpointArtifact
|
||||
warnings []contracts.Warning
|
||||
accepted bool
|
||||
serialized CheckpointArtifact
|
||||
warnings []contracts.Warning
|
||||
accepted bool
|
||||
reuseEligible bool
|
||||
}
|
||||
|
||||
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, output *RunOutput) (normalizeStageResult, error) {
|
||||
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
|
||||
var stageResult normalizeStageResult
|
||||
lane, typed := prepared.resolved, prepared.typed
|
||||
normalizeReferences := operationReferenceSet(input, lane.NormalizeReferences)
|
||||
stageResult.reuseEligible = upstreamReuseEligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
|
||||
normalizeDeps := append(artifactCheckpointDigests([]CheckpointArtifact{serializedMerge}), generatedReferenceDependencies(normalizeReferences)...)
|
||||
normalizeDeps = normalizeCheckpointFingerprints(normalizeDeps)
|
||||
normalizeCP, normalizeDecision := loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps)
|
||||
var normalizeCP NormalizeCheckpoint
|
||||
normalizeDecision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
|
||||
if stageResult.reuseEligible {
|
||||
normalizeCP, normalizeDecision = loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps)
|
||||
}
|
||||
normalizeResolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, normalizeDecision, typed.codec, []CheckpointArtifact{normalizeCP.Output})
|
||||
if err != nil {
|
||||
return stageResult, err
|
||||
@@ -392,8 +415,10 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
normalizeWarnings = cloneWarnings(normalizeCP.Warnings)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
} else {
|
||||
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
|
||||
return stageResult, err
|
||||
if stageResult.reuseEligible {
|
||||
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
}
|
||||
terminalResult, runErr := runProducerAttempts(ctx, producerAttemptConfig{Retries: lane.Normalize.Retries, Policy: lane.NormalizeValidationPolicy, AllowStructuralRetry: lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked}, func(attemptCtx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
started := time.Now().UTC()
|
||||
@@ -444,23 +469,19 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
if validationErr != nil {
|
||||
return report, candidate.terminal.record(payload, validationErr)
|
||||
}
|
||||
if report.FirstRejection() != nil {
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
}
|
||||
if lane.NormalizeValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
|
||||
if failure := firstIncompleteValidation(report); failure != nil {
|
||||
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
return report, candidate.terminal.record(payload, nil)
|
||||
})
|
||||
terminalSummary := validationSummary(terminalResult, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, "", 0)
|
||||
if debugErr := writeProducerTerminalDebug(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "terminal.json"), terminalResult, lane.NormalizeValidationPolicy, terminalSummary); debugErr != nil {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
|
||||
}
|
||||
return stageResult, debugErr
|
||||
}
|
||||
if runErr != nil {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
|
||||
}
|
||||
return stageResult, runErr
|
||||
}
|
||||
if terminalResult.Action == producerTerminalRejected {
|
||||
@@ -473,8 +494,10 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(terminalSummary))
|
||||
output.Warnings = append(output.Warnings, terminalResult.Warnings...)
|
||||
output.Rejected = append(output.Rejected, *rejected)
|
||||
if err := checkpointNormalizeRejected(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, *rejected); err != nil {
|
||||
return stageResult, err
|
||||
if stageResult.reuseEligible {
|
||||
if err := checkpointNormalizeRejected(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, *rejected); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
}
|
||||
return stageResult, nil
|
||||
}
|
||||
@@ -489,18 +512,25 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
}
|
||||
if encodeErr != nil {
|
||||
attemptErr := fmt.Errorf("serialize accepted normalize output for lane %q: %w", lane.ID, encodeErr)
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, attemptErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, attemptErr)
|
||||
}
|
||||
return stageResult, candidate.terminal.record(payload, attemptErr)
|
||||
}
|
||||
if debugErr := candidate.terminal.record(payload, nil); debugErr != nil {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
|
||||
if stageResult.reuseEligible {
|
||||
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
|
||||
}
|
||||
return stageResult, debugErr
|
||||
}
|
||||
serializedNormalize = stored
|
||||
normalizeWarnings = cloneWarnings(terminalResult.Warnings)
|
||||
output.ValidationSummaries = append(output.ValidationSummaries, terminalSummary)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if !terminalResult.ValidationIncomplete {
|
||||
if terminalResult.ValidationIncomplete {
|
||||
stageResult.reuseEligible = false
|
||||
}
|
||||
if stageResult.reuseEligible {
|
||||
if err := recordNormalize(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
|
||||
return stageResult, err
|
||||
}
|
||||
@@ -616,20 +646,6 @@ func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEnt
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
report, err := r.validateTypedReport(ctx, codec, target, chain, attempt, debug)
|
||||
if err != nil {
|
||||
return report.Warnings(), nil, err
|
||||
}
|
||||
if failure := report.FirstFailure(); failure != nil {
|
||||
return report.Warnings(), nil, validatorFailureError(*failure)
|
||||
}
|
||||
if rejected := typedRejection(report, target, attempt); rejected != nil {
|
||||
return report.Warnings(), rejected, nil
|
||||
}
|
||||
return report.Warnings(), nil, nil
|
||||
}
|
||||
|
||||
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
|
||||
rejection := report.FirstRejection()
|
||||
if rejection == nil {
|
||||
|
||||
@@ -18,7 +18,7 @@ const (
|
||||
validationSkipped validationOutcome = "skipped"
|
||||
)
|
||||
|
||||
const defaultCorrectionGuidance = "Correct the candidate to satisfy the validator requirements."
|
||||
const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"
|
||||
|
||||
// validationRecord captures the settled result of one configured validator.
|
||||
// Its fields remain private so reports cannot expose mutable warning storage.
|
||||
@@ -82,33 +82,39 @@ func (report validationReport) FirstFailure() *validationRecord {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (report validationReport) CorrectionGuidance() string {
|
||||
func (report validationReport) CorrectionRequest() (string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
parts := make([]string, 0, len(report.records))
|
||||
used := 0
|
||||
for _, record := range report.records {
|
||||
if record.outcome != validationRejected {
|
||||
continue
|
||||
}
|
||||
guidance := strings.TrimSpace(record.correctionGuidance)
|
||||
if guidance == "" {
|
||||
guidance = defaultCorrectionGuidance
|
||||
return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
|
||||
}
|
||||
if _, exists := seen[guidance]; exists {
|
||||
continue
|
||||
}
|
||||
separator := 0
|
||||
if len(parts) > 0 {
|
||||
separator = 1
|
||||
}
|
||||
if used+separator+len(guidance) > contracts.MaxCorrectionGuidanceBytes {
|
||||
break
|
||||
}
|
||||
seen[guidance] = struct{}{}
|
||||
parts = append(parts, guidance)
|
||||
used += separator + len(guidance)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
if len(parts) == 0 {
|
||||
return "", errors.New("validation report contains no correction guidance")
|
||||
}
|
||||
var builder strings.Builder
|
||||
builder.WriteString(correctionRequestIntroduction)
|
||||
for index, guidance := range parts {
|
||||
fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
|
||||
if index+1 < len(parts) {
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
request := builder.String()
|
||||
if len(request) > contracts.MaxCorrectionGuidanceBytes {
|
||||
return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type validationInvocation struct {
|
||||
@@ -184,15 +190,11 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, warnings: cloneWarnings(invocation.result.Warnings), diagnosticPath: invocation.result.DiagnosticArtifactPath})
|
||||
break
|
||||
}
|
||||
reason := invocation.result.ReasonCode
|
||||
if reason == "" {
|
||||
reason = "output_rejected"
|
||||
}
|
||||
message := invocation.result.Message
|
||||
if message == "" {
|
||||
message = "output rejected"
|
||||
}
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: reason, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, warnings: cloneWarnings(invocation.result.Warnings), correctionGuidance: invocation.result.CorrectionGuidance})
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, warnings: cloneWarnings(invocation.result.Warnings), correctionGuidance: invocation.result.CorrectionGuidance})
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
invoke validationInvoker
|
||||
want []validationOutcome
|
||||
wantCalls []string
|
||||
wantGuidance string
|
||||
wantGuidance []string
|
||||
wantWarnings []string
|
||||
}{
|
||||
{
|
||||
@@ -41,7 +41,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
|
||||
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
|
||||
wantGuidance: "repair shape\nrepair references",
|
||||
wantGuidance: []string{"repair shape", "repair references"},
|
||||
},
|
||||
{
|
||||
name: "rejection and failure both settle",
|
||||
@@ -52,7 +52,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationFailed},
|
||||
wantCalls: []string{"shape:1", "remote:1"},
|
||||
wantGuidance: "repair shape",
|
||||
wantGuidance: []string{"repair shape"},
|
||||
},
|
||||
{
|
||||
name: "failure only has no correction guidance",
|
||||
@@ -103,8 +103,19 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
|
||||
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
|
||||
}
|
||||
if guidance := report.CorrectionGuidance(); guidance != test.wantGuidance {
|
||||
t.Fatalf("CorrectionGuidance() = %q, want %q", guidance, test.wantGuidance)
|
||||
if len(test.wantGuidance) > 0 {
|
||||
guidance, err := report.CorrectionRequest()
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionRequest() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(guidance, "complete corrected replacement") {
|
||||
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
|
||||
}
|
||||
for _, item := range test.wantGuidance {
|
||||
if strings.Count(guidance, item) != 1 {
|
||||
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
warnings := report.Warnings()
|
||||
var warningCodes []string
|
||||
@@ -139,7 +150,7 @@ func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationReportBoundsGuidanceAndOwnsRecords(t *testing.T) {
|
||||
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
|
||||
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
|
||||
report := validationReport{records: make([]validationRecord, 17)}
|
||||
for index := range report.records {
|
||||
@@ -148,8 +159,8 @@ func TestValidationReportBoundsGuidanceAndOwnsRecords(t *testing.T) {
|
||||
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
|
||||
}
|
||||
report.records[0].warnings = []contracts.Warning{{ReasonCode: "warning"}}
|
||||
if got := report.CorrectionGuidance(); len(got) > contracts.MaxCorrectionGuidanceBytes || !strings.Contains(got, string([]byte{byte('a')})) || strings.Contains(got, string([]byte{byte('q')})) {
|
||||
t.Fatalf("CorrectionGuidance() length = %d, want bounded ordered guidance", len(got))
|
||||
if _, err := report.CorrectionRequest(); err == nil {
|
||||
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
|
||||
}
|
||||
records := report.Records()
|
||||
records[0].warnings[0].ReasonCode = "changed"
|
||||
|
||||
Reference in New Issue
Block a user