Strengthen validation retries and checkpoint safety

This commit is contained in:
2026-08-27 11:50:37 +00:00
parent b2e83bd6e7
commit e6b2ae88d2
73 changed files with 934 additions and 1560 deletions

View File

@@ -124,8 +124,8 @@ func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
}
correction := extractor.correctionSnapshot()
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || correction.UserGuidance != "use a known spell name" {
t.Fatalf("extract correction = %#v, want exact rejected model response and validator guidance", correction)
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
}
}

View File

@@ -126,11 +126,17 @@ func (candidate ModelCandidate) Validate() error {
}
func ValidateValidationResult(result ValidationResult) error {
if !result.Approved && result.ReasonCode == "" {
return errors.New("validation rejection reason code must not be empty")
}
if result.ReasonCode != "" {
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
return err
}
}
if !result.Approved && result.CorrectionGuidance == "" {
return errors.New("validation rejection correction guidance must not be empty")
}
if result.CorrectionGuidance != "" {
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
return err

View File

@@ -59,13 +59,21 @@ func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
{"oversized reason code", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason}) }},
{"blank reason code", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: " \t"}) }},
{"missing rejection reason code", func() error {
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
}},
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
{"oversized reason code", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
}},
{"blank reason code", func() error {
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
}},
{"invalid correction guidance utf8", func() error {
return ValidateValidationResult(ValidationResult{CorrectionGuidance: string([]byte{0xff})})
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
}},
{"oversized correction guidance", func() error {
return ValidateValidationResult(ValidationResult{CorrectionGuidance: tooLongValidationGuidance})
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
}},
} {
t.Run(test.name, func(t *testing.T) {

View File

@@ -89,6 +89,7 @@ const (
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
)
type CheckpointDecision struct {
@@ -160,6 +161,8 @@ func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
return "accepted normalized artifact is reusable"
case CheckpointReasonRecomputeStep:
return "selected step requires execution"
case CheckpointReasonValidationIncompleteLineage:
return "checkpoint reuse is disabled by validation-incomplete input lineage"
default:
return "checkpoint decision"
}

View File

@@ -114,7 +114,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) {
prepared.Steps[1].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
prepared.Steps[1].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput

View File

@@ -41,6 +41,54 @@ func operationReferenceSet(input RunInput, target ResolvedReferenceTarget) contr
return CloneReferenceSet(target.ReferenceSet)
}
// referenceTargetReuseEligible reports whether every generated artifact in a
// stage's reference set descends exclusively from fully validated work. Static
// references and callers that do not supply lineage metadata are reusable.
func referenceTargetReuseEligible(input RunInput, target ResolvedReferenceTarget) bool {
if input.referenceReuseEligibility == nil {
return true
}
eligible, ok := input.referenceReuseEligibility[keyForReferenceTarget(target)]
return !ok || eligible
}
func laneReferencesReuseEligible(input RunInput, lane ResolvedArtifactLane) bool {
return referenceTargetReuseEligible(input, lane.ExtractReferences) &&
referenceTargetReuseEligible(input, lane.MergeReferences) &&
referenceTargetReuseEligible(input, lane.NormalizeReferences)
}
// buildStepReferenceReuseEligibility carries validation completeness alongside
// generated references without exposing the internal lineage flag in artifact
// payloads. A target becomes ineligible when any generated input is ineligible.
func buildStepReferenceReuseEligibility(step PreparedPipelineStep, outputs map[generatedOutputKey]bool) map[referenceTargetKey]bool {
eligibility := make(map[referenceTargetKey]bool)
for _, prepared := range step.lanes {
lane := prepared.resolved
for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
generated := false
eligible := true
for _, binding := range target.Bindings {
if binding.Artifact == nil {
continue
}
generated = true
producer := generatedOutputKeyFor(binding.Artifact.Step, binding.Artifact.Lane)
if reusable, ok := outputs[producer]; ok && !reusable {
eligible = false
}
}
if generated {
eligibility[keyForReferenceTarget(target)] = eligible
}
}
}
if len(eligibility) == 0 {
return nil
}
return eligibility
}
// buildStepReferenceSets resolves every generated binding for a step before
// any lane in that step is allowed to start. Each returned set is a fresh
// operation-time view; prepared reference sets are never modified.

View File

@@ -232,7 +232,12 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
}
}
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, report.CorrectionGuidance())
correctionRequest, guidanceErr := report.CorrectionRequest()
if guidanceErr != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
}
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)

View File

@@ -61,10 +61,11 @@ type RunInput struct {
// single worker so direct framework callers retain deterministic behavior.
ExtractWorkers int
pipeline ResolvedPipeline
llmClient contracts.StructuredLLMClient
stepID string
references map[referenceTargetKey]contracts.ReferenceSet
pipeline ResolvedPipeline
llmClient contracts.StructuredLLMClient
stepID string
references map[referenceTargetKey]contracts.ReferenceSet
referenceReuseEligibility map[referenceTargetKey]bool
}
type RunOutput struct {
@@ -76,6 +77,8 @@ type RunOutput struct {
OutputFiles []contracts.OutputFile `json:"-"`
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
normalizeReuseEligibility map[generatedOutputKey]bool
}
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
@@ -425,6 +428,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
}
stepInput.references = stepReferences
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
if err := mergeLaneOutput(output, laneOutput); err != nil {

View File

@@ -237,7 +237,7 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate"}, nil
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}
switch stage {
@@ -280,11 +280,12 @@ func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedWarningsOnly(t *te
func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
tests := []struct {
name string
configure func(*PreparedPipeline)
path string
wantError string
wantBody string
name string
configure func(*PreparedPipeline)
path string
wantError string
wantBody string
attemptError bool
}{
{
name: "merge module error",
@@ -293,8 +294,9 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
return erasedTypedResult{}, errors.New("merge exploded")
}
},
path: "merge/notes/attempt-01.json",
wantError: "merge exploded",
path: "merge/notes/attempt-01.json",
wantError: "merge exploded",
attemptError: true,
},
{
name: "normalize validator error",
@@ -318,7 +320,7 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
},
@@ -332,8 +334,9 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
return erasedTypedResult{Value: "wrong artifact type"}, nil
}
},
path: "normalize/notes/attempt-01.json",
wantError: "serialize normalize candidate",
path: "normalize/notes/attempt-01.json",
wantError: "serialize normalize candidate",
attemptError: true,
},
}
@@ -345,9 +348,15 @@ func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
envelope := debug.envelope(t, tc.path)
if tc.wantError != "" {
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else if runErr != nil {
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
}

View File

@@ -160,7 +160,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
validator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("candidate-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator"}, nil
return contracts.ValidationResult{Approved: approved, ReasonCode: "candidate_rejected", Message: "candidate rejected by validator", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}
switch target {

View File

@@ -72,10 +72,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
result.setValidation(report.Warnings(), nil, err)
return result, err
}
if report.FirstRejection() == nil {
if incomplete := firstIncompleteValidation(report); incomplete != nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
return result, validatorFailureError(*incomplete)
}
rejection := report.FirstRejection()
incomplete := firstIncompleteValidation(report)
if rejection == nil && incomplete == nil {
if err := result.setCandidate(record, "reused"); err != nil {
return result, fmt.Errorf("clone reused chunk plan record: %w", err)
}
@@ -85,18 +84,21 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
result.accepted = true
result.setValidation(report.Warnings(), nil, nil)
cachedTerminal := producerAttemptTerminal{Action: producerTerminalAccepted, Validation: report}
if firstIncompleteValidation(report) != nil {
result.summary.ValidationStatus = "incomplete"
cachedTerminal.Action = producerTerminalIncompleteAccepted
cachedTerminal.ValidationIncomplete = true
}
summary := validationSummary(cachedTerminal, StageChunk, "", "", chunker.Key(), "", 0)
result.validation = &summary
return result, nil
}
if incomplete != nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
failure := validatorFailureError(*incomplete)
result.setValidation(report.Warnings(), nil, failure)
failedTerminal := producerAttemptTerminal{Action: producerTerminalFailed, Validation: report, ValidationIncomplete: true}
summary := validationSummary(failedTerminal, StageChunk, "", "", chunker.Key(), "", 0)
result.validation = &summary
return result, failure
}
// A cache hit is not model material. Its rejection is discarded and
// generation begins with the ordinary initial request below.
result.setValidation(report.Warnings(), chunkRejection(report, 1, chunker.Key()), nil)
// generation begins with the ordinary initial request below. Warnings
// from this discarded candidate are intentionally not promoted.
}
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
result.summary.LookupStatus = "invalid"
@@ -176,11 +178,6 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if validationErr != nil {
return report, candidate.terminal.record(payload, validationErr)
}
if report.FirstRejection() == nil && input.pipeline.ChunkValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
if failure := firstIncompleteValidation(report); failure != nil {
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
}
}
return report, candidate.terminal.record(payload, nil)
})
terminalSummary := validationSummary(terminal, StageChunk, "", "", chunker.Key(), "", 0)

View File

@@ -293,7 +293,7 @@ func TestRunnerOmitsChunkMapForRejectedPlan(t *testing.T) {
prepared.output = encoder
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/chunk"), Target: ValidatorTargetChunk},
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}},
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable chunk plan"}},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
@@ -351,6 +351,92 @@ func TestRunnerDoesNotPublishValidationIncompleteChunkPlan(t *testing.T) {
}
}
func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validatorCalls := 0
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("cache-then-generated"), Target: ValidatorTargetChunk},
chunk: chunkValidationFunc{name: "cache-then-generated", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validatorCalls++
if validatorCalls == 1 {
return contracts.ValidationResult{}, errors.New("cached candidate could not be validated")
}
return contracts.ValidationResult{Approved: true}, nil
}},
}}
record := chunkPlanRecord(t, prepared, plan)
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validatorCalls != 2 || store.saves != 1 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 1", calls, validatorCalls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "approved" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Warnings) != 0 {
t.Fatalf("warnings = %#v, want discarded cache warnings omitted", output.Warnings)
}
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
}
func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validator.calls != 2 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "incomplete" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("warnings = %#v, want only generated incomplete warning", output.Warnings)
}
if !reflect.DeepEqual(store.record, record) {
t.Fatal("discarded incomplete candidates mutated the stored record")
}
}
func TestRunnerFailsOnValidationIncompleteChunkPlanHitUnderFailRun(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err == nil || !strings.Contains(err.Error(), "validator unavailable") {
t.Fatalf("Run() error = %v, want cached validation failure", err)
}
if calls != 0 || validator.calls != 1 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 0 1 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "error" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
}
func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
@@ -358,7 +444,7 @@ func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/lane"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
@@ -392,7 +478,7 @@ func TestRunnerChunkMapRequestDoesNotAliasStoredPlan(t *testing.T) {
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no"}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no", CorrectionGuidance: "return a policy-compliant chunk plan"}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
if err != nil {
@@ -508,7 +594,7 @@ func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
wantReject bool
}{
{name: "warning", result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "current", ReasonCode: "observed", Message: "current warning"}}}},
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit"}, wantReject: true},
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit", CorrectionGuidance: "return an acceptable chunk plan"}, wantReject: true},
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
}
for _, tc := range tests {
@@ -573,7 +659,7 @@ func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
wantReject bool
}{
{name: "module error", moduleErr: errors.New("generation failed"), wantCalls: 2},
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan"}, wantCalls: 1, wantReject: true},
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan", CorrectionGuidance: "return an acceptable chunk plan"}, wantCalls: 1, wantReject: true},
{name: "cancellation", cancel: true},
} {
t.Run(tc.name, func(t *testing.T) {

View File

@@ -568,7 +568,7 @@ func TestRunnerKeepsSuccessfulChunksWhenAnotherExtractIsRejected(t *testing.T) {
validator := &prepared.Steps[0].lanes[0].extractValidators.validators[0]
validator.typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
if target.chunk != nil && target.chunk.Index == 0 {
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test", CorrectionGuidance: "return an acceptable candidate"}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -21,6 +21,7 @@ type laneExtractState struct {
prepared preparedLaneExecutor
deps []CheckpointFingerprint
decision CheckpointDecision
reuseEligible bool
values []erasedExtractArtifact
serialized []CheckpointArtifact
warnings []contracts.Warning
@@ -42,6 +43,7 @@ type finalizedExtractResults struct {
incomplete []int
validationSummaries []artifacts.ValidationSummary
decision CheckpointDecision
reuseEligible bool
}
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
@@ -123,6 +125,9 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
return states, err
}
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
if !laneReferencesReuseEligible(input, prepared.resolved) {
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
}
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
states[i] = state
if err != nil {
@@ -134,7 +139,7 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
if err != nil {
return states, err
}
if !state.decision.Reused {
if !state.decision.Reused && state.reuseEligible {
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return states, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
@@ -303,7 +308,9 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
if result.err != nil {
state.failed = true
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
if state.reuseEligible {
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
}
e.cancel()
} else {
state.results[result.chunkIndex] = result
@@ -341,7 +348,7 @@ func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.So
}
resolution, err := resolveCheckpointDecision(&local, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
decision = resolution.decision
state := &laneExtractState{index: index, prepared: prepared, decision: decision, terminal: true, output: local}
state := &laneExtractState{index: index, prepared: prepared, decision: decision, reuseEligible: true, terminal: true, output: local}
if err != nil {
return state, err
}
@@ -354,6 +361,7 @@ func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.So
SourceID: doc.ID,
Artifact: contracts.CloneSerializedArtifact(hydrated.Artifact),
})
local.normalizeReuseEligibility = map[generatedOutputKey]bool{generatedOutputKeyFor(input.stepID, lane.ID): true}
state.output = local
return state, nil
}
@@ -387,7 +395,12 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
}
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
if !state.reuseEligible {
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
return state, nil
}
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
if err != nil {
@@ -460,14 +473,6 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
if validationErr != nil {
return report, candidate.terminal.record(payload, validationErr)
}
if report.FirstRejection() != nil {
return report, candidate.terminal.record(payload, nil)
}
if report.FirstRejection() == nil && lane.ExtractValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
if failure := firstIncompleteValidation(report); failure != nil {
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
}
}
return report, candidate.terminal.record(payload, nil)
})
result.err = err
@@ -540,7 +545,10 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
sort.Ints(state.incomplete)
if !state.decision.Reused && len(state.incomplete) == 0 {
if len(state.incomplete) > 0 {
state.reuseEligible = false
}
if !state.decision.Reused && state.reuseEligible {
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected, state.warnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
@@ -559,6 +567,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints C
incomplete: state.incomplete,
validationSummaries: state.validationSummaries,
decision: state.decision,
reuseEligible: state.reuseEligible,
}
local.Warnings = append(local.Warnings, cloneWarnings(results.warnings)...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
@@ -634,6 +643,14 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
dst.Warnings = append(dst.Warnings, cloneWarnings(src.Warnings)...)
dst.CheckpointEvents = append(dst.CheckpointEvents, src.CheckpointEvents...)
dst.ValidationSummaries = append(dst.ValidationSummaries, cloneValidationSummaries(src.ValidationSummaries)...)
if len(src.normalizeReuseEligibility) > 0 {
if dst.normalizeReuseEligibility == nil {
dst.normalizeReuseEligibility = make(map[generatedOutputKey]bool, len(src.normalizeReuseEligibility))
}
for key, eligible := range src.normalizeReuseEligibility {
dst.normalizeReuseEligibility[key] = eligible
}
}
for i := range dst.Manifest.ArtifactLanes {
for j := range src.Manifest.ArtifactLanes {
if dst.Manifest.ArtifactLanes[i].ID == src.Manifest.ArtifactLanes[j].ID && dst.Manifest.ArtifactLanes[i].StepID == src.Manifest.ArtifactLanes[j].StepID && src.Manifest.ArtifactLanes[j].Metadata != nil {

View File

@@ -65,8 +65,8 @@ func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
}
for index := 0; index < 2; index++ {
correction := corrections[index]
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || correction.UserGuidance != fmt.Sprintf("correct chunk %d", index) {
t.Fatalf("chunk %d correction = %#v, want its exact initial response and guidance", index, correction)
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || !strings.Contains(correction.UserGuidance, fmt.Sprintf("correct chunk %d", index)) || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "incorrect_extract") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
t.Fatalf("chunk %d correction = %#v, want its exact initial response and semantic replacement guidance only", index, correction)
}
}
terminal := string(debug.json["extract/notes/chunk-000001/terminal.json"])

View File

@@ -230,7 +230,7 @@ func TestRunnerPromotesOnlyAcceptedExtractRetryWarnings(t *testing.T) {
validatorCalls := 0
prepared.Steps[0].lanes[0].extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract"}, nil
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract", CorrectionGuidance: "return an acceptable extract"}, nil
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})

View File

@@ -70,7 +70,7 @@ func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
validator: &preparedValidator{
resolved: ResolvedValidator{Binding: Binding("reject-final-fallback"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback", CorrectionGuidance: "return an acceptable normalized artifact"}, nil
},
},
wantCalls: 2,

View File

@@ -50,7 +50,7 @@ func TestRunnerPromotesOnlyTerminalRejectionWarnings(t *testing.T) {
return contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: fmt.Sprintf("validator-%d", attempts), ReasonCode: "validator", Message: "validator warning"}}}
}
reject := func() contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected"}
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected", CorrectionGuidance: "return an acceptable candidate"}
}
debug := newCapturedDebugRecorder()
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}

View File

@@ -0,0 +1,238 @@
package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type reuseLineageCheckpointSpy struct {
CheckpointLoader
CheckpointRecorder
loads map[string]int
writes map[string]int
forbid map[string]struct{}
}
func newReuseLineageCheckpointSpy() *reuseLineageCheckpointSpy {
return &reuseLineageCheckpointSpy{
CheckpointLoader: NoopCheckpointLoader(),
CheckpointRecorder: NoopCheckpointRecorder(),
loads: make(map[string]int),
writes: make(map[string]int),
forbid: make(map[string]struct{}),
}
}
func (s *reuseLineageCheckpointSpy) Enabled() bool { return true }
func (s *reuseLineageCheckpointSpy) load(stage, laneID string) {
key := stage + "/" + laneID
if _, forbidden := s.forbid[key]; forbidden {
panic("checkpoint lookup crossed validation-incomplete lineage: " + key)
}
s.loads[key]++
}
func (s *reuseLineageCheckpointSpy) write(stage, action, laneID string) {
s.writes[stage+"/"+action+"/"+laneID]++
}
func (s *reuseLineageCheckpointSpy) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
s.load("extract", laneID)
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
s.load("merge", laneID)
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
s.load("normalize", laneID)
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) ExtractRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("extract", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) ExtractSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ []CheckpointArtifact, _ []contracts.RejectedOutput, _ []contracts.Warning) error {
s.write("extract", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) ExtractFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("extract", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("merge", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact, _ []contracts.Warning) error {
s.write("merge", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
s.write("merge", "rejected", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("merge", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("normalize", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact, _ []contracts.Warning) error {
s.write("normalize", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
s.write("normalize", "rejected", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("normalize", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) writesFor(stage, laneID string) int {
total := 0
needle := stage + "/"
suffix := "/" + laneID
for key, count := range s.writes {
if strings.HasPrefix(key, needle) && strings.HasSuffix(key, suffix) {
total += count
}
}
return total
}
func unavailableTypedValidator(kind contracts.ArtifactKind) preparedValidator {
return preparedValidator{
resolved: ResolvedValidator{Binding: Binding("typed/check"), Target: ValidatorTargetTyped, ArtifactKind: kind},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
},
}
}
func TestValidationIncompleteExtractDisablesDownstreamCheckpointIO(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
}
if spy.loads["merge/notes"] != 0 || spy.loads["normalize/notes"] != 0 {
t.Fatalf("downstream loads = %#v, want none", spy.loads)
}
if spy.writesFor("merge", "notes") != 0 || spy.writesFor("normalize", "notes") != 0 {
t.Fatalf("downstream writes = %#v, want none", spy.writes)
}
if spy.writes["extract/succeeded/notes"] != 0 {
t.Fatalf("extract writes = %#v, want no reusable success", spy.writes)
}
}
func TestValidationIncompleteMergeDisablesNormalizeCheckpointIO(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.MergeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.mergeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
}
if spy.loads["merge/notes"] != 1 || spy.loads["normalize/notes"] != 0 {
t.Fatalf("loads = %#v, want merge lookup only", spy.loads)
}
if spy.writes["merge/succeeded/notes"] != 0 || spy.writesFor("normalize", "notes") != 0 {
t.Fatalf("writes = %#v, want no reusable merge or normalize state", spy.writes)
}
}
func TestValidationIncompleteNormalizeIsNotPublished(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 || spy.writes["normalize/succeeded/notes"] != 0 {
t.Fatalf("output/writes = %d / %#v, want in-memory output without normalize publication", len(output.NormalizeOutputs), spy.writes)
}
}
func TestGeneratedReferenceFromIncompleteValidationDisablesDependentCheckpointIO(t *testing.T) {
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"producer"}})
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
producer.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
producer.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(producer.resolved.ArtifactKind)}
referenceItems := 0
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
referenceItems = len(request.References.Slots["producer-output"].Items)
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
spy := newReuseLineageCheckpointSpy()
for _, stage := range []string{"extract", "merge", "normalize"} {
spy.forbid[stage+"/score"] = struct{}{}
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if referenceItems != 1 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("current-run handoff = items %d outputs %d, want one generated item and two outputs", referenceItems, len(output.NormalizeOutputs))
}
for _, stage := range []string{"extract", "merge", "normalize"} {
if spy.writesFor(stage, "score") != 0 {
t.Fatalf("dependent writes = %#v, want none for %s", spy.writes, stage)
}
}
}

View File

@@ -3,6 +3,7 @@ package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -61,8 +62,8 @@ func TestRunnerCorrectsRejectedMergeAndNormalizeCandidates(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || correction.UserGuidance != "produce the accepted value" {
t.Fatalf("%s correction = %#v, want exact rejected response and guidance", target, correction)
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || !strings.Contains(correction.UserGuidance, "produce the accepted value") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "invalid_candidate") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
t.Fatalf("%s correction = %#v, want exact rejected response and semantic replacement guidance only", target, correction)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted output", output)
@@ -84,7 +85,7 @@ func TestRunnerRejectsDeterministicMergeCandidateWithoutCorrection(t *testing.T)
lane.mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}

View File

@@ -157,10 +157,11 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
validator terminalChunkValidator
wantError string
wantRejection bool
attemptError bool
}{
{name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}},
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed"},
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected"}}, wantRejection: true},
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed", attemptError: true},
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected", CorrectionGuidance: "return an acceptable chunk plan"}}, wantRejection: true},
{name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"},
}
for _, tc := range tests {
@@ -174,9 +175,15 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
envelope := debug.envelope(t, "chunk/attempt-01.json")
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -300,12 +307,13 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
candidateFail bool
finalFail bool
wantError string
attemptError bool
}{
{name: "terminal rejection", reject: true},
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed"},
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed", attemptError: true},
{name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"},
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate"},
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output"},
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate", attemptError: true},
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output", attemptError: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
@@ -333,7 +341,7 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected"}, tc.validatorErr
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected", CorrectionGuidance: "return an acceptable extract"}, tc.validatorErr
},
}}
debug := newCapturedDebugRecorder()
@@ -343,9 +351,15 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
envelope := debug.envelope(t, attemptPath)
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else {
if err != nil {
t.Fatalf("Run() error = %v, want nil rejection", err)

View File

@@ -193,6 +193,7 @@ type mergeStageResult struct {
serialized CheckpointArtifact
terminal bool
validationIncomplete bool
reuseEligible bool
}
type mergeAttemptValue struct {
@@ -223,12 +224,16 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
if merged.terminal {
return nil
}
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, output)
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, merged.reuseEligible, output)
if err != nil {
return &laneRunError{stage: StageNormalize, err: err}
}
if normalized.accepted {
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized.Artifact)})
if output.normalizeReuseEligibility == nil {
output.normalizeReuseEligibility = make(map[generatedOutputKey]bool)
}
output.normalizeReuseEligibility[generatedOutputKeyFor(input.stepID, lane.ID)] = normalized.reuseEligible
}
return nil
}
@@ -241,9 +246,14 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
}
mergeReferences := operationReferenceSet(input, lane.MergeReferences)
stageResult.reuseEligible = extracts.reuseEligible && referenceTargetReuseEligible(input, lane.MergeReferences)
mergeDeps := append(artifactCheckpointDigests(extracts.serialized), generatedReferenceDependencies(mergeReferences)...)
mergeDeps = normalizeCheckpointFingerprints(mergeDeps)
mergeCP, mergeDecision := loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, mergeDeps)
var mergeCP MergeCheckpoint
mergeDecision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if stageResult.reuseEligible {
mergeCP, mergeDecision = loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, mergeDeps)
}
mergeResolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageMerge, input.stepID, lane.ID, lane.Merge.Module, mergeDecision, typed.codec, []CheckpointArtifact{mergeCP.Output})
if err != nil {
return stageResult, err
@@ -261,8 +271,10 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
mergeWarnings = cloneWarnings(mergeCP.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps); err != nil {
return stageResult, err
if stageResult.reuseEligible {
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps); err != nil {
return stageResult, err
}
}
terminalResult, runErr := runProducerAttempts(ctx, producerAttemptConfig{Retries: lane.Merge.Retries, Policy: lane.MergeValidationPolicy, AllowStructuralRetry: lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked}, func(attemptCtx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
started := time.Now().UTC()
@@ -295,23 +307,19 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
if validationErr != nil {
return report, candidate.terminal.record(payload, validationErr)
}
if report.FirstRejection() != nil {
return report, candidate.terminal.record(payload, nil)
}
if lane.MergeValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
if failure := firstIncompleteValidation(report); failure != nil {
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
}
}
return report, nil
return report, candidate.terminal.record(payload, nil)
})
terminalSummary := validationSummary(terminalResult, StageMerge, input.stepID, lane.ID, lane.Merge.Module, "", 0)
if debugErr := writeProducerTerminalDebug(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "terminal.json"), terminalResult, lane.MergeValidationPolicy, terminalSummary); debugErr != nil {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
if stageResult.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
}
return stageResult, debugErr
}
if runErr != nil {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, runErr)
if stageResult.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, runErr)
}
return stageResult, runErr
}
if terminalResult.Action == producerTerminalRejected {
@@ -324,8 +332,10 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(terminalSummary))
output.Warnings = append(output.Warnings, terminalResult.Warnings...)
output.Rejected = append(output.Rejected, *rejected)
if err := checkpointMergeRejected(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, *rejected); err != nil {
return stageResult, err
if stageResult.reuseEligible {
if err := checkpointMergeRejected(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, *rejected); err != nil {
return stageResult, err
}
}
stageResult.terminal = true
return stageResult, nil
@@ -338,19 +348,26 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
payload := map[string]any{"output": debugCheckpointArtifact(candidate.candidate), "warnings": debugWarningEnvelopes(terminalResult.Warnings), "rejection": debugRejectedOutputPtr(nil)}
if encodeErr != nil {
attemptErr := fmt.Errorf("serialize accepted merge output for lane %q: %w", lane.ID, encodeErr)
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, attemptErr)
if stageResult.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, attemptErr)
}
return stageResult, candidate.terminal.record(payload, attemptErr)
}
if debugErr := candidate.terminal.record(payload, nil); debugErr != nil {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
if stageResult.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, debugErr)
}
return stageResult, debugErr
}
merged, serializedMerge = candidate.artifact, stored
mergeWarnings = cloneWarnings(terminalResult.Warnings)
output.ValidationSummaries = append(output.ValidationSummaries, terminalSummary)
stageResult.validationIncomplete = terminalResult.ValidationIncomplete
if stageResult.validationIncomplete {
stageResult.reuseEligible = false
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if !stageResult.validationIncomplete {
if stageResult.reuseEligible {
if err := recordMerge(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
return stageResult, err
}
@@ -365,18 +382,24 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
}
type normalizeStageResult struct {
serialized CheckpointArtifact
warnings []contracts.Warning
accepted bool
serialized CheckpointArtifact
warnings []contracts.Warning
accepted bool
reuseEligible bool
}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, output *RunOutput) (normalizeStageResult, error) {
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
var stageResult normalizeStageResult
lane, typed := prepared.resolved, prepared.typed
normalizeReferences := operationReferenceSet(input, lane.NormalizeReferences)
stageResult.reuseEligible = upstreamReuseEligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
normalizeDeps := append(artifactCheckpointDigests([]CheckpointArtifact{serializedMerge}), generatedReferenceDependencies(normalizeReferences)...)
normalizeDeps = normalizeCheckpointFingerprints(normalizeDeps)
normalizeCP, normalizeDecision := loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps)
var normalizeCP NormalizeCheckpoint
normalizeDecision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if stageResult.reuseEligible {
normalizeCP, normalizeDecision = loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps)
}
normalizeResolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, normalizeDecision, typed.codec, []CheckpointArtifact{normalizeCP.Output})
if err != nil {
return stageResult, err
@@ -392,8 +415,10 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
normalizeWarnings = cloneWarnings(normalizeCP.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
return stageResult, err
if stageResult.reuseEligible {
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
return stageResult, err
}
}
terminalResult, runErr := runProducerAttempts(ctx, producerAttemptConfig{Retries: lane.Normalize.Retries, Policy: lane.NormalizeValidationPolicy, AllowStructuralRetry: lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked}, func(attemptCtx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
started := time.Now().UTC()
@@ -444,23 +469,19 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
if validationErr != nil {
return report, candidate.terminal.record(payload, validationErr)
}
if report.FirstRejection() != nil {
return report, candidate.terminal.record(payload, nil)
}
if lane.NormalizeValidationPolicy.ValidatorFailure == ValidatorFailureFailRun {
if failure := firstIncompleteValidation(report); failure != nil {
return report, candidate.terminal.record(payload, validatorFailureError(*failure))
}
}
return report, nil
return report, candidate.terminal.record(payload, nil)
})
terminalSummary := validationSummary(terminalResult, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, "", 0)
if debugErr := writeProducerTerminalDebug(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "terminal.json"), terminalResult, lane.NormalizeValidationPolicy, terminalSummary); debugErr != nil {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
if stageResult.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
}
return stageResult, debugErr
}
if runErr != nil {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
if stageResult.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
}
return stageResult, runErr
}
if terminalResult.Action == producerTerminalRejected {
@@ -473,8 +494,10 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(terminalSummary))
output.Warnings = append(output.Warnings, terminalResult.Warnings...)
output.Rejected = append(output.Rejected, *rejected)
if err := checkpointNormalizeRejected(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, *rejected); err != nil {
return stageResult, err
if stageResult.reuseEligible {
if err := checkpointNormalizeRejected(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, *rejected); err != nil {
return stageResult, err
}
}
return stageResult, nil
}
@@ -489,18 +512,25 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
}
if encodeErr != nil {
attemptErr := fmt.Errorf("serialize accepted normalize output for lane %q: %w", lane.ID, encodeErr)
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, attemptErr)
if stageResult.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, attemptErr)
}
return stageResult, candidate.terminal.record(payload, attemptErr)
}
if debugErr := candidate.terminal.record(payload, nil); debugErr != nil {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
if stageResult.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, debugErr)
}
return stageResult, debugErr
}
serializedNormalize = stored
normalizeWarnings = cloneWarnings(terminalResult.Warnings)
output.ValidationSummaries = append(output.ValidationSummaries, terminalSummary)
output.Warnings = append(output.Warnings, normalizeWarnings...)
if !terminalResult.ValidationIncomplete {
if terminalResult.ValidationIncomplete {
stageResult.reuseEligible = false
}
if stageResult.reuseEligible {
if err := recordNormalize(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
return stageResult, err
}
@@ -616,20 +646,6 @@ func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEnt
return report, nil
}
func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
report, err := r.validateTypedReport(ctx, codec, target, chain, attempt, debug)
if err != nil {
return report.Warnings(), nil, err
}
if failure := report.FirstFailure(); failure != nil {
return report.Warnings(), nil, validatorFailureError(*failure)
}
if rejected := typedRejection(report, target, attempt); rejected != nil {
return report.Warnings(), rejected, nil
}
return report.Warnings(), nil, nil
}
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
rejection := report.FirstRejection()
if rejection == nil {

View File

@@ -18,7 +18,7 @@ const (
validationSkipped validationOutcome = "skipped"
)
const defaultCorrectionGuidance = "Correct the candidate to satisfy the validator requirements."
const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"
// validationRecord captures the settled result of one configured validator.
// Its fields remain private so reports cannot expose mutable warning storage.
@@ -82,33 +82,39 @@ func (report validationReport) FirstFailure() *validationRecord {
return nil
}
func (report validationReport) CorrectionGuidance() string {
func (report validationReport) CorrectionRequest() (string, error) {
seen := make(map[string]struct{})
parts := make([]string, 0, len(report.records))
used := 0
for _, record := range report.records {
if record.outcome != validationRejected {
continue
}
guidance := strings.TrimSpace(record.correctionGuidance)
if guidance == "" {
guidance = defaultCorrectionGuidance
return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
}
if _, exists := seen[guidance]; exists {
continue
}
separator := 0
if len(parts) > 0 {
separator = 1
}
if used+separator+len(guidance) > contracts.MaxCorrectionGuidanceBytes {
break
}
seen[guidance] = struct{}{}
parts = append(parts, guidance)
used += separator + len(guidance)
}
return strings.Join(parts, "\n")
if len(parts) == 0 {
return "", errors.New("validation report contains no correction guidance")
}
var builder strings.Builder
builder.WriteString(correctionRequestIntroduction)
for index, guidance := range parts {
fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
if index+1 < len(parts) {
builder.WriteByte('\n')
}
}
request := builder.String()
if len(request) > contracts.MaxCorrectionGuidanceBytes {
return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
}
return request, nil
}
type validationInvocation struct {
@@ -184,15 +190,11 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, warnings: cloneWarnings(invocation.result.Warnings), diagnosticPath: invocation.result.DiagnosticArtifactPath})
break
}
reason := invocation.result.ReasonCode
if reason == "" {
reason = "output_rejected"
}
message := invocation.result.Message
if message == "" {
message = "output rejected"
}
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: reason, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, warnings: cloneWarnings(invocation.result.Warnings), correctionGuidance: invocation.result.CorrectionGuidance})
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, warnings: cloneWarnings(invocation.result.Warnings), correctionGuidance: invocation.result.CorrectionGuidance})
break
}
}

View File

@@ -17,7 +17,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
invoke validationInvoker
want []validationOutcome
wantCalls []string
wantGuidance string
wantGuidance []string
wantWarnings []string
}{
{
@@ -41,7 +41,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
}),
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
wantGuidance: "repair shape\nrepair references",
wantGuidance: []string{"repair shape", "repair references"},
},
{
name: "rejection and failure both settle",
@@ -52,7 +52,7 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
}),
want: []validationOutcome{validationRejected, validationFailed},
wantCalls: []string{"shape:1", "remote:1"},
wantGuidance: "repair shape",
wantGuidance: []string{"repair shape"},
},
{
name: "failure only has no correction guidance",
@@ -103,8 +103,19 @@ func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
}
if guidance := report.CorrectionGuidance(); guidance != test.wantGuidance {
t.Fatalf("CorrectionGuidance() = %q, want %q", guidance, test.wantGuidance)
if len(test.wantGuidance) > 0 {
guidance, err := report.CorrectionRequest()
if err != nil {
t.Fatalf("CorrectionRequest() error = %v", err)
}
if !strings.Contains(guidance, "complete corrected replacement") {
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
}
for _, item := range test.wantGuidance {
if strings.Count(guidance, item) != 1 {
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
}
}
}
warnings := report.Warnings()
var warningCodes []string
@@ -139,7 +150,7 @@ func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
}
}
func TestValidationReportBoundsGuidanceAndOwnsRecords(t *testing.T) {
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
report := validationReport{records: make([]validationRecord, 17)}
for index := range report.records {
@@ -148,8 +159,8 @@ func TestValidationReportBoundsGuidanceAndOwnsRecords(t *testing.T) {
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
}
report.records[0].warnings = []contracts.Warning{{ReasonCode: "warning"}}
if got := report.CorrectionGuidance(); len(got) > contracts.MaxCorrectionGuidanceBytes || !strings.Contains(got, string([]byte{byte('a')})) || strings.Contains(got, string([]byte{byte('q')})) {
t.Fatalf("CorrectionGuidance() length = %d, want bounded ordered guidance", len(got))
if _, err := report.CorrectionRequest(); err == nil {
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
}
records := report.Records()
records[0].warnings[0].ReasonCode = "changed"

View File

@@ -40,7 +40,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.CombatTurnList]) (contracts.ValidationResult, error) {
if err := Validate(req.Source, req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return combat turns in transcript order, without duplicate turns, using consistent contextual combatant names and evidence for each turn."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -100,5 +100,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return a complete combat-turn list with every required field present, valid combatant names, and valid source references."}
}

View File

@@ -41,6 +41,9 @@ func TestValidateRejectsEveryOwnedShapeBoundary(t *testing.T) {
if err != nil || result.Approved || result.ReasonCode != ReasonCode {
t.Fatalf("Validator result = %#v, %v; want shape rejection", result, err)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
})
}
}

View File

@@ -49,9 +49,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid combat turn source references", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid combat turn source references", issues),
CorrectionGuidance: "Return combat turns whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each turn.",
}, nil
}

View File

@@ -42,6 +42,9 @@ func TestValidatorRejectsInvalidSourceIdentityExistenceAndOrder(t *testing.T) {
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, test.want) {
t.Fatalf("Validate() = %#v, %v; want source-reference rejection containing %q", result, err, test.want)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
})
}
}

View File

@@ -0,0 +1,180 @@
package validate_test
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
combatshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
combatsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs"
combatsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness"
npcregistryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcoccurrences/registry"
npcidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/identity"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
)
func TestRepresentativeProductionValidatorResultsSatisfyCorrectionContract(t *testing.T) {
document := &source.SourceDocument{
ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session",
Units: []source.SourceUnit{{ID: 1, Kind: "message", Text: "Mira Thorn waits while Aria acts."}},
}
npcReferences := npcRegistryReferences(t, "Mira Thorn")
npcRegistryValidator, err := npcregistryvalidator.New(npcregistryvalidator.Options{}, npcReferences)
if err != nil {
t.Fatal(err)
}
unknownNPCID := "npc:sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
tests := []struct {
name string
validate func() (contracts.ValidationResult, error)
wantApproved bool
wantWarningCount int
}{
{
name: "shape rejection",
validate: func() (contracts.ValidationResult, error) {
return combatshape.New(combatshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{
Value: dnd.CombatTurnList{},
})
},
},
{
name: "source reference rejection",
validate: func() (contracts.ValidationResult, error) {
return combatsourcerefs.New(combatsourcerefs.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{
Source: document,
Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{{
Actor: "Aria", TurnKind: dnd.CombatTurnKindTurn,
SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}},
}}},
})
},
},
{
name: "registry identity rejection",
validate: func() (contracts.ValidationResult, error) {
return npcidentity.New(npcidentity.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{
Value: dnd.NPCRegistry{NPCs: []dnd.NPC{{
ID: "not-an-id", Name: "Mira Thorn",
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}},
})
},
},
{
name: "catalog membership rejection",
validate: func() (contracts.ValidationResult, error) {
validator, buildErr := spellcatalog.New(spellcatalog.Options{})
if buildErr != nil {
return contracts.ValidationResult{}, buildErr
}
return validator.Validate(context.Background(), contracts.TypedValidationRequest[dnd.SpellList]{
Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
Caster: "Aria", Spell: "Definitely Not A Spell",
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}},
})
},
},
{
name: "registry membership rejection",
validate: func() (contracts.ValidationResult, error) {
return npcRegistryValidator.Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{
References: npcReferences,
Value: dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{{
NPCID: unknownNPCID, Name: "Mira Thorn", Kind: dnd.NPCOccurrenceKindDialogue,
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}},
})
},
},
{
name: "source relatedness advisory",
validate: func() (contracts.ValidationResult, error) {
return combatsourcerelatedness.New(combatsourcerelatedness.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{
Source: document,
Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{{
Actor: "Unmentioned Actor", TurnKind: dnd.CombatTurnKindTurn,
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}},
})
},
wantApproved: true,
wantWarningCount: 1,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := test.validate()
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
if result.Approved != test.wantApproved {
t.Fatalf("Validate() approved = %t, want %t: %#v", result.Approved, test.wantApproved, result)
}
if len(result.Warnings) != test.wantWarningCount {
t.Fatalf("Validate() warnings = %d, want %d: %#v", len(result.Warnings), test.wantWarningCount, result)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
if !result.Approved && strings.TrimSpace(result.CorrectionGuidance) == "" {
t.Fatalf("rejection lacks actionable correction guidance: %#v", result)
}
})
}
}
func TestRegistryCorrectionGuidanceUsesContextualNamesNotOpaqueIDs(t *testing.T) {
references := npcRegistryReferences(t, "Mira Thorn")
validator, err := npcregistryvalidator.New(npcregistryvalidator.Options{}, references)
if err != nil {
t.Fatal(err)
}
opaqueID := "npc:sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
result, err := validator.Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{
References: references,
Value: dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{{
NPCID: opaqueID, Name: "Mira Thorn", Kind: dnd.NPCOccurrenceKindDialogue,
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}},
})
if err != nil || result.Approved {
t.Fatalf("Validate() = %#v, %v; want rejection", result, err)
}
if !strings.Contains(result.CorrectionGuidance, "contextual NPC names") {
t.Fatalf("correction guidance = %q, want contextual-name instruction", result.CorrectionGuidance)
}
if strings.Contains(result.CorrectionGuidance, opaqueID) || strings.Contains(result.CorrectionGuidance, "npc:sha256:") {
t.Fatalf("correction guidance leaked opaque durable ID: %q", result.CorrectionGuidance)
}
}
func npcRegistryReferences(t *testing.T, name string) contracts.ReferenceSet {
t.Helper()
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{{
ID: identity.DeriveID(name), Name: name,
SourceRefs: []source.SourceRef{{SourceID: "prior-session", StartUnitID: 1, EndUnitID: 1}},
}}})
if err != nil {
t.Fatal(err)
}
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
npcregistry.ReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: npcregistry.ReferenceSlot},
Items: []contracts.ReferenceItem{{
SlotName: npcregistry.ReferenceSlot,
MediaType: npccodec.MediaType,
Content: content,
}},
},
}}
}

View File

@@ -54,6 +54,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
Message: diagnostics.Aggregate("duplicate enemy engagement", []string{
fmt.Sprintf("subject %s has more than one engagement in one combat scene", diagnostics.Quote(event.Name)),
}),
CorrectionGuidance: "Return at most one engagement event for each contextual enemy name within the same combat scene.",
}, nil
}
seen[identity] = struct{}{}

View File

@@ -90,13 +90,13 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{}, fmt.Errorf("resolve NPC registry: %w", err)
}
if !registry.Bound() {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "invalid enemy event normalization: NPC registry reference is required"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "invalid enemy event normalization: NPC registry reference is required", CorrectionGuidance: "Return enemy events using contextual NPC names that match the supplied NPC registry."}, nil
}
issues := issuesFor(shared.NewSourceRefOrderFromIndex(index), req.Value, registry)
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid enemy event normalization", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid enemy event normalization", issues), CorrectionGuidance: "Return enemy events in canonical transcript order, without duplicate events, using contextual NPC names that match the supplied registry and evidence."}, nil
}
func allSourceRefsValid(index source.DocumentIndex, value dnd.EnemyEventList) bool {

View File

@@ -36,7 +36,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.EnemyEventList]) (contracts.ValidationResult, error) {
if err := Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return a complete enemy-event list with every required field present, valid contextual NPC names, supported event values, and valid source references."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -50,7 +50,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid enemy event source references", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid enemy event source references", issues), CorrectionGuidance: "Return enemy events whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each event."}, nil
}
func sourceRefIssues(index source.DocumentIndex, doc *source.SourceDocument, coverage *chunkCoverage, value dnd.EnemyEventList) []string {

View File

@@ -38,7 +38,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemOccurrenceList]) (contracts.ValidationResult, error) {
if err := Validate(req.Source, req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return item occurrences in canonical transcript order, without duplicate occurrences, using consistent contextual item names and valid event transitions."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -96,7 +96,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
func rejection(issues []string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item occurrence registry", issues)}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item occurrence registry", issues), CorrectionGuidance: "Return item occurrences using contextual item names that match an item in the supplied registry; omit occurrences that cannot be matched unambiguously."}
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}

View File

@@ -36,7 +36,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemOccurrenceList]) (contracts.ValidationResult, error) {
if err := Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return a complete item-occurrence list with every required field present, valid contextual item names, supported event kinds and holder combinations, positive quantities, and valid source references."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -61,7 +61,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item occurrence source references", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item occurrence source references", issues), CorrectionGuidance: "Return item occurrences whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each occurrence."}, nil
}
type chunkCoverage struct {

View File

@@ -46,7 +46,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
for index, issue := range identityIssues {
issues[index] = fmt.Sprintf("items[%d] %s: %s", issue.RecordIndex, issue.Code, diagnostics.Quote(issue.Value))
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item identity", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid item identity", issues), CorrectionGuidance: "Return one canonical registry entry per distinct properly named item, combining duplicate mentions under the same transcript-supported name."}, nil
}
func Spec() pipeline.ValidatorSpec {

View File

@@ -93,5 +93,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return a complete item registry containing only properly named items with valid source references."}
}

View File

@@ -97,5 +97,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return registry items whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each named item."}
}

View File

@@ -50,7 +50,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence normalization", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence normalization", issues), CorrectionGuidance: "Return location occurrences in canonical transcript order, without duplicates, using consistent proper location names and supported occurrence kinds."}, nil
}
func allSourceRefsValid(index source.DocumentIndex, value dnd.LocationOccurrenceList) bool {

View File

@@ -96,7 +96,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
func rejection(issues []string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence registry", issues)}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence registry", issues), CorrectionGuidance: "Return location occurrences using proper contextual location names that match a location in the supplied registry; omit occurrences that cannot be matched unambiguously."}
}
func Spec() pipeline.ValidatorSpec {

View File

@@ -35,7 +35,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.LocationOccurrenceList]) (contracts.ValidationResult, error) {
if err := Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return a complete location-occurrence list with every required field present, valid contextual location names, supported occurrence kinds, and valid source references."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -57,7 +57,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence source references", issues)}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: diagnostics.Aggregate("invalid location occurrence source references", issues), CorrectionGuidance: "Return location occurrences whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each occurrence."}, nil
}
func chunkContainsRef(chunk *source.Chunk, ref source.SourceRef) bool {

View File

@@ -47,8 +47,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
issues[index] = fmt.Sprintf("locations[%d] %s: %s", issue.RecordIndex, issue.Code, diagnostics.Quote(issue.Value))
}
return contracts.ValidationResult{
Approved: false, ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid location identity", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid location identity", issues),
CorrectionGuidance: "Return one canonical registry entry per distinct proper location name, combining duplicate mentions of the same location.",
}, nil
}

View File

@@ -93,5 +93,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return a complete location registry containing only properly named locations with valid source references."}
}

View File

@@ -97,5 +97,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return registry locations whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each proper location name."}
}

View File

@@ -92,16 +92,17 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{}, fmt.Errorf("resolve NPC registry: %w", err)
}
if !npcRegistry.Bound() {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "invalid NPC occurrence normalization: NPC registry reference is required"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "invalid NPC occurrence normalization: NPC registry reference is required", CorrectionGuidance: "Return NPC occurrences using contextual NPC names that match the supplied NPC registry."}, nil
}
issues := issuesFor(shared.NewSourceRefOrderFromIndex(index), req.Value, npcRegistry)
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence normalization", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence normalization", issues),
CorrectionGuidance: "Return NPC occurrences in canonical transcript order, without duplicates, using contextual NPC names that match the supplied registry and supported interaction kinds.",
}, nil
}

View File

@@ -105,9 +105,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
func rejection(issues []string) contracts.ValidationResult {
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence registry", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence registry", issues),
CorrectionGuidance: "Return NPC occurrences using contextual NPC names that match an NPC in the supplied registry; omit occurrences that cannot be matched unambiguously.",
}
}

View File

@@ -29,6 +29,12 @@ func TestValidatorRecognizesExactRegistryPairsAndRejectsUnknownIDs(t *testing.T)
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "occurrences[0].npc_id") {
t.Fatalf("unknown result = %#v, %v", result, err)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
if !strings.Contains(result.CorrectionGuidance, "contextual NPC names") || strings.Contains(result.CorrectionGuidance, value.Occurrences[0].NPCID) {
t.Fatalf("correction guidance = %q, want contextual names without opaque IDs", result.CorrectionGuidance)
}
value = validList("Mira Thorn")
value.Occurrences[0].Name = "Hooded Guard"
result, err = validator.Validate(context.Background(), request(references, value))

View File

@@ -35,7 +35,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCOccurrenceList]) (contracts.ValidationResult, error) {
if err := Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return a complete NPC-occurrence list with every required field present, valid contextual NPC names, supported interaction kinds, and valid source references."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -61,9 +61,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence source references", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC occurrence source references", issues),
CorrectionGuidance: "Return NPC occurrences whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each occurrence.",
}, nil
}

View File

@@ -51,9 +51,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
issues[index] = fmt.Sprintf("%s %s: %s", location, issue.Code, diagnostics.Quote(issue.Value))
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC identity", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid NPC identity", issues),
CorrectionGuidance: "Return one canonical registry entry per distinct properly named NPC, combining duplicate mentions under the same transcript-supported name.",
}, nil
}

View File

@@ -57,6 +57,9 @@ func TestValidatorDefersShapeAndRejectsIdentityIssues(t *testing.T) {
if err != nil || result.Approved || result.ReasonCode != ReasonCode {
t.Fatalf("identity result = %#v, error = %v, want rejection", result, err)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
for _, want := range []string{"invalid_id", "duplicate_canonical_identity", "duplicate_id", "npcs[0]", "npcs[1]"} {
if !strings.Contains(result.Message, want) {
t.Fatalf("identity message %q missing %q", result.Message, want)

View File

@@ -92,5 +92,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return a complete NPC registry containing only properly named NPCs with valid source references."}
}

View File

@@ -117,5 +117,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return registry NPCs whose source references identify valid transcript ranges within the supplied extraction chunk and directly support each proper NPC name."}
}

View File

@@ -48,8 +48,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false, ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid scene description normalization", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid scene description normalization", issues),
CorrectionGuidance: "Return scene descriptions in canonical source order, with one unique scene per source range and no duplicate scene identity.",
}, nil
}

View File

@@ -35,7 +35,7 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
if err := ValidateForStage(req.Value, req.Stage); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error()}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: err.Error(), CorrectionGuidance: "Return a complete scene-description list with the required number of scenes, supported scene kinds, nonblank titles and summaries, and valid source references."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -62,8 +62,10 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false, ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid scene description source references", issues),
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("invalid scene description source references", issues),
CorrectionGuidance: "Return exactly one scene description for the supplied chunk, using that chunk's scene identity and complete source range.",
}, nil
}

View File

@@ -98,7 +98,7 @@ func rejection(unknown []unknownSpell) contracts.ValidationResult {
issues = append(issues, issue)
}
message := rejectionMessage(issues, len(unknown)-len(issues))
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Use recognized D&D spell names supported by the supplied transcript evidence, and omit any candidate that is not a spell cast."}
}
func truncateDisplayedName(name string) string {

View File

@@ -54,6 +54,9 @@ func TestValidatorRejectsMultipleUnknownCastsInStableOrder(t *testing.T) {
if result.Approved || result.ReasonCode != ReasonCode {
t.Fatalf("Validate() = %#v, want unknown-spell rejection", result)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult() error = %v for %#v", err, result)
}
if want := `spell_casts[0].spell "Unknown First", spell_casts[2].spell "Unknown Second"`; !strings.Contains(result.Message, want) {
t.Fatalf("message = %q, want %q", result.Message, want)
}

View File

@@ -75,5 +75,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return a complete spell-cast list with a nonblank spell name and caster plus valid source references for every cast."}
}

View File

@@ -111,5 +111,5 @@ func DecodeOptions(options map[string]any) (Options, error) {
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Return spell casts whose source references identify valid transcript ranges within the supplied extraction chunk and directly support the named spell and caster."}
}

View File

@@ -18,7 +18,7 @@ func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} }
func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} }
func rejection() contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "output rejected by always-reject validator"}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: "output rejected by always-reject validator", CorrectionGuidance: "Return one complete replacement response that satisfies the configured validation requirements."}
}
func (v *ChunkValidator) Name() string { return Key }
func (v *ChunkValidator) ExecutionClass() contracts.ExecutionClass {

View File

@@ -31,7 +31,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
func validate(content []byte) contracts.ValidationResult {
if !json.Valid(content) {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON", CorrectionGuidance: "Return one complete replacement response encoded as valid JSON."}
}
return contracts.ValidationResult{Approved: true}
}

View File

@@ -56,14 +56,14 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
}
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Content))
if err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON", CorrectionGuidance: "Return one complete replacement response encoded as valid JSON."}, nil
}
schema, err := v.compiledSchema(req.Schema)
if err != nil {
return contracts.ValidationResult{}, err
}
if err := schema.Validate(instance); err != nil {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeSchemaInvalid, Message: "payload does not conform to response schema"}, nil
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeSchemaInvalid, Message: "payload does not conform to response schema", CorrectionGuidance: "Return one complete JSON replacement that conforms exactly to the supplied response schema."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}