Execute complete validator chains

This commit is contained in:
2026-08-27 00:18:16 +00:00
parent 04ba87e174
commit 8dd7a4324d
8 changed files with 524 additions and 68 deletions

View File

@@ -37,6 +37,11 @@ type terminalChunkValidator struct {
err error
}
type retryingLLMChunkValidator struct {
requests []contracts.ChunkValidationRequest
calls int
}
type observingChunkValidator struct {
request contracts.ChunkValidationRequest
}
@@ -73,6 +78,24 @@ func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidat
return v.result, v.err
}
func (*retryingLLMChunkValidator) Name() string { return "retrying/llm-chunk-validator" }
func (*retryingLLMChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (validator *retryingLLMChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, request)
if validator.calls == 1 && len(request.Chunks) > 0 {
request.Chunks[0].ID = "mutated"
}
if validator.calls == 1 {
return contracts.ValidationResult{}, errors.New("temporary validator failure")
}
return contracts.ValidationResult{Approved: true}, nil
}
func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) {
t.Helper()
marker := strings.TrimSuffix(prefix, "/") + "/attempt-"
@@ -152,6 +175,28 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
}
}
func TestRunnerRetriesLLMValidatorsWithoutRegeneratingChunkCandidate(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
chunkCalls := 0
prepared.resolved.Chunk.Retries = 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &chunkCalls}
validator := &retryingLLMChunkValidator{}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: ModuleBinding{Module: validator.Name(), Retries: 1}, ExecutionClass: contracts.ExecutionClassLLMBacked, Target: ValidatorTargetChunk},
chunk: validator,
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v", err)
}
if chunkCalls != 1 || validator.calls != 2 {
t.Fatalf("producer calls = %d validator calls = %d, want 1 and 2", chunkCalls, validator.calls)
}
if len(validator.requests) != 2 || len(validator.requests[0].Chunks) == 0 || len(validator.requests[1].Chunks) == 0 || validator.requests[1].Chunks[0].ID == "mutated" {
t.Fatalf("validator requests = %#v, want independently owned immutable candidates", validator.requests)
}
}
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}