Execute complete validator chains
This commit is contained in:
@@ -37,6 +37,11 @@ type terminalChunkValidator struct {
|
||||
err error
|
||||
}
|
||||
|
||||
type retryingLLMChunkValidator struct {
|
||||
requests []contracts.ChunkValidationRequest
|
||||
calls int
|
||||
}
|
||||
|
||||
type observingChunkValidator struct {
|
||||
request contracts.ChunkValidationRequest
|
||||
}
|
||||
@@ -73,6 +78,24 @@ func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidat
|
||||
return v.result, v.err
|
||||
}
|
||||
|
||||
func (*retryingLLMChunkValidator) Name() string { return "retrying/llm-chunk-validator" }
|
||||
|
||||
func (*retryingLLMChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (validator *retryingLLMChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
validator.calls++
|
||||
validator.requests = append(validator.requests, request)
|
||||
if validator.calls == 1 && len(request.Chunks) > 0 {
|
||||
request.Chunks[0].ID = "mutated"
|
||||
}
|
||||
if validator.calls == 1 {
|
||||
return contracts.ValidationResult{}, errors.New("temporary validator failure")
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) {
|
||||
t.Helper()
|
||||
marker := strings.TrimSuffix(prefix, "/") + "/attempt-"
|
||||
@@ -152,6 +175,28 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetriesLLMValidatorsWithoutRegeneratingChunkCandidate(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
chunkCalls := 0
|
||||
prepared.resolved.Chunk.Retries = 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &chunkCalls}
|
||||
validator := &retryingLLMChunkValidator{}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: ModuleBinding{Module: validator.Name(), Retries: 1}, ExecutionClass: contracts.ExecutionClassLLMBacked, Target: ValidatorTargetChunk},
|
||||
chunk: validator,
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if chunkCalls != 1 || validator.calls != 2 {
|
||||
t.Fatalf("producer calls = %d validator calls = %d, want 1 and 2", chunkCalls, validator.calls)
|
||||
}
|
||||
if len(validator.requests) != 2 || len(validator.requests[0].Chunks) == 0 || len(validator.requests[1].Chunks) == 0 || validator.requests[1].Chunks[0].ID == "mutated" {
|
||||
t.Fatalf("validator requests = %#v, want independently owned immutable candidates", validator.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
|
||||
|
||||
Reference in New Issue
Block a user