eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make combat scene validation more reliable

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 11:39:02 +00:00
parent 7e626753bf
commit f208dbe954
18 changed files with 146 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/framework/pipeline/runner_attempt_debug_test.go
View File

@@ -107,8 +107,12 @@ func (attemptDebugLLM) CompleteStructured(_ context.Context, request contracts.S
}
func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
return preparedAttemptDebugPipelineWithChunks(t, 1)
}
func preparedAttemptDebugPipelineWithChunks(t *testing.T, chunkCount int) *PreparedPipeline {
t.Helper()
prepared := preparedConcurrentPipeline(t, 1)
prepared := preparedConcurrentPipeline(t, chunkCount)
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]

29
internal/framework/pipeline/runner_terminal_debug_test.go
View File

@@ -3,6 +3,7 @@ package pipeline
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"strconv"
@@ -427,7 +428,7 @@ func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
}
func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedAttemptDebugPipelineWithChunks(t, 2)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
@@ -449,15 +450,23 @@ func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
module := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/01-llm-check-attempt-01.json")
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/chunk-000001/attempt-01/") {
t.Fatalf("module LLM calls = %#v, want extract module call only", module.LLMCalls)
validatorResponses := make(map[string]struct{})
for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ {
chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber)
module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json")
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json")
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") {
t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls)
}
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") {
t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls)
}
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls)
}
validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{}
}
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/") {
t.Fatalf("validator LLM calls = %#v, want validator call only", validator.LLMCalls)
}
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
t.Fatalf("module and validator attempts share LLM call %#v", module.LLMCalls)
if len(validatorResponses) != 2 {
t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses)
}
}

7
internal/framework/pipeline/runner_typed.go
View File

@@ -596,7 +596,12 @@ func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEnt
var result contracts.ValidationResult
var err error
started := time.Now().UTC()
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)}
if target.chunk != nil {
pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1))
}
pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt))
attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt)
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
requestTarget := target
requestTarget.sourceInput = target.sourceInput.Clone()