eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make combat scene validation more reliable

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 11:39:02 +00:00
parent 7e626753bf
commit f208dbe954
18 changed files with 146 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
internal/framework/pipeline/runner_terminal_debug_test.go
View File

@@ -3,6 +3,7 @@ package pipeline
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"strconv"
@@ -427,7 +428,7 @@ func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
}
func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedAttemptDebugPipelineWithChunks(t, 2)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
@@ -449,15 +450,23 @@ func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
module := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/01-llm-check-attempt-01.json")
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/chunk-000001/attempt-01/") {
t.Fatalf("module LLM calls = %#v, want extract module call only", module.LLMCalls)
validatorResponses := make(map[string]struct{})
for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ {
chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber)
module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json")
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json")
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") {
t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls)
}
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") {
t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls)
}
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls)
}
validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{}
}
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/") {
t.Fatalf("validator LLM calls = %#v, want validator call only", validator.LLMCalls)
}
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
t.Fatalf("module and validator attempts share LLM call %#v", module.LLMCalls)
if len(validatorResponses) != 2 {
t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses)
}
}