package pipeline import ( "context" "errors" "reflect" "sort" "strconv" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) type terminalChunker struct { key string chunks []source.Chunk warnings []contracts.Warning err error calls *int } func (c terminalChunker) Key() string { return c.key } func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (c terminalChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) { if c.calls != nil { (*c.calls)++ } return contracts.ChunkResult{Chunks: cloneSourceChunks(c.chunks), Warnings: cloneWarnings(c.warnings)}, c.err } type terminalChunkValidator struct { result contracts.ValidationResult err error } func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" } func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { return v.result, v.err } func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) { t.Helper() marker := strings.TrimSuffix(prefix, "/") + "/attempt-" var got []int for _, name := range debug.names() { if !strings.HasPrefix(name, marker) || !strings.HasSuffix(name, ".json") { continue } remainder := strings.TrimSuffix(strings.TrimPrefix(name, marker), ".json") if strings.Contains(remainder, "/") { continue } value, err := strconv.Atoi(remainder) if err != nil { t.Fatalf("parse attempt index from %q: %v", name, err) } got = append(got, value) } sort.Ints(got) if !reflect.DeepEqual(got, attempts) { t.Fatalf("attempt envelopes under %q = %#v, want %#v; names = %#v", prefix, got, attempts, debug.names()) } } func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, []source.Chunk) { t.Helper() prepared := preparedAttemptDebugPipeline(t) chunker, ok := prepared.chunker.(*typedTestChunker) if !ok { t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker) } return prepared, cloneSourceChunks(chunker.chunks) } func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) { tests := []struct { name string moduleError error validator terminalChunkValidator wantError string wantRejection bool }{ {name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}}, {name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed"}, {name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected"}}, wantRejection: true}, {name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { prepared, chunks := preparedTerminalDebugPipeline(t) prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError} prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}} debug := newCapturedDebugRecorder() output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) assertAttemptEnvelopeSequence(t, debug, "chunk", 1) envelope := debug.envelope(t, "chunk/attempt-01.json") if tc.wantError != "" { if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) { t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError) } } else if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if tc.wantRejection { if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "chunk_rejected" { t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected) } } }) } } func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) { tests := []struct { name string moduleError error validatorErr error reject bool candidateFail bool finalFail bool wantError string }{ {name: "terminal rejection", reject: true}, {name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed"}, {name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"}, {name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate"}, {name: "final codec error", finalFail: true, wantError: "serialize accepted extract output"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { prepared := preparedAttemptDebugPipeline(t) value := "extract-terminal" codec := &observedNotesCodec{} if tc.candidateFail { codec.candidateError = value } if tc.finalFail { codec.finalError = value } installObservedNotesCodec(t, prepared, codec) installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) { if tc.moduleError != nil { return erasedTypedResult{}, tc.moduleError } return erasedTypedResult{Value: codecNotes{Items: []string{value}}, Warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "observed", Message: "extract warning"}}}, nil }) prepared.lanes[0].extractValidators.validators = []preparedValidator{{ resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected"}, tc.validatorErr }, }} debug := newCapturedDebugRecorder() output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1) attemptPath := "extract/notes/chunk-000001/attempt-01.json" envelope := debug.envelope(t, attemptPath) if tc.wantError != "" { if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) { t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError) } } else { if err != nil { t.Fatalf("Run() error = %v, want nil rejection", err) } if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "extract_rejected" { t.Fatalf("extract rejection = envelope %#v, outputs %#v", envelope, output.Rejected) } } if tc.moduleError == nil && !strings.Contains(string(debug.json[attemptPath]), "extract warning") { t.Fatalf("attempt envelope = %s, want attempt warning", debug.json[attemptPath]) } }) } } func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) { t.Run("chunk", func(t *testing.T) { prepared, chunks := preparedTerminalDebugPipeline(t) primary := errors.New("chunk operation failed") prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, err: primary} debug := newCapturedDebugRecorder() debug.failPath = "chunk/attempt-01.json" _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { t.Fatalf("Run() error = %v, want joined operation and debug errors", err) } }) t.Run("extract", func(t *testing.T) { prepared := preparedAttemptDebugPipeline(t) primary := errors.New("extract operation failed") installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) { return erasedTypedResult{}, primary }) debug := newCapturedDebugRecorder() debug.failPath = "extract/notes/chunk-000001/attempt-01.json" _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write extract attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { t.Fatalf("Run() error = %v, want joined operation and debug errors", err) } }) } func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) { prepared, chunks := preparedTerminalDebugPipeline(t) prepared.resolved.Chunk.Retries = 1 calls := 0 prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, calls: &calls} prepared.chunkValidators.validators = []preparedValidator{{ resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}, }} debug := newCapturedDebugRecorder() debug.failPath = "chunk/attempt-01.json" _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) if err == nil || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { t.Fatalf("Run() error = %v, want terminal attempt debug failure", err) } if calls != 1 { t.Fatalf("chunk calls = %d, want one attempt without retry", calls) } if debug.has("chunk/attempt-02.json") { t.Fatal("second chunk attempt envelope exists after non-retryable debug persistence failure") } } func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) { prepared := preparedAttemptDebugPipeline(t) debug := newCapturedDebugRecorder() client := WithDebugLLMRecording(attemptDebugLLM{}, debug) installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { if err := callAttemptDebugLLM(ctx, client, "extract-module"); err != nil { return erasedTypedResult{}, err } return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil }) prepared.lanes[0].extractValidators.validators = []preparedValidator{{ resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) { if err := callAttemptDebugLLM(ctx, client, "extract-validator"); err != nil { return contracts.ValidationResult{}, err } return contracts.ValidationResult{Approved: true}, nil }, }} if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil { t.Fatalf("Run() error = %v, want nil", err) } module := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json") validator := debug.envelope(t, "validate/extract/notes/typed~2fextract-notes/01-llm-check-attempt-01.json") if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/chunk-000001/attempt-01/") { t.Fatalf("module LLM calls = %#v, want extract module call only", module.LLMCalls) } if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/") { t.Fatalf("validator LLM calls = %#v, want validator call only", validator.LLMCalls) } if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID { t.Fatalf("module and validator attempts share LLM call %#v", module.LLMCalls) } }