notarius/internal/framework/pipeline/runner_terminal_debug_test.go

package pipeline

import (
	"context"
	"errors"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)

type terminalChunker struct {
	key      string
	plan     source.ChunkPlan
	warnings []contracts.Warning
	err      error
	calls    *int
}

func (c terminalChunker) Key() string { return c.key }

func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }

func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
	if c.calls != nil {
		(*c.calls)++
	}
	return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Warnings: cloneWarnings(c.warnings)}, c.err
}

type terminalChunkValidator struct {
	result contracts.ValidationResult
	err    error
}

type observingChunkValidator struct {
	request contracts.ChunkValidationRequest
}

func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
	return contracts.ExecutionClassDeterministic
}
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
	v.request = request
	return contracts.ValidationResult{Approved: true}, nil
}

func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }

func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
	return contracts.ExecutionClassDeterministic
}

func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
	return v.result, v.err
}

func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) {
	t.Helper()
	marker := strings.TrimSuffix(prefix, "/") + "/attempt-"
	var got []int
	for _, name := range debug.names() {
		if !strings.HasPrefix(name, marker) || !strings.HasSuffix(name, ".json") {
			continue
		}
		remainder := strings.TrimSuffix(strings.TrimPrefix(name, marker), ".json")
		if strings.Contains(remainder, "/") {
			continue
		}
		value, err := strconv.Atoi(remainder)
		if err != nil {
			t.Fatalf("parse attempt index from %q: %v", name, err)
		}
		got = append(got, value)
	}
	sort.Ints(got)
	if !reflect.DeepEqual(got, attempts) {
		t.Fatalf("attempt envelopes under %q = %#v, want %#v; names = %#v", prefix, got, attempts, debug.names())
	}
}

func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) {
	t.Helper()
	prepared := preparedAttemptDebugPipeline(t)
	chunker, ok := prepared.chunker.(*typedTestChunker)
	if !ok {
		t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
	}
	return prepared, source.CloneChunkPlan(chunker.plan)
}

func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
	tests := []struct {
		name          string
		moduleError   error
		validator     terminalChunkValidator
		wantError     string
		wantRejection bool
	}{
		{name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}},
		{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed"},
		{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected"}}, wantRejection: true},
		{name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			prepared, plan := preparedTerminalDebugPipeline(t)
			prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError}
			prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
			debug := newCapturedDebugRecorder()

			output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
			assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
			envelope := debug.envelope(t, "chunk/attempt-01.json")
			if tc.wantError != "" {
				if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
					t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
				}
			} else if err != nil {
				t.Fatalf("Run() error = %v, want nil", err)
			}
			if tc.wantRejection {
				if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "chunk_rejected" {
					t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
				}
			}
			if tc.name == "accepted" {
				encoded := string(debug.json["chunk/attempt-01.json"])
				if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) {
					t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded)
				}
			}
		})
	}
}

func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
	prepared, plan := preparedTerminalDebugPipeline(t)
	plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
	plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
	validator := &observingChunkValidator{}
	prepared.chunkValidators.validators = []preparedValidator{{
		resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
		chunk:    validator,
	}}

	if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
		t.Fatalf("Run() error = %v, want nil", err)
	}
	if len(validator.request.Chunks) != 1 {
		t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
	}
	chunk := validator.request.Chunks[0]
	if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
		t.Fatalf("materialized chunk identity = %#v", chunk)
	}
	if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
		t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
	}
	if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
		t.Fatalf("materialized metadata = %#v", chunk.Metadata)
	}
}

func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
	prepared, plan := preparedTerminalDebugPipeline(t)
	plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
	prepared.resolved.Chunk.Retries = 1
	calls := 0
	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
	debug := newCapturedDebugRecorder()

	_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
	if err == nil || !strings.Contains(err.Error(), "valid JSON") {
		t.Fatalf("Run() error = %v, want malformed annotation error", err)
	}
	if calls != 2 {
		t.Fatalf("plan calls = %d, want two attempts", calls)
	}
	assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2)
}

func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
	tests := []struct {
		name          string
		moduleError   error
		validatorErr  error
		reject        bool
		candidateFail bool
		finalFail     bool
		wantError     string
	}{
		{name: "terminal rejection", reject: true},
		{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed"},
		{name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"},
		{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate"},
		{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output"},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			prepared := preparedAttemptDebugPipeline(t)
			value := "extract-terminal"
			codec := &observedNotesCodec{}
			if tc.candidateFail {
				codec.candidateError = value
			}
			if tc.finalFail {
				codec.finalError = value
			}
			installObservedNotesCodec(t, prepared, codec)
			installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
				if tc.moduleError != nil {
					return erasedTypedResult{}, tc.moduleError
				}
				return erasedTypedResult{Value: codecNotes{Items: []string{value}}, Warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "observed", Message: "extract warning"}}}, nil
			})
			prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
				resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
				typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
					return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected"}, tc.validatorErr
				},
			}}
			debug := newCapturedDebugRecorder()

			output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
			assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1)
			attemptPath := "extract/notes/chunk-000001/attempt-01.json"
			envelope := debug.envelope(t, attemptPath)
			if tc.wantError != "" {
				if err == nil || !strings.Contains(err.Error(), tc.wantError) || !strings.Contains(envelope.Error, tc.wantError) {
					t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
				}
			} else {
				if err != nil {
					t.Fatalf("Run() error = %v, want nil rejection", err)
				}
				if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "extract_rejected" {
					t.Fatalf("extract rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
				}
			}
			if tc.moduleError == nil && !strings.Contains(string(debug.json[attemptPath]), "extract warning") {
				t.Fatalf("attempt envelope = %s, want attempt warning", debug.json[attemptPath])
			}
		})
	}
}

func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
	t.Run("chunk", func(t *testing.T) {
		prepared, plan := preparedTerminalDebugPipeline(t)
		primary := errors.New("chunk operation failed")
		prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary}
		debug := newCapturedDebugRecorder()
		debug.failPath = "chunk/attempt-01.json"

		_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
		if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
			t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
		}
	})

	t.Run("extract", func(t *testing.T) {
		prepared := preparedAttemptDebugPipeline(t)
		primary := errors.New("extract operation failed")
		installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
			return erasedTypedResult{}, primary
		})
		debug := newCapturedDebugRecorder()
		debug.failPath = "extract/notes/chunk-000001/attempt-01.json"

		_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
		if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write extract attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
			t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
		}
	})
}

func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
	prepared, plan := preparedTerminalDebugPipeline(t)
	prepared.resolved.Chunk.Retries = 1
	calls := 0
	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
	prepared.chunkValidators.validators = []preparedValidator{{
		resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
		chunk:    terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
	}}
	debug := newCapturedDebugRecorder()
	debug.failPath = "chunk/attempt-01.json"

	_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
	if err == nil || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
		t.Fatalf("Run() error = %v, want terminal attempt debug failure", err)
	}
	if calls != 1 {
		t.Fatalf("chunk calls = %d, want one attempt without retry", calls)
	}
	if debug.has("chunk/attempt-02.json") {
		t.Fatal("second chunk attempt envelope exists after non-retryable debug persistence failure")
	}
}

func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
	prepared := preparedAttemptDebugPipeline(t)
	debug := newCapturedDebugRecorder()
	client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
	installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
		if err := callAttemptDebugLLM(ctx, client, "extract-module"); err != nil {
			return erasedTypedResult{}, err
		}
		return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
	})
	prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
		resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
		typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
			if err := callAttemptDebugLLM(ctx, client, "extract-validator"); err != nil {
				return contracts.ValidationResult{}, err
			}
			return contracts.ValidationResult{Approved: true}, nil
		},
	}}

	if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
		t.Fatalf("Run() error = %v, want nil", err)
	}
	module := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
	validator := debug.envelope(t, "validate/extract/notes/typed~2fextract-notes/01-llm-check-attempt-01.json")
	if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/chunk-000001/attempt-01/") {
		t.Fatalf("module LLM calls = %#v, want extract module call only", module.LLMCalls)
	}
	if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/") {
		t.Fatalf("validator LLM calls = %#v, want validator call only", validator.LLMCalls)
	}
	if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
		t.Fatalf("module and validator attempts share LLM call %#v", module.LLMCalls)
	}
}