scriptorium/internal/usecase/runner_test.go

package usecase

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"regexp"
	"testing"

	"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
	"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)

type fakePromptRepo struct {
	def         *domain.PromptDefinition
	err         error
	lastID      string
	lastVersion string
}

func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
	f.lastID = id
	f.lastVersion = version
	if f.err != nil {
		return nil, f.err
	}
	return f.def, nil
}

type fakeArtifactReader struct {
	artifactsByURI map[string]*domain.Artifact
	errByURI       map[string]error
}

func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
	if err, ok := f.errByURI[ref.URI]; ok {
		return nil, err
	}
	if art, ok := f.artifactsByURI[ref.URI]; ok {
		cp := *art
		return &cp, nil
	}
	return nil, errors.New("artifact not found")
}

type fakeRenderer struct {
	rendered *domain.RenderedPrompt
	err      error
}

func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
	if f.err != nil {
		return nil, f.err
	}
	return f.rendered, nil
}

type fakeLLM struct {
	resp    *domain.GenerateResponse
	err     error
	lastReq domain.GenerateRequest
}

func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
	f.lastReq = req
	if f.err != nil {
		return nil, f.err
	}
	return f.resp, nil
}

type fakeValidator struct {
	result domain.ValidationResult
	err    error
}

func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
	if f.err != nil {
		return domain.ValidationResult{}, f.err
	}
	return f.result, nil
}

type fakeRepairer struct {
	responses []*domain.GenerateResponse
	err       error
	calls     int
}

func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) {
	f.calls++
	if f.err != nil {
		return nil, f.err
	}
	if len(f.responses) == 0 {
		return nil, errors.New("no repair response configured")
	}
	idx := f.calls - 1
	if idx >= len(f.responses) {
		idx = len(f.responses) - 1
	}
	return f.responses[idx], nil
}

func TestRunnerRunSuccessful(t *testing.T) {
	repo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
	reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
		"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
		"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
	}}
	renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
	llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}

	runner := NewRunner(repo, reader, renderer, llmClient, nil)
	res, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:      "p",
		PromptVersion: "1",
		ProfileID:     "exec",
		Inputs: map[string]domain.ArtifactRef{
			"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
			"glossary":   {Type: domain.ArtifactRefFile, URI: "a://g"},
		},
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
	})
	if err != nil {
		t.Fatalf("expected no error, got %v", err)
	}
	if res.PromptID != "p" || res.PromptVersion != "1" {
		t.Fatalf("unexpected prompt metadata: %+v", res)
	}
	if res.SelectedProfileID != "exec" {
		t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID)
	}
	if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
		t.Fatalf("invalid run id: %q", res.RunID)
	}
	if res.PromptHash == "" || res.RenderedPromptHash == "" {
		t.Fatal("expected prompt hashes")
	}
	if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
		t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
	}
	if res.RawOutput != "# recap" {
		t.Fatalf("expected raw output, got %q", res.RawOutput)
	}
	if res.Validation.Status != domain.ValidationSkipped {
		t.Fatalf("expected skipped validation, got %q", res.Validation.Status)
	}
	if llmClient.lastReq.Target.TimeoutSeconds != 90 {
		t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
	}
}

func TestRunnerRunPromptLoadFailure(t *testing.T) {
	runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
	_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
	if !errors.Is(err, ErrProfileLoad) {
		t.Fatalf("expected ErrProfileLoad, got %v", err)
	}
}

func TestRunnerRunMissingProfileSelection(t *testing.T) {
	repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
	runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
	_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
	if !errors.Is(err, ErrInvalidRequest) {
		t.Fatalf("expected invalid request, got %v", err)
	}
}

func TestRunnerRunMissingExecutionOverride(t *testing.T) {
	repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
	runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
	_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
	if !errors.Is(err, ErrInvalidRequest) {
		t.Fatalf("expected invalid request, got %v", err)
	}
}

func TestRunnerRunArtifactLoadFailure(t *testing.T) {
	runner := NewRunner(
		&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
		&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
		&fakeRenderer{rendered: &domain.RenderedPrompt{}},
		&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
		nil,
	)

	_, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:  "p",
		ProfileID: "exec",
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
		Inputs:    map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
	})
	if !errors.Is(err, ErrArtifactLoad) {
		t.Fatalf("expected ErrArtifactLoad, got %v", err)
	}
}

func TestRunnerRunPromptRenderFailure(t *testing.T) {
	runner := NewRunner(
		&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
		&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
		&fakeRenderer{err: errors.New("render failed")},
		&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
		nil,
	)
	_, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:  "p",
		ProfileID: "exec",
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
		Inputs:    map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
	})
	if !errors.Is(err, ErrPromptRender) {
		t.Fatalf("expected ErrPromptRender, got %v", err)
	}
}

func TestRunnerRunLLMFailure(t *testing.T) {
	runner := NewRunner(
		&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
		&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
		&fakeRenderer{rendered: &domain.RenderedPrompt{}},
		&fakeLLM{err: errors.New("llm failed")},
		nil,
	)
	_, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:  "p",
		ProfileID: "exec",
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
		Inputs:    map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
	})
	if !errors.Is(err, ErrLLMGenerate) {
		t.Fatalf("expected ErrLLMGenerate, got %v", err)
	}
}

func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
	validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
	runner := NewRunner(
		&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
		&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
		&fakeRenderer{rendered: &domain.RenderedPrompt{}},
		&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
		validator,
	)
	res, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:  "p",
		ProfileID: "exec",
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
		Inputs:    map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
	})
	if err != nil {
		t.Fatalf("expected no error, got %v", err)
	}
	if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" {
		t.Fatalf("unexpected validation/raw output: %+v", res)
	}
}

func TestRunnerRunRepairBounded(t *testing.T) {
	repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
	runner := NewRunnerWithRepairer(
		&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
		&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
		&fakeRenderer{rendered: &domain.RenderedPrompt{}},
		&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
		validate.NewStandardValidator("."),
		repairer,
	)
	res, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID:  "p",
		ProfileID: "exec",
		Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
		Inputs:    map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
	})
	if err != nil {
		t.Fatalf("expected no error, got %v", err)
	}
	if repairer.calls != 1 || res.Validation.RepairAttempts != 1 {
		t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts)
	}
}

func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition {
	return &domain.PromptDefinition{
		ID:             "p",
		Version:        "1",
		DefaultProfile: "exec",
		Inputs:         []domain.PromptInput{{Name: "transcript", Required: true}},
		Templates:      []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
		OutputFormat:   format,
		Validation: domain.OutputContract{
			ValidationMode: mode,
			RepairAttempts: attempts,
			Format:         format,
		},
	}
}

func hashString(s string) string {
	sum := sha256.Sum256([]byte(s))
	return hex.EncodeToString(sum[:])
}