package usecase import ( "context" "crypto/sha256" "encoding/hex" "errors" "regexp" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/validate" ) type fakePromptRepo struct { def *domain.PromptDefinition err error lastID string lastVersion string } type fakeExecutionProfileRepo struct { profile *domain.ExecutionProfile err error lastID string } func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { f.lastID = id if f.err != nil { return nil, f.err } if f.profile == nil { return nil, errors.New("profile not found") } return f.profile, nil } func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { f.lastID = id f.lastVersion = version if f.err != nil { return nil, f.err } return f.def, nil } type fakeArtifactReader struct { artifactsByURI map[string]*domain.Artifact errByURI map[string]error } func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { if err, ok := f.errByURI[ref.URI]; ok { return nil, err } if art, ok := f.artifactsByURI[ref.URI]; ok { cp := *art return &cp, nil } return nil, errors.New("artifact not found") } type fakeRenderer struct { rendered *domain.RenderedPrompt err error } func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { if f.err != nil { return nil, f.err } return f.rendered, nil } type fakeLLM struct { resp *domain.GenerateResponse err error lastReq domain.GenerateRequest } func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { f.lastReq = req if f.err != nil { return nil, f.err } return f.resp, nil } type fakeValidator struct { result domain.ValidationResult err error } func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { if f.err != nil { return domain.ValidationResult{}, f.err } return f.result, nil } type fakeRepairer struct { responses []*domain.GenerateResponse err error calls int } func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { f.calls++ if f.err != nil { return nil, f.err } if len(f.responses) == 0 { return nil, errors.New("no repair response configured") } idx := f.calls - 1 if idx >= len(f.responses) { idx = len(f.responses) - 1 } return f.responses[idx], nil } func TestRunnerRunSuccessful(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, }, Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.PromptID != "p" || res.PromptVersion != "1" { t.Fatalf("unexpected prompt metadata: %+v", res) } if res.SelectedProfileID != "exec" { t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID) } if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok { t.Fatalf("invalid run id: %q", res.RunID) } if res.PromptHash == "" || res.RenderedPromptHash == "" { t.Fatal("expected prompt hashes") } if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" { t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams) } if res.RawOutput != "# recap" { t.Fatalf("expected raw output, got %q", res.RawOutput) } if res.Validation.Status != domain.ValidationSkipped { t.Fatalf("expected skipped validation, got %q", res.Validation.Status) } if llmClient.lastReq.Target.TimeoutSeconds != 90 { t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) } } func TestRunnerRunPromptLoadFailure(t *testing.T) { runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } func TestRunnerRunMissingProfileSelection(t *testing.T) { repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}} runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrInvalidRequest) { t.Fatalf("expected invalid request, got %v", err) } } func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected profile load failure, got %v", err) } } func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}}, }) if !errors.Is(err, ErrArtifactLoad) { t.Fatalf("expected ErrArtifactLoad, got %v", err) } } func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{err: errors.New("render failed")}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) } } func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{err: errors.New("llm failed")}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if !errors.Is(err, ErrLLMGenerate) { t.Fatalf("expected ErrLLMGenerate, got %v", err) } } func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) { validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, validator, ) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" { t.Fatalf("unexpected validation/raw output: %+v", res) } } func TestRunnerRunRepairBounded(t *testing.T) { repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}} runner := NewRunnerWithRepairer( &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}}, validate.NewStandardValidator("."), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if repairer.calls != 1 || res.Validation.RepairAttempts != 1 { t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts) } } func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition { return &domain.PromptDefinition{ ID: "p", Version: "1", DefaultProfile: "exec", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: format, Validation: domain.OutputContract{ ValidationMode: mode, RepairAttempts: attempts, Format: format, }, } } func hashString(s string) string { sum := sha256.Sum256([]byte(s)) return hex.EncodeToString(sum[:]) } func defaultExecutionProfile() *domain.ExecutionProfile { return &domain.ExecutionProfile{ ID: "exec", Endpoint: "http://llm/v1", Model: "model-from-profile", } }