Implement the core Run use case with a fake LLM path
This commit is contained in:
398
internal/usecase/runner_test.go
Normal file
398
internal/usecase/runner_test.go
Normal file
@@ -0,0 +1,398 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
type fakeProfileRepo struct {
|
||||
profile *domain.PromptProfile
|
||||
err error
|
||||
lastID string
|
||||
lastVersion string
|
||||
}
|
||||
|
||||
func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) {
|
||||
f.lastID = id
|
||||
f.lastVersion = version
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.profile, nil
|
||||
}
|
||||
|
||||
type fakeArtifactReader struct {
|
||||
artifactsByURI map[string]*domain.Artifact
|
||||
errByURI map[string]error
|
||||
}
|
||||
|
||||
func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
if err, ok := f.errByURI[ref.URI]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if art, ok := f.artifactsByURI[ref.URI]; ok {
|
||||
copy := *art
|
||||
return ©, nil
|
||||
}
|
||||
return nil, errors.New("artifact not found")
|
||||
}
|
||||
|
||||
type fakeRenderer struct {
|
||||
rendered *domain.RenderedPrompt
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.rendered, nil
|
||||
}
|
||||
|
||||
type fakeLLM struct {
|
||||
resp *domain.GenerateResponse
|
||||
err error
|
||||
lastReq domain.GenerateRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
f.lastReq = req
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.resp, nil
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
result domain.ValidationResult
|
||||
err error
|
||||
called bool
|
||||
lastContract domain.OutputContract
|
||||
}
|
||||
|
||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||
f.called = true
|
||||
f.lastContract = contract
|
||||
if f.err != nil {
|
||||
return domain.ValidationResult{}, f.err
|
||||
}
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
func TestRunnerRunSuccessful(t *testing.T) {
|
||||
repo := &fakeProfileRepo{
|
||||
profile: &domain.PromptProfile{
|
||||
ID: "p1",
|
||||
Version: "1.0.0",
|
||||
OutputFormat: domain.FormatMarkdown,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep1",
|
||||
Model: "model-default",
|
||||
Temperature: 0.4,
|
||||
MaxTokens: 200,
|
||||
TopP: 0.9,
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||
"a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")},
|
||||
"a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")},
|
||||
}}
|
||||
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "System context"},
|
||||
{Role: "user", Content: "Please summarize"},
|
||||
}}}
|
||||
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{
|
||||
Content: "# recap\n- item",
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 20,
|
||||
TotalTokens: 30,
|
||||
},
|
||||
}}
|
||||
|
||||
runner := NewRunner(repo, reader, renderer, llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p1",
|
||||
ProfileVersion: "1.0.0",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||
},
|
||||
Model: &domain.ModelTarget{
|
||||
Model: "model-override",
|
||||
Temperature: 0,
|
||||
MaxTokens: 0,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
|
||||
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
|
||||
}
|
||||
if res.ModelName != "model-override" {
|
||||
t.Fatalf("expected model override to apply, got %q", res.ModelName)
|
||||
}
|
||||
if res.Endpoint != "ep1" {
|
||||
t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint)
|
||||
}
|
||||
if res.Artifact.ContentType != "text/markdown" {
|
||||
t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType)
|
||||
}
|
||||
if string(res.Artifact.Body) != "# recap\n- item" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body))
|
||||
}
|
||||
if res.RawOutput != "# recap\n- item" {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationSkipped {
|
||||
t.Fatalf("expected skipped validation, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.Validation.Mode != domain.ValidationBasic {
|
||||
t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode)
|
||||
}
|
||||
if res.PromptHash == "" {
|
||||
t.Fatal("expected non-empty prompt hash")
|
||||
}
|
||||
if res.Usage.TotalTokens != 30 {
|
||||
t.Fatalf("expected usage to propagate, got %+v", res.Usage)
|
||||
}
|
||||
if res.StartTime.IsZero() || res.EndTime.IsZero() {
|
||||
t.Fatal("expected start and end times")
|
||||
}
|
||||
if res.EndTime.Before(res.StartTime) {
|
||||
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
|
||||
}
|
||||
|
||||
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
|
||||
t.Fatalf("unexpected transcript hash: %q", got)
|
||||
}
|
||||
if got := res.InputHashes["glossary"]; got != hashString("glossary body") {
|
||||
t.Fatalf("unexpected glossary hash: %q", got)
|
||||
}
|
||||
|
||||
if llmClient.lastReq.Target.Temperature != 0.4 {
|
||||
t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunProfileLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{err: errors.New("boom")},
|
||||
&fakeArtifactReader{},
|
||||
&fakeRenderer{},
|
||||
&fakeLLM{},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrArtifactLoad) {
|
||||
t.Fatalf("expected ErrArtifactLoad, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{err: errors.New("render failed")},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrPromptRender) {
|
||||
t.Fatalf("expected ErrPromptRender, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{err: errors.New("llm failed")},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("expected ErrLLMGenerate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationFailureNonError(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationBasic,
|
||||
Errors: []string{"bad output"},
|
||||
IsValid: false,
|
||||
}}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationFailed {
|
||||
t.Fatalf("expected validation failed result, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.RawOutput != "raw output" {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
if !validator.called {
|
||||
t.Fatal("expected validator to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationRuntimeError(t *testing.T) {
|
||||
validator := &fakeValidator{err: errors.New("validator unavailable")}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("expected ErrValidation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["name"],
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
}`), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: &domain.PromptProfile{
|
||||
ID: "p-json",
|
||||
Version: "1",
|
||||
OutputFormat: domain.FormatJSON,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep",
|
||||
Model: "m",
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
Format: domain.FormatJSON,
|
||||
},
|
||||
}},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
|
||||
validate.NewStandardValidator(tmp),
|
||||
)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p-json",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationFailed {
|
||||
t.Fatalf("expected validation failed, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.RawOutput != `{"count":1}` {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func minimalProfile() *domain.PromptProfile {
|
||||
return &domain.PromptProfile{
|
||||
ID: "p",
|
||||
Version: "1",
|
||||
OutputFormat: domain.FormatText,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep",
|
||||
Model: "m",
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func hashString(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user