Implement the core Run use case with a fake LLM path
This commit is contained in:
201
internal/usecase/runner.go
Normal file
201
internal/usecase/runner.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrProfileLoad = errors.New("failed to load profile")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Runner executes the Analyzer core use case.
|
||||
type Runner struct {
|
||||
profiles profile.Repository
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
validator validate.Validator
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
profiles profile.Repository,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
) *Runner {
|
||||
return &Runner{
|
||||
profiles: profiles,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
validator: validator,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
||||
if strings.TrimSpace(req.ProfileID) == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrProfileLoad, err)
|
||||
}
|
||||
|
||||
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
|
||||
effectiveContract := resolveOutputContract(prof, req.Validation)
|
||||
|
||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
||||
inputHashes := make(map[string]string, len(req.Inputs))
|
||||
for name, ref := range req.Inputs {
|
||||
art, readErr := r.artifacts.Read(ctx, ref)
|
||||
if readErr != nil {
|
||||
return nil, fmt.Errorf("%w: input %q: %v", ErrArtifactLoad, name, readErr)
|
||||
}
|
||||
if art.Name == "" {
|
||||
art.Name = name
|
||||
}
|
||||
resolvedInputs[name] = art
|
||||
inputHashes[name] = art.Hash
|
||||
}
|
||||
|
||||
renderedPrompt, err := r.renderer.Render(ctx, prof, resolvedInputs, req.Vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPromptRender, err)
|
||||
}
|
||||
|
||||
promptHash := hashRenderedPrompt(*renderedPrompt)
|
||||
|
||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: *renderedPrompt,
|
||||
Target: effectiveModel,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrLLMGenerate, err)
|
||||
}
|
||||
|
||||
outputArtifact := buildOutputArtifact(genResp.Content, effectiveContract.Format)
|
||||
|
||||
validationResult := domain.ValidationResult{
|
||||
Status: domain.ValidationSkipped,
|
||||
Mode: effectiveContract.ValidationMode,
|
||||
SchemaPath: effectiveContract.SchemaPath,
|
||||
RepairAttempts: effectiveContract.RepairAttempts,
|
||||
IsValid: true,
|
||||
}
|
||||
if r.validator != nil && effectiveContract.ValidationMode != domain.ValidationNone {
|
||||
validationResult, err = r.validator.Validate(ctx, &outputArtifact, effectiveContract)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrValidation, err)
|
||||
}
|
||||
}
|
||||
|
||||
end := time.Now().UTC()
|
||||
|
||||
return &domain.RunResult{
|
||||
Artifact: outputArtifact,
|
||||
RawOutput: genResp.Content,
|
||||
Validation: validationResult,
|
||||
ProfileID: prof.ID,
|
||||
ProfileVersion: prof.Version,
|
||||
ModelName: effectiveModel.Model,
|
||||
Endpoint: effectiveModel.Endpoint,
|
||||
InputHashes: inputHashes,
|
||||
PromptHash: promptHash,
|
||||
Usage: genResp.Usage,
|
||||
StartTime: start,
|
||||
EndTime: end,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget {
|
||||
if override == nil {
|
||||
return base
|
||||
}
|
||||
|
||||
out := base
|
||||
if override.Endpoint != "" {
|
||||
out.Endpoint = override.Endpoint
|
||||
}
|
||||
if override.Model != "" {
|
||||
out.Model = override.Model
|
||||
}
|
||||
if override.Temperature != 0 {
|
||||
out.Temperature = override.Temperature
|
||||
}
|
||||
if override.MaxTokens != 0 {
|
||||
out.MaxTokens = override.MaxTokens
|
||||
}
|
||||
if override.TopP != 0 {
|
||||
out.TopP = override.TopP
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveOutputContract(prof *domain.PromptProfile, override *domain.OutputContract) domain.OutputContract {
|
||||
contract := prof.Validation
|
||||
if contract.Format == "" {
|
||||
contract.Format = prof.OutputFormat
|
||||
}
|
||||
if override != nil {
|
||||
contract = *override
|
||||
}
|
||||
if contract.Format == "" {
|
||||
contract.Format = domain.FormatText
|
||||
}
|
||||
return contract
|
||||
}
|
||||
|
||||
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
||||
var b strings.Builder
|
||||
for _, msg := range p.Messages {
|
||||
b.WriteString(msg.Role)
|
||||
b.WriteByte('\n')
|
||||
b.WriteString(msg.Content)
|
||||
b.WriteString("\n---\n")
|
||||
}
|
||||
h := sha256.Sum256([]byte(b.String()))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact {
|
||||
body := []byte(content)
|
||||
hash := sha256.Sum256(body)
|
||||
|
||||
contentType := "text/plain"
|
||||
switch format {
|
||||
case domain.FormatMarkdown:
|
||||
contentType = "text/markdown"
|
||||
case domain.FormatJSON:
|
||||
contentType = "application/json"
|
||||
}
|
||||
|
||||
return domain.Artifact{
|
||||
Name: "output",
|
||||
ContentType: contentType,
|
||||
Body: body,
|
||||
Size: int64(len(body)),
|
||||
Hash: hex.EncodeToString(hash[:]),
|
||||
}
|
||||
}
|
||||
398
internal/usecase/runner_test.go
Normal file
398
internal/usecase/runner_test.go
Normal file
@@ -0,0 +1,398 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
type fakeProfileRepo struct {
|
||||
profile *domain.PromptProfile
|
||||
err error
|
||||
lastID string
|
||||
lastVersion string
|
||||
}
|
||||
|
||||
func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) {
|
||||
f.lastID = id
|
||||
f.lastVersion = version
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.profile, nil
|
||||
}
|
||||
|
||||
type fakeArtifactReader struct {
|
||||
artifactsByURI map[string]*domain.Artifact
|
||||
errByURI map[string]error
|
||||
}
|
||||
|
||||
func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
if err, ok := f.errByURI[ref.URI]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if art, ok := f.artifactsByURI[ref.URI]; ok {
|
||||
copy := *art
|
||||
return ©, nil
|
||||
}
|
||||
return nil, errors.New("artifact not found")
|
||||
}
|
||||
|
||||
type fakeRenderer struct {
|
||||
rendered *domain.RenderedPrompt
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.rendered, nil
|
||||
}
|
||||
|
||||
type fakeLLM struct {
|
||||
resp *domain.GenerateResponse
|
||||
err error
|
||||
lastReq domain.GenerateRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
f.lastReq = req
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.resp, nil
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
result domain.ValidationResult
|
||||
err error
|
||||
called bool
|
||||
lastContract domain.OutputContract
|
||||
}
|
||||
|
||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||
f.called = true
|
||||
f.lastContract = contract
|
||||
if f.err != nil {
|
||||
return domain.ValidationResult{}, f.err
|
||||
}
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
func TestRunnerRunSuccessful(t *testing.T) {
|
||||
repo := &fakeProfileRepo{
|
||||
profile: &domain.PromptProfile{
|
||||
ID: "p1",
|
||||
Version: "1.0.0",
|
||||
OutputFormat: domain.FormatMarkdown,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep1",
|
||||
Model: "model-default",
|
||||
Temperature: 0.4,
|
||||
MaxTokens: 200,
|
||||
TopP: 0.9,
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||
"a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")},
|
||||
"a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")},
|
||||
}}
|
||||
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "System context"},
|
||||
{Role: "user", Content: "Please summarize"},
|
||||
}}}
|
||||
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{
|
||||
Content: "# recap\n- item",
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 20,
|
||||
TotalTokens: 30,
|
||||
},
|
||||
}}
|
||||
|
||||
runner := NewRunner(repo, reader, renderer, llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p1",
|
||||
ProfileVersion: "1.0.0",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||
},
|
||||
Model: &domain.ModelTarget{
|
||||
Model: "model-override",
|
||||
Temperature: 0,
|
||||
MaxTokens: 0,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
|
||||
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
|
||||
}
|
||||
if res.ModelName != "model-override" {
|
||||
t.Fatalf("expected model override to apply, got %q", res.ModelName)
|
||||
}
|
||||
if res.Endpoint != "ep1" {
|
||||
t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint)
|
||||
}
|
||||
if res.Artifact.ContentType != "text/markdown" {
|
||||
t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType)
|
||||
}
|
||||
if string(res.Artifact.Body) != "# recap\n- item" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body))
|
||||
}
|
||||
if res.RawOutput != "# recap\n- item" {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationSkipped {
|
||||
t.Fatalf("expected skipped validation, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.Validation.Mode != domain.ValidationBasic {
|
||||
t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode)
|
||||
}
|
||||
if res.PromptHash == "" {
|
||||
t.Fatal("expected non-empty prompt hash")
|
||||
}
|
||||
if res.Usage.TotalTokens != 30 {
|
||||
t.Fatalf("expected usage to propagate, got %+v", res.Usage)
|
||||
}
|
||||
if res.StartTime.IsZero() || res.EndTime.IsZero() {
|
||||
t.Fatal("expected start and end times")
|
||||
}
|
||||
if res.EndTime.Before(res.StartTime) {
|
||||
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
|
||||
}
|
||||
|
||||
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
|
||||
t.Fatalf("unexpected transcript hash: %q", got)
|
||||
}
|
||||
if got := res.InputHashes["glossary"]; got != hashString("glossary body") {
|
||||
t.Fatalf("unexpected glossary hash: %q", got)
|
||||
}
|
||||
|
||||
if llmClient.lastReq.Target.Temperature != 0.4 {
|
||||
t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunProfileLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{err: errors.New("boom")},
|
||||
&fakeArtifactReader{},
|
||||
&fakeRenderer{},
|
||||
&fakeLLM{},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrArtifactLoad) {
|
||||
t.Fatalf("expected ErrArtifactLoad, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{err: errors.New("render failed")},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrPromptRender) {
|
||||
t.Fatalf("expected ErrPromptRender, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{err: errors.New("llm failed")},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("expected ErrLLMGenerate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationFailureNonError(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationBasic,
|
||||
Errors: []string{"bad output"},
|
||||
IsValid: false,
|
||||
}}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationFailed {
|
||||
t.Fatalf("expected validation failed result, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.RawOutput != "raw output" {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
if !validator.called {
|
||||
t.Fatal("expected validator to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationRuntimeError(t *testing.T) {
|
||||
validator := &fakeValidator{err: errors.New("validator unavailable")}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: minimalProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("expected ErrValidation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["name"],
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
}`), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
runner := NewRunner(
|
||||
&fakeProfileRepo{profile: &domain.PromptProfile{
|
||||
ID: "p-json",
|
||||
Version: "1",
|
||||
OutputFormat: domain.FormatJSON,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep",
|
||||
Model: "m",
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
Format: domain.FormatJSON,
|
||||
},
|
||||
}},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
|
||||
validate.NewStandardValidator(tmp),
|
||||
)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
ProfileID: "p-json",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Validation.Status != domain.ValidationFailed {
|
||||
t.Fatalf("expected validation failed, got %q", res.Validation.Status)
|
||||
}
|
||||
if res.RawOutput != `{"count":1}` {
|
||||
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func minimalProfile() *domain.PromptProfile {
|
||||
return &domain.PromptProfile{
|
||||
ID: "p",
|
||||
Version: "1",
|
||||
OutputFormat: domain.FormatText,
|
||||
ModelDefaults: domain.ModelTarget{
|
||||
Endpoint: "ep",
|
||||
Model: "m",
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func hashString(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user