Refactor: split prompt definition from execution settings and migrate run contracts to prompt_* + execution_target

This commit is contained in:
2026-05-05 10:09:31 -05:00
parent fdfd8641f5
commit a633c67538
28 changed files with 712 additions and 1021 deletions

View File

@@ -44,7 +44,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "generic.structured_events",
PromptID: "generic.structured_events",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "test-model",
},
Inputs: map[string]domain.ArtifactRef{
"transcript": {
Type: domain.ArtifactRefFile,
@@ -60,17 +65,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
t.Fatalf("expected no error, got %v", err)
}
if res.ProfileID != "generic.structured_events" {
t.Fatalf("unexpected profile id: %q", res.ProfileID)
if res.PromptID != "generic.structured_events" {
t.Fatalf("unexpected prompt id: %q", res.PromptID)
}
if res.RunID == "" {
t.Fatal("expected run id")
}
if res.ProfileHash == "" {
t.Fatal("expected profile hash")
if res.PromptHash == "" {
t.Fatal("expected prompt hash")
}
if res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile version: %q", res.ProfileVersion)
if res.PromptVersion != "1.0.0" {
t.Fatalf("unexpected prompt version: %q", res.PromptVersion)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)

View File

@@ -17,7 +17,7 @@ type OutputRepairer interface {
type RepairRequest struct {
PreviousOutput string
ValidationErrors []string
Target domain.ModelTarget
Target domain.ExecutionTarget
Attempt int
MaxAttempts int
Mode domain.ValidationMode

View File

@@ -21,7 +21,7 @@ import (
var (
ErrInvalidRequest = errors.New("invalid run request")
ErrProfileLoad = errors.New("failed to load profile")
ErrProfileLoad = errors.New("failed to load prompt definition")
ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
@@ -67,8 +67,8 @@ func NewRunnerWithRepairer(
}
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if strings.TrimSpace(req.ProfileID) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
runID, err := newRunID()
@@ -78,17 +78,32 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
profileHash, err := hashProfile(prof)
promptDefinitionHash, err := hashPromptDefinition(def)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err)
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
}
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
effectiveContract := resolveOutputContract(prof, req.Validation)
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
}
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
}
if req.Execution == nil {
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
}
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
}
effectiveContract := resolveOutputContract(def, req.Validation)
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
inputHashes := make(map[string]string, len(req.Inputs))
@@ -104,12 +119,12 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
inputHashes[name] = art.Hash
}
renderedPrompt, err := r.renderer.Render(ctx, prof, resolvedInputs, req.Vars)
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
}
promptHash := hashRenderedPrompt(*renderedPrompt)
renderedPromptHash := hashRenderedPrompt(*renderedPrompt)
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: *renderedPrompt,
@@ -158,22 +173,23 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
ProfileID: prof.ID,
ProfileVersion: prof.Version,
ProfileHash: profileHash,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
ModelParams: effectiveModel,
InputHashes: inputHashes,
PromptHash: promptHash,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
PromptID: def.ID,
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
RenderedPromptHash: renderedPromptHash,
SelectedProfileID: selectedProfileID,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
EffectiveModelParams: effectiveModel,
InputHashes: inputHashes,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
@@ -209,7 +225,7 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema
}
func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget {
func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.ExecutionTarget) domain.ExecutionTarget {
if override == nil {
return base
}
@@ -233,13 +249,26 @@ func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) dom
if override.TimeoutSeconds != 0 {
out.TimeoutSeconds = override.TimeoutSeconds
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if len(override.ExtraParams) > 0 {
cp := make(map[string]string, len(override.ExtraParams))
for k, v := range override.ExtraParams {
cp[k] = v
}
out.ExtraParams = cp
}
return out
}
func resolveOutputContract(prof *domain.PromptProfile, override *domain.OutputContract) domain.OutputContract {
contract := prof.Validation
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation
if contract.Format == "" {
contract.Format = prof.OutputFormat
contract.Format = def.OutputFormat
}
if override != nil {
contract = *override
@@ -283,8 +312,8 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
}
}
func hashProfile(prof *domain.PromptProfile) (string, error) {
b, err := json.Marshal(prof)
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
b, err := json.Marshal(def)
if err != nil {
return "", err
}

View File

@@ -5,8 +5,6 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"os"
"path/filepath"
"regexp"
"testing"
@@ -14,20 +12,20 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
type fakeProfileRepo struct {
profile *domain.PromptProfile
type fakePromptRepo struct {
def *domain.PromptDefinition
err error
lastID string
lastVersion string
}
func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) {
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
f.lastID = id
f.lastVersion = version
if f.err != nil {
return nil, f.err
}
return f.profile, nil
return f.def, nil
}
type fakeArtifactReader struct {
@@ -40,8 +38,8 @@ func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (
return nil, err
}
if art, ok := f.artifactsByURI[ref.URI]; ok {
copy := *art
return &copy, nil
cp := *art
return &cp, nil
}
return nil, errors.New("artifact not found")
}
@@ -51,7 +49,7 @@ type fakeRenderer struct {
err error
}
func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
if f.err != nil {
return nil, f.err
}
@@ -73,15 +71,11 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do
}
type fakeValidator struct {
result domain.ValidationResult
err error
called bool
lastContract domain.OutputContract
result domain.ValidationResult
err error
}
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
f.called = true
f.lastContract = contract
if f.err != nil {
return domain.ValidationResult{}, f.err
}
@@ -92,12 +86,10 @@ type fakeRepairer struct {
responses []*domain.GenerateResponse
err error
calls int
lastReq RepairRequest
}
func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) {
f.calls++
f.lastReq = req
if f.err != nil {
return nil, f.err
}
@@ -112,156 +104,83 @@ func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.G
}
func TestRunnerRunSuccessful(t *testing.T) {
repo := &fakeProfileRepo{
profile: &domain.PromptProfile{
ID: "p1",
Version: "1.0.0",
OutputFormat: domain.FormatMarkdown,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep1",
Model: "model-default",
Temperature: 0.4,
MaxTokens: 200,
TopP: 0.9,
TimeoutSeconds: 90,
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationBasic,
},
},
}
repo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")},
"a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "System context"},
{Role: "user", Content: "Please summarize"},
}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{
Content: "# recap\n- item",
Usage: domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 20,
TotalTokens: 30,
},
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(repo, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p1",
ProfileVersion: "1.0.0",
PromptID: "p",
PromptVersion: "1",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Model: &domain.ModelTarget{
Model: "model-override",
Temperature: 0,
MaxTokens: 0,
TimeoutSeconds: 0,
},
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
if res.PromptID != "p" || res.PromptVersion != "1" {
t.Fatalf("unexpected prompt metadata: %+v", res)
}
if res.SelectedProfileID != "exec" {
t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID)
}
if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
t.Fatalf("expected UUIDv4 run id, got %q", res.RunID)
t.Fatalf("invalid run id: %q", res.RunID)
}
if res.ProfileHash == "" {
t.Fatal("expected non-empty profile hash")
if res.PromptHash == "" || res.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if res.ModelName != "model-override" {
t.Fatalf("expected model override to apply, got %q", res.ModelName)
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
}
if res.Endpoint != "ep1" {
t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint)
}
if res.Artifact.ContentType != "text/markdown" {
t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType)
}
if string(res.Artifact.Body) != "# recap\n- item" {
t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body))
}
if res.RawOutput != "# recap\n- item" {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
if res.RawOutput != "# recap" {
t.Fatalf("expected raw output, got %q", res.RawOutput)
}
if res.Validation.Status != domain.ValidationSkipped {
t.Fatalf("expected skipped validation, got %q", res.Validation.Status)
}
if res.Validation.Mode != domain.ValidationBasic {
t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode)
}
if res.PromptHash == "" {
t.Fatal("expected non-empty prompt hash")
}
if res.Usage.TotalTokens != 30 {
t.Fatalf("expected usage to propagate, got %+v", res.Usage)
}
if res.StartTime.IsZero() || res.EndTime.IsZero() {
t.Fatal("expected start and end times")
}
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
t.Fatalf("unexpected transcript hash: %q", got)
}
if got := res.InputHashes["glossary"]; got != hashString("glossary body") {
t.Fatalf("unexpected glossary hash: %q", got)
}
if llmClient.lastReq.Target.Temperature != 0.4 {
t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature)
}
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" {
t.Fatalf("expected effective model params in result, got %+v", res.ModelParams)
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
}
func TestRunnerRunProfileLoadFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{err: errors.New("boom")},
&fakeArtifactReader{},
&fakeRenderer{},
&fakeLLM{},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"})
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
}
func TestMergeModelTargetTimeoutOverride(t *testing.T) {
base := domain.ModelTarget{TimeoutSeconds: 30}
override := &domain.ModelTarget{TimeoutSeconds: 75}
func TestRunnerRunMissingProfileSelection(t *testing.T) {
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
got := mergeModelTarget(base, override)
if got.TimeoutSeconds != 75 {
t.Fatalf("expected timeout override to apply, got %d", got.TimeoutSeconds)
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -269,10 +188,10 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
})
if !errors.Is(err, ErrArtifactLoad) {
t.Fatalf("expected ErrArtifactLoad, got %v", err)
@@ -281,18 +200,17 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrPromptRender) {
t.Fatalf("expected ErrPromptRender, got %v", err)
@@ -301,405 +219,82 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{err: errors.New("llm failed")},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("expected ErrLLMGenerate, got %v", err)
}
}
func TestRunnerRunValidationFailureNonError(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{
Status: domain.ValidationFailed,
Mode: domain.ValidationBasic,
Errors: []string{"bad output"},
IsValid: false,
}}
func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected validation failed result, got %q", res.Validation.Status)
}
if res.RawOutput != "raw output" {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
}
if !validator.called {
t.Fatal("expected validator to be called")
if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" {
t.Fatalf("unexpected validation/raw output: %+v", res)
}
}
func TestRunnerRunValidationRuntimeError(t *testing.T) {
validator := &fakeValidator{err: errors.New("validator unavailable")}
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
}
func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) {
tmp := t.TempDir()
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
}
}`), 0644); err != nil {
t.Fatal(err)
}
runner := NewRunner(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
Format: domain.FormatJSON,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
validate.NewStandardValidator(tmp),
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected validation failed, got %q", res.Validation.Status)
}
if res.RawOutput != `{"count":1}` {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
}
}
func TestRunnerRunNoRepairWhenDisabled(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"ok":true}`}},
}
func TestRunnerRunRepairBounded(t *testing.T) {
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 0,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 0 {
t.Fatalf("expected no repair calls, got %d", repairer.calls)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected failed validation, got %q", res.Validation.Status)
}
if res.RawOutput != `{"broken":` {
t.Fatalf("expected original output preserved, got %q", res.RawOutput)
}
if res.Validation.RepairAttempts != 0 {
t.Fatalf("expected repair attempts 0, got %d", res.Validation.RepairAttempts)
}
}
func TestRunnerRunSuccessfulRepairAfterInvalidJSON(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"ok":true}`, Usage: domain.TokenUsage{TotalTokens: 5}}},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`, Usage: domain.TokenUsage{TotalTokens: 3}}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 1 {
t.Fatalf("expected one repair call, got %d", repairer.calls)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"ok":true}` {
t.Fatalf("expected repaired output, got %q", res.RawOutput)
}
}
func TestRunnerRunSuccessfulRepairAfterSchemaFailure(t *testing.T) {
tmp := t.TempDir()
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
}
}`), 0644); err != nil {
t.Fatal(err)
}
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"name":"eris"}`}},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
validate.NewStandardValidator(tmp),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"name":"eris"}` {
t.Fatalf("expected repaired output, got %q", res.RawOutput)
}
}
func TestRunnerRunFailedRepairPreservesRawOutputAndErrors(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{
{Content: `{"repair1":`},
{Content: `{"repair2":`},
},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 2,
},
}},
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
validate.NewStandardValidator(t.TempDir()),
validate.NewStandardValidator("."),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected failed validation, got %q", res.Validation.Status)
}
if len(res.Validation.Errors) == 0 {
t.Fatal("expected validation errors after failed repair")
}
if res.Validation.RepairAttempts != 2 {
t.Fatalf("expected repair attempts 2, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"repair2":` {
t.Fatalf("expected final repaired output preserved, got %q", res.RawOutput)
if repairer.calls != 1 || res.Validation.RepairAttempts != 1 {
t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts)
}
}
func TestRunnerRunRepairAttemptsBounded(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{
{Content: `{"repair1":`},
{Content: `{"repair2":`},
{Content: `{"repair3":`},
},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 1 {
t.Fatalf("expected repair calls bounded to 1, got %d", repairer.calls)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
}
func minimalProfile() *domain.PromptProfile {
return &domain.PromptProfile{
ID: "p",
Version: "1",
OutputFormat: domain.FormatText,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition {
return &domain.PromptDefinition{
ID: "p",
Version: "1",
DefaultProfile: "exec",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
OutputFormat: format,
Validation: domain.OutputContract{
ValidationMode: domain.ValidationBasic,
ValidationMode: mode,
RepairAttempts: attempts,
Format: format,
},
}
}