eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Refactor: load execution profiles from YAML and split prompt definitions into promptdef repository

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-05 10:18:28 -05:00
parent a633c67538
commit 7fffdaede3
25 changed files with 567 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/usecase/runner_test.go
View File

@@ -19,6 +19,23 @@ type fakePromptRepo struct {
lastVersion string
}
type fakeExecutionProfileRepo struct {
profile *domain.ExecutionProfile
err error
lastID string
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
return nil, f.err
}
if f.profile == nil {
return nil, errors.New("profile not found")
}
return f.profile, nil
}
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
f.lastID = id
f.lastVersion = version
@@ -112,7 +129,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(repo, reader, renderer, llmClient, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -121,7 +138,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -138,7 +155,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.PromptHash == "" || res.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" {
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
}
if res.RawOutput != "# recap" {
@@ -153,7 +170,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
@@ -162,25 +179,26 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunMissingProfileSelection(t *testing.T) {
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected profile load failure, got %v", err)
}
}
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -190,7 +208,6 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
})
if !errors.Is(err, ErrArtifactLoad) {
@@ -201,6 +218,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -209,7 +227,6 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrPromptRender) {
@@ -220,6 +237,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{err: errors.New("llm failed")},
@@ -228,7 +246,6 @@ func TestRunnerRunLLMFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrLLMGenerate) {
@@ -240,6 +257,7 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
@@ -248,7 +266,6 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -263,6 +280,7 @@ func TestRunnerRunRepairBounded(t *testing.T) {
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
@@ -272,7 +290,6 @@ func TestRunnerRunRepairBounded(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -303,3 +320,11 @@ func hashString(s string) string {
sum := sha256.Sum256([]byte(s))
return hex.EncodeToString(sum[:])
}
func defaultExecutionProfile() *domain.ExecutionProfile {
return &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://llm/v1",
Model: "model-from-profile",
}
}