eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Refactor: load execution profiles from YAML and split prompt definitions into promptdef repository

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-05 10:18:28 -05:00
parent a633c67538
commit 7fffdaede3
25 changed files with 567 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/usecase/integration_test.go
View File

@@ -2,6 +2,7 @@ package usecase
import (
"context"
"os"
"path/filepath"
"testing"
@@ -9,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -32,11 +34,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
}
profilesDir := filepath.Join(root, "profiles")
execProfilesDir := t.TempDir()
schemasDir := filepath.Join(root, "schemas")
fixturesDir := filepath.Join(root, "examples", "fixtures")
if err := os.WriteFile(filepath.Join(execProfilesDir, "local-default.yaml"), []byte(
"id: local-default\nendpoint: http://llm/v1\nmodel: test-model\n"), 0644); err != nil {
t.Fatalf("failed to write execution profile fixture: %v", err)
}
runner := NewRunner(
profile.NewFilesystemRepository(profilesDir),
promptdef.NewFilesystemRepository(profilesDir),
profile.NewFilesystemRepository(execProfilesDir),
artifact.NewCompositeReader(),
prompt.NewGoRenderer(),
&integrationLLM{},
@@ -45,11 +53,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "generic.structured_events",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "test-model",
},
ProfileID: "local-default",
Inputs: map[string]domain.ArtifactRef{
"transcript": {
Type: domain.ArtifactRefFile,

64
internal/usecase/runner.go
View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -30,25 +31,28 @@ var (
// Runner executes the Scriptorium core use case.
type Runner struct {
profiles profile.Repository
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
validator validate.Validator
repairer OutputRepairer
promptDefs promptdef.Repository
profiles profile.Repository
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
validator validate.Validator
repairer OutputRepairer
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
) *Runner {
return NewRunnerWithRepairer(profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
artifacts artifact.Reader,
renderer prompt.Renderer,
@@ -57,12 +61,13 @@ func NewRunnerWithRepairer(
repairer OutputRepairer,
) *Runner {
return &Runner{
profiles: profiles,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
validator: validator,
repairer: repairer,
promptDefs: promptDefs,
profiles: profiles,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
validator: validator,
repairer: repairer,
}
}
@@ -78,7 +83,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
@@ -93,10 +98,11 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
}
if req.Execution == nil {
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
effectiveModel := mergeExecutionTarget(executionProfileToTarget(execProfile), req.Execution)
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
@@ -265,6 +271,30 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.Executio
return out
}
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
if p == nil {
return domain.ExecutionTarget{}
}
cp := map[string]string(nil)
if len(p.ExtraParams) > 0 {
cp = make(map[string]string, len(p.ExtraParams))
for k, v := range p.ExtraParams {
cp[k] = v
}
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
TopP: p.TopP,
TimeoutSeconds: p.TimeoutSeconds,
ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv,
ExtraParams: cp,
}
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation
if contract.Format == "" {

55
internal/usecase/runner_test.go
View File

@@ -19,6 +19,23 @@ type fakePromptRepo struct {
lastVersion string
}
type fakeExecutionProfileRepo struct {
profile *domain.ExecutionProfile
err error
lastID string
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
return nil, f.err
}
if f.profile == nil {
return nil, errors.New("profile not found")
}
return f.profile, nil
}
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
f.lastID = id
f.lastVersion = version
@@ -112,7 +129,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(repo, reader, renderer, llmClient, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -121,7 +138,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -138,7 +155,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.PromptHash == "" || res.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" {
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
}
if res.RawOutput != "# recap" {
@@ -153,7 +170,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
@@ -162,25 +179,26 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunMissingProfileSelection(t *testing.T) {
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected profile load failure, got %v", err)
}
}
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -190,7 +208,6 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
})
if !errors.Is(err, ErrArtifactLoad) {
@@ -201,6 +218,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -209,7 +227,6 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrPromptRender) {
@@ -220,6 +237,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{err: errors.New("llm failed")},
@@ -228,7 +246,6 @@ func TestRunnerRunLLMFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrLLMGenerate) {
@@ -240,6 +257,7 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
@@ -248,7 +266,6 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -263,6 +280,7 @@ func TestRunnerRunRepairBounded(t *testing.T) {
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
@@ -272,7 +290,6 @@ func TestRunnerRunRepairBounded(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -303,3 +320,11 @@ func hashString(s string) string {
sum := sha256.Sum256([]byte(s))
return hex.EncodeToString(sum[:])
}
func defaultExecutionProfile() *domain.ExecutionProfile {
return &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://llm/v1",
Model: "model-from-profile",
}
}