Refactor: load execution profiles from YAML and split prompt definitions into promptdef repository

This commit is contained in:
2026-05-05 10:18:28 -05:00
parent a633c67538
commit 7fffdaede3
25 changed files with 567 additions and 183 deletions

View File

@@ -18,6 +18,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -125,6 +126,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
}
runner := usecase.NewRunner(
promptdef.NewFilesystemRepository(cfg.profileDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
@@ -183,6 +185,7 @@ func serveCommand(args []string, stderr io.Writer) int {
}
runner := usecase.NewRunner(
promptdef.NewFilesystemRepository(cfg.profileDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
)
@@ -147,8 +148,10 @@ func mapValidation(v domain.ValidationResult) validationDTO {
func mapRunError(err error) (int, string, string) {
switch {
case errors.Is(err, profile.ErrProfileNotFound):
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
case errors.Is(err, profile.ErrProfileNotFound):
return http.StatusNotFound, "profile_not_found", "execution profile not found"
case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, usecase.ErrProfileLoad):

View File

@@ -5,17 +5,19 @@ import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gopkg.in/yaml.v3"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gopkg.in/yaml.v3"
)
var (
ErrProfileNotFound = errors.New("prompt definition not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidProfile = errors.New("invalid prompt definition configuration")
ErrProfileNotFound = errors.New("execution profile not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidProfile = errors.New("invalid execution profile configuration")
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
)
type filesystemRepository struct {
@@ -26,9 +28,9 @@ func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidProfile)
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
files, err := os.ReadDir(r.dir)
@@ -53,97 +55,60 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err)
}
var prof domain.PromptDefinition
var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil {
if strings.Contains(err.Error(), "field api_key not found") {
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, file.Name())
}
continue
}
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
}
continue
}
if prof.ID == id {
if version == "" || prof.Version == version {
if err := validateProfile(&prof); err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
}
return &prof, nil
}
if prof.ID != id {
continue
}
if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, file.Name())
}
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
}
return &prof, nil
}
return nil, ErrProfileNotFound
}
func validateProfile(p *domain.PromptDefinition) error {
if p.ID == "" {
return errors.New("prompt id is required")
func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required")
}
if p.Version == "" {
return errors.New("prompt version is required")
if strings.TrimSpace(p.Endpoint) == "" {
return errors.New("endpoint is required")
}
if len(p.Templates) == 0 {
return errors.New("at least one prompt template message is required")
if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required")
}
if len(p.Inputs) == 0 {
return errors.New("at least one prompt input is required")
if p.Temperature < 0 || p.Temperature > 2 {
return errors.New("temperature must be between 0 and 2")
}
for i, input := range p.Inputs {
if strings.TrimSpace(input.Name) == "" {
return fmt.Errorf("input %d has empty name", i)
}
if p.MaxTokens < 0 {
return errors.New("max_tokens must be greater than or equal to 0")
}
for i, t := range p.Templates {
if !isValidMessageRole(t.Role) {
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
}
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
return fmt.Errorf("template message %d must provide content or content_file", i)
}
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
return fmt.Errorf("template message %d cannot set both content and content_file", i)
}
if p.TopP < 0 || p.TopP > 1 {
return errors.New("top_p must be between 0 and 1")
}
if !isValidOutputFormat(p.OutputFormat) {
return fmt.Errorf("invalid output format: %s", p.OutputFormat)
}
if !isValidValidationMode(p.Validation.ValidationMode) {
return fmt.Errorf("invalid validation mode: %s", p.Validation.ValidationMode)
}
if p.Validation.RepairAttempts < 0 {
return errors.New("validation.repair_attempts must be greater than or equal to 0")
}
if p.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(p.Validation.SchemaPath) == "" {
return errors.New("validation.schema_path is required when validation_mode is json_schema")
}
if p.Validation.Format != "" && p.Validation.Format != p.OutputFormat {
return fmt.Errorf("validation format %q does not match output format %q", p.Validation.Format, p.OutputFormat)
if p.TimeoutSeconds < 0 {
return errors.New("timeout_seconds must be greater than or equal to 0")
}
return nil
}
func isValidOutputFormat(f domain.OutputFormat) bool {
switch f {
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
return true
}
return false
}
func isValidValidationMode(m domain.ValidationMode) bool {
switch m {
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
return true
}
return false
}
func isValidMessageRole(role string) bool {
switch role {
case "system", "user", "assistant", "developer":
return true
}
return false
}

View File

@@ -2,12 +2,11 @@ package profile
import (
"context"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
// Repository is a transitional prompt-definition repository.
// It currently lives in internal/profile until package responsibilities
// are split in a follow-up refactor.
// Repository loads execution profiles.
type Repository interface {
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
}

View File

@@ -6,25 +6,21 @@ import (
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "profile_test")
func TestFilesystemRepository_GetProfile(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
testDataDir := "testdata"
files, err := os.ReadDir(testDataDir)
files, err := os.ReadDir("testdata")
if err != nil {
t.Fatalf("failed to read testdata: %v", err)
}
for _, f := range files {
src := filepath.Join(testDataDir, f.Name())
src := filepath.Join("testdata", f.Name())
dst := filepath.Join(tmpDir, f.Name())
data, err := os.ReadFile(src)
if err != nil {
@@ -38,69 +34,78 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
repo := NewFilesystemRepository(tmpDir)
ctx := context.Background()
t.Run("valid prompt definition", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
t.Run("valid local profile", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-default")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p == nil || p.ID != "test-profile" {
t.Errorf("expected prompt definition test-profile, got %v", p)
if p.ID != "local-default" {
t.Fatalf("unexpected id: %q", p.ID)
}
if p.Version != "1.0.0" {
t.Fatalf("expected version 1.0.0, got %q", p.Version)
}
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
t.Fatalf("unexpected inputs: %#v", p.Inputs)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
}
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
t.Fatalf("unexpected template roles: %#v", p.Templates)
}
if p.OutputFormat != domain.FormatMarkdown {
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
}
if p.Validation.ValidationMode != domain.ValidationBasic {
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
}
if p.DefaultProfile != "test-exec" {
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
if p.Endpoint == "" || p.Model == "" {
t.Fatalf("expected endpoint/model to be set: %+v", p)
}
})
t.Run("invalid YAML", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
}
if p.ReasoningEffort != "medium" {
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
}
})
t.Run("invalid yaml", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "invalid_yaml")
if !errors.Is(err, ErrInvalidYAML) {
t.Errorf("expected ErrInvalidYAML, got %v", err)
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("missing ID", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
t.Run("missing id", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing_id")
if !errors.Is(err, ErrProfileNotFound) {
t.Errorf("expected ErrProfileNotFound for profile with missing ID, got %v", err)
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
t.Run("no templates", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
t.Run("missing endpoint", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing-endpoint")
if !errors.Is(err, ErrInvalidProfile) {
t.Errorf("expected ErrInvalidProfile for profile with no templates, got %v", err)
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
t.Run("json schema mode missing schema path", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
t.Run("missing model", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing-model")
if !errors.Is(err, ErrInvalidProfile) {
t.Errorf("expected ErrInvalidProfile for json_schema profile without schema_path, got %v", err)
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
t.Run("prompt definition not found", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
t.Run("unknown field", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "unknown_field")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
}
})
t.Run("raw api_key rejected", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "raw_api_key")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
})
t.Run("profile not found", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "does-not-exist")
if !errors.Is(err, ErrProfileNotFound) {
t.Errorf("expected ErrProfileNotFound, got %v", err)
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
}

View File

@@ -1,5 +1,3 @@
id: invalid-yaml
version: 1.0.0
templates:
- role: system
content: [unclosed bracket
id: invalid_yaml
endpoint: http://localhost:8000/v1
model: [broken

View File

@@ -0,0 +1,2 @@
id: missing-endpoint
model: gpt-4o-mini

View File

@@ -1,11 +1,2 @@
version: 1.0.0
description: Missing ID
inputs:
- name: transcript
required: true
templates:
- role: system
content: Hello
output_format: text
validation:
validation_mode: none
endpoint: http://localhost:8000/v1
model: gpt-4o-mini

View File

@@ -0,0 +1,2 @@
id: missing-model
endpoint: http://localhost:8000/v1

View File

@@ -0,0 +1,4 @@
id: raw-api-key
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key: super-secret-should-not-be-here

View File

@@ -0,0 +1,4 @@
id: unknown-field
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
foo: bar

View File

@@ -0,0 +1,7 @@
id: local-default
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
temperature: 0.2
max_tokens: 700
top_p: 1.0
timeout_seconds: 120

View File

@@ -0,0 +1,7 @@
id: local-secure
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key_env: SCRIPTORIUM_API_KEY
reasoning_effort: medium
extra_params:
provider: local

View File

@@ -0,0 +1,151 @@
package promptdef
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gopkg.in/yaml.v3"
)
var (
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
)
type filesystemRepository struct {
dir string
}
func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
files, err := os.ReadDir(r.dir)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
for _, file := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) {
continue
}
fullPath := filepath.Join(r.dir, file.Name())
data, err := os.ReadFile(fullPath)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", file.Name(), err)
}
var def domain.PromptDefinition
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&def); err != nil {
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
}
continue
}
if def.ID != id {
continue
}
if version != "" && def.Version != version {
continue
}
if err := validatePromptDefinition(&def); err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err)
}
return &def, nil
}
return nil, ErrPromptDefinitionNotFound
}
func validatePromptDefinition(d *domain.PromptDefinition) error {
if d.ID == "" {
return errors.New("prompt id is required")
}
if d.Version == "" {
return errors.New("prompt version is required")
}
if len(d.Templates) == 0 {
return errors.New("at least one prompt template message is required")
}
if len(d.Inputs) == 0 {
return errors.New("at least one prompt input is required")
}
for i, input := range d.Inputs {
if strings.TrimSpace(input.Name) == "" {
return fmt.Errorf("input %d has empty name", i)
}
}
for i, t := range d.Templates {
if !isValidMessageRole(t.Role) {
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
}
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
return fmt.Errorf("template message %d must provide content or content_file", i)
}
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
return fmt.Errorf("template message %d cannot set both content and content_file", i)
}
}
if !isValidOutputFormat(d.OutputFormat) {
return fmt.Errorf("invalid output format: %s", d.OutputFormat)
}
if !isValidValidationMode(d.Validation.ValidationMode) {
return fmt.Errorf("invalid validation mode: %s", d.Validation.ValidationMode)
}
if d.Validation.RepairAttempts < 0 {
return errors.New("validation.repair_attempts must be greater than or equal to 0")
}
if d.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(d.Validation.SchemaPath) == "" {
return errors.New("validation.schema_path is required when validation_mode is json_schema")
}
if d.Validation.Format != "" && d.Validation.Format != d.OutputFormat {
return fmt.Errorf("validation format %q does not match output format %q", d.Validation.Format, d.OutputFormat)
}
return nil
}
func isValidOutputFormat(f domain.OutputFormat) bool {
switch f {
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
return true
}
return false
}
func isValidValidationMode(m domain.ValidationMode) bool {
switch m {
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
return true
}
return false
}
func isValidMessageRole(role string) bool {
switch role {
case "system", "user", "assistant", "developer":
return true
}
return false
}

View File

@@ -0,0 +1,12 @@
package promptdef
import (
"context"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
// Repository loads prompt definitions.
type Repository interface {
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
}

View File

@@ -0,0 +1,106 @@
package promptdef
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "promptdef_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
testDataDir := "testdata"
files, err := os.ReadDir(testDataDir)
if err != nil {
t.Fatalf("failed to read testdata: %v", err)
}
for _, f := range files {
src := filepath.Join(testDataDir, f.Name())
dst := filepath.Join(tmpDir, f.Name())
data, err := os.ReadFile(src)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(dst, data, 0644); err != nil {
t.Fatal(err)
}
}
repo := NewFilesystemRepository(tmpDir)
ctx := context.Background()
t.Run("valid prompt definition", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p == nil || p.ID != "test-profile" {
t.Errorf("expected prompt definition test-profile, got %v", p)
}
if p.Version != "1.0.0" {
t.Fatalf("expected version 1.0.0, got %q", p.Version)
}
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
t.Fatalf("unexpected inputs: %#v", p.Inputs)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
}
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
t.Fatalf("unexpected template roles: %#v", p.Templates)
}
if p.OutputFormat != domain.FormatMarkdown {
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
}
if p.Validation.ValidationMode != domain.ValidationBasic {
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
}
if p.DefaultProfile != "test-exec" {
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
}
})
t.Run("invalid YAML", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Errorf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("missing ID", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Errorf("expected ErrPromptDefinitionNotFound for profile with missing ID, got %v", err)
}
})
t.Run("no templates", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Errorf("expected ErrInvalidPromptDefinition for profile with no templates, got %v", err)
}
})
t.Run("json schema mode missing schema path", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Errorf("expected ErrInvalidPromptDefinition for json_schema profile without schema_path, got %v", err)
}
})
t.Run("prompt definition not found", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Errorf("expected ErrPromptDefinitionNotFound, got %v", err)
}
})
}

View File

@@ -0,0 +1,5 @@
id: invalid-yaml
version: 1.0.0
templates:
- role: system
content: [unclosed bracket

View File

@@ -0,0 +1,11 @@
id: json-schema-missing-path
version: "1.0.0"
inputs:
- name: transcript
required: true
templates:
- role: user
content: "Return JSON"
output_format: json
validation:
validation_mode: json_schema

View File

@@ -0,0 +1,11 @@
version: 1.0.0
description: Missing ID
inputs:
- name: transcript
required: true
templates:
- role: system
content: Hello
output_format: text
validation:
validation_mode: none

View File

@@ -0,0 +1,11 @@
id: negative-timeout
version: "1.0.0"
inputs:
- name: transcript
required: true
templates:
- role: user
content: "Say hi"
output_format: text
validation:
validation_mode: none

View File

@@ -0,0 +1,9 @@
id: no-templates
version: 1.0.0
inputs:
- name: transcript
required: true
templates: []
output_format: text
validation:
validation_mode: none

19
internal/promptdef/testdata/valid.yaml vendored Normal file
View File

@@ -0,0 +1,19 @@
id: test-profile
version: "1.0.0"
default_profile: test-exec
description: A valid test prompt definition
inputs:
- name: transcript
required: true
content_type: text/markdown
- name: glossary
required: false
content_type: text/yaml
templates:
- role: system
content: "You are a helpful assistant."
- role: user
content: 'Analyze this: {{input "transcript"}}'
output_format: markdown
validation:
validation_mode: basic

View File

@@ -2,6 +2,7 @@ package usecase
import (
"context"
"os"
"path/filepath"
"testing"
@@ -9,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -32,11 +34,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
}
profilesDir := filepath.Join(root, "profiles")
execProfilesDir := t.TempDir()
schemasDir := filepath.Join(root, "schemas")
fixturesDir := filepath.Join(root, "examples", "fixtures")
if err := os.WriteFile(filepath.Join(execProfilesDir, "local-default.yaml"), []byte(
"id: local-default\nendpoint: http://llm/v1\nmodel: test-model\n"), 0644); err != nil {
t.Fatalf("failed to write execution profile fixture: %v", err)
}
runner := NewRunner(
profile.NewFilesystemRepository(profilesDir),
promptdef.NewFilesystemRepository(profilesDir),
profile.NewFilesystemRepository(execProfilesDir),
artifact.NewCompositeReader(),
prompt.NewGoRenderer(),
&integrationLLM{},
@@ -45,11 +53,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "generic.structured_events",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "test-model",
},
ProfileID: "local-default",
Inputs: map[string]domain.ArtifactRef{
"transcript": {
Type: domain.ArtifactRefFile,

View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -30,25 +31,28 @@ var (
// Runner executes the Scriptorium core use case.
type Runner struct {
profiles profile.Repository
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
validator validate.Validator
repairer OutputRepairer
promptDefs promptdef.Repository
profiles profile.Repository
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
validator validate.Validator
repairer OutputRepairer
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
) *Runner {
return NewRunnerWithRepairer(profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
artifacts artifact.Reader,
renderer prompt.Renderer,
@@ -57,12 +61,13 @@ func NewRunnerWithRepairer(
repairer OutputRepairer,
) *Runner {
return &Runner{
profiles: profiles,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
validator: validator,
repairer: repairer,
promptDefs: promptDefs,
profiles: profiles,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
validator: validator,
repairer: repairer,
}
}
@@ -78,7 +83,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
@@ -93,10 +98,11 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
}
if req.Execution == nil {
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
effectiveModel := mergeExecutionTarget(executionProfileToTarget(execProfile), req.Execution)
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
@@ -265,6 +271,30 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.Executio
return out
}
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
if p == nil {
return domain.ExecutionTarget{}
}
cp := map[string]string(nil)
if len(p.ExtraParams) > 0 {
cp = make(map[string]string, len(p.ExtraParams))
for k, v := range p.ExtraParams {
cp[k] = v
}
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
TopP: p.TopP,
TimeoutSeconds: p.TimeoutSeconds,
ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv,
ExtraParams: cp,
}
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation
if contract.Format == "" {

View File

@@ -19,6 +19,23 @@ type fakePromptRepo struct {
lastVersion string
}
type fakeExecutionProfileRepo struct {
profile *domain.ExecutionProfile
err error
lastID string
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
return nil, f.err
}
if f.profile == nil {
return nil, errors.New("profile not found")
}
return f.profile, nil
}
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
f.lastID = id
f.lastVersion = version
@@ -112,7 +129,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(repo, reader, renderer, llmClient, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -121,7 +138,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -138,7 +155,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.PromptHash == "" || res.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" {
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
}
if res.RawOutput != "# recap" {
@@ -153,7 +170,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
@@ -162,25 +179,26 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunMissingProfileSelection(t *testing.T) {
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected profile load failure, got %v", err)
}
}
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -190,7 +208,6 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
})
if !errors.Is(err, ErrArtifactLoad) {
@@ -201,6 +218,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -209,7 +227,6 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrPromptRender) {
@@ -220,6 +237,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{err: errors.New("llm failed")},
@@ -228,7 +246,6 @@ func TestRunnerRunLLMFailure(t *testing.T) {
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrLLMGenerate) {
@@ -240,6 +257,7 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
@@ -248,7 +266,6 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -263,6 +280,7 @@ func TestRunnerRunRepairBounded(t *testing.T) {
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
@@ -272,7 +290,6 @@ func TestRunnerRunRepairBounded(t *testing.T) {
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
@@ -303,3 +320,11 @@ func hashString(s string) string {
sum := sha256.Sum256([]byte(s))
return hex.EncodeToString(sum[:])
}
func defaultExecutionProfile() *domain.ExecutionProfile {
return &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://llm/v1",
Model: "model-from-profile",
}
}