Refactor: load execution profiles from YAML and split prompt definitions into promptdef repository
This commit is contained in:
@@ -18,6 +18,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
@@ -125,6 +126,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
|
||||
runner := usecase.NewRunner(
|
||||
promptdef.NewFilesystemRepository(cfg.profileDir),
|
||||
profile.NewFilesystemRepository(cfg.profileDir),
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
@@ -183,6 +185,7 @@ func serveCommand(args []string, stderr io.Writer) int {
|
||||
}
|
||||
|
||||
runner := usecase.NewRunner(
|
||||
promptdef.NewFilesystemRepository(cfg.profileDir),
|
||||
profile.NewFilesystemRepository(cfg.profileDir),
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
)
|
||||
|
||||
@@ -147,8 +148,10 @@ func mapValidation(v domain.ValidationResult) validationDTO {
|
||||
|
||||
func mapRunError(err error) (int, string, string) {
|
||||
switch {
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
|
||||
@@ -5,17 +5,19 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gopkg.in/yaml.v3"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrProfileNotFound = errors.New("prompt definition not found")
|
||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||
ErrInvalidProfile = errors.New("invalid prompt definition configuration")
|
||||
ErrProfileNotFound = errors.New("execution profile not found")
|
||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||
ErrInvalidProfile = errors.New("invalid execution profile configuration")
|
||||
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
|
||||
)
|
||||
|
||||
type filesystemRepository struct {
|
||||
@@ -26,9 +28,9 @@ func NewFilesystemRepository(dir string) Repository {
|
||||
return &filesystemRepository{dir: dir}
|
||||
}
|
||||
|
||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidProfile)
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||
}
|
||||
|
||||
files, err := os.ReadDir(r.dir)
|
||||
@@ -53,97 +55,60 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
||||
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err)
|
||||
}
|
||||
|
||||
var prof domain.PromptDefinition
|
||||
var prof domain.ExecutionProfile
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&prof); err != nil {
|
||||
if strings.Contains(err.Error(), "field api_key not found") {
|
||||
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, file.Name())
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if prof.ID == id {
|
||||
if version == "" || prof.Version == version {
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
|
||||
}
|
||||
return &prof, nil
|
||||
}
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, file.Name())
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
|
||||
}
|
||||
return &prof, nil
|
||||
}
|
||||
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
|
||||
func validateProfile(p *domain.PromptDefinition) error {
|
||||
if p.ID == "" {
|
||||
return errors.New("prompt id is required")
|
||||
func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if p.Version == "" {
|
||||
return errors.New("prompt version is required")
|
||||
if strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
}
|
||||
if len(p.Templates) == 0 {
|
||||
return errors.New("at least one prompt template message is required")
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
}
|
||||
if len(p.Inputs) == 0 {
|
||||
return errors.New("at least one prompt input is required")
|
||||
|
||||
if p.Temperature < 0 || p.Temperature > 2 {
|
||||
return errors.New("temperature must be between 0 and 2")
|
||||
}
|
||||
for i, input := range p.Inputs {
|
||||
if strings.TrimSpace(input.Name) == "" {
|
||||
return fmt.Errorf("input %d has empty name", i)
|
||||
}
|
||||
if p.MaxTokens < 0 {
|
||||
return errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
for i, t := range p.Templates {
|
||||
if !isValidMessageRole(t.Role) {
|
||||
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
|
||||
}
|
||||
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
|
||||
return fmt.Errorf("template message %d must provide content or content_file", i)
|
||||
}
|
||||
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
|
||||
return fmt.Errorf("template message %d cannot set both content and content_file", i)
|
||||
}
|
||||
if p.TopP < 0 || p.TopP > 1 {
|
||||
return errors.New("top_p must be between 0 and 1")
|
||||
}
|
||||
if !isValidOutputFormat(p.OutputFormat) {
|
||||
return fmt.Errorf("invalid output format: %s", p.OutputFormat)
|
||||
}
|
||||
if !isValidValidationMode(p.Validation.ValidationMode) {
|
||||
return fmt.Errorf("invalid validation mode: %s", p.Validation.ValidationMode)
|
||||
}
|
||||
if p.Validation.RepairAttempts < 0 {
|
||||
return errors.New("validation.repair_attempts must be greater than or equal to 0")
|
||||
}
|
||||
if p.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(p.Validation.SchemaPath) == "" {
|
||||
return errors.New("validation.schema_path is required when validation_mode is json_schema")
|
||||
}
|
||||
if p.Validation.Format != "" && p.Validation.Format != p.OutputFormat {
|
||||
return fmt.Errorf("validation format %q does not match output format %q", p.Validation.Format, p.OutputFormat)
|
||||
if p.TimeoutSeconds < 0 {
|
||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||
switch f {
|
||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
||||
switch m {
|
||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isValidMessageRole(role string) bool {
|
||||
switch role {
|
||||
case "system", "user", "assistant", "developer":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -2,12 +2,11 @@ package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
// Repository is a transitional prompt-definition repository.
|
||||
// It currently lives in internal/profile until package responsibilities
|
||||
// are split in a follow-up refactor.
|
||||
// Repository loads execution profiles.
|
||||
type Repository interface {
|
||||
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||
}
|
||||
|
||||
@@ -6,25 +6,21 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "profile_test")
|
||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
testDataDir := "testdata"
|
||||
files, err := os.ReadDir(testDataDir)
|
||||
files, err := os.ReadDir("testdata")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read testdata: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
src := filepath.Join(testDataDir, f.Name())
|
||||
src := filepath.Join("testdata", f.Name())
|
||||
dst := filepath.Join(tmpDir, f.Name())
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
@@ -38,69 +34,78 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||
repo := NewFilesystemRepository(tmpDir)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("valid prompt definition", func(t *testing.T) {
|
||||
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
|
||||
t.Run("valid local profile", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-default")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p == nil || p.ID != "test-profile" {
|
||||
t.Errorf("expected prompt definition test-profile, got %v", p)
|
||||
if p.ID != "local-default" {
|
||||
t.Fatalf("unexpected id: %q", p.ID)
|
||||
}
|
||||
if p.Version != "1.0.0" {
|
||||
t.Fatalf("expected version 1.0.0, got %q", p.Version)
|
||||
}
|
||||
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
|
||||
t.Fatalf("unexpected inputs: %#v", p.Inputs)
|
||||
}
|
||||
if len(p.Templates) != 2 {
|
||||
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
|
||||
}
|
||||
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
|
||||
t.Fatalf("unexpected template roles: %#v", p.Templates)
|
||||
}
|
||||
if p.OutputFormat != domain.FormatMarkdown {
|
||||
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
|
||||
}
|
||||
if p.Validation.ValidationMode != domain.ValidationBasic {
|
||||
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
|
||||
}
|
||||
if p.DefaultProfile != "test-exec" {
|
||||
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
|
||||
if p.Endpoint == "" || p.Model == "" {
|
||||
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid YAML", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
|
||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-secure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
||||
}
|
||||
if p.ReasoningEffort != "medium" {
|
||||
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid yaml", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Errorf("expected ErrInvalidYAML, got %v", err)
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing ID", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
|
||||
t.Run("missing id", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing_id")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Errorf("expected ErrProfileNotFound for profile with missing ID, got %v", err)
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no templates", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
|
||||
t.Run("missing endpoint", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Errorf("expected ErrInvalidProfile for profile with no templates, got %v", err)
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("json schema mode missing schema path", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
|
||||
t.Run("missing model", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing-model")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Errorf("expected ErrInvalidProfile for json_schema profile without schema_path, got %v", err)
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("prompt definition not found", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
|
||||
t.Run("unknown field", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile not found", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "does-not-exist")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Errorf("expected ErrProfileNotFound, got %v", err)
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
8
internal/profile/testdata/invalid_yaml.yaml
vendored
8
internal/profile/testdata/invalid_yaml.yaml
vendored
@@ -1,5 +1,3 @@
|
||||
id: invalid-yaml
|
||||
version: 1.0.0
|
||||
templates:
|
||||
- role: system
|
||||
content: [unclosed bracket
|
||||
id: invalid_yaml
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: [broken
|
||||
|
||||
2
internal/profile/testdata/missing_endpoint.yaml
vendored
Normal file
2
internal/profile/testdata/missing_endpoint.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
id: missing-endpoint
|
||||
model: gpt-4o-mini
|
||||
13
internal/profile/testdata/missing_id.yaml
vendored
13
internal/profile/testdata/missing_id.yaml
vendored
@@ -1,11 +1,2 @@
|
||||
version: 1.0.0
|
||||
description: Missing ID
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
templates:
|
||||
- role: system
|
||||
content: Hello
|
||||
output_format: text
|
||||
validation:
|
||||
validation_mode: none
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
|
||||
2
internal/profile/testdata/missing_model.yaml
vendored
Normal file
2
internal/profile/testdata/missing_model.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
id: missing-model
|
||||
endpoint: http://localhost:8000/v1
|
||||
4
internal/profile/testdata/raw_api_key.yaml
vendored
Normal file
4
internal/profile/testdata/raw_api_key.yaml
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
id: raw-api-key
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
api_key: super-secret-should-not-be-here
|
||||
4
internal/profile/testdata/unknown_field.yaml
vendored
Normal file
4
internal/profile/testdata/unknown_field.yaml
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
id: unknown-field
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
foo: bar
|
||||
7
internal/profile/testdata/valid_local_profile.yaml
vendored
Normal file
7
internal/profile/testdata/valid_local_profile.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
id: local-default
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
temperature: 0.2
|
||||
max_tokens: 700
|
||||
top_p: 1.0
|
||||
timeout_seconds: 120
|
||||
7
internal/profile/testdata/valid_with_api_key_env.yaml
vendored
Normal file
7
internal/profile/testdata/valid_with_api_key_env.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
id: local-secure
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
api_key_env: SCRIPTORIUM_API_KEY
|
||||
reasoning_effort: medium
|
||||
extra_params:
|
||||
provider: local
|
||||
151
internal/promptdef/filesystem_repository.go
Normal file
151
internal/promptdef/filesystem_repository.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package promptdef
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
|
||||
)
|
||||
|
||||
type filesystemRepository struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func NewFilesystemRepository(dir string) Repository {
|
||||
return &filesystemRepository{dir: dir}
|
||||
}
|
||||
|
||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||
}
|
||||
|
||||
files, err := os.ReadDir(r.dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(r.dir, file.Name())
|
||||
data, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", file.Name(), err)
|
||||
}
|
||||
|
||||
var def domain.PromptDefinition
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&def); err != nil {
|
||||
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if def.ID != id {
|
||||
continue
|
||||
}
|
||||
if version != "" && def.Version != version {
|
||||
continue
|
||||
}
|
||||
if err := validatePromptDefinition(&def); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err)
|
||||
}
|
||||
return &def, nil
|
||||
}
|
||||
|
||||
return nil, ErrPromptDefinitionNotFound
|
||||
}
|
||||
|
||||
func validatePromptDefinition(d *domain.PromptDefinition) error {
|
||||
if d.ID == "" {
|
||||
return errors.New("prompt id is required")
|
||||
}
|
||||
if d.Version == "" {
|
||||
return errors.New("prompt version is required")
|
||||
}
|
||||
if len(d.Templates) == 0 {
|
||||
return errors.New("at least one prompt template message is required")
|
||||
}
|
||||
if len(d.Inputs) == 0 {
|
||||
return errors.New("at least one prompt input is required")
|
||||
}
|
||||
for i, input := range d.Inputs {
|
||||
if strings.TrimSpace(input.Name) == "" {
|
||||
return fmt.Errorf("input %d has empty name", i)
|
||||
}
|
||||
}
|
||||
for i, t := range d.Templates {
|
||||
if !isValidMessageRole(t.Role) {
|
||||
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
|
||||
}
|
||||
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
|
||||
return fmt.Errorf("template message %d must provide content or content_file", i)
|
||||
}
|
||||
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
|
||||
return fmt.Errorf("template message %d cannot set both content and content_file", i)
|
||||
}
|
||||
}
|
||||
if !isValidOutputFormat(d.OutputFormat) {
|
||||
return fmt.Errorf("invalid output format: %s", d.OutputFormat)
|
||||
}
|
||||
if !isValidValidationMode(d.Validation.ValidationMode) {
|
||||
return fmt.Errorf("invalid validation mode: %s", d.Validation.ValidationMode)
|
||||
}
|
||||
if d.Validation.RepairAttempts < 0 {
|
||||
return errors.New("validation.repair_attempts must be greater than or equal to 0")
|
||||
}
|
||||
if d.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(d.Validation.SchemaPath) == "" {
|
||||
return errors.New("validation.schema_path is required when validation_mode is json_schema")
|
||||
}
|
||||
if d.Validation.Format != "" && d.Validation.Format != d.OutputFormat {
|
||||
return fmt.Errorf("validation format %q does not match output format %q", d.Validation.Format, d.OutputFormat)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||
switch f {
|
||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
||||
switch m {
|
||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isValidMessageRole(role string) bool {
|
||||
switch role {
|
||||
case "system", "user", "assistant", "developer":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
12
internal/promptdef/repository.go
Normal file
12
internal/promptdef/repository.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package promptdef
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
// Repository loads prompt definitions.
|
||||
type Repository interface {
|
||||
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
||||
}
|
||||
106
internal/promptdef/repository_test.go
Normal file
106
internal/promptdef/repository_test.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package promptdef
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "promptdef_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
testDataDir := "testdata"
|
||||
files, err := os.ReadDir(testDataDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read testdata: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
src := filepath.Join(testDataDir, f.Name())
|
||||
dst := filepath.Join(tmpDir, f.Name())
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
repo := NewFilesystemRepository(tmpDir)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("valid prompt definition", func(t *testing.T) {
|
||||
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p == nil || p.ID != "test-profile" {
|
||||
t.Errorf("expected prompt definition test-profile, got %v", p)
|
||||
}
|
||||
if p.Version != "1.0.0" {
|
||||
t.Fatalf("expected version 1.0.0, got %q", p.Version)
|
||||
}
|
||||
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
|
||||
t.Fatalf("unexpected inputs: %#v", p.Inputs)
|
||||
}
|
||||
if len(p.Templates) != 2 {
|
||||
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
|
||||
}
|
||||
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
|
||||
t.Fatalf("unexpected template roles: %#v", p.Templates)
|
||||
}
|
||||
if p.OutputFormat != domain.FormatMarkdown {
|
||||
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
|
||||
}
|
||||
if p.Validation.ValidationMode != domain.ValidationBasic {
|
||||
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
|
||||
}
|
||||
if p.DefaultProfile != "test-exec" {
|
||||
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid YAML", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Errorf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing ID", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
|
||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("expected ErrPromptDefinitionNotFound for profile with missing ID, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no templates", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
|
||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||
t.Errorf("expected ErrInvalidPromptDefinition for profile with no templates, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("json schema mode missing schema path", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
|
||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||
t.Errorf("expected ErrInvalidPromptDefinition for json_schema profile without schema_path, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("prompt definition not found", func(t *testing.T) {
|
||||
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
|
||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("expected ErrPromptDefinitionNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
5
internal/promptdef/testdata/invalid_yaml.yaml
vendored
Normal file
5
internal/promptdef/testdata/invalid_yaml.yaml
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
id: invalid-yaml
|
||||
version: 1.0.0
|
||||
templates:
|
||||
- role: system
|
||||
content: [unclosed bracket
|
||||
11
internal/promptdef/testdata/json_schema_missing_path.yaml
vendored
Normal file
11
internal/promptdef/testdata/json_schema_missing_path.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
id: json-schema-missing-path
|
||||
version: "1.0.0"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
templates:
|
||||
- role: user
|
||||
content: "Return JSON"
|
||||
output_format: json
|
||||
validation:
|
||||
validation_mode: json_schema
|
||||
11
internal/promptdef/testdata/missing_id.yaml
vendored
Normal file
11
internal/promptdef/testdata/missing_id.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
version: 1.0.0
|
||||
description: Missing ID
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
templates:
|
||||
- role: system
|
||||
content: Hello
|
||||
output_format: text
|
||||
validation:
|
||||
validation_mode: none
|
||||
11
internal/promptdef/testdata/negative_timeout.yaml
vendored
Normal file
11
internal/promptdef/testdata/negative_timeout.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
id: negative-timeout
|
||||
version: "1.0.0"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
templates:
|
||||
- role: user
|
||||
content: "Say hi"
|
||||
output_format: text
|
||||
validation:
|
||||
validation_mode: none
|
||||
9
internal/promptdef/testdata/no_templates.yaml
vendored
Normal file
9
internal/promptdef/testdata/no_templates.yaml
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
id: no-templates
|
||||
version: 1.0.0
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
templates: []
|
||||
output_format: text
|
||||
validation:
|
||||
validation_mode: none
|
||||
19
internal/promptdef/testdata/valid.yaml
vendored
Normal file
19
internal/promptdef/testdata/valid.yaml
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
id: test-profile
|
||||
version: "1.0.0"
|
||||
default_profile: test-exec
|
||||
description: A valid test prompt definition
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: text/markdown
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/yaml
|
||||
templates:
|
||||
- role: system
|
||||
content: "You are a helpful assistant."
|
||||
- role: user
|
||||
content: 'Analyze this: {{input "transcript"}}'
|
||||
output_format: markdown
|
||||
validation:
|
||||
validation_mode: basic
|
||||
@@ -2,6 +2,7 @@ package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
@@ -32,11 +34,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
|
||||
}
|
||||
|
||||
profilesDir := filepath.Join(root, "profiles")
|
||||
execProfilesDir := t.TempDir()
|
||||
schemasDir := filepath.Join(root, "schemas")
|
||||
fixturesDir := filepath.Join(root, "examples", "fixtures")
|
||||
if err := os.WriteFile(filepath.Join(execProfilesDir, "local-default.yaml"), []byte(
|
||||
"id: local-default\nendpoint: http://llm/v1\nmodel: test-model\n"), 0644); err != nil {
|
||||
t.Fatalf("failed to write execution profile fixture: %v", err)
|
||||
}
|
||||
|
||||
runner := NewRunner(
|
||||
profile.NewFilesystemRepository(profilesDir),
|
||||
promptdef.NewFilesystemRepository(profilesDir),
|
||||
profile.NewFilesystemRepository(execProfilesDir),
|
||||
artifact.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&integrationLLM{},
|
||||
@@ -45,11 +53,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "generic.structured_events",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "test-model",
|
||||
},
|
||||
ProfileID: "local-default",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {
|
||||
Type: domain.ArtifactRefFile,
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
@@ -30,25 +31,28 @@ var (
|
||||
|
||||
// Runner executes the Scriptorium core use case.
|
||||
type Runner struct {
|
||||
profiles profile.Repository
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
validator validate.Validator
|
||||
repairer OutputRepairer
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
validator validate.Validator
|
||||
repairer OutputRepairer
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
) *Runner {
|
||||
return NewRunnerWithRepairer(profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
}
|
||||
|
||||
func NewRunnerWithRepairer(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
@@ -57,12 +61,13 @@ func NewRunnerWithRepairer(
|
||||
repairer OutputRepairer,
|
||||
) *Runner {
|
||||
return &Runner{
|
||||
profiles: profiles,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
validator: validator,
|
||||
repairer: repairer,
|
||||
promptDefs: promptDefs,
|
||||
profiles: profiles,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
validator: validator,
|
||||
repairer: repairer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +83,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
@@ -93,10 +98,11 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
if selectedProfileID == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
|
||||
}
|
||||
if req.Execution == nil {
|
||||
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
|
||||
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
|
||||
effectiveModel := mergeExecutionTarget(executionProfileToTarget(execProfile), req.Execution)
|
||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||
}
|
||||
@@ -265,6 +271,30 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.Executio
|
||||
return out
|
||||
}
|
||||
|
||||
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
|
||||
if p == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
cp := map[string]string(nil)
|
||||
if len(p.ExtraParams) > 0 {
|
||||
cp = make(map[string]string, len(p.ExtraParams))
|
||||
for k, v := range p.ExtraParams {
|
||||
cp[k] = v
|
||||
}
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
Endpoint: p.Endpoint,
|
||||
Model: p.Model,
|
||||
Temperature: p.Temperature,
|
||||
MaxTokens: p.MaxTokens,
|
||||
TopP: p.TopP,
|
||||
TimeoutSeconds: p.TimeoutSeconds,
|
||||
ReasoningEffort: p.ReasoningEffort,
|
||||
APIKeyEnv: p.APIKeyEnv,
|
||||
ExtraParams: cp,
|
||||
}
|
||||
}
|
||||
|
||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
||||
contract := def.Validation
|
||||
if contract.Format == "" {
|
||||
|
||||
@@ -19,6 +19,23 @@ type fakePromptRepo struct {
|
||||
lastVersion string
|
||||
}
|
||||
|
||||
type fakeExecutionProfileRepo struct {
|
||||
profile *domain.ExecutionProfile
|
||||
err error
|
||||
lastID string
|
||||
}
|
||||
|
||||
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
f.lastID = id
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if f.profile == nil {
|
||||
return nil, errors.New("profile not found")
|
||||
}
|
||||
return f.profile, nil
|
||||
}
|
||||
|
||||
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
f.lastID = id
|
||||
f.lastVersion = version
|
||||
@@ -112,7 +129,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||
|
||||
runner := NewRunner(repo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil)
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
@@ -121,7 +138,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||
},
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -138,7 +155,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
if res.PromptHash == "" || res.RenderedPromptHash == "" {
|
||||
t.Fatal("expected prompt hashes")
|
||||
}
|
||||
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
|
||||
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" {
|
||||
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
|
||||
}
|
||||
if res.RawOutput != "# recap" {
|
||||
@@ -153,7 +170,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
@@ -162,25 +179,26 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
|
||||
func TestRunnerRunMissingProfileSelection(t *testing.T) {
|
||||
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
|
||||
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected invalid request, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
|
||||
func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) {
|
||||
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected invalid request, got %v", err)
|
||||
runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected profile load failure, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
@@ -190,7 +208,6 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
|
||||
})
|
||||
if !errors.Is(err, ErrArtifactLoad) {
|
||||
@@ -201,6 +218,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{err: errors.New("render failed")},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
@@ -209,7 +227,6 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||
})
|
||||
if !errors.Is(err, ErrPromptRender) {
|
||||
@@ -220,6 +237,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{err: errors.New("llm failed")},
|
||||
@@ -228,7 +246,6 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||
})
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
@@ -240,6 +257,7 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
@@ -248,7 +266,6 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -263,6 +280,7 @@ func TestRunnerRunRepairBounded(t *testing.T) {
|
||||
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
|
||||
@@ -272,7 +290,6 @@ func TestRunnerRunRepairBounded(t *testing.T) {
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -303,3 +320,11 @@ func hashString(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func defaultExecutionProfile() *domain.ExecutionProfile {
|
||||
return &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "model-from-profile",
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user