Refactor: load execution profiles from YAML and split prompt definitions into promptdef repository
This commit is contained in:
@@ -18,6 +18,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -125,6 +126,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
runner := usecase.NewRunner(
|
runner := usecase.NewRunner(
|
||||||
|
promptdef.NewFilesystemRepository(cfg.profileDir),
|
||||||
profile.NewFilesystemRepository(cfg.profileDir),
|
profile.NewFilesystemRepository(cfg.profileDir),
|
||||||
artifactadapter.NewCompositeReader(),
|
artifactadapter.NewCompositeReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
@@ -183,6 +185,7 @@ func serveCommand(args []string, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
runner := usecase.NewRunner(
|
runner := usecase.NewRunner(
|
||||||
|
promptdef.NewFilesystemRepository(cfg.profileDir),
|
||||||
profile.NewFilesystemRepository(cfg.profileDir),
|
profile.NewFilesystemRepository(cfg.profileDir),
|
||||||
artifactadapter.NewCompositeReader(),
|
artifactadapter.NewCompositeReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -147,8 +148,10 @@ func mapValidation(v domain.ValidationResult) validationDTO {
|
|||||||
|
|
||||||
func mapRunError(err error) (int, string, string) {
|
func mapRunError(err error) (int, string, string) {
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, profile.ErrProfileNotFound):
|
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||||
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
||||||
|
case errors.Is(err, profile.ErrProfileNotFound):
|
||||||
|
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
case errors.Is(err, usecase.ErrProfileLoad):
|
case errors.Is(err, usecase.ErrProfileLoad):
|
||||||
|
|||||||
@@ -5,17 +5,19 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrProfileNotFound = errors.New("prompt definition not found")
|
ErrProfileNotFound = errors.New("execution profile not found")
|
||||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||||
ErrInvalidProfile = errors.New("invalid prompt definition configuration")
|
ErrInvalidProfile = errors.New("invalid execution profile configuration")
|
||||||
|
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
|
||||||
)
|
)
|
||||||
|
|
||||||
type filesystemRepository struct {
|
type filesystemRepository struct {
|
||||||
@@ -26,9 +28,9 @@ func NewFilesystemRepository(dir string) Repository {
|
|||||||
return &filesystemRepository{dir: dir}
|
return &filesystemRepository{dir: dir}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidProfile)
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := os.ReadDir(r.dir)
|
files, err := os.ReadDir(r.dir)
|
||||||
@@ -53,97 +55,60 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err)
|
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var prof domain.PromptDefinition
|
var prof domain.ExecutionProfile
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
decoder.KnownFields(true)
|
decoder.KnownFields(true)
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
if err := decoder.Decode(&prof); err != nil {
|
||||||
|
if strings.Contains(err.Error(), "field api_key not found") {
|
||||||
|
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, file.Name())
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if prof.ID == id {
|
if prof.ID != id {
|
||||||
if version == "" || prof.Version == version {
|
continue
|
||||||
if err := validateProfile(&prof); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
|
|
||||||
}
|
|
||||||
return &prof, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if err := validateProfile(&prof); err != nil {
|
||||||
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
|
return nil, fmt.Errorf("%w: %s", err, file.Name())
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err)
|
||||||
|
}
|
||||||
|
return &prof, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, ErrProfileNotFound
|
return nil, ErrProfileNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProfile(p *domain.PromptDefinition) error {
|
func validateProfile(p *domain.ExecutionProfile) error {
|
||||||
if p.ID == "" {
|
if strings.TrimSpace(p.ID) == "" {
|
||||||
return errors.New("prompt id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if p.Version == "" {
|
if strings.TrimSpace(p.Endpoint) == "" {
|
||||||
return errors.New("prompt version is required")
|
return errors.New("endpoint is required")
|
||||||
}
|
}
|
||||||
if len(p.Templates) == 0 {
|
if strings.TrimSpace(p.Model) == "" {
|
||||||
return errors.New("at least one prompt template message is required")
|
return errors.New("model is required")
|
||||||
}
|
}
|
||||||
if len(p.Inputs) == 0 {
|
|
||||||
return errors.New("at least one prompt input is required")
|
if p.Temperature < 0 || p.Temperature > 2 {
|
||||||
|
return errors.New("temperature must be between 0 and 2")
|
||||||
}
|
}
|
||||||
for i, input := range p.Inputs {
|
if p.MaxTokens < 0 {
|
||||||
if strings.TrimSpace(input.Name) == "" {
|
return errors.New("max_tokens must be greater than or equal to 0")
|
||||||
return fmt.Errorf("input %d has empty name", i)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for i, t := range p.Templates {
|
if p.TopP < 0 || p.TopP > 1 {
|
||||||
if !isValidMessageRole(t.Role) {
|
return errors.New("top_p must be between 0 and 1")
|
||||||
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
|
|
||||||
return fmt.Errorf("template message %d must provide content or content_file", i)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
|
|
||||||
return fmt.Errorf("template message %d cannot set both content and content_file", i)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if !isValidOutputFormat(p.OutputFormat) {
|
if p.TimeoutSeconds < 0 {
|
||||||
return fmt.Errorf("invalid output format: %s", p.OutputFormat)
|
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
}
|
|
||||||
if !isValidValidationMode(p.Validation.ValidationMode) {
|
|
||||||
return fmt.Errorf("invalid validation mode: %s", p.Validation.ValidationMode)
|
|
||||||
}
|
|
||||||
if p.Validation.RepairAttempts < 0 {
|
|
||||||
return errors.New("validation.repair_attempts must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
if p.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(p.Validation.SchemaPath) == "" {
|
|
||||||
return errors.New("validation.schema_path is required when validation_mode is json_schema")
|
|
||||||
}
|
|
||||||
if p.Validation.Format != "" && p.Validation.Format != p.OutputFormat {
|
|
||||||
return fmt.Errorf("validation format %q does not match output format %q", p.Validation.Format, p.OutputFormat)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
|
||||||
switch f {
|
|
||||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
|
||||||
switch m {
|
|
||||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidMessageRole(role string) bool {
|
|
||||||
switch role {
|
|
||||||
case "system", "user", "assistant", "developer":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,12 +2,11 @@ package profile
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Repository is a transitional prompt-definition repository.
|
// Repository loads execution profiles.
|
||||||
// It currently lives in internal/profile until package responsibilities
|
|
||||||
// are split in a follow-up refactor.
|
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,25 +6,21 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||||
tmpDir, err := os.MkdirTemp("", "profile_test")
|
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(tmpDir)
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
testDataDir := "testdata"
|
files, err := os.ReadDir("testdata")
|
||||||
files, err := os.ReadDir(testDataDir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to read testdata: %v", err)
|
t.Fatalf("failed to read testdata: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
src := filepath.Join(testDataDir, f.Name())
|
src := filepath.Join("testdata", f.Name())
|
||||||
dst := filepath.Join(tmpDir, f.Name())
|
dst := filepath.Join(tmpDir, f.Name())
|
||||||
data, err := os.ReadFile(src)
|
data, err := os.ReadFile(src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -38,69 +34,78 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
|||||||
repo := NewFilesystemRepository(tmpDir)
|
repo := NewFilesystemRepository(tmpDir)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
t.Run("valid prompt definition", func(t *testing.T) {
|
t.Run("valid local profile", func(t *testing.T) {
|
||||||
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
|
p, err := repo.GetProfile(ctx, "local-default")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
if p == nil || p.ID != "test-profile" {
|
if p.ID != "local-default" {
|
||||||
t.Errorf("expected prompt definition test-profile, got %v", p)
|
t.Fatalf("unexpected id: %q", p.ID)
|
||||||
}
|
}
|
||||||
if p.Version != "1.0.0" {
|
if p.Endpoint == "" || p.Model == "" {
|
||||||
t.Fatalf("expected version 1.0.0, got %q", p.Version)
|
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
||||||
}
|
|
||||||
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
|
|
||||||
t.Fatalf("unexpected inputs: %#v", p.Inputs)
|
|
||||||
}
|
|
||||||
if len(p.Templates) != 2 {
|
|
||||||
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
|
|
||||||
}
|
|
||||||
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
|
|
||||||
t.Fatalf("unexpected template roles: %#v", p.Templates)
|
|
||||||
}
|
|
||||||
if p.OutputFormat != domain.FormatMarkdown {
|
|
||||||
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
|
|
||||||
}
|
|
||||||
if p.Validation.ValidationMode != domain.ValidationBasic {
|
|
||||||
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
|
|
||||||
}
|
|
||||||
if p.DefaultProfile != "test-exec" {
|
|
||||||
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid YAML", func(t *testing.T) {
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
|
p, err := repo.GetProfile(ctx, "local-secure")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||||
|
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
||||||
|
}
|
||||||
|
if p.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid yaml", func(t *testing.T) {
|
||||||
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
t.Errorf("expected ErrInvalidYAML, got %v", err)
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("missing ID", func(t *testing.T) {
|
t.Run("missing id", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
|
_, err := repo.GetProfile(ctx, "missing_id")
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
t.Errorf("expected ErrProfileNotFound for profile with missing ID, got %v", err)
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("no templates", func(t *testing.T) {
|
t.Run("missing endpoint", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
|
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
t.Errorf("expected ErrInvalidProfile for profile with no templates, got %v", err)
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("json schema mode missing schema path", func(t *testing.T) {
|
t.Run("missing model", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
|
_, err := repo.GetProfile(ctx, "missing-model")
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
t.Errorf("expected ErrInvalidProfile for json_schema profile without schema_path, got %v", err)
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("prompt definition not found", func(t *testing.T) {
|
t.Run("unknown field", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
|
_, err := repo.GetProfile(ctx, "unknown_field")
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||||
|
_, err := repo.GetProfile(ctx, "raw_api_key")
|
||||||
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("profile not found", func(t *testing.T) {
|
||||||
|
_, err := repo.GetProfile(ctx, "does-not-exist")
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
t.Errorf("expected ErrProfileNotFound, got %v", err)
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
8
internal/profile/testdata/invalid_yaml.yaml
vendored
8
internal/profile/testdata/invalid_yaml.yaml
vendored
@@ -1,5 +1,3 @@
|
|||||||
id: invalid-yaml
|
id: invalid_yaml
|
||||||
version: 1.0.0
|
endpoint: http://localhost:8000/v1
|
||||||
templates:
|
model: [broken
|
||||||
- role: system
|
|
||||||
content: [unclosed bracket
|
|
||||||
|
|||||||
2
internal/profile/testdata/missing_endpoint.yaml
vendored
Normal file
2
internal/profile/testdata/missing_endpoint.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
id: missing-endpoint
|
||||||
|
model: gpt-4o-mini
|
||||||
13
internal/profile/testdata/missing_id.yaml
vendored
13
internal/profile/testdata/missing_id.yaml
vendored
@@ -1,11 +1,2 @@
|
|||||||
version: 1.0.0
|
endpoint: http://localhost:8000/v1
|
||||||
description: Missing ID
|
model: gpt-4o-mini
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates:
|
|
||||||
- role: system
|
|
||||||
content: Hello
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
|
||||||
|
|||||||
2
internal/profile/testdata/missing_model.yaml
vendored
Normal file
2
internal/profile/testdata/missing_model.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
id: missing-model
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
4
internal/profile/testdata/raw_api_key.yaml
vendored
Normal file
4
internal/profile/testdata/raw_api_key.yaml
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
id: raw-api-key
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: gpt-4o-mini
|
||||||
|
api_key: super-secret-should-not-be-here
|
||||||
4
internal/profile/testdata/unknown_field.yaml
vendored
Normal file
4
internal/profile/testdata/unknown_field.yaml
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
id: unknown-field
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: gpt-4o-mini
|
||||||
|
foo: bar
|
||||||
7
internal/profile/testdata/valid_local_profile.yaml
vendored
Normal file
7
internal/profile/testdata/valid_local_profile.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: local-default
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: gpt-4o-mini
|
||||||
|
temperature: 0.2
|
||||||
|
max_tokens: 700
|
||||||
|
top_p: 1.0
|
||||||
|
timeout_seconds: 120
|
||||||
7
internal/profile/testdata/valid_with_api_key_env.yaml
vendored
Normal file
7
internal/profile/testdata/valid_with_api_key_env.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: local-secure
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: gpt-4o-mini
|
||||||
|
api_key_env: SCRIPTORIUM_API_KEY
|
||||||
|
reasoning_effort: medium
|
||||||
|
extra_params:
|
||||||
|
provider: local
|
||||||
151
internal/promptdef/filesystem_repository.go
Normal file
151
internal/promptdef/filesystem_repository.go
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
package promptdef
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
||||||
|
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||||
|
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
|
||||||
|
)
|
||||||
|
|
||||||
|
type filesystemRepository struct {
|
||||||
|
dir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFilesystemRepository(dir string) Repository {
|
||||||
|
return &filesystemRepository{dir: dir}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := os.ReadDir(r.dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fullPath := filepath.Join(r.dir, file.Name())
|
||||||
|
data, err := os.ReadFile(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", file.Name(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var def domain.PromptDefinition
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.KnownFields(true)
|
||||||
|
if err := decoder.Decode(&def); err != nil {
|
||||||
|
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if def.ID != id {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if version != "" && def.Version != version {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := validatePromptDefinition(&def); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err)
|
||||||
|
}
|
||||||
|
return &def, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, ErrPromptDefinitionNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePromptDefinition(d *domain.PromptDefinition) error {
|
||||||
|
if d.ID == "" {
|
||||||
|
return errors.New("prompt id is required")
|
||||||
|
}
|
||||||
|
if d.Version == "" {
|
||||||
|
return errors.New("prompt version is required")
|
||||||
|
}
|
||||||
|
if len(d.Templates) == 0 {
|
||||||
|
return errors.New("at least one prompt template message is required")
|
||||||
|
}
|
||||||
|
if len(d.Inputs) == 0 {
|
||||||
|
return errors.New("at least one prompt input is required")
|
||||||
|
}
|
||||||
|
for i, input := range d.Inputs {
|
||||||
|
if strings.TrimSpace(input.Name) == "" {
|
||||||
|
return fmt.Errorf("input %d has empty name", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, t := range d.Templates {
|
||||||
|
if !isValidMessageRole(t.Role) {
|
||||||
|
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
|
||||||
|
return fmt.Errorf("template message %d must provide content or content_file", i)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
|
||||||
|
return fmt.Errorf("template message %d cannot set both content and content_file", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isValidOutputFormat(d.OutputFormat) {
|
||||||
|
return fmt.Errorf("invalid output format: %s", d.OutputFormat)
|
||||||
|
}
|
||||||
|
if !isValidValidationMode(d.Validation.ValidationMode) {
|
||||||
|
return fmt.Errorf("invalid validation mode: %s", d.Validation.ValidationMode)
|
||||||
|
}
|
||||||
|
if d.Validation.RepairAttempts < 0 {
|
||||||
|
return errors.New("validation.repair_attempts must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if d.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(d.Validation.SchemaPath) == "" {
|
||||||
|
return errors.New("validation.schema_path is required when validation_mode is json_schema")
|
||||||
|
}
|
||||||
|
if d.Validation.Format != "" && d.Validation.Format != d.OutputFormat {
|
||||||
|
return fmt.Errorf("validation format %q does not match output format %q", d.Validation.Format, d.OutputFormat)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||||
|
switch f {
|
||||||
|
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isValidValidationMode(m domain.ValidationMode) bool {
|
||||||
|
switch m {
|
||||||
|
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isValidMessageRole(role string) bool {
|
||||||
|
switch role {
|
||||||
|
case "system", "user", "assistant", "developer":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
12
internal/promptdef/repository.go
Normal file
12
internal/promptdef/repository.go
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
package promptdef
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Repository loads prompt definitions.
|
||||||
|
type Repository interface {
|
||||||
|
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
||||||
|
}
|
||||||
106
internal/promptdef/repository_test.go
Normal file
106
internal/promptdef/repository_test.go
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
package promptdef
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||||
|
tmpDir, err := os.MkdirTemp("", "promptdef_test")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
testDataDir := "testdata"
|
||||||
|
files, err := os.ReadDir(testDataDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read testdata: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range files {
|
||||||
|
src := filepath.Join(testDataDir, f.Name())
|
||||||
|
dst := filepath.Join(tmpDir, f.Name())
|
||||||
|
data, err := os.ReadFile(src)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(dst, data, 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := NewFilesystemRepository(tmpDir)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("valid prompt definition", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p == nil || p.ID != "test-profile" {
|
||||||
|
t.Errorf("expected prompt definition test-profile, got %v", p)
|
||||||
|
}
|
||||||
|
if p.Version != "1.0.0" {
|
||||||
|
t.Fatalf("expected version 1.0.0, got %q", p.Version)
|
||||||
|
}
|
||||||
|
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
|
||||||
|
t.Fatalf("unexpected inputs: %#v", p.Inputs)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
|
||||||
|
t.Fatalf("unexpected template roles: %#v", p.Templates)
|
||||||
|
}
|
||||||
|
if p.OutputFormat != domain.FormatMarkdown {
|
||||||
|
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
|
||||||
|
}
|
||||||
|
if p.Validation.ValidationMode != domain.ValidationBasic {
|
||||||
|
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
|
||||||
|
}
|
||||||
|
if p.DefaultProfile != "test-exec" {
|
||||||
|
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid YAML", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Errorf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing ID", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
|
||||||
|
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||||
|
t.Errorf("expected ErrPromptDefinitionNotFound for profile with missing ID, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no templates", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
|
||||||
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
|
t.Errorf("expected ErrInvalidPromptDefinition for profile with no templates, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("json schema mode missing schema path", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
|
||||||
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
|
t.Errorf("expected ErrInvalidPromptDefinition for json_schema profile without schema_path, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("prompt definition not found", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
|
||||||
|
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||||
|
t.Errorf("expected ErrPromptDefinitionNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
5
internal/promptdef/testdata/invalid_yaml.yaml
vendored
Normal file
5
internal/promptdef/testdata/invalid_yaml.yaml
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
id: invalid-yaml
|
||||||
|
version: 1.0.0
|
||||||
|
templates:
|
||||||
|
- role: system
|
||||||
|
content: [unclosed bracket
|
||||||
11
internal/promptdef/testdata/json_schema_missing_path.yaml
vendored
Normal file
11
internal/promptdef/testdata/json_schema_missing_path.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
id: json-schema-missing-path
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
templates:
|
||||||
|
- role: user
|
||||||
|
content: "Return JSON"
|
||||||
|
output_format: json
|
||||||
|
validation:
|
||||||
|
validation_mode: json_schema
|
||||||
11
internal/promptdef/testdata/missing_id.yaml
vendored
Normal file
11
internal/promptdef/testdata/missing_id.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
version: 1.0.0
|
||||||
|
description: Missing ID
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
templates:
|
||||||
|
- role: system
|
||||||
|
content: Hello
|
||||||
|
output_format: text
|
||||||
|
validation:
|
||||||
|
validation_mode: none
|
||||||
11
internal/promptdef/testdata/negative_timeout.yaml
vendored
Normal file
11
internal/promptdef/testdata/negative_timeout.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
id: negative-timeout
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
templates:
|
||||||
|
- role: user
|
||||||
|
content: "Say hi"
|
||||||
|
output_format: text
|
||||||
|
validation:
|
||||||
|
validation_mode: none
|
||||||
9
internal/promptdef/testdata/no_templates.yaml
vendored
Normal file
9
internal/promptdef/testdata/no_templates.yaml
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: no-templates
|
||||||
|
version: 1.0.0
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
templates: []
|
||||||
|
output_format: text
|
||||||
|
validation:
|
||||||
|
validation_mode: none
|
||||||
19
internal/promptdef/testdata/valid.yaml
vendored
Normal file
19
internal/promptdef/testdata/valid.yaml
vendored
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
id: test-profile
|
||||||
|
version: "1.0.0"
|
||||||
|
default_profile: test-exec
|
||||||
|
description: A valid test prompt definition
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: text/markdown
|
||||||
|
- name: glossary
|
||||||
|
required: false
|
||||||
|
content_type: text/yaml
|
||||||
|
templates:
|
||||||
|
- role: system
|
||||||
|
content: "You are a helpful assistant."
|
||||||
|
- role: user
|
||||||
|
content: 'Analyze this: {{input "transcript"}}'
|
||||||
|
output_format: markdown
|
||||||
|
validation:
|
||||||
|
validation_mode: basic
|
||||||
@@ -2,6 +2,7 @@ package usecase
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -9,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -32,11 +34,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
profilesDir := filepath.Join(root, "profiles")
|
profilesDir := filepath.Join(root, "profiles")
|
||||||
|
execProfilesDir := t.TempDir()
|
||||||
schemasDir := filepath.Join(root, "schemas")
|
schemasDir := filepath.Join(root, "schemas")
|
||||||
fixturesDir := filepath.Join(root, "examples", "fixtures")
|
fixturesDir := filepath.Join(root, "examples", "fixtures")
|
||||||
|
if err := os.WriteFile(filepath.Join(execProfilesDir, "local-default.yaml"), []byte(
|
||||||
|
"id: local-default\nendpoint: http://llm/v1\nmodel: test-model\n"), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write execution profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
profile.NewFilesystemRepository(profilesDir),
|
promptdef.NewFilesystemRepository(profilesDir),
|
||||||
|
profile.NewFilesystemRepository(execProfilesDir),
|
||||||
artifact.NewCompositeReader(),
|
artifact.NewCompositeReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
&integrationLLM{},
|
&integrationLLM{},
|
||||||
@@ -45,11 +53,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
|
|||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "generic.structured_events",
|
PromptID: "generic.structured_events",
|
||||||
ProfileID: "exec",
|
ProfileID: "local-default",
|
||||||
Execution: &domain.ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "test-model",
|
|
||||||
},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{
|
Inputs: map[string]domain.ArtifactRef{
|
||||||
"transcript": {
|
"transcript": {
|
||||||
Type: domain.ArtifactRefFile,
|
Type: domain.ArtifactRefFile,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -30,25 +31,28 @@ var (
|
|||||||
|
|
||||||
// Runner executes the Scriptorium core use case.
|
// Runner executes the Scriptorium core use case.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
profiles profile.Repository
|
promptDefs promptdef.Repository
|
||||||
artifacts artifact.Reader
|
profiles profile.Repository
|
||||||
renderer prompt.Renderer
|
artifacts artifact.Reader
|
||||||
llm llm.Client
|
renderer prompt.Renderer
|
||||||
validator validate.Validator
|
llm llm.Client
|
||||||
repairer OutputRepairer
|
validator validate.Validator
|
||||||
|
repairer OutputRepairer
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRunner(
|
func NewRunner(
|
||||||
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.Validator,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return NewRunnerWithRepairer(profiles, artifacts, renderer, llmClient, validator, nil)
|
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRunnerWithRepairer(
|
func NewRunnerWithRepairer(
|
||||||
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
@@ -57,12 +61,13 @@ func NewRunnerWithRepairer(
|
|||||||
repairer OutputRepairer,
|
repairer OutputRepairer,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return &Runner{
|
return &Runner{
|
||||||
profiles: profiles,
|
promptDefs: promptDefs,
|
||||||
artifacts: artifacts,
|
profiles: profiles,
|
||||||
renderer: renderer,
|
artifacts: artifacts,
|
||||||
llm: llmClient,
|
renderer: renderer,
|
||||||
validator: validator,
|
llm: llmClient,
|
||||||
repairer: repairer,
|
validator: validator,
|
||||||
|
repairer: repairer,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,7 +83,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
|
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
|
|
||||||
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
@@ -93,10 +98,11 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
if selectedProfileID == "" {
|
if selectedProfileID == "" {
|
||||||
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
if req.Execution == nil {
|
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
||||||
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
|
effectiveModel := mergeExecutionTarget(executionProfileToTarget(execProfile), req.Execution)
|
||||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
@@ -265,6 +271,30 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.Executio
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
|
||||||
|
if p == nil {
|
||||||
|
return domain.ExecutionTarget{}
|
||||||
|
}
|
||||||
|
cp := map[string]string(nil)
|
||||||
|
if len(p.ExtraParams) > 0 {
|
||||||
|
cp = make(map[string]string, len(p.ExtraParams))
|
||||||
|
for k, v := range p.ExtraParams {
|
||||||
|
cp[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return domain.ExecutionTarget{
|
||||||
|
Endpoint: p.Endpoint,
|
||||||
|
Model: p.Model,
|
||||||
|
Temperature: p.Temperature,
|
||||||
|
MaxTokens: p.MaxTokens,
|
||||||
|
TopP: p.TopP,
|
||||||
|
TimeoutSeconds: p.TimeoutSeconds,
|
||||||
|
ReasoningEffort: p.ReasoningEffort,
|
||||||
|
APIKeyEnv: p.APIKeyEnv,
|
||||||
|
ExtraParams: cp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
|
|||||||
@@ -19,6 +19,23 @@ type fakePromptRepo struct {
|
|||||||
lastVersion string
|
lastVersion string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeExecutionProfileRepo struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
err error
|
||||||
|
lastID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
f.lastID = id
|
||||||
|
if f.err != nil {
|
||||||
|
return nil, f.err
|
||||||
|
}
|
||||||
|
if f.profile == nil {
|
||||||
|
return nil, errors.New("profile not found")
|
||||||
|
}
|
||||||
|
return f.profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
f.lastID = id
|
f.lastID = id
|
||||||
f.lastVersion = version
|
f.lastVersion = version
|
||||||
@@ -112,7 +129,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||||
|
|
||||||
runner := NewRunner(repo, reader, renderer, llmClient, nil)
|
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, reader, renderer, llmClient, nil)
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
@@ -121,7 +138,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -138,7 +155,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
if res.PromptHash == "" || res.RenderedPromptHash == "" {
|
if res.PromptHash == "" || res.RenderedPromptHash == "" {
|
||||||
t.Fatal("expected prompt hashes")
|
t.Fatal("expected prompt hashes")
|
||||||
}
|
}
|
||||||
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
|
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" {
|
||||||
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
|
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
|
||||||
}
|
}
|
||||||
if res.RawOutput != "# recap" {
|
if res.RawOutput != "# recap" {
|
||||||
@@ -153,7 +170,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrProfileLoad) {
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
@@ -162,25 +179,26 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunnerRunMissingProfileSelection(t *testing.T) {
|
func TestRunnerRunMissingProfileSelection(t *testing.T) {
|
||||||
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
|
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
|
||||||
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
runner := NewRunner(repo, &fakeExecutionProfileRepo{profile: defaultExecutionProfile()}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected invalid request, got %v", err)
|
t.Fatalf("expected invalid request, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
|
func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) {
|
||||||
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
runner := NewRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefInline, Body: "x"}}})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
t.Fatalf("expected invalid request, got %v", err)
|
t.Fatalf("expected profile load failure, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
@@ -190,7 +208,6 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
|||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
|
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrArtifactLoad) {
|
if !errors.Is(err, ErrArtifactLoad) {
|
||||||
@@ -201,6 +218,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
|||||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||||
&fakeRenderer{err: errors.New("render failed")},
|
&fakeRenderer{err: errors.New("render failed")},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
@@ -209,7 +227,6 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
|||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrPromptRender) {
|
if !errors.Is(err, ErrPromptRender) {
|
||||||
@@ -220,6 +237,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
|||||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{err: errors.New("llm failed")},
|
&fakeLLM{err: errors.New("llm failed")},
|
||||||
@@ -228,7 +246,6 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
|||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrLLMGenerate) {
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
@@ -240,6 +257,7 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
|
|||||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||||
@@ -248,7 +266,6 @@ func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
|
|||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -263,6 +280,7 @@ func TestRunnerRunRepairBounded(t *testing.T) {
|
|||||||
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
|
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
|
&fakeExecutionProfileRepo{profile: defaultExecutionProfile()},
|
||||||
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
|
||||||
@@ -272,7 +290,6 @@ func TestRunnerRunRepairBounded(t *testing.T) {
|
|||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
|
|
||||||
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -303,3 +320,11 @@ func hashString(s string) string {
|
|||||||
sum := sha256.Sum256([]byte(s))
|
sum := sha256.Sum256([]byte(s))
|
||||||
return hex.EncodeToString(sum[:])
|
return hex.EncodeToString(sum[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func defaultExecutionProfile() *domain.ExecutionProfile {
|
||||||
|
return &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "model-from-profile",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user