Refactor: enforce canonical messages/output YAML, normalize content_file, and expand fixture-based validation tests
This commit is contained in:
@@ -1,11 +0,0 @@
|
|||||||
id: json-schema-missing-path
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates:
|
|
||||||
- role: user
|
|
||||||
content: "Return JSON"
|
|
||||||
output_format: json
|
|
||||||
validation:
|
|
||||||
validation_mode: json_schema
|
|
||||||
11
internal/profile/testdata/negative_timeout.yaml
vendored
11
internal/profile/testdata/negative_timeout.yaml
vendored
@@ -1,11 +0,0 @@
|
|||||||
id: negative-timeout
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates:
|
|
||||||
- role: user
|
|
||||||
content: "Say hi"
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
|
||||||
9
internal/profile/testdata/no_templates.yaml
vendored
9
internal/profile/testdata/no_templates.yaml
vendored
@@ -1,9 +0,0 @@
|
|||||||
id: no-templates
|
|
||||||
version: 1.0.0
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates: []
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
|
||||||
19
internal/profile/testdata/valid.yaml
vendored
19
internal/profile/testdata/valid.yaml
vendored
@@ -1,19 +0,0 @@
|
|||||||
id: test-profile
|
|
||||||
version: "1.0.0"
|
|
||||||
default_profile: test-exec
|
|
||||||
description: A valid test prompt definition
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
content_type: text/markdown
|
|
||||||
- name: glossary
|
|
||||||
required: false
|
|
||||||
content_type: text/yaml
|
|
||||||
templates:
|
|
||||||
- role: system
|
|
||||||
content: "You are a helpful assistant."
|
|
||||||
- role: user
|
|
||||||
content: 'Analyze this: {{input "transcript"}}'
|
|
||||||
output_format: markdown
|
|
||||||
validation:
|
|
||||||
validation_mode: basic
|
|
||||||
@@ -64,9 +64,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse and execute template
|
// Parse and execute template
|
||||||
if tmplMsg.ContentFile != "" {
|
|
||||||
return nil, fmt.Errorf("%w: message %d: content_file is not implemented yet", ErrRenderFailure, i)
|
|
||||||
}
|
|
||||||
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
|
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
|
||||||
|
|||||||
@@ -23,6 +23,36 @@ type filesystemRepository struct {
|
|||||||
dir string
|
dir string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptDefinitionFile struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
DefaultProfile *string `yaml:"default_profile"`
|
||||||
|
Description string `yaml:"description"`
|
||||||
|
Inputs []promptInputFile `yaml:"inputs"`
|
||||||
|
Messages []promptMessageFile `yaml:"messages"`
|
||||||
|
Output promptOutputContractFile `yaml:"output"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptInputFile struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Required bool `yaml:"required"`
|
||||||
|
ContentType string `yaml:"content_type"`
|
||||||
|
Description string `yaml:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptMessageFile struct {
|
||||||
|
Role string `yaml:"role"`
|
||||||
|
Content string `yaml:"content"`
|
||||||
|
ContentFile string `yaml:"content_file"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptOutputContractFile struct {
|
||||||
|
Format domain.OutputFormat `yaml:"format"`
|
||||||
|
ValidationMode domain.ValidationMode `yaml:"validation_mode"`
|
||||||
|
SchemaPath string `yaml:"schema_path"`
|
||||||
|
RepairAttempts int `yaml:"repair_attempts"`
|
||||||
|
}
|
||||||
|
|
||||||
func NewFilesystemRepository(dir string) Repository {
|
func NewFilesystemRepository(dir string) Repository {
|
||||||
return &filesystemRepository{dir: dir}
|
return &filesystemRepository{dir: dir}
|
||||||
}
|
}
|
||||||
@@ -44,22 +74,25 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) {
|
if file.IsDir() || !isYAMLFile(file.Name()) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fullPath := filepath.Join(r.dir, file.Name())
|
fullPath := filepath.Join(r.dir, file.Name())
|
||||||
data, err := os.ReadFile(fullPath)
|
fileMatch := promptIDFromFileName(file.Name()) == id
|
||||||
|
|
||||||
|
raw, err := loadPromptDefinitionFile(fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", file.Name(), err)
|
if fileMatch {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
var def domain.PromptDefinition
|
def, err := normalizePromptDefinition(raw, fullPath)
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
if err != nil {
|
||||||
decoder.KnownFields(true)
|
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
||||||
if err := decoder.Decode(&def); err != nil {
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err)
|
||||||
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
|
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -70,82 +103,166 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
if version != "" && def.Version != version {
|
if version != "" && def.Version != version {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := validatePromptDefinition(&def); err != nil {
|
return def, nil
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err)
|
|
||||||
}
|
|
||||||
return &def, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, ErrPromptDefinitionNotFound
|
return nil, ErrPromptDefinitionNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
func validatePromptDefinition(d *domain.PromptDefinition) error {
|
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
|
||||||
if d.ID == "" {
|
data, err := os.ReadFile(path)
|
||||||
return errors.New("prompt id is required")
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition file: %w", err)
|
||||||
}
|
}
|
||||||
if d.Version == "" {
|
|
||||||
return errors.New("prompt version is required")
|
var raw promptDefinitionFile
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.KnownFields(true)
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
if len(d.Templates) == 0 {
|
return &raw, nil
|
||||||
return errors.New("at least one prompt template message is required")
|
}
|
||||||
|
|
||||||
|
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
||||||
|
if raw == nil {
|
||||||
|
return nil, errors.New("prompt definition is nil")
|
||||||
}
|
}
|
||||||
if len(d.Inputs) == 0 {
|
|
||||||
return errors.New("at least one prompt input is required")
|
id := strings.TrimSpace(raw.ID)
|
||||||
|
if id == "" {
|
||||||
|
return nil, errors.New("id is required")
|
||||||
}
|
}
|
||||||
for i, input := range d.Inputs {
|
|
||||||
if strings.TrimSpace(input.Name) == "" {
|
version := strings.TrimSpace(raw.Version)
|
||||||
return fmt.Errorf("input %d has empty name", i)
|
if version == "" {
|
||||||
|
return nil, errors.New("version is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(raw.Messages) == 0 {
|
||||||
|
return nil, errors.New("at least one message is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
inputs := make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||||
|
seenInputNames := make(map[string]struct{}, len(raw.Inputs))
|
||||||
|
for i, in := range raw.Inputs {
|
||||||
|
name := strings.TrimSpace(in.Name)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("input %d has empty name", i)
|
||||||
|
}
|
||||||
|
if _, exists := seenInputNames[name]; exists {
|
||||||
|
return nil, fmt.Errorf("duplicate input name %q", name)
|
||||||
|
}
|
||||||
|
seenInputNames[name] = struct{}{}
|
||||||
|
|
||||||
|
inputs = append(inputs, domain.PromptInput{
|
||||||
|
Name: name,
|
||||||
|
Required: in.Required,
|
||||||
|
ContentType: strings.TrimSpace(in.ContentType),
|
||||||
|
Description: strings.TrimSpace(in.Description),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||||
|
promptDir := filepath.Dir(sourcePath)
|
||||||
|
for i, msg := range raw.Messages {
|
||||||
|
role := strings.TrimSpace(msg.Role)
|
||||||
|
if role == "" {
|
||||||
|
return nil, fmt.Errorf("message %d role is required", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
hasContent := strings.TrimSpace(msg.Content) != ""
|
||||||
|
hasContentFile := strings.TrimSpace(msg.ContentFile) != ""
|
||||||
|
if hasContent == hasContentFile {
|
||||||
|
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
||||||
|
}
|
||||||
|
|
||||||
|
templateContent := msg.Content
|
||||||
|
resolvedContentFile := ""
|
||||||
|
if hasContentFile {
|
||||||
|
resolvedPath := strings.TrimSpace(msg.ContentFile)
|
||||||
|
if !filepath.IsAbs(resolvedPath) {
|
||||||
|
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
||||||
|
}
|
||||||
|
resolvedPath = filepath.Clean(resolvedPath)
|
||||||
|
|
||||||
|
body, err := os.ReadFile(resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
|
||||||
|
}
|
||||||
|
templateContent = string(body)
|
||||||
|
resolvedContentFile = resolvedPath
|
||||||
|
}
|
||||||
|
|
||||||
|
templates = append(templates, domain.PromptMessageTemplate{
|
||||||
|
Role: role,
|
||||||
|
Content: templateContent,
|
||||||
|
ContentFile: resolvedContentFile,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isValidOutputFormat(raw.Output.Format) {
|
||||||
|
return nil, fmt.Errorf("invalid output format: %q", raw.Output.Format)
|
||||||
|
}
|
||||||
|
if !isValidValidationMode(raw.Output.ValidationMode) {
|
||||||
|
return nil, fmt.Errorf("invalid validation mode: %q", raw.Output.ValidationMode)
|
||||||
|
}
|
||||||
|
if raw.Output.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(raw.Output.SchemaPath) == "" {
|
||||||
|
return nil, errors.New("output.schema_path is required when output.validation_mode is json_schema")
|
||||||
|
}
|
||||||
|
if raw.Output.RepairAttempts < 0 {
|
||||||
|
return nil, errors.New("output.repair_attempts must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
defaultProfile := ""
|
||||||
|
if raw.DefaultProfile != nil {
|
||||||
|
defaultProfile = strings.TrimSpace(*raw.DefaultProfile)
|
||||||
|
if defaultProfile == "" {
|
||||||
|
return nil, errors.New("default_profile must be a non-empty string when set")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for i, t := range d.Templates {
|
|
||||||
if !isValidMessageRole(t.Role) {
|
return &domain.PromptDefinition{
|
||||||
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
|
ID: id,
|
||||||
}
|
Version: version,
|
||||||
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
|
DefaultProfile: defaultProfile,
|
||||||
return fmt.Errorf("template message %d must provide content or content_file", i)
|
Description: strings.TrimSpace(raw.Description),
|
||||||
}
|
Inputs: inputs,
|
||||||
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
|
Templates: templates,
|
||||||
return fmt.Errorf("template message %d cannot set both content and content_file", i)
|
OutputFormat: raw.Output.Format,
|
||||||
}
|
Validation: domain.OutputContract{
|
||||||
}
|
Format: raw.Output.Format,
|
||||||
if !isValidOutputFormat(d.OutputFormat) {
|
ValidationMode: raw.Output.ValidationMode,
|
||||||
return fmt.Errorf("invalid output format: %s", d.OutputFormat)
|
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
|
||||||
}
|
RepairAttempts: raw.Output.RepairAttempts,
|
||||||
if !isValidValidationMode(d.Validation.ValidationMode) {
|
},
|
||||||
return fmt.Errorf("invalid validation mode: %s", d.Validation.ValidationMode)
|
}, nil
|
||||||
}
|
}
|
||||||
if d.Validation.RepairAttempts < 0 {
|
|
||||||
return errors.New("validation.repair_attempts must be greater than or equal to 0")
|
func isYAMLFile(name string) bool {
|
||||||
}
|
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||||
if d.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(d.Validation.SchemaPath) == "" {
|
}
|
||||||
return errors.New("validation.schema_path is required when validation_mode is json_schema")
|
|
||||||
}
|
func promptIDFromFileName(name string) string {
|
||||||
if d.Validation.Format != "" && d.Validation.Format != d.OutputFormat {
|
name = strings.TrimSuffix(name, ".yaml")
|
||||||
return fmt.Errorf("validation format %q does not match output format %q", d.Validation.Format, d.OutputFormat)
|
name = strings.TrimSuffix(name, ".yml")
|
||||||
}
|
return name
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||||
switch f {
|
switch f {
|
||||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||||
return true
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
func isValidValidationMode(m domain.ValidationMode) bool {
|
||||||
switch m {
|
switch m {
|
||||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||||
return true
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidMessageRole(role string) bool {
|
|
||||||
switch role {
|
|
||||||
case "system", "user", "assistant", "developer":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,104 +3,143 @@ package promptdef
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||||
tmpDir, err := os.MkdirTemp("", "promptdef_test")
|
tmpDir := t.TempDir()
|
||||||
if err != nil {
|
if err := copyTree("testdata", tmpDir); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatalf("failed to copy testdata: %v", err)
|
||||||
}
|
|
||||||
defer os.RemoveAll(tmpDir)
|
|
||||||
|
|
||||||
testDataDir := "testdata"
|
|
||||||
files, err := os.ReadDir(testDataDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read testdata: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
src := filepath.Join(testDataDir, f.Name())
|
|
||||||
dst := filepath.Join(tmpDir, f.Name())
|
|
||||||
data, err := os.ReadFile(src)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
repo := NewFilesystemRepository(tmpDir)
|
repo := NewFilesystemRepository(tmpDir)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
t.Run("valid prompt definition", func(t *testing.T) {
|
t.Run("valid inline prompt", func(t *testing.T) {
|
||||||
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
|
p, err := repo.GetPromptDefinition(ctx, "valid-inline", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
if p == nil || p.ID != "test-profile" {
|
if p.ID != "valid-inline" {
|
||||||
t.Errorf("expected prompt definition test-profile, got %v", p)
|
t.Fatalf("unexpected id: %q", p.ID)
|
||||||
}
|
}
|
||||||
if p.Version != "1.0.0" {
|
if p.Version != "1.0.0" {
|
||||||
t.Fatalf("expected version 1.0.0, got %q", p.Version)
|
t.Fatalf("unexpected version: %q", p.Version)
|
||||||
}
|
|
||||||
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
|
|
||||||
t.Fatalf("unexpected inputs: %#v", p.Inputs)
|
|
||||||
}
|
|
||||||
if len(p.Templates) != 2 {
|
|
||||||
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
|
|
||||||
}
|
|
||||||
if p.Templates[0].Role != "system" || p.Templates[1].Role != "user" {
|
|
||||||
t.Fatalf("unexpected template roles: %#v", p.Templates)
|
|
||||||
}
|
}
|
||||||
if p.OutputFormat != domain.FormatMarkdown {
|
if p.OutputFormat != domain.FormatMarkdown {
|
||||||
t.Fatalf("expected output format markdown, got %q", p.OutputFormat)
|
t.Fatalf("unexpected output format: %q", p.OutputFormat)
|
||||||
}
|
}
|
||||||
if p.Validation.ValidationMode != domain.ValidationBasic {
|
if p.Validation.ValidationMode != domain.ValidationBasic {
|
||||||
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
|
t.Fatalf("unexpected validation mode: %q", p.Validation.ValidationMode)
|
||||||
}
|
}
|
||||||
if p.DefaultProfile != "test-exec" {
|
if len(p.Templates) != 2 {
|
||||||
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid YAML", func(t *testing.T) {
|
t.Run("valid file-backed prompt", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
|
p, err := repo.GetPromptDefinition(ctx, "valid-file-backed", "")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if err != nil {
|
||||||
t.Errorf("expected ErrInvalidYAML, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
if !strings.Contains(p.Templates[1].Content, "{{input \"transcript\"}}") {
|
||||||
|
t.Fatalf("expected content_file template body to be loaded, got %q", p.Templates[1].Content)
|
||||||
|
}
|
||||||
|
if p.Templates[1].ContentFile == "" {
|
||||||
|
t.Fatal("expected ContentFile source metadata to be preserved")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(p.Templates[1].ContentFile) {
|
||||||
|
t.Fatalf("expected resolved content_file path to be absolute, got %q", p.Templates[1].ContentFile)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("missing ID", func(t *testing.T) {
|
t.Run("prompt with default_profile", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
|
p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.DefaultProfile != "local-default" {
|
||||||
|
t.Fatalf("unexpected default profile: %q", p.DefaultProfile)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("version lookup", func(t *testing.T) {
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9")
|
||||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||||
t.Errorf("expected ErrPromptDefinitionNotFound for profile with missing ID, got %v", err)
|
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("no templates", func(t *testing.T) {
|
cases := []struct {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
|
name string
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
id string
|
||||||
t.Errorf("expected ErrInvalidPromptDefinition for profile with no templates, got %v", err)
|
targetErr error
|
||||||
}
|
errSubstrs []string
|
||||||
})
|
}{
|
||||||
|
{name: "invalid YAML", id: "invalid_yaml", targetErr: ErrInvalidYAML},
|
||||||
|
{name: "missing id", id: "missing_id", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"id is required"}},
|
||||||
|
{name: "no messages", id: "no_messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}},
|
||||||
|
{name: "both content and content_file", id: "both_content_and_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
||||||
|
{name: "neither content nor content_file", id: "neither_content_nor_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
||||||
|
{name: "missing content_file", id: "missing_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}},
|
||||||
|
{name: "duplicate input names", id: "duplicate_input_names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}},
|
||||||
|
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
||||||
|
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("json schema mode missing schema path", func(t *testing.T) {
|
for _, tc := range cases {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
_, err := repo.GetPromptDefinition(ctx, tc.id, "")
|
||||||
t.Errorf("expected ErrInvalidPromptDefinition for json_schema profile without schema_path, got %v", err)
|
if !errors.Is(err, tc.targetErr) {
|
||||||
}
|
t.Fatalf("expected %v, got %v", tc.targetErr, err)
|
||||||
})
|
}
|
||||||
|
for _, sub := range tc.errSubstrs {
|
||||||
|
if !strings.Contains(err.Error(), sub) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", sub, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("prompt definition not found", func(t *testing.T) {
|
t.Run("prompt definition not found", func(t *testing.T) {
|
||||||
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
|
_, err := repo.GetPromptDefinition(ctx, "does-not-exist", "")
|
||||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||||
t.Errorf("expected ErrPromptDefinitionNotFound, got %v", err)
|
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyTree(src, dst string) error {
|
||||||
|
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(src, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if rel == "." {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
target := filepath.Join(dst, rel)
|
||||||
|
if d.IsDir() {
|
||||||
|
return os.MkdirAll(target, 0o755)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(target, data, 0o644)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
10
internal/promptdef/testdata/both_content_and_content_file.yaml
vendored
Normal file
10
internal/promptdef/testdata/both_content_and_content_file.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
id: both-content-and-content-file
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "Hi"
|
||||||
|
content_file: ./messages/user_prompt.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
14
internal/promptdef/testdata/duplicate_input_names.yaml
vendored
Normal file
14
internal/promptdef/testdata/duplicate_input_names.yaml
vendored
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
id: duplicate-input-names
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
- name: transcript
|
||||||
|
required: false
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "Hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
9
internal/promptdef/testdata/invalid_validation_mode.yaml
vendored
Normal file
9
internal/promptdef/testdata/invalid_validation_mode.yaml
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: invalid-validation-mode
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "Hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: nope
|
||||||
|
repair_attempts: 0
|
||||||
12
internal/promptdef/testdata/invalid_yaml.yaml
vendored
12
internal/promptdef/testdata/invalid_yaml.yaml
vendored
@@ -1,5 +1,9 @@
|
|||||||
id: invalid-yaml
|
id: invalid-yaml
|
||||||
version: 1.0.0
|
version: "1.0.0"
|
||||||
templates:
|
messages:
|
||||||
- role: system
|
- role: user
|
||||||
content: [unclosed bracket
|
content: [broken
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
id: json-schema-missing-path
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates:
|
|
||||||
- role: user
|
|
||||||
content: "Return JSON"
|
|
||||||
output_format: json
|
|
||||||
validation:
|
|
||||||
validation_mode: json_schema
|
|
||||||
9
internal/promptdef/testdata/json_schema_without_schema_path.yaml
vendored
Normal file
9
internal/promptdef/testdata/json_schema_without_schema_path.yaml
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: json-schema-without-schema-path
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "Return JSON"
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
repair_attempts: 0
|
||||||
2
internal/promptdef/testdata/messages/user_prompt.tmpl
vendored
Normal file
2
internal/promptdef/testdata/messages/user_prompt.tmpl
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
Use transcript:
|
||||||
|
{{input "transcript"}}
|
||||||
9
internal/promptdef/testdata/missing_content_file.yaml
vendored
Normal file
9
internal/promptdef/testdata/missing_content_file.yaml
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: missing-content-file
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: ./messages/does_not_exist.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
17
internal/promptdef/testdata/missing_id.yaml
vendored
17
internal/promptdef/testdata/missing_id.yaml
vendored
@@ -1,11 +1,8 @@
|
|||||||
version: 1.0.0
|
version: "1.0.0"
|
||||||
description: Missing ID
|
messages:
|
||||||
inputs:
|
- role: user
|
||||||
- name: transcript
|
content: "Hi"
|
||||||
required: true
|
output:
|
||||||
templates:
|
format: text
|
||||||
- role: system
|
|
||||||
content: Hello
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
id: negative-timeout
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates:
|
|
||||||
- role: user
|
|
||||||
content: "Say hi"
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
|
||||||
8
internal/promptdef/testdata/neither_content_nor_content_file.yaml
vendored
Normal file
8
internal/promptdef/testdata/neither_content_nor_content_file.yaml
vendored
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
id: neither-content-nor-content-file
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
6
internal/promptdef/testdata/no_messages.yaml
vendored
Normal file
6
internal/promptdef/testdata/no_messages.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
id: no-messages
|
||||||
|
version: "1.0.0"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: no-templates
|
|
||||||
version: 1.0.0
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
templates: []
|
|
||||||
output_format: text
|
|
||||||
validation:
|
|
||||||
validation_mode: none
|
|
||||||
19
internal/promptdef/testdata/valid.yaml
vendored
19
internal/promptdef/testdata/valid.yaml
vendored
@@ -1,19 +0,0 @@
|
|||||||
id: test-profile
|
|
||||||
version: "1.0.0"
|
|
||||||
default_profile: test-exec
|
|
||||||
description: A valid test prompt definition
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
content_type: text/markdown
|
|
||||||
- name: glossary
|
|
||||||
required: false
|
|
||||||
content_type: text/yaml
|
|
||||||
templates:
|
|
||||||
- role: system
|
|
||||||
content: "You are a helpful assistant."
|
|
||||||
- role: user
|
|
||||||
content: 'Analyze this: {{input "transcript"}}'
|
|
||||||
output_format: markdown
|
|
||||||
validation:
|
|
||||||
validation_mode: basic
|
|
||||||
14
internal/promptdef/testdata/valid_file_backed.yaml
vendored
Normal file
14
internal/promptdef/testdata/valid_file_backed.yaml
vendored
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
id: valid-file-backed
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Return markdown."
|
||||||
|
- role: user
|
||||||
|
content_file: ./messages/user_prompt.tmpl
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
18
internal/promptdef/testdata/valid_inline.yaml
vendored
Normal file
18
internal/promptdef/testdata/valid_inline.yaml
vendored
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
id: valid-inline
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: text/markdown
|
||||||
|
description: Transcript content
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "You are concise."
|
||||||
|
- role: user
|
||||||
|
content: |
|
||||||
|
Summarize:
|
||||||
|
{{input "transcript"}}
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
13
internal/promptdef/testdata/with_default_profile.yaml
vendored
Normal file
13
internal/promptdef/testdata/with_default_profile.yaml
vendored
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
id: with-default-profile
|
||||||
|
version: "1.0.0"
|
||||||
|
default_profile: local-default
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "Write output"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
@@ -9,7 +9,7 @@ inputs:
|
|||||||
- name: glossary
|
- name: glossary
|
||||||
required: false
|
required: false
|
||||||
content_type: text/yaml
|
content_type: text/yaml
|
||||||
templates:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
You create concise tabletop RPG session recaps.
|
You create concise tabletop RPG session recaps.
|
||||||
@@ -26,6 +26,7 @@ templates:
|
|||||||
|
|
||||||
Glossary:
|
Glossary:
|
||||||
{{input "glossary"}}
|
{{input "glossary"}}
|
||||||
output_format: markdown
|
output:
|
||||||
validation:
|
format: markdown
|
||||||
validation_mode: basic
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ inputs:
|
|||||||
required: false
|
required: false
|
||||||
content_type: text/yaml
|
content_type: text/yaml
|
||||||
description: Optional glossary context
|
description: Optional glossary context
|
||||||
templates:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
You are a concise analysis assistant.
|
You are a concise analysis assistant.
|
||||||
@@ -25,6 +25,7 @@ templates:
|
|||||||
Reference glossary:
|
Reference glossary:
|
||||||
|
|
||||||
{{input "glossary"}}
|
{{input "glossary"}}
|
||||||
output_format: markdown
|
output:
|
||||||
validation:
|
format: markdown
|
||||||
validation_mode: basic
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ inputs:
|
|||||||
- name: glossary
|
- name: glossary
|
||||||
required: false
|
required: false
|
||||||
content_type: text/yaml
|
content_type: text/yaml
|
||||||
templates:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
Return only JSON following the requested schema.
|
Return only JSON following the requested schema.
|
||||||
@@ -23,8 +23,7 @@ templates:
|
|||||||
|
|
||||||
Glossary:
|
Glossary:
|
||||||
{{input "glossary"}}
|
{{input "glossary"}}
|
||||||
output_format: json
|
output:
|
||||||
validation:
|
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: structured_events.schema.json
|
schema_path: structured_events.schema.json
|
||||||
|
|||||||
Reference in New Issue
Block a user