360 lines
11 KiB
Go
360 lines
11 KiB
Go
// Package testfixture keeps existing runtime regression scenarios executable
|
|
// while their source setup is expressed through the new configuration boundary.
|
|
package testfixture
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// Config carries the existing regression fixtures' construction settings.
|
|
type Config struct {
|
|
PromptDir string
|
|
ProfileDir string
|
|
SchemaDir string
|
|
Timeout time.Duration
|
|
HTTPClient *http.Client
|
|
TransientRetries *int
|
|
}
|
|
|
|
// Profile describes an existing regression fixture's execution settings.
|
|
type Profile struct {
|
|
ID, BaseProfileID, BackendID, Endpoint, Model string
|
|
Temperature, TopP float64
|
|
MaxTokens, TimeoutSeconds int
|
|
ServiceTier, ReasoningEffort string
|
|
APIKeyRequired bool
|
|
ExtraParams map[string]any
|
|
}
|
|
|
|
// OpenAICompatibleProfileConfig describes a profile fixture.
|
|
type OpenAICompatibleProfileConfig = Profile
|
|
|
|
// OpenAICompatibleProfile produces a profile fixture.
|
|
func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile {
|
|
profile := Profile(config)
|
|
if config.ExtraParams != nil {
|
|
profile.ExtraParams = make(map[string]any, len(config.ExtraParams))
|
|
for key, value := range config.ExtraParams {
|
|
profile.ExtraParams[key] = value
|
|
}
|
|
}
|
|
return profile
|
|
}
|
|
|
|
// Backend describes an existing regression fixture's backend settings.
|
|
type Backend struct {
|
|
ID, Endpoint, APIKeyEnv string
|
|
ConcurrencyLimit int
|
|
QueueCapacity *int
|
|
ExtraParams map[string]any
|
|
}
|
|
|
|
// LocalBackend returns a conventional local backend fixture.
|
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
|
return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit}
|
|
}
|
|
|
|
type option func(*choices) error
|
|
|
|
type choices struct {
|
|
promptFS, profileFS, fallbackFS fs.FS
|
|
promptRoot, profileRoot, fallbackRoot string
|
|
promptFile, profileFile string
|
|
profiles []Profile
|
|
profilesSet bool
|
|
backends []Backend
|
|
}
|
|
|
|
// WithPromptFS selects prompt fixture files.
|
|
func WithPromptFS(fsys fs.FS, root string) option {
|
|
return func(c *choices) error {
|
|
if err := validSourceFS(fsys, root); err != nil {
|
|
return err
|
|
}
|
|
c.promptFS, c.promptRoot, c.promptFile = fsys, root, ""
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// WithPromptFile selects one prompt fixture file.
|
|
func WithPromptFile(name string) option {
|
|
return func(c *choices) error {
|
|
if strings.TrimSpace(name) == "" {
|
|
return fmt.Errorf("prompt file is required")
|
|
}
|
|
c.promptFS, c.promptFile = nil, name
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// WithProfileFS selects profile fixture files.
|
|
func WithProfileFS(fsys fs.FS, root string) option {
|
|
return func(c *choices) error {
|
|
if err := validSourceFS(fsys, root); err != nil {
|
|
return err
|
|
}
|
|
c.profileFS, c.profileRoot, c.profileFile = fsys, root, ""
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// WithProfileFile selects one profile fixture file.
|
|
func WithProfileFile(name string) option {
|
|
return func(c *choices) error {
|
|
if strings.TrimSpace(name) == "" {
|
|
return fmt.Errorf("profile file is required")
|
|
}
|
|
c.profileFS, c.profileFile = nil, name
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// WithFallbackProfileFS selects lower-precedence profile fixture files.
|
|
func WithFallbackProfileFS(fsys fs.FS, root string) option {
|
|
return func(c *choices) error {
|
|
if err := validSourceFS(fsys, root); err != nil {
|
|
return err
|
|
}
|
|
c.fallbackFS, c.fallbackRoot = fsys, root
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func validSourceFS(filesystem fs.FS, root string) error {
|
|
if filesystem == nil || strings.TrimSpace(root) == "" {
|
|
return fmt.Errorf("source filesystem and root are required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// WithProfiles selects in-memory profile fixtures.
|
|
func WithProfiles(profiles ...Profile) option {
|
|
return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil }
|
|
}
|
|
|
|
// WithBackend adds one backend fixture.
|
|
func WithBackend(backend Backend) option {
|
|
if backend.QueueCapacity != nil {
|
|
capacity := *backend.QueueCapacity
|
|
backend.QueueCapacity = &capacity
|
|
}
|
|
return func(c *choices) error { c.backends = append(c.backends, backend); return nil }
|
|
}
|
|
|
|
// NewEngine expresses existing regression scenarios through ConfigKit sources.
|
|
func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) {
|
|
var choices choices
|
|
var runtime []promptkit.Option
|
|
for _, candidate := range opts {
|
|
switch value := candidate.(type) {
|
|
case nil:
|
|
case option:
|
|
if err := value(&choices); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
case promptkit.Option:
|
|
runtime = append(runtime, value)
|
|
default:
|
|
return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate)
|
|
}
|
|
}
|
|
var underlays []configkit.Underlay
|
|
add := func(inputs []configkit.Input, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(inputs) > 0 {
|
|
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
|
|
}
|
|
return nil
|
|
}
|
|
if choices.fallbackFS != nil {
|
|
if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
}
|
|
if choices.profileFile != "" {
|
|
if err := add(singleInput(choices.profileFile, "profiles")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
} else if choices.profileFS != nil {
|
|
if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
} else if config.ProfileDir != "" {
|
|
if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
}
|
|
if choices.profilesSet {
|
|
profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles))
|
|
for _, value := range choices.profiles {
|
|
if _, exists := profiles[value.ID]; exists {
|
|
return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID)
|
|
}
|
|
profiles[value.ID] = profileConfig(value)
|
|
}
|
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}})
|
|
}
|
|
for index, value := range choices.backends {
|
|
config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv,
|
|
ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams}
|
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}})
|
|
}
|
|
if choices.promptFile != "" {
|
|
if err := add(singleInput(choices.promptFile, "prompts")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
} else if choices.promptFS != nil {
|
|
if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
} else if config.PromptDir != "" {
|
|
if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil {
|
|
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
|
}
|
|
}
|
|
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays},
|
|
SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient,
|
|
TransientRetries: config.TransientRetries}, runtime...)
|
|
}
|
|
|
|
func profileConfig(value Profile) promptkit.ProfileConfig {
|
|
config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams}
|
|
if value.BaseProfileID != "" {
|
|
config.Extends = &value.BaseProfileID
|
|
}
|
|
if value.BackendID != "" {
|
|
config.Backend = &value.BackendID
|
|
}
|
|
if value.Endpoint != "" {
|
|
config.Endpoint = &value.Endpoint
|
|
}
|
|
if value.Model != "" {
|
|
config.Model = &value.Model
|
|
}
|
|
if value.Temperature != 0 {
|
|
config.Temperature = &value.Temperature
|
|
}
|
|
if value.TopP != 0 {
|
|
config.TopP = &value.TopP
|
|
}
|
|
if value.MaxTokens != 0 {
|
|
config.MaxTokens = &value.MaxTokens
|
|
}
|
|
if value.TimeoutSeconds != 0 {
|
|
config.TimeoutSeconds = &value.TimeoutSeconds
|
|
}
|
|
if value.ServiceTier != "" {
|
|
config.ServiceTier = &value.ServiceTier
|
|
}
|
|
if value.ReasoningEffort != "" {
|
|
config.ReasoningEffort = &value.ReasoningEffort
|
|
}
|
|
if value.APIKeyRequired {
|
|
config.APIKeyRequired = &value.APIKeyRequired
|
|
}
|
|
return config
|
|
}
|
|
|
|
func directoryInputs(directory, collection string) ([]configkit.Input, error) {
|
|
if _, err := os.Stat(directory); err != nil {
|
|
return nil, err
|
|
}
|
|
return sourceInputs(directory, os.DirFS(directory), ".", collection)
|
|
}
|
|
|
|
func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) {
|
|
return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root)
|
|
}
|
|
|
|
func singleInput(name, collection string) ([]configkit.Input, error) {
|
|
info, err := os.Stat(name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if info.IsDir() {
|
|
return nil, fmt.Errorf("source %s is a directory", name)
|
|
}
|
|
filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection}
|
|
namespace := configkit.NewFilesystem(name, filesystem)
|
|
return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil
|
|
}
|
|
|
|
type convertingFS struct {
|
|
fs.FS
|
|
collection string
|
|
}
|
|
|
|
func (source convertingFS) Open(name string) (fs.File, error) {
|
|
file, err := source.FS.Open(name)
|
|
if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) {
|
|
return file, err
|
|
}
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
file.Close()
|
|
return nil, err
|
|
}
|
|
data, err := io.ReadAll(file)
|
|
closeErr := file.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if closeErr != nil {
|
|
return nil, closeErr
|
|
}
|
|
converted := convertDocument(data, source.collection)
|
|
return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil
|
|
}
|
|
|
|
type memoryFile struct {
|
|
*bytes.Reader
|
|
info fs.FileInfo
|
|
}
|
|
|
|
func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil }
|
|
func (file *memoryFile) Close() error { return nil }
|
|
|
|
func convertDocument(data []byte, collection string) []byte {
|
|
var document map[string]any
|
|
if err := yaml.Unmarshal(data, &document); err != nil || document == nil {
|
|
return data
|
|
}
|
|
if _, keyed := document[collection]; keyed {
|
|
return data
|
|
}
|
|
id, idOK := document["id"]
|
|
if !idOK {
|
|
return data
|
|
}
|
|
delete(document, "id")
|
|
if collection == "profiles" {
|
|
if base, exists := document["base_profile"]; exists {
|
|
delete(document, "base_profile")
|
|
document["extends"] = base
|
|
}
|
|
data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}})
|
|
return data
|
|
}
|
|
version, versionOK := document["version"]
|
|
if !versionOK {
|
|
return data
|
|
}
|
|
delete(document, "version")
|
|
data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}})
|
|
return data
|
|
}
|