Cut over public engine construction to ConfigKit sources
This commit is contained in:
359
testfixture/engine.go
Normal file
359
testfixture/engine.go
Normal file
@@ -0,0 +1,359 @@
|
||||
// Package testfixture keeps existing runtime regression scenarios executable
|
||||
// while their source setup is expressed through the new configuration boundary.
|
||||
package testfixture
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Config carries the existing regression fixtures' construction settings.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// Profile describes an existing regression fixture's execution settings.
|
||||
type Profile struct {
|
||||
ID, BaseProfileID, BackendID, Endpoint, Model string
|
||||
Temperature, TopP float64
|
||||
MaxTokens, TimeoutSeconds int
|
||||
ServiceTier, ReasoningEffort string
|
||||
APIKeyRequired bool
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// OpenAICompatibleProfileConfig describes a profile fixture.
|
||||
type OpenAICompatibleProfileConfig = Profile
|
||||
|
||||
// OpenAICompatibleProfile produces a profile fixture.
|
||||
func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile {
|
||||
profile := Profile(config)
|
||||
if config.ExtraParams != nil {
|
||||
profile.ExtraParams = make(map[string]any, len(config.ExtraParams))
|
||||
for key, value := range config.ExtraParams {
|
||||
profile.ExtraParams[key] = value
|
||||
}
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
// Backend describes an existing regression fixture's backend settings.
|
||||
type Backend struct {
|
||||
ID, Endpoint, APIKeyEnv string
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity *int
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// LocalBackend returns a conventional local backend fixture.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||
return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit}
|
||||
}
|
||||
|
||||
type option func(*choices) error
|
||||
|
||||
type choices struct {
|
||||
promptFS, profileFS, fallbackFS fs.FS
|
||||
promptRoot, profileRoot, fallbackRoot string
|
||||
promptFile, profileFile string
|
||||
profiles []Profile
|
||||
profilesSet bool
|
||||
backends []Backend
|
||||
}
|
||||
|
||||
// WithPromptFS selects prompt fixture files.
|
||||
func WithPromptFS(fsys fs.FS, root string) option {
|
||||
return func(c *choices) error {
|
||||
if err := validSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
c.promptFS, c.promptRoot, c.promptFile = fsys, root, ""
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithPromptFile selects one prompt fixture file.
|
||||
func WithPromptFile(name string) option {
|
||||
return func(c *choices) error {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return fmt.Errorf("prompt file is required")
|
||||
}
|
||||
c.promptFS, c.promptFile = nil, name
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithProfileFS selects profile fixture files.
|
||||
func WithProfileFS(fsys fs.FS, root string) option {
|
||||
return func(c *choices) error {
|
||||
if err := validSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
c.profileFS, c.profileRoot, c.profileFile = fsys, root, ""
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithProfileFile selects one profile fixture file.
|
||||
func WithProfileFile(name string) option {
|
||||
return func(c *choices) error {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return fmt.Errorf("profile file is required")
|
||||
}
|
||||
c.profileFS, c.profileFile = nil, name
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithFallbackProfileFS selects lower-precedence profile fixture files.
|
||||
func WithFallbackProfileFS(fsys fs.FS, root string) option {
|
||||
return func(c *choices) error {
|
||||
if err := validSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
c.fallbackFS, c.fallbackRoot = fsys, root
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func validSourceFS(filesystem fs.FS, root string) error {
|
||||
if filesystem == nil || strings.TrimSpace(root) == "" {
|
||||
return fmt.Errorf("source filesystem and root are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WithProfiles selects in-memory profile fixtures.
|
||||
func WithProfiles(profiles ...Profile) option {
|
||||
return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil }
|
||||
}
|
||||
|
||||
// WithBackend adds one backend fixture.
|
||||
func WithBackend(backend Backend) option {
|
||||
if backend.QueueCapacity != nil {
|
||||
capacity := *backend.QueueCapacity
|
||||
backend.QueueCapacity = &capacity
|
||||
}
|
||||
return func(c *choices) error { c.backends = append(c.backends, backend); return nil }
|
||||
}
|
||||
|
||||
// NewEngine expresses existing regression scenarios through ConfigKit sources.
|
||||
func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) {
|
||||
var choices choices
|
||||
var runtime []promptkit.Option
|
||||
for _, candidate := range opts {
|
||||
switch value := candidate.(type) {
|
||||
case nil:
|
||||
case option:
|
||||
if err := value(&choices); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
case promptkit.Option:
|
||||
runtime = append(runtime, value)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate)
|
||||
}
|
||||
}
|
||||
var underlays []configkit.Underlay
|
||||
add := func(inputs []configkit.Input, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(inputs) > 0 {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if choices.fallbackFS != nil {
|
||||
if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
if choices.profileFile != "" {
|
||||
if err := add(singleInput(choices.profileFile, "profiles")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
} else if choices.profileFS != nil {
|
||||
if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
} else if config.ProfileDir != "" {
|
||||
if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
if choices.profilesSet {
|
||||
profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles))
|
||||
for _, value := range choices.profiles {
|
||||
if _, exists := profiles[value.ID]; exists {
|
||||
return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID)
|
||||
}
|
||||
profiles[value.ID] = profileConfig(value)
|
||||
}
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}})
|
||||
}
|
||||
for index, value := range choices.backends {
|
||||
config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv,
|
||||
ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams}
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}})
|
||||
}
|
||||
if choices.promptFile != "" {
|
||||
if err := add(singleInput(choices.promptFile, "prompts")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
} else if choices.promptFS != nil {
|
||||
if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
} else if config.PromptDir != "" {
|
||||
if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays},
|
||||
SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient,
|
||||
TransientRetries: config.TransientRetries}, runtime...)
|
||||
}
|
||||
|
||||
func profileConfig(value Profile) promptkit.ProfileConfig {
|
||||
config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams}
|
||||
if value.BaseProfileID != "" {
|
||||
config.Extends = &value.BaseProfileID
|
||||
}
|
||||
if value.BackendID != "" {
|
||||
config.Backend = &value.BackendID
|
||||
}
|
||||
if value.Endpoint != "" {
|
||||
config.Endpoint = &value.Endpoint
|
||||
}
|
||||
if value.Model != "" {
|
||||
config.Model = &value.Model
|
||||
}
|
||||
if value.Temperature != 0 {
|
||||
config.Temperature = &value.Temperature
|
||||
}
|
||||
if value.TopP != 0 {
|
||||
config.TopP = &value.TopP
|
||||
}
|
||||
if value.MaxTokens != 0 {
|
||||
config.MaxTokens = &value.MaxTokens
|
||||
}
|
||||
if value.TimeoutSeconds != 0 {
|
||||
config.TimeoutSeconds = &value.TimeoutSeconds
|
||||
}
|
||||
if value.ServiceTier != "" {
|
||||
config.ServiceTier = &value.ServiceTier
|
||||
}
|
||||
if value.ReasoningEffort != "" {
|
||||
config.ReasoningEffort = &value.ReasoningEffort
|
||||
}
|
||||
if value.APIKeyRequired {
|
||||
config.APIKeyRequired = &value.APIKeyRequired
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
func directoryInputs(directory, collection string) ([]configkit.Input, error) {
|
||||
if _, err := os.Stat(directory); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sourceInputs(directory, os.DirFS(directory), ".", collection)
|
||||
}
|
||||
|
||||
func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) {
|
||||
return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root)
|
||||
}
|
||||
|
||||
func singleInput(name, collection string) ([]configkit.Input, error) {
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("source %s is a directory", name)
|
||||
}
|
||||
filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection}
|
||||
namespace := configkit.NewFilesystem(name, filesystem)
|
||||
return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil
|
||||
}
|
||||
|
||||
type convertingFS struct {
|
||||
fs.FS
|
||||
collection string
|
||||
}
|
||||
|
||||
func (source convertingFS) Open(name string) (fs.File, error) {
|
||||
file, err := source.FS.Open(name)
|
||||
if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) {
|
||||
return file, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(file)
|
||||
closeErr := file.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
converted := convertDocument(data, source.collection)
|
||||
return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil
|
||||
}
|
||||
|
||||
type memoryFile struct {
|
||||
*bytes.Reader
|
||||
info fs.FileInfo
|
||||
}
|
||||
|
||||
func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil }
|
||||
func (file *memoryFile) Close() error { return nil }
|
||||
|
||||
func convertDocument(data []byte, collection string) []byte {
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal(data, &document); err != nil || document == nil {
|
||||
return data
|
||||
}
|
||||
if _, keyed := document[collection]; keyed {
|
||||
return data
|
||||
}
|
||||
id, idOK := document["id"]
|
||||
if !idOK {
|
||||
return data
|
||||
}
|
||||
delete(document, "id")
|
||||
if collection == "profiles" {
|
||||
if base, exists := document["base_profile"]; exists {
|
||||
delete(document, "base_profile")
|
||||
document["extends"] = base
|
||||
}
|
||||
data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}})
|
||||
return data
|
||||
}
|
||||
version, versionOK := document["version"]
|
||||
if !versionOK {
|
||||
return data
|
||||
}
|
||||
delete(document, "version")
|
||||
data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}})
|
||||
return data
|
||||
}
|
||||
Reference in New Issue
Block a user