// Package testfixture keeps existing runtime regression scenarios executable // while their source setup is expressed through the new configuration boundary. package testfixture import ( "bytes" "fmt" "io" "io/fs" "net/http" "os" "path/filepath" "strings" "time" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" "gopkg.in/yaml.v3" ) // Config carries the existing regression fixtures' construction settings. type Config struct { PromptDir string ProfileDir string SchemaDir string Timeout time.Duration HTTPClient *http.Client TransientRetries *int } // Profile describes an existing regression fixture's execution settings. type Profile struct { ID, BaseProfileID, BackendID, Endpoint, Model string Temperature, TopP float64 MaxTokens, TimeoutSeconds int ServiceTier, ReasoningEffort string APIKeyRequired bool ExtraParams map[string]any } // OpenAICompatibleProfileConfig describes a profile fixture. type OpenAICompatibleProfileConfig = Profile // OpenAICompatibleProfile produces a profile fixture. func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile { profile := Profile(config) if config.ExtraParams != nil { profile.ExtraParams = make(map[string]any, len(config.ExtraParams)) for key, value := range config.ExtraParams { profile.ExtraParams[key] = value } } return profile } // Backend describes an existing regression fixture's backend settings. type Backend struct { ID, Endpoint, APIKeyEnv string ConcurrencyLimit int QueueCapacity *int ExtraParams map[string]any } // LocalBackend returns a conventional local backend fixture. func LocalBackend(endpoint string, concurrencyLimit int) Backend { return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit} } type option func(*choices) error type choices struct { promptFS, profileFS, fallbackFS fs.FS promptRoot, profileRoot, fallbackRoot string promptFile, profileFile string profiles []Profile profilesSet bool backends []Backend } // WithPromptFS selects prompt fixture files. func WithPromptFS(fsys fs.FS, root string) option { return func(c *choices) error { if err := validSourceFS(fsys, root); err != nil { return err } c.promptFS, c.promptRoot, c.promptFile = fsys, root, "" return nil } } // WithPromptFile selects one prompt fixture file. func WithPromptFile(name string) option { return func(c *choices) error { if strings.TrimSpace(name) == "" { return fmt.Errorf("prompt file is required") } c.promptFS, c.promptFile = nil, name return nil } } // WithProfileFS selects profile fixture files. func WithProfileFS(fsys fs.FS, root string) option { return func(c *choices) error { if err := validSourceFS(fsys, root); err != nil { return err } c.profileFS, c.profileRoot, c.profileFile = fsys, root, "" return nil } } // WithProfileFile selects one profile fixture file. func WithProfileFile(name string) option { return func(c *choices) error { if strings.TrimSpace(name) == "" { return fmt.Errorf("profile file is required") } c.profileFS, c.profileFile = nil, name return nil } } // WithFallbackProfileFS selects lower-precedence profile fixture files. func WithFallbackProfileFS(fsys fs.FS, root string) option { return func(c *choices) error { if err := validSourceFS(fsys, root); err != nil { return err } c.fallbackFS, c.fallbackRoot = fsys, root return nil } } func validSourceFS(filesystem fs.FS, root string) error { if filesystem == nil || strings.TrimSpace(root) == "" { return fmt.Errorf("source filesystem and root are required") } return nil } // WithProfiles selects in-memory profile fixtures. func WithProfiles(profiles ...Profile) option { return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil } } // WithBackend adds one backend fixture. func WithBackend(backend Backend) option { if backend.QueueCapacity != nil { capacity := *backend.QueueCapacity backend.QueueCapacity = &capacity } return func(c *choices) error { c.backends = append(c.backends, backend); return nil } } // NewEngine expresses existing regression scenarios through ConfigKit sources. func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) { var choices choices var runtime []promptkit.Option for _, candidate := range opts { switch value := candidate.(type) { case nil: case option: if err := value(&choices); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } case promptkit.Option: runtime = append(runtime, value) default: return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate) } } var underlays []configkit.Underlay add := func(inputs []configkit.Input, err error) error { if err != nil { return err } if len(inputs) > 0 { underlays = append(underlays, configkit.Underlay{Inputs: inputs}) } return nil } if choices.fallbackFS != nil { if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } if choices.profileFile != "" { if err := add(singleInput(choices.profileFile, "profiles")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } else if choices.profileFS != nil { if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } else if config.ProfileDir != "" { if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } if choices.profilesSet { profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles)) for _, value := range choices.profiles { if _, exists := profiles[value.ID]; exists { return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID) } profiles[value.ID] = profileConfig(value) } underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}}) } for index, value := range choices.backends { config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv, ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams} underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}}) } if choices.promptFile != "" { if err := add(singleInput(choices.promptFile, "prompts")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } else if choices.promptFS != nil { if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } else if config.PromptDir != "" { if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil { return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) } } return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}, SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient, TransientRetries: config.TransientRetries}, runtime...) } func profileConfig(value Profile) promptkit.ProfileConfig { config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams} if value.BaseProfileID != "" { config.Extends = &value.BaseProfileID } if value.BackendID != "" { config.Backend = &value.BackendID } if value.Endpoint != "" { config.Endpoint = &value.Endpoint } if value.Model != "" { config.Model = &value.Model } if value.Temperature != 0 { config.Temperature = &value.Temperature } if value.TopP != 0 { config.TopP = &value.TopP } if value.MaxTokens != 0 { config.MaxTokens = &value.MaxTokens } if value.TimeoutSeconds != 0 { config.TimeoutSeconds = &value.TimeoutSeconds } if value.ServiceTier != "" { config.ServiceTier = &value.ServiceTier } if value.ReasoningEffort != "" { config.ReasoningEffort = &value.ReasoningEffort } if value.APIKeyRequired { config.APIKeyRequired = &value.APIKeyRequired } return config } func directoryInputs(directory, collection string) ([]configkit.Input, error) { if _, err := os.Stat(directory); err != nil { return nil, err } return sourceInputs(directory, os.DirFS(directory), ".", collection) } func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) { return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root) } func singleInput(name, collection string) ([]configkit.Input, error) { info, err := os.Stat(name) if err != nil { return nil, err } if info.IsDir() { return nil, fmt.Errorf("source %s is a directory", name) } filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection} namespace := configkit.NewFilesystem(name, filesystem) return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil } type convertingFS struct { fs.FS collection string } func (source convertingFS) Open(name string) (fs.File, error) { file, err := source.FS.Open(name) if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) { return file, err } info, err := file.Stat() if err != nil { file.Close() return nil, err } data, err := io.ReadAll(file) closeErr := file.Close() if err != nil { return nil, err } if closeErr != nil { return nil, closeErr } converted := convertDocument(data, source.collection) return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil } type memoryFile struct { *bytes.Reader info fs.FileInfo } func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil } func (file *memoryFile) Close() error { return nil } func convertDocument(data []byte, collection string) []byte { var document map[string]any if err := yaml.Unmarshal(data, &document); err != nil || document == nil { return data } if _, keyed := document[collection]; keyed { return data } id, idOK := document["id"] if !idOK { return data } delete(document, "id") if collection == "profiles" { if base, exists := document["base_profile"]; exists { delete(document, "base_profile") document["extends"] = base } data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}}) return data } version, versionOK := document["version"] if !versionOK { return data } delete(document, "version") data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}}) return data }