Integrate released catalog sources with effective configuration
This commit is contained in:
@@ -1,249 +1,52 @@
|
||||
// Package catalog validates immutable maintained backend catalog assets.
|
||||
// Package catalog selects the maintained configuration assets supplied with
|
||||
// PromptKit. ConfigKit owns their composition and effective validation.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
// Source identifies one immutable backend catalog asset tree.
|
||||
type Source struct {
|
||||
Name string
|
||||
ExpectedBackendID string
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
// Set is the validated maintained backend and raw profile catalog.
|
||||
type Set struct {
|
||||
Backends []domain.Backend
|
||||
Profiles profile.Repository
|
||||
profileIDs []string
|
||||
}
|
||||
|
||||
// Load validates and combines immutable catalog sources in source order.
|
||||
func Load(sources ...Source) (Set, error) {
|
||||
if len(sources) == 0 {
|
||||
return Set{}, errors.New("at least one catalog source is required")
|
||||
}
|
||||
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
|
||||
names := map[string]bool{}
|
||||
backendIDs := map[string]bool{}
|
||||
profileIDs := map[string]bool{}
|
||||
for _, source := range sources {
|
||||
if err := validateSource(source, names); err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
names[source.Name] = true
|
||||
definition, err := loadBackend(source)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
if backendIDs[definition.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
|
||||
}
|
||||
backendIDs[definition.ID] = true
|
||||
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
|
||||
}
|
||||
resolvingRepository := profile.NewResolvingRepository(repository)
|
||||
for _, entry := range metadata {
|
||||
if strings.HasPrefix(entry.ID, "promptkit-") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path)
|
||||
}
|
||||
if profileIDs[entry.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
|
||||
}
|
||||
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
|
||||
}
|
||||
value, err := repository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
|
||||
}
|
||||
if err := rejectSecretKeys(value.ExtraParams); err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
|
||||
}
|
||||
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
|
||||
}
|
||||
if resolved.BackendID != definition.ID {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
|
||||
}
|
||||
profileIDs[entry.ID] = true
|
||||
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
|
||||
}
|
||||
if loaded.Profiles == nil {
|
||||
loaded.Profiles = repository
|
||||
} else {
|
||||
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
|
||||
}
|
||||
loaded.Backends = append(loaded.Backends, definition)
|
||||
}
|
||||
sort.Strings(loaded.profileIDs)
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func validateSource(source Source, names map[string]bool) error {
|
||||
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
|
||||
return errors.New("catalog source name must not be blank")
|
||||
}
|
||||
if names[source.Name] {
|
||||
return fmt.Errorf("duplicate catalog source name %q", source.Name)
|
||||
}
|
||||
if source.FS == nil {
|
||||
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
|
||||
}
|
||||
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
|
||||
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(source.ExpectedBackendID) == "" {
|
||||
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
|
||||
}
|
||||
return validateLayout(source)
|
||||
}
|
||||
|
||||
func validateLayout(source Source) error {
|
||||
manifestPath := path.Join(source.Root, "backend.json")
|
||||
profilesRoot := path.Join(source.Root, "profiles")
|
||||
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if assetPath == source.Root {
|
||||
if !entry.IsDir() {
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
if assetPath == manifestPath && entry.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
})
|
||||
}
|
||||
|
||||
func loadBackend(source Source) (domain.Backend, error) {
|
||||
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
|
||||
// Underlays selects external catalogs followed by PromptKit-owned profiles.
|
||||
// Neither group is interpreted before the final effective configuration loads.
|
||||
func Underlays() ([]configkit.Underlay, error) {
|
||||
openrouterInputs, err := assetInputs("OpenRouter catalog", openrouter.FS(), openrouter.Root)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
|
||||
return nil, err
|
||||
}
|
||||
type manifest struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
ID *string `json:"id"`
|
||||
Endpoint *string `json:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit"`
|
||||
QueueCapacity *int `json:"queue_capacity"`
|
||||
ExtraParams json.RawMessage `json:"extra_params"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var value manifest
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
|
||||
}
|
||||
if *value.ID != source.ExpectedBackendID {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(*value.APIKeyEnv) == "" {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
|
||||
}
|
||||
extraParams, err := decodeExtraParams(value.ExtraParams)
|
||||
rakestrawhomeInputs, err := assetInputs("Rakestrawhome catalog", rakestrawhome.FS(), rakestrawhome.Root)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
|
||||
return nil, err
|
||||
}
|
||||
if err := rejectSecretKeys(extraParams); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
|
||||
}
|
||||
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
|
||||
catalogInputs := append(openrouterInputs, rakestrawhomeInputs...)
|
||||
owned := configkit.Input{Source: configkit.Mapping("PromptKit-owned profiles", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"promptkit-openrouter-standard": map[string]any{"extends": "gemini-flash-latest"},
|
||||
"promptkit-openrouter-auto": map[string]any{"backend": backend.OpenRouterID, "model": "openrouter/auto"},
|
||||
},
|
||||
})}
|
||||
return []configkit.Underlay{{Inputs: catalogInputs}, {Inputs: []configkit.Input{owned}}}, nil
|
||||
}
|
||||
|
||||
func assetInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
|
||||
return nil, fmt.Errorf("discover %s configuration: %w", label, err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func decodeExtraParams(data []byte) (map[string]any, error) {
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var value map[string]any
|
||||
if err := decoder.Decode(&value); err != nil || value == nil {
|
||||
return nil, errors.New("extra parameters must be an object or null")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("extra parameters must contain exactly one JSON value")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func containsField(fields []string, target string) bool {
|
||||
for _, field := range fields {
|
||||
if field == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func rejectSecretKeys(value map[string]any) error {
|
||||
return rejectSecretValue(value)
|
||||
}
|
||||
|
||||
func rejectSecretValue(value any) error {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range value {
|
||||
switch strings.ToLower(key) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return errors.New("prohibited key")
|
||||
}
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, child := range value {
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
namespace := configkit.NewFilesystem(label, filesystem)
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, name := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: namespace.File(name),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user