eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load backend smoke-test metadata

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-13 15:53:27 +00:00
parent f91a02548e
commit ece764f19e
9 changed files with 473 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
internal/configuration/load.go
View File

@@ -4,6 +4,7 @@ package configuration
import (
"fmt"
"net/http"
"regexp"
"sort"
"strings"
@@ -22,11 +23,27 @@ type document struct {
}
type backendConfig struct {
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
}
type backendSmokeTestConfig struct {
DefaultProfile *string `yaml:"default_profile"`
Reachability *reachabilityProbeConfig `yaml:"reachability"`
Authentication *authenticationProbeConfig `yaml:"authentication"`
}
type reachabilityProbeConfig struct {
Endpoint *string `yaml:"endpoint"`
}
type authenticationProbeConfig struct {
Kind *string `yaml:"kind"`
Endpoint *string `yaml:"endpoint"`
}
type profileConfig struct {
@@ -104,7 +121,7 @@ type loaded struct {
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params"}
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
// Load captures one immutable effective configuration for internal consumers.
@@ -242,6 +259,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
if backend.APIKeyEnv != nil {
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
}
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.Reachability != nil {
normalizeDiagnosticEndpoint(&smokeTests.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
}
if smokeTests.Authentication != nil {
if smokeTests.Authentication.Kind != nil {
normalizeString(&smokeTests.Authentication.Kind,
smokePath.Key("authentication").Key("kind"), resolution,
strings.ToLower(strings.TrimSpace(*smokeTests.Authentication.Kind)))
}
normalizeDiagnosticEndpoint(&smokeTests.Authentication.Endpoint,
smokePath.Key("authentication").Key("endpoint"), resolution)
}
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
capacity := 1024
if resolution.IsExplicit(base.Key("queue_capacity")) {
@@ -254,6 +286,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
}
}
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
if value == nil || *value == nil {
return
}
normalized := strings.TrimSpace(**value)
request, err := http.NewRequest(http.MethodGet, normalized, nil)
if normalized == "" || err != nil || request.URL == nil ||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
request.URL.Hostname() == "" {
resolution.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
return
}
normalizeString(value, path, resolution, normalized)
}
func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Resolution) {
for _, id := range sortedKeys(profiles) {
profile := profiles[id]
@@ -309,6 +356,20 @@ func validateEffective(config *document, validation *configkit.Validation) {
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.DefaultProfile != nil && strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
}
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
}
}
}
}
for _, id := range sortedKeys(config.Profiles) {
profile := config.Profiles[id]

187
internal/configuration/load_test.go
View File

@@ -78,6 +78,159 @@ func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct {
name string
smokeTests any
include bool
inherited bool
}{
{name: "omitted"},
{name: "null", smokeTests: nil, include: true, inherited: true},
} {
t.Run(test.name, func(t *testing.T) {
backend := map[string]any{"endpoint": "https://backend.example/v1"}
if test.include {
backend["smoke_tests"] = test.smokeTests
}
main := mapping("backend", map[string]any{
"backends": map[string]any{"local": backend},
})
var underlays []configkit.Underlay
if test.inherited {
underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
}},
})}}}
}
loaded, err := load(main, underlays)
if err != nil {
t.Fatal(err)
}
if got := loaded.result.Config.Backends["local"].SmokeTests; got != nil {
t.Fatalf("smoke_tests = %+v, want nil", got)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests != nil {
t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
}
})
}
}
func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
base := mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "catalog-profile",
"reachability": map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
"authentication": map[string]any{
"kind": " OPENROUTER_KEY ",
"endpoint": "https://probe.example/key?scope=current",
},
},
}},
})
overlay := mapping("operator", map[string]any{
"backends": map[string]any{"local": map[string]any{
"smoke_tests": map[string]any{
"default_profile": "not-yet-listed",
"authentication": map[string]any{"kind": " NONE ", "endpoint": nil},
},
}},
})
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
{Inputs: []configkit.Input{base}},
{Inputs: []configkit.Input{overlay}},
})
if err != nil {
t.Fatalf("load effective diagnostic metadata: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
metadata := selected.SmokeTests
if selected.Endpoint != "https://backend.example/v1" {
t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
}
if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
t.Fatalf("default profile overlay was not retained: %+v", metadata)
}
if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
}
if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
metadata.Authentication.Endpoint != nil {
t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
}
}
func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
for _, test := range []struct {
name string
data map[string]any
path string
}{
{
name: "blank default profile",
data: map[string]any{"default_profile": " \t "},
path: "/backends/local/smoke_tests/default_profile",
},
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/backends/local/smoke_tests/authentication/kind",
},
{
name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
path: "/backends/local/smoke_tests/authentication/kind",
},
{
name: "non-http reachability URL",
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
path: "/backends/local/smoke_tests/reachability/endpoint",
},
{
name: "blank authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
path: "/backends/local/smoke_tests/authentication/endpoint",
},
{
name: "relative authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
path: "/backends/local/smoke_tests/authentication/endpoint",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("backend", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": test.data,
}},
}), nil)
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected final-value validation error, got %v", err)
}
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Path.String() == test.path {
return
}
}
t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
})
}
}
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
@@ -128,6 +281,40 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
}
}
func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
underlay := mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": " ",
"reachability": map[string]any{"endpoint": "ftp://invalid.example/"},
"authentication": map[string]any{"kind": "unsupported"},
},
}},
})
main := mapping("operator", map[string]any{
"backends": map[string]any{"local": map[string]any{
"smoke_tests": map[string]any{
"default_profile": "dangling-profile",
"reachability": map[string]any{"endpoint": nil},
"authentication": map[string]any{"kind": "none"},
},
}},
})
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("replaced lower metadata should not fail: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
*selected.SmokeTests.DefaultProfile != "dangling-profile" {
t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
}
}
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})

17
internal/configuration/targets.go
View File

@@ -15,11 +15,20 @@ func (state *loaded) buildExecutionProfiles() error {
definitions := make(map[string]domain.Backend, len(state.result.Config.Backends))
for _, id := range sortedKeys(state.result.Config.Backends) {
config := state.result.Config.Backends[id]
params, err := jsonvalue.CopyMap(config.ExtraParams)
if err != nil {
return fmt.Errorf("backend %q extra parameters: %w", id, err)
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
}
if config.SmokeTests.Authentication != nil {
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
Kind: config.SmokeTests.Authentication.Kind,
Endpoint: config.SmokeTests.Authentication.Endpoint,
}
}
definition.SmokeTests = smokeTests
}
definition := domain.Backend{ID: id, ExtraParams: params}
if config.Endpoint != nil {
definition.Endpoint = *config.Endpoint
}