Load backend smoke-test metadata

2026-10-13 15:53:27 +00:00
parent f91a02548e
commit ece764f19e
9 changed files with 473 additions and 26 deletions
--- a/internal/backend/registry.go
+++ b/internal/backend/registry.go
@@ -7,7 +7,6 @@ import (
 	"fmt"

 	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
-	"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
 )

 const (
@@ -31,13 +30,12 @@ type Registry struct {
 func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
 	registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
 	for id, definition := range definitions {
-		params, err := jsonvalue.CopyMap(definition.ExtraParams)
-		if err != nil {
-			return nil, fmt.Errorf("backend %q extra parameters: %w", id, err)
-		}
 		definition.ID = id
-		definition.ExtraParams = params
-		registry.backends[id] = definition
+		cloned, err := domain.CloneBackend(definition)
+		if err != nil {
+			return nil, fmt.Errorf("backend %q: %w", id, err)
+		}
+		registry.backends[id] = cloned
 	}
 	return registry, nil
 }
@@ -51,12 +49,11 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
 	if !ok {
 		return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
 	}
-	extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
+	cloned, err := domain.CloneBackend(definition)
 	if err != nil {
 		return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
 	}
-	definition.ExtraParams = extraParams
-	return definition, nil
+	return cloned, nil
 }

 // CapacityPolicies returns a copy of the normalized policies for limited
--- a/internal/backend/registry_test.go
+++ b/internal/backend/registry_test.go
@@ -11,25 +11,61 @@ import (

 func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
 	params := map[string]any{"nested": map[string]any{"enabled": true}}
+	defaultProfile := "catalog-profile"
+	reachabilityEndpoint := "https://probe.example/ready"
+	authenticationKind := "openrouter_key"
+	authenticationEndpoint := "https://probe.example/key"
+	smokeTests := &domain.BackendSmokeTestConfig{
+		DefaultProfile: &defaultProfile,
+		Reachability:   &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
+		Authentication: &domain.AuthenticationProbeConfig{
+			Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
+		},
+	}
 	registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
 		" local.v1 ": {
 			Endpoint: "https://backend.example/v1", ExtraParams: params,
 			ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
+			SmokeTests: smokeTests,
 		},
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	params["nested"].(map[string]any)["enabled"] = false
+	defaultProfile = "mutated-profile"
+	reachabilityEndpoint = "https://mutated.example/ready"
+	authenticationKind = "none"
+	authenticationEndpoint = "https://mutated.example/key"
 	selected, err := registry.GetBackend(" local.v1 ")
 	if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
 		t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
 	}
+	if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
+		*selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
+		selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
+		selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
+		*selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
+		*selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
+		t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
+	}
 	selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
+	*selected.SmokeTests.DefaultProfile = "mutated-return"
+	*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
+	*selected.SmokeTests.Authentication.Kind = "none"
+	*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
 	again, err := registry.GetBackend(" local.v1 ")
 	if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
 		t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
 	}
+	if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
+		*again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
+		again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
+		again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
+		*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
+		*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
+		t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
+	}
 	policies := registry.CapacityPolicies()
 	if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
 		" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
--- a/internal/configuration/load.go
+++ b/internal/configuration/load.go
@@ -4,6 +4,7 @@ package configuration

 import (
 	"fmt"
+	"net/http"
 	"regexp"
 	"sort"
 	"strings"
@@ -22,11 +23,27 @@ type document struct {
 }

 type backendConfig struct {
-	Endpoint         *string        `yaml:"endpoint"`
-	APIKeyEnv        *string        `yaml:"api_key_env"`
-	ConcurrencyLimit *int           `yaml:"concurrency_limit"`
-	QueueCapacity    *int           `yaml:"queue_capacity"`
-	ExtraParams      map[string]any `yaml:"extra_params"`
+	Endpoint         *string                 `yaml:"endpoint"`
+	APIKeyEnv        *string                 `yaml:"api_key_env"`
+	ConcurrencyLimit *int                    `yaml:"concurrency_limit"`
+	QueueCapacity    *int                    `yaml:"queue_capacity"`
+	ExtraParams      map[string]any          `yaml:"extra_params"`
+	SmokeTests       *backendSmokeTestConfig `yaml:"smoke_tests"`
+}
+
+type backendSmokeTestConfig struct {
+	DefaultProfile *string                    `yaml:"default_profile"`
+	Reachability   *reachabilityProbeConfig   `yaml:"reachability"`
+	Authentication *authenticationProbeConfig `yaml:"authentication"`
+}
+
+type reachabilityProbeConfig struct {
+	Endpoint *string `yaml:"endpoint"`
+}
+
+type authenticationProbeConfig struct {
+	Kind     *string `yaml:"kind"`
+	Endpoint *string `yaml:"endpoint"`
 }

 type profileConfig struct {
@@ -104,7 +121,7 @@ type loaded struct {

 var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)

-var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params"}
+var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
 var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}

 // Load captures one immutable effective configuration for internal consumers.
@@ -242,6 +259,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
 		if backend.APIKeyEnv != nil {
 			normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
 		}
+		if smokeTests := backend.SmokeTests; smokeTests != nil {
+			smokePath := base.Key("smoke_tests")
+			if smokeTests.Reachability != nil {
+				normalizeDiagnosticEndpoint(&smokeTests.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
+			}
+			if smokeTests.Authentication != nil {
+				if smokeTests.Authentication.Kind != nil {
+					normalizeString(&smokeTests.Authentication.Kind,
+						smokePath.Key("authentication").Key("kind"), resolution,
+						strings.ToLower(strings.TrimSpace(*smokeTests.Authentication.Kind)))
+				}
+				normalizeDiagnosticEndpoint(&smokeTests.Authentication.Endpoint,
+					smokePath.Key("authentication").Key("endpoint"), resolution)
+			}
+		}
 		if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
 			capacity := 1024
 			if resolution.IsExplicit(base.Key("queue_capacity")) {
@@ -254,6 +286,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
 	}
 }

+func normalizeDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
+	if value == nil || *value == nil {
+		return
+	}
+	normalized := strings.TrimSpace(**value)
+	request, err := http.NewRequest(http.MethodGet, normalized, nil)
+	if normalized == "" || err != nil || request.URL == nil ||
+		(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
+		request.URL.Hostname() == "" {
+		resolution.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
+		return
+	}
+	normalizeString(value, path, resolution, normalized)
+}
+
 func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Resolution) {
 	for _, id := range sortedKeys(profiles) {
 		profile := profiles[id]
@@ -309,6 +356,20 @@ func validateEffective(config *document, validation *configkit.Validation) {
 			}
 		}
 		validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
+		if smokeTests := backend.SmokeTests; smokeTests != nil {
+			smokePath := base.Key("smoke_tests")
+			if smokeTests.DefaultProfile != nil && strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
+				validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
+			}
+			if authentication := smokeTests.Authentication; authentication != nil {
+				kindPath := smokePath.Key("authentication").Key("kind")
+				if authentication.Kind == nil {
+					validation.Error(kindPath, "authentication kind is required")
+				} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
+					validation.Error(kindPath, "authentication kind must be none or openrouter_key")
+				}
+			}
+		}
 	}
 	for _, id := range sortedKeys(config.Profiles) {
 		profile := config.Profiles[id]
--- a/internal/configuration/load_test.go
+++ b/internal/configuration/load_test.go
@@ -78,6 +78,159 @@ func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
 	}
 }

+func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
+	for _, test := range []struct {
+		name       string
+		smokeTests any
+		include    bool
+		inherited  bool
+	}{
+		{name: "omitted"},
+		{name: "null", smokeTests: nil, include: true, inherited: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			backend := map[string]any{"endpoint": "https://backend.example/v1"}
+			if test.include {
+				backend["smoke_tests"] = test.smokeTests
+			}
+			main := mapping("backend", map[string]any{
+				"backends": map[string]any{"local": backend},
+			})
+			var underlays []configkit.Underlay
+			if test.inherited {
+				underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
+					"backends": map[string]any{"local": map[string]any{
+						"endpoint":    "https://backend.example/v1",
+						"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
+					}},
+				})}}}
+			}
+			loaded, err := load(main, underlays)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := loaded.result.Config.Backends["local"].SmokeTests; got != nil {
+				t.Fatalf("smoke_tests = %+v, want nil", got)
+			}
+			selected, err := loaded.registry.GetBackend("local")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if selected.SmokeTests != nil {
+				t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
+			}
+		})
+	}
+}
+
+func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
+	base := mapping("defaults", map[string]any{
+		"backends": map[string]any{"local": map[string]any{
+			"endpoint": "https://backend.example/v1",
+			"smoke_tests": map[string]any{
+				"default_profile": "catalog-profile",
+				"reachability":    map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
+				"authentication": map[string]any{
+					"kind":     " OPENROUTER_KEY ",
+					"endpoint": "https://probe.example/key?scope=current",
+				},
+			},
+		}},
+	})
+	overlay := mapping("operator", map[string]any{
+		"backends": map[string]any{"local": map[string]any{
+			"smoke_tests": map[string]any{
+				"default_profile": "not-yet-listed",
+				"authentication":  map[string]any{"kind": " NONE ", "endpoint": nil},
+			},
+		}},
+	})
+	loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
+		{Inputs: []configkit.Input{base}},
+		{Inputs: []configkit.Input{overlay}},
+	})
+	if err != nil {
+		t.Fatalf("load effective diagnostic metadata: %v", err)
+	}
+	selected, err := loaded.registry.GetBackend("local")
+	if err != nil {
+		t.Fatal(err)
+	}
+	metadata := selected.SmokeTests
+	if selected.Endpoint != "https://backend.example/v1" {
+		t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
+	}
+	if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
+		t.Fatalf("default profile overlay was not retained: %+v", metadata)
+	}
+	if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
+		*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
+		t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
+	}
+	if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
+		metadata.Authentication.Endpoint != nil {
+		t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
+	}
+}
+
+func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		data map[string]any
+		path string
+	}{
+		{
+			name: "blank default profile",
+			data: map[string]any{"default_profile": " \t "},
+			path: "/backends/local/smoke_tests/default_profile",
+		},
+		{
+			name: "authentication kind required",
+			data: map[string]any{"authentication": map[string]any{}},
+			path: "/backends/local/smoke_tests/authentication/kind",
+		},
+		{
+			name: "unknown authentication kind",
+			data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
+			path: "/backends/local/smoke_tests/authentication/kind",
+		},
+		{
+			name: "non-http reachability URL",
+			data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
+			path: "/backends/local/smoke_tests/reachability/endpoint",
+		},
+		{
+			name: "blank authentication URL",
+			data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "  "}},
+			path: "/backends/local/smoke_tests/authentication/endpoint",
+		},
+		{
+			name: "relative authentication URL",
+			data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
+			path: "/backends/local/smoke_tests/authentication/endpoint",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			_, err := load(mapping("backend", map[string]any{
+				"backends": map[string]any{"local": map[string]any{
+					"endpoint":    "https://backend.example/v1",
+					"smoke_tests": test.data,
+				}},
+			}), nil)
+			var loadError *configkit.LoadError
+			if !errors.As(err, &loadError) {
+				t.Fatalf("expected final-value validation error, got %v", err)
+			}
+			for _, diagnostic := range loadError.Diagnostics() {
+				if diagnostic.Path.String() == test.path {
+					return
+				}
+			}
+			t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
+		})
+	}
+}
+
 func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
 	loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
 		"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
@@ -128,6 +281,40 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
 	}
 }

+func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
+	underlay := mapping("defaults", map[string]any{
+		"backends": map[string]any{"local": map[string]any{
+			"endpoint": "https://backend.example/v1",
+			"smoke_tests": map[string]any{
+				"default_profile": " ",
+				"reachability":    map[string]any{"endpoint": "ftp://invalid.example/"},
+				"authentication":  map[string]any{"kind": "unsupported"},
+			},
+		}},
+	})
+	main := mapping("operator", map[string]any{
+		"backends": map[string]any{"local": map[string]any{
+			"smoke_tests": map[string]any{
+				"default_profile": "dangling-profile",
+				"reachability":    map[string]any{"endpoint": nil},
+				"authentication":  map[string]any{"kind": "none"},
+			},
+		}},
+	})
+	loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
+	if err != nil {
+		t.Fatalf("replaced lower metadata should not fail: %v", err)
+	}
+	selected, err := loaded.registry.GetBackend("local")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
+		*selected.SmokeTests.DefaultProfile != "dangling-profile" {
+		t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
+	}
+}
+
 func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
 	_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
 		[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
--- a/internal/configuration/targets.go
+++ b/internal/configuration/targets.go
@@ -15,11 +15,20 @@ func (state *loaded) buildExecutionProfiles() error {
 	definitions := make(map[string]domain.Backend, len(state.result.Config.Backends))
 	for _, id := range sortedKeys(state.result.Config.Backends) {
 		config := state.result.Config.Backends[id]
-		params, err := jsonvalue.CopyMap(config.ExtraParams)
-		if err != nil {
-			return fmt.Errorf("backend %q extra parameters: %w", id, err)
+		definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
+		if config.SmokeTests != nil {
+			smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
+			if config.SmokeTests.Reachability != nil {
+				smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
+			}
+			if config.SmokeTests.Authentication != nil {
+				smokeTests.Authentication = &domain.AuthenticationProbeConfig{
+					Kind:     config.SmokeTests.Authentication.Kind,
+					Endpoint: config.SmokeTests.Authentication.Endpoint,
+				}
+			}
+			definition.SmokeTests = smokeTests
 		}
-		definition := domain.Backend{ID: id, ExtraParams: params}
 		if config.Endpoint != nil {
 			definition.Endpoint = *config.Endpoint
 		}