From adc700f96e7cb24caada0625189f345986f6c2d5 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:13:17 +0000 Subject: [PATCH] Capture provider policy in backend execution targets --- convert.go | 2 +- docs/roadmap/implementation.md | 2 ++ .../configuration/bundled/openrouter.yaml | 1 + internal/configuration/bundled_test.go | 3 +- internal/configuration/interpretation_test.go | 29 +++++++++++++++++++ internal/configuration/ownership.go | 2 ++ internal/configuration/resources.go | 4 ++- internal/configuration/targets.go | 6 +++- internal/domain/domain.go | 4 +++ internal/llm/provider.go | 8 +++++ internal/usecase/prepared_execution_test.go | 6 +++- internal/usecase/resolved_targets_test.go | 3 +- resource_inspection.go | 2 ++ resource_serialization_test.go | 6 ++-- 14 files changed, 69 insertions(+), 9 deletions(-) diff --git a/convert.go b/convert.go index 3050905..e3982be 100644 --- a/convert.go +++ b/convert.go @@ -265,7 +265,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection { } func fromBackendResource(v *configuration.BackendResource) *BackendInspection { - out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)} + out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)} if v.ExtraParams != nil { data := copyAnyMap(v.ExtraParams) out.ExtraParams = &data diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index df86dee..04eec4a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -151,6 +151,8 @@ Validation: focused configuration/settings/source-helper tests. ## Stage 4 — Runtime Capture, Freezing, And Backend Inspection +**Completed.** + Touch `internal/configuration/targets.go`, `ownership.go`, `resources.go`, domain backend/target values and clones, and `bundled/openrouter.yaml` under that package. diff --git a/internal/configuration/bundled/openrouter.yaml b/internal/configuration/bundled/openrouter.yaml index a487031..f144a79 100644 --- a/internal/configuration/bundled/openrouter.yaml +++ b/internal/configuration/bundled/openrouter.yaml @@ -1,5 +1,6 @@ backends: - id: openrouter + provider: openrouter endpoint: https://openrouter.ai/api/v1 api_key_env: OPENROUTER_API_KEY smoke_tests: diff --git a/internal/configuration/bundled_test.go b/internal/configuration/bundled_test.go index 55dce2b..a642b78 100644 --- a/internal/configuration/bundled_test.go +++ b/internal/configuration/bundled_test.go @@ -17,7 +17,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) { t.Fatal(err) } backend, err := state.GetBackend("openrouter") - if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" { + if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" { t.Fatalf("backend = %+v, %v", backend, err) } if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 { @@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) { t.Fatalf("profile %s = %+v, %v", id, profile, err) } want := defaults.ExecutionTargetDefault() + want.Provider = "openrouter" want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY" if tier, ok := strings.CutPrefix(id, "auto-"); ok { want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}} diff --git a/internal/configuration/interpretation_test.go b/internal/configuration/interpretation_test.go index 3d84a59..31103e2 100644 --- a/internal/configuration/interpretation_test.go +++ b/internal/configuration/interpretation_test.go @@ -59,6 +59,35 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) { } } +func TestProviderPolicyCaptureAndInspection(t *testing.T) { + state, err := load(mapping("provider policy", map[string]any{ + "collections": map[string]any{"team": map[string]any{ + "backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}}, + "profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}}, + }}, + "profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}}, + }), nil) + if err != nil { + t.Fatal(err) + } + backend, err := state.GetBackend("team/child") + if err != nil || backend.Provider != "openrouter" || backend.ZeroCost { + t.Fatalf("backend: %+v, %v", backend, err) + } + profile, err := state.GetProfile(context.Background(), "team/selected") + if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" { + t.Fatalf("profile: %+v, %v", profile, err) + } + profile, err = state.GetProfile(context.Background(), "generic") + if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost { + t.Fatalf("generic: %+v, %v", profile, err) + } + resource, err := state.InspectBackend(context.Background(), "team/child") + if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost { + t.Fatalf("inspection: %+v, %v", resource, err) + } +} + func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) { definition := map[string]any{ "id": "example", "inputs": []any{map[string]any{"name": " document "}}, diff --git a/internal/configuration/ownership.go b/internal/configuration/ownership.go index 8df54d7..83ad649 100644 --- a/internal/configuration/ownership.go +++ b/internal/configuration/ownership.go @@ -21,6 +21,8 @@ func copyProviderMap(value map[string]any) map[string]any { } func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings { + in.Provider = copyPointer(in.Provider) + in.ZeroCost = copyPointer(in.ZeroCost) in.Endpoint = copyPointer(in.Endpoint) in.APIKeyEnv = copyPointer(in.APIKeyEnv) in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit) diff --git a/internal/configuration/resources.go b/internal/configuration/resources.go index c36de34..ff900b2 100644 --- a/internal/configuration/resources.go +++ b/internal/configuration/resources.go @@ -26,6 +26,8 @@ type ProfileResource struct { // Its fields borrow immutable configuration captured by the adapter. type BackendResource struct { ID, Endpoint string + Provider *string + ZeroCost *bool APIKeyEnv *string ConcurrencyLimit, QueueCapacity *int ExtraParams map[string]any @@ -131,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id) } return &BackendResource{ - ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, + ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost, ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity, ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests, }, nil diff --git a/internal/configuration/targets.go b/internal/configuration/targets.go index d3cbedc..6bc7ede 100644 --- a/internal/configuration/targets.go +++ b/internal/configuration/targets.go @@ -8,6 +8,7 @@ import ( "go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/jsonvalue" + "go.maximumdirect.net/promptkit/internal/llm" profilepkg "go.maximumdirect.net/promptkit/internal/profile" "go.maximumdirect.net/promptkit/internal/settings" ) @@ -36,7 +37,7 @@ func (state *loaded) buildExecutionProfiles(config *settings.Settings) error { } func captureBackend(id string, config backendConfig) domain.Backend { - definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams} + definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams} if config.SmokeTests != nil { smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile} if config.SmokeTests.Reachability != nil { @@ -67,6 +68,7 @@ func captureBackend(id string, config backendConfig) domain.Backend { func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) { target := defaults.ExecutionTargetDefault() + target.Provider = llm.EffectiveProvider("") if config.Backend != nil && *config.Backend != "" { // These defaults are capture-owned and immutable. Public and runtime // lookups still detach mutable values; profile overlays copy before merging. @@ -75,6 +77,8 @@ func captureExecutionProfile(id string, config profileConfig, backends map[strin return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend) } target.BackendID = selected.ID + target.Provider = selected.Provider + target.ZeroCost = selected.ZeroCost target.Endpoint = selected.Endpoint target.APIKeyEnv = selected.APIKeyEnv target.ExtraParams = selected.ExtraParams diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 3fc7dcd..d2c2ab9 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -188,6 +188,8 @@ type PromptMessageTemplate struct { // diagnostic metadata. type Backend struct { ID string + Provider string + ZeroCost bool Endpoint string APIKeyEnv string ExtraParams map[string]any @@ -236,6 +238,8 @@ type ExecutionTargetPresence struct { // ExecutionTarget represents effective model runtime settings for a run. type ExecutionTarget struct { + Provider string `yaml:"-" json:"-"` + ZeroCost bool `yaml:"-" json:"-"` BackendID string `yaml:"backend" json:"backend_id,omitempty"` Endpoint string `yaml:"endpoint" json:"endpoint"` Model string `yaml:"model" json:"model"` diff --git a/internal/llm/provider.go b/internal/llm/provider.go index 8468015..3eeaf88 100644 --- a/internal/llm/provider.go +++ b/internal/llm/provider.go @@ -1,5 +1,13 @@ package llm +// EffectiveProvider resolves an omitted provider to the generic conventions. +func EffectiveProvider(provider string) string { + if provider == "" { + return "openai-compatible" + } + return provider +} + // SupportedProvider reports whether a normalized provider selection is supported. // An empty selection uses the generic OpenAI-compatible conventions. func SupportedProvider(provider string) bool { diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index 152c6bc..88634ca 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -95,6 +95,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing. } admitter := &fakeRunAdmitter{} profile := defaultExecutionProfile() + profile.ResolvedTarget.Provider = "vllm" + profile.ResolvedTarget.ZeroCost = true profile.ResolvedTarget.ExtraParams = map[string]any{ "metadata": map[string]any{"source": "original"}, } @@ -144,9 +146,11 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing. } renderer.rendered.Messages[0].Content = "source mutation" + profile.ResolvedTarget.Provider = "openrouter" + profile.ResolvedTarget.ZeroCost = false second := prepared.Details() - if second != first || second.Messages[0].Content != "original message" || + if second != first || second.Messages[0].Content != "original message" || second.EffectiveModelParams.Provider != "vllm" || !second.EffectiveModelParams.ZeroCost || second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" || second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" { t.Fatalf("borrowed details did not preserve the snapshot: %#v", second) diff --git a/internal/usecase/resolved_targets_test.go b/internal/usecase/resolved_targets_test.go index f61edfd..c7bf115 100644 --- a/internal/usecase/resolved_targets_test.go +++ b/internal/usecase/resolved_targets_test.go @@ -11,6 +11,7 @@ import ( func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) { profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{ + Provider: "openrouter", ZeroCost: true, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds, Temperature: 0, ExtraParams: map[string]any{"configured": true}, }, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}} @@ -35,7 +36,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) t.Fatalf("request did not complete partial target: %v", err) } if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" || - prepared.EffectiveModelParams.Model != "runtime-model" || !prepared.TargetPresence.Temperature || + prepared.EffectiveModelParams.Model != "runtime-model" || prepared.EffectiveModelParams.Provider != "openrouter" || !prepared.EffectiveModelParams.ZeroCost || !prepared.TargetPresence.Temperature || prepared.EffectiveModelParams.ExtraParams["request"] != true { t.Fatalf("request overrides were not applied: %+v", prepared) } diff --git a/resource_inspection.go b/resource_inspection.go index 2bf728c..e9977e4 100644 --- a/resource_inspection.go +++ b/resource_inspection.go @@ -29,6 +29,8 @@ type ProfileInspection struct { // explicit zero values and empty collections in JSON and YAML. type BackendInspection struct { BackendID string `json:"id" yaml:"id"` + Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"` + ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"` Endpoint string `json:"endpoint" yaml:"endpoint"` APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"` ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"` diff --git a/resource_serialization_test.go b/resource_serialization_test.go index e9673ad..aabc9b3 100644 --- a/resource_serialization_test.go +++ b/resource_serialization_test.go @@ -13,7 +13,7 @@ import ( func TestResourceInspectionGenericEncoding(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{ - "backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}}, + "backends": []any{map[string]any{"id": "local", "provider": " VLLM ", "zero_cost": false, "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}}, "profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}}, "prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, })}}}) @@ -77,7 +77,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) { } } b := decode(backend) - if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" { + if b["id"] != "local" || b["provider"] != "vllm" || b["zero_cost"] != false || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" { t.Errorf("backend = %#v", b) } params := b["extra_params"].(map[string]any) @@ -92,7 +92,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) { t.Errorf("reachability = %#v", reach) } absent(b, "extends") - absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit") + absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit", "provider", "zero_cost") p := decode(profile) if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 { t.Errorf("profile = %#v", p)