From 5525fe875e47e65bc6045ff15f8241a5609e1d42 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 01:10:10 +0000 Subject: [PATCH] Verify native profile runtime behavior --- docs/roadmap/implementation.md | 9 +- .../framework/llm/adapter_capacity_test.go | 11 ++ internal/framework/llm/adapter_test.go | 122 ++++++++++++++++-- .../framework/llm/runtime_assembly_test.go | 40 ++++++ .../framework/llm/runtime_inspection_test.go | 27 ++++ 5 files changed, 198 insertions(+), 11 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 068d230..89ed0b5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,7 +1,7 @@ # Implementation Plan: PromptKit v0.12.0 Adoption **Last updated:** 2026-10-04 -**Status:** In progress; Stages 1–6 completed. +**Status:** In progress; Stages 1–7 completed. **Audience:** Luna coding agent. ## Scope and Execution @@ -407,6 +407,13 @@ integration tests where their established behavior is affected. and `git diff --check`. Review whether each new test addresses a distinct changed boundary. Commit the native-profile and runtime verification unit. +**Completion marker:** Stage 7 complete. Native profile inheritance, control +presence, provider-map overlays, application backend authority, selected +credential rejection, catalog-ID handling, shared backend capacity, and safe +inspection behavior are covered. The six targeted packages passed, along with +focused wire, timeout, capacity, credential, and safe-view tests; `git diff +--check` passed. + ## Stage 8 — Document the Migration and Update Attribution **Primary files:** `docs/config.md`, `docs/internal/llm-runtime.md`, diff --git a/internal/framework/llm/adapter_capacity_test.go b/internal/framework/llm/adapter_capacity_test.go index 8e9c19e..c174902 100644 --- a/internal/framework/llm/adapter_capacity_test.go +++ b/internal/framework/llm/adapter_capacity_test.go @@ -24,6 +24,7 @@ import ( func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret") + t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret") entered := make(chan struct{}, 3) release := make(chan struct{}) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -51,6 +52,16 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { t.Fatal(err) } } + operatorBackend := `backends: + shared: + endpoint: https://operator.invalid/v1 + concurrency_limit: 77 + queue_capacity: 88 + api_key_env: OPERATOR_BACKEND_KEY +` + if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil { + t.Fatal(err) + } loaded, err := config.Load(ck.File(main), "") if err != nil { t.Fatal(err) diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 3516abb..fa9a497 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -30,10 +30,22 @@ type reply struct { } func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) { - return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "") + return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{ + "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", + }) } func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) { + documents := map[string]string{ + "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", + } + if operatorPrompt != "" { + documents["operator-prompt.yaml"] = operatorPrompt + } + return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents) +} + +func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) { t.Helper() var count atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -69,15 +81,18 @@ func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs in })) t.Cleanup(server.Close) t.Setenv("ADAPTER_KEY", "adapter-secret") - base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs) - dir := profileLayout(t, base, "") - main := filepath.Join(dir, "config.yml") - profile := "profiles:\n base:\n backend: offline\n model: full-model\n" - if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil { + dir := t.TempDir() + profileDirectory := filepath.Join(dir, "profiles") + if err := os.Mkdir(profileDirectory, 0700); err != nil { t.Fatal(err) } - if operatorPrompt != "" { - if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(operatorPrompt), 0600); err != nil { + base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs) + main := filepath.Join(dir, "config.yml") + if err := os.WriteFile(main, []byte(base), 0600); err != nil { + t.Fatal(err) + } + for name, content := range documents { + if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil { t.Fatal(err) } } @@ -388,6 +403,91 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing } } +func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) { + t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used") + cases := []struct { + name string + parentTemperature string + childTemperature string + wantTemperature bool + }{ + {name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true}, + {name: "omitted", wantTemperature: false}, + {name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature + + " base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature + + " extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n" + operatorBackend := `backends: + openrouter: + endpoint: https://operator.invalid/v1 + concurrency_limit: 91 + queue_capacity: 92 + api_key_env: CATALOG_TRAP_KEY + extra_params: + routing: + inherited: backend + overridden: backend + cleared: remove-me +` + documents := map[string]string{ + "unrelated-profile-file.yml": profileDocument, + "provider-tuning.yaml": operatorBackend, + } + var wire map[string]json.RawMessage + runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" { + t.Errorf("application credential was not authoritative: %q", got) + } + if err := json.NewDecoder(r.Body).Decode(&wire); err != nil { + t.Error(err) + } + }, "openrouter", documents) + profile := runtime.Profiles["proposal"] + if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 { + t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams) + } + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + var out map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) + if err != nil { + t.Fatal(err) + } + if count.Load() != 1 { + t.Fatalf("provider calls = %d, want 1", count.Load()) + } + temperature, hasTemperature := wire["temperature"] + if hasTemperature != tc.wantTemperature { + t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire) + } + if tc.wantTemperature { + var value float64 + if err := json.Unmarshal(temperature, &value); err != nil || value != 0 { + t.Fatalf("explicit zero temperature = %s (%v)", temperature, err) + } + } + var routing map[string]any + if err := json.Unmarshal(wire["routing"], &routing); err != nil { + t.Fatalf("provider routing fields missing: %s (%v)", wire, err) + } + if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" { + t.Fatalf("recursively overlaid provider fields = %#v", routing) + } + if value, exists := routing["cleared"]; !exists || value != nil { + t.Fatalf("null did not replace inherited provider field: %#v", routing) + } + if response.Model != "full-model" || response.Prepared == nil { + t.Fatalf("execution did not use selected profile: %+v", response) + } + }) + } +} + func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) { var posted []struct { Model string `json:"model"` @@ -409,7 +509,7 @@ func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testin t.Fatal(err) } profile := "profiles:\n base:\n backend: offline\n model: edited-model\n" - if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil { t.Fatal(err) } updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.") @@ -480,7 +580,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) { blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) } - runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking) + runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{ + "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n", + }) runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { diff --git a/internal/framework/llm/runtime_assembly_test.go b/internal/framework/llm/runtime_assembly_test.go index c98e09b..99dc41f 100644 --- a/internal/framework/llm/runtime_assembly_test.go +++ b/internal/framework/llm/runtime_assembly_test.go @@ -124,6 +124,46 @@ func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) { } } +func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) { + cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{ + "unrelated-name.yaml": `profiles: + "parent~/shared": + backend: offline + temperature: 0.4 + top_p: 0.7 + extra_params: + routing: + retained: from-parent + overridden: parent + cleared: remove-me + "child profile": + extends: "parent~/shared" + model: child-model + extra_params: + routing: + overridden: child + child_only: present + cleared: null +`, + }) + runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}}) + if err != nil { + t.Fatal(err) + } + profile := runtime.Profiles["proposal"] + target := profile.EffectiveModelParams + if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 { + t.Fatalf("inherited profile target = %+v", target) + } + routing, ok := target.ExtraParams["routing"].(map[string]any) + if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" { + t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams) + } + if value, exists := routing["cleared"]; !exists || value != nil { + t.Fatalf("null did not replace the inherited provider value: %#v", routing) + } +} + func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) { cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", diff --git a/internal/framework/llm/runtime_inspection_test.go b/internal/framework/llm/runtime_inspection_test.go index 96732a6..d93670b 100644 --- a/internal/framework/llm/runtime_inspection_test.go +++ b/internal/framework/llm/runtime_inspection_test.go @@ -113,6 +113,33 @@ func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) { } } +func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) { + const providerValue = "private-provider-value" + const promptValue = "private-operator-prompt-text" + runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{ + "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n", + "operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue), + }) + profiles, err := runtime.SelectedProfilesYAML() + if err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "") + if err != nil { + t.Fatal(err) + } + explain, err := ExplanationYAML(loaded) + if err != nil { + t.Fatal(err) + } + combined := string(profiles) + string(explain) + for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} { + if strings.Contains(combined, value) { + t.Fatalf("safe views exposed %q: %s", value, combined) + } + } +} + func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) { t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print") text := `version: 2