eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Verify native profile runtime behavior

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-04 01:10:10 +00:00
parent b81bd28865
commit 5525fe875e
5 changed files with 198 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/roadmap/implementation.md
View File

@@ -1,7 +1,7 @@
# Implementation Plan: PromptKit v0.12.0 Adoption
**Last updated:** 2026-10-04
**Status:** In progress; Stages 1–6 completed.
**Status:** In progress; Stages 1–7 completed.
**Audience:** Luna coding agent.
## Scope and Execution
@@ -407,6 +407,13 @@ integration tests where their established behavior is affected.
and `git diff --check`. Review whether each new test addresses a distinct changed
boundary. Commit the native-profile and runtime verification unit.
**Completion marker:** Stage 7 complete. Native profile inheritance, control
presence, provider-map overlays, application backend authority, selected
credential rejection, catalog-ID handling, shared backend capacity, and safe
inspection behavior are covered. The six targeted packages passed, along with
focused wire, timeout, capacity, credential, and safe-view tests; `git diff
--check` passed.
## Stage 8 — Document the Migration and Update Attribution
**Primary files:** `docs/config.md`, `docs/internal/llm-runtime.md`,

11
internal/framework/llm/adapter_capacity_test.go
View File

@@ -24,6 +24,7 @@ import (
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
entered := make(chan struct{}, 3)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -51,6 +52,16 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Fatal(err)
}
}
operatorBackend := `backends:
shared:
endpoint: https://operator.invalid/v1
concurrency_limit: 77
queue_capacity: 88
api_key_env: OPERATOR_BACKEND_KEY
`
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)

122
internal/framework/llm/adapter_test.go
View File

@@ -30,10 +30,22 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
})
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
documents := map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
}
if operatorPrompt != "" {
documents["operator-prompt.yaml"] = operatorPrompt
}
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
}
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -69,15 +81,18 @@ func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs in
}))
t.Cleanup(server.Close)
t.Setenv("ADAPTER_KEY", "adapter-secret")
base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs)
dir := profileLayout(t, base, "")
main := filepath.Join(dir, "config.yml")
profile := "profiles:\n base:\n backend: offline\n model: full-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
dir := t.TempDir()
profileDirectory := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDirectory, 0700); err != nil {
t.Fatal(err)
}
if operatorPrompt != "" {
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(operatorPrompt), 0600); err != nil {
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
@@ -388,6 +403,91 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
}
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
cases := []struct {
name string
parentTemperature string
childTemperature string
wantTemperature bool
}{
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
{name: "omitted", wantTemperature: false},
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
operatorBackend := `backends:
openrouter:
endpoint: https://operator.invalid/v1
concurrency_limit: 91
queue_capacity: 92
api_key_env: CATALOG_TRAP_KEY
extra_params:
routing:
inherited: backend
overridden: backend
cleared: remove-me
`
documents := map[string]string{
"unrelated-profile-file.yml": profileDocument,
"provider-tuning.yaml": operatorBackend,
}
var wire map[string]json.RawMessage
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
t.Errorf("application credential was not authoritative: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
t.Error(err)
}
}, "openrouter", documents)
profile := runtime.Profiles["proposal"]
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
}
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if count.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", count.Load())
}
temperature, hasTemperature := wire["temperature"]
if hasTemperature != tc.wantTemperature {
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
}
if tc.wantTemperature {
var value float64
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
}
}
var routing map[string]any
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
}
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
t.Fatalf("recursively overlaid provider fields = %#v", routing)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace inherited provider field: %#v", routing)
}
if response.Model != "full-model" || response.Prepared == nil {
t.Fatalf("execution did not use selected profile: %+v", response)
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
@@ -409,7 +509,7 @@ func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testin
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
@@ -480,7 +580,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
})
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {

40
internal/framework/llm/runtime_assembly_test.go
View File

@@ -124,6 +124,46 @@ func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
}
}
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
"unrelated-name.yaml": `profiles:
"parent~/shared":
backend: offline
temperature: 0.4
top_p: 0.7
extra_params:
routing:
retained: from-parent
overridden: parent
cleared: remove-me
"child profile":
extends: "parent~/shared"
model: child-model
extra_params:
routing:
overridden: child
child_only: present
cleared: null
`,
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
if err != nil {
t.Fatal(err)
}
profile := runtime.Profiles["proposal"]
target := profile.EffectiveModelParams
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
t.Fatalf("inherited profile target = %+v", target)
}
routing, ok := target.ExtraParams["routing"].(map[string]any)
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",

27
internal/framework/llm/runtime_inspection_test.go
View File

@@ -113,6 +113,33 @@ func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
}
}
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
const providerValue = "private-provider-value"
const promptValue = "private-operator-prompt-text"
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
})
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
if strings.Contains(combined, value) {
t.Fatalf("safe views exposed %q: %s", value, combined)
}
}
}
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
text := `version: 2