diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cabc62d..068d230 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,7 +1,7 @@ # Implementation Plan: PromptKit v0.12.0 Adoption **Last updated:** 2026-10-04 -**Status:** In progress; Stages 1–5 completed. +**Status:** In progress; Stages 1–6 completed. **Audience:** Luna coding agent. ## Scope and Execution @@ -352,6 +352,14 @@ tests. Change `adapter.go` only for an actual integration defect. prepared-adapter tests. Compare semantic messages and selected targets, not old hash values or exact YAML formatting. Check whitespace and commit. +**Completion marker:** Stage 6 complete. Native operator messages reach both +the prepared snapshot and provider request, with and without shared-variable +references. The adapter's per-call schema and repair budget remain effective +under a valid operator output-contract override. Existing runtimes retain +captured prompt/profile values after source edits, and a new runtime observes +valid edits; the target guard and preparation failure remain covered. The +`internal/framework/llm` tests and `git diff --check` passed. + ## Stage 7 — Verify Native Profiles, Backend Authority, and Resilience **Primary files:** Runtime inspection and adapter tests, diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 5f567b0..3516abb 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -30,6 +30,10 @@ type reply struct { } func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) { + return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "") +} + +func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) { t.Helper() var count atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -72,6 +76,11 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil { t.Fatal(err) } + if operatorPrompt != "" { + if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(operatorPrompt), 0600); err != nil { + t.Fatal(err) + } + } loaded, err := config.Load(ck.File(main), "") if err != nil { t.Fatal(err) @@ -83,6 +92,10 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu return runtime, &count, dir } +func operatorPromptDocument(system, user string) string { + return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user) +} + func adapterRequest(schema responseschema.Key) structuredoutput.Request { id := prompts.PromptIDModuleGlossaryProposal vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"} @@ -286,10 +299,153 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { if !found { t.Fatal("hardening policy missing") } + backend := runtime.Config.Backends["offline"] + backend.Endpoint = "http://127.0.0.1:1/v1" + runtime.Config.Backends["offline"] = backend + if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 { + t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load()) + } +} + +func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) { + cases := []struct { + name string + system string + user string + replies []reply + repairs int + wantSystem string + wantUser string + }{ + { + name: "operator text without shared references", + system: "Operator system instructions.", + user: "Operator request without variables.", + replies: []reply{{200, `{"wrong":true}`}, {200, `{"corrections":[]}`}}, + repairs: 1, + wantSystem: "Operator system instructions.", + wantUser: "Operator request without variables.", + }, + { + name: "operator text with shared references", + system: "Policy: {{.HardeningPolicy}}", + user: "Section: {{.SectionJSON}}", + replies: []reply{{200, `{"corrections":[]}`}}, + wantSystem: "Policy: " + prompts.HardeningText(), + wantUser: "Section: []", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var posted []struct { + Messages []structuredoutput.Message `json:"messages"` + } + runtime, count, _ := adapterFixtureWithPrompt(t, tc.replies, 0, tc.repairs, func(r *http.Request) { + var envelope struct { + Messages []structuredoutput.Message `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil { + t.Error(err) + return + } + posted = append(posted, envelope) + }, operatorPromptDocument(tc.system, tc.user)) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + req := adapterRequest(responseschema.CorrectionSetKey) + if tc.name == "operator text with shared references" { + req.Variables["HardeningPolicy"] = "attacker-controlled" + } + var out map[string]any + response, err := client.CompleteStructured(context.Background(), req, &out) + if err != nil { + t.Fatal(err) + } + if int(count.Load()) != len(tc.replies) || response.Execution.StructuralRepairs != tc.repairs { + t.Fatalf("schema/repair execution = %d calls, %+v; want %d calls and %d repair", count.Load(), response.Execution, len(tc.replies), tc.repairs) + } + if response.Prepared == nil || len(posted) == 0 || len(posted[0].Messages) != len(response.Prepared.Messages) { + t.Fatal("prepared operator prompt was not dispatched") + } + for i := range response.Prepared.Messages { + if response.Prepared.Messages[i] != posted[0].Messages[i] { + t.Fatal("prepared operator messages differ from the first dispatched request") + } + } + if len(response.Prepared.Messages) != 2 || response.Prepared.Messages[0].Content != tc.wantSystem || response.Prepared.Messages[1].Content != tc.wantUser { + t.Fatalf("prepared messages = %+v", response.Prepared.Messages) + } + if tc.name == "operator text with shared references" { + if strings.Contains(response.Prepared.Messages[0].Content, "attacker-controlled") || !strings.Contains(response.Prepared.Messages[0].Content, prompts.HardeningText()) { + t.Fatal("caller variables replaced the reserved shared value") + } + } else if strings.Contains(response.Prepared.Messages[0].Content+response.Prepared.Messages[1].Content, prompts.HardeningText()) { + t.Fatal("shared text was injected into operator messages that omit its reference") + } + }) + } +} + +func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) { + var posted []struct { + Model string `json:"model"` + Messages []structuredoutput.Message `json:"messages"` + } + runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { + var envelope struct { + Model string `json:"model"` + Messages []structuredoutput.Message `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil { + t.Error(err) + return + } + posted = append(posted, envelope) + }, operatorPromptDocument("Initial operator instructions.", "Initial payload.")) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + profile := "profiles:\n base:\n backend: offline\n model: edited-model\n" + if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil { + t.Fatal(err) + } + updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.") + if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil { + t.Fatal(err) + } + + var out map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) + if err != nil { + t.Fatal(err) + } + if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" { + t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0]) + } + + main := filepath.Join(dir, "config.yml") + newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient) + if err != nil { + t.Fatal(err) + } + newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) + if err != nil { + t.Fatal(err) + } + if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" { + t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1]) + } } func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) { - runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil) + runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil) client, err := runtime.NewRoleClient("proposal", CallHooks{}) if err != nil { t.Fatal(err) @@ -300,15 +456,6 @@ func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) { if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 { t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load()) } - // PrepareExecution reloads the profile, so a changed target must be blocked. - if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n endpoint: http://127.0.0.1:1/v1\n model: full-model\n"), 0600); err != nil { - t.Fatal(err) - } - req = adapterRequest(responseschema.CorrectionSetKey) - _, err = client.CompleteStructured(context.Background(), req, &out) - if err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 0 { - t.Fatalf("target drift dispatched: %v, %d", err, count.Load()) - } } func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {