eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Verify native prompt overrides and snapshots

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-04 01:00:46 +00:00
parent 9b82eb7da2
commit b81bd28865
2 changed files with 166 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/roadmap/implementation.md
View File

@@ -1,7 +1,7 @@
# Implementation Plan: PromptKit v0.12.0 Adoption
**Last updated:** 2026-10-04
**Status:** In progress; Stages 1–5 completed.
**Status:** In progress; Stages 1–6 completed.
**Audience:** Luna coding agent.
## Scope and Execution
@@ -352,6 +352,14 @@ tests. Change `adapter.go` only for an actual integration defect.
prepared-adapter tests. Compare semantic messages and selected targets, not old
hash values or exact YAML formatting. Check whitespace and commit.
**Completion marker:** Stage 6 complete. Native operator messages reach both
the prepared snapshot and provider request, with and without shared-variable
references. The adapter's per-call schema and repair budget remain effective
under a valid operator output-contract override. Existing runtimes retain
captured prompt/profile values after source edits, and a new runtime observes
valid edits; the target guard and preparation failure remain covered. The
`internal/framework/llm` tests and `git diff --check` passed.
## Stage 7 — Verify Native Profiles, Backend Authority, and Resilience
**Primary files:** Runtime inspection and adapter tests,

167
internal/framework/llm/adapter_test.go
View File

@@ -30,6 +30,10 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -72,6 +76,11 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
if operatorPrompt != "" {
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(operatorPrompt), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
@@ -83,6 +92,10 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
return runtime, &count, dir
}
func operatorPromptDocument(system, user string) string {
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
}
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id := prompts.PromptIDModuleGlossaryProposal
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
@@ -286,10 +299,153 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
if !found {
t.Fatal("hardening policy missing")
}
backend := runtime.Config.Backends["offline"]
backend.Endpoint = "http://127.0.0.1:1/v1"
runtime.Config.Backends["offline"] = backend
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
}
}
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
cases := []struct {
name string
system string
user string
replies []reply
repairs int
wantSystem string
wantUser string
}{
{
name: "operator text without shared references",
system: "Operator system instructions.",
user: "Operator request without variables.",
replies: []reply{{200, `{"wrong":true}`}, {200, `{"corrections":[]}`}},
repairs: 1,
wantSystem: "Operator system instructions.",
wantUser: "Operator request without variables.",
},
{
name: "operator text with shared references",
system: "Policy: {{.HardeningPolicy}}",
user: "Section: {{.SectionJSON}}",
replies: []reply{{200, `{"corrections":[]}`}},
wantSystem: "Policy: " + prompts.HardeningText(),
wantUser: "Section: []",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var posted []struct {
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, _ := adapterFixtureWithPrompt(t, tc.replies, 0, tc.repairs, func(r *http.Request) {
var envelope struct {
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument(tc.system, tc.user))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
req := adapterRequest(responseschema.CorrectionSetKey)
if tc.name == "operator text with shared references" {
req.Variables["HardeningPolicy"] = "attacker-controlled"
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), req, &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != len(tc.replies) || response.Execution.StructuralRepairs != tc.repairs {
t.Fatalf("schema/repair execution = %d calls, %+v; want %d calls and %d repair", count.Load(), response.Execution, len(tc.replies), tc.repairs)
}
if response.Prepared == nil || len(posted) == 0 || len(posted[0].Messages) != len(response.Prepared.Messages) {
t.Fatal("prepared operator prompt was not dispatched")
}
for i := range response.Prepared.Messages {
if response.Prepared.Messages[i] != posted[0].Messages[i] {
t.Fatal("prepared operator messages differ from the first dispatched request")
}
}
if len(response.Prepared.Messages) != 2 || response.Prepared.Messages[0].Content != tc.wantSystem || response.Prepared.Messages[1].Content != tc.wantUser {
t.Fatalf("prepared messages = %+v", response.Prepared.Messages)
}
if tc.name == "operator text with shared references" {
if strings.Contains(response.Prepared.Messages[0].Content, "attacker-controlled") || !strings.Contains(response.Prepared.Messages[0].Content, prompts.HardeningText()) {
t.Fatal("caller variables replaced the reserved shared value")
}
} else if strings.Contains(response.Prepared.Messages[0].Content+response.Prepared.Messages[1].Content, prompts.HardeningText()) {
t.Fatal("shared text was injected into operator messages that omit its reference")
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
var envelope struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
}
main := filepath.Join(dir, "config.yml")
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
if err != nil {
t.Fatal(err)
}
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
}
}
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
@@ -300,15 +456,6 @@ func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 {
t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load())
}
// PrepareExecution reloads the profile, so a changed target must be blocked.
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n endpoint: http://127.0.0.1:1/v1\n model: full-model\n"), 0600); err != nil {
t.Fatal(err)
}
req = adapterRequest(responseschema.CorrectionSetKey)
_, err = client.CompleteStructured(context.Background(), req, &out)
if err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 0 {
t.Fatalf("target drift dispatched: %v, %d", err, count.Load())
}
}
func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {