Execute PromptKit requests from prepared snapshots

docs/integrations/pkg-promptkit.md

@@ -14,8 +14,9 @@ Notarius relies on the root `promptkit` package to:
 
 - construct an `Engine` with filesystem-backed prompt, schema, and optional
   profile sources;
-- prepare and run a `RunRequest` with named inline artifacts, variables,
+- prepare one frozen execution from a `RunRequest` with named inline artifacts,
-  a direct session ID, prompt identity, and profile selection;
+  variables, a direct session ID, prompt identity, and profile selection, then
+  record credential-redacted details and run that exact execution;
 - return rendered debug material, validated structured output, selected
   profile, backend, effective model metadata, and token usage;
 - register the optional conventional `local` backend through `BackendLocal`,
@@ -58,8 +59,9 @@ Notarius retains its application-wide scheduled client around the PromptKit
 adapter. PromptKit may apply a narrower limit for the selected backend;
 endpoint-only profiles have no such backend limit. The adapter translates
 PromptKit capacity rejection into the provider-neutral Notarius
-`ErrLLMCapacityExceeded` contract and leaves retries to the calling pipeline
+`ErrLLMCapacityExceeded` contract. It may include the normalized selected
-stage.
+backend ID in safe diagnostic context, without exposing PromptKit's capacity
+error type, and leaves retries to the calling pipeline stage.
 
 ## Notarius Ownership
 

docs/internal/llm.md

@@ -28,13 +28,14 @@ durable schemas. Those responsibilities remain with the module and its
 named material to a PromptKit inline artifact while preserving its origin URI,
 maps the trimmed request session to PromptKit's direct per-run session field,
 retains the same value as the `session_id` prompt variable for maintained
-prompt compatibility, forwards profile selection, then prepares and runs the
+prompt compatibility, and forwards profile selection. It then creates one
-prompt. The direct field is authoritative for provider session behavior. A
+frozen prepared execution, captures its caller-owned credential-redacted
-session ID is a stable, non-secret correlation identifier and may be exposed
+details for debug material, and executes that exact snapshot. The direct field
-to providers and provider observability. The adapter returns PromptKit’s
+is authoritative for provider session behavior. A session ID is a stable,
-validated raw bytes rather than re-encoding the decoded target. An empty
+non-secret correlation identifier and may be exposed to providers and provider
-optional material is represented as one space so its named input is retained
+observability. The adapter returns PromptKit’s validated raw bytes rather than
-by PromptKit.
+re-encoding the decoded target. An empty optional material is represented as
+one space so its named input is retained by PromptKit.
 
 Client construction may also receive a run-wide reasoning-effort override from
 the CLI factory boundary. The adapter copies the caller-owned pointer and
@@ -155,10 +156,12 @@ than output-validation failures.
 When PromptKit rejects backend admission before generation, the adapter maps
 `promptkit.ErrCapacityExceeded` to
 `contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted
-upstream diagnostic without exposing the PromptKit sentinel as a framework
+upstream diagnostic without exposing the PromptKit sentinel or capacity-error
-contract. A canceled caller context takes precedence. The adapter does not
+type as a framework contract. When supplied, the normalized selected backend
-retry capacity failures; the pipeline's existing binding attempt policy sees
+ID appears only in that safe application-owned diagnostic context. A canceled
-the operational error and decides whether to rerun the complete operation.
+caller context takes precedence. The adapter does not retry capacity failures;
+the pipeline's existing binding attempt policy sees the operational error and
+decides whether to rerun the complete operation.
 
 Prompt-declared repair is executed within PromptKit’s structured-output flow.
 The current production D&D prompt manifests set repair attempts to zero. That

internal/framework/llm/promptkit_client.go

@@ -139,19 +139,31 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 		Vars:          promptKitVars(req, sessionID),
 		Execution:     execution,
 	}
-	prepared, err := c.engine.Prepare(ctx, runReq)
+	prepared, err := c.engine.PrepareExecution(ctx, runReq)
 	if err != nil {
 		if ctxErr := ctx.Err(); ctxErr != nil {
 			return contracts.StructuredCompletionResponse{}, ctxErr
 		}
 		return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
 	}
-	result, err := c.engine.Run(ctx, runReq)
+	defer prepared.Discard()
+	preparedDetails := prepared.Details()
+	result, err := c.engine.RunPrepared(ctx, prepared)
 	if err != nil {
 		if ctxErr := ctx.Err(); ctxErr != nil {
 			return contracts.StructuredCompletionResponse{}, ctxErr
 		}
 		if errors.Is(err, promptkit.ErrCapacityExceeded) {
+			var capacityErr *promptkit.CapacityError
+			if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
+				return contracts.StructuredCompletionResponse{}, fmt.Errorf(
+					"run PromptKit prompt %q on backend %q: %w: %v",
+					promptID,
+					strings.TrimSpace(capacityErr.BackendID),
+					contracts.ErrLLMCapacityExceeded,
+					redactPromptKitError(err),
+				)
+			}
 			return contracts.StructuredCompletionResponse{}, fmt.Errorf(
 				"run PromptKit prompt %q: %w: %v",
 				promptID,
@@ -164,7 +176,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 	if result == nil {
 		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
 	}
-	response := c.responseFromResult(result, prepared)
+	response := c.responseFromResult(result, &preparedDetails)
 	if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
 		return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
 	}

internal/framework/llm/promptkit_client_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/fs"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -114,6 +115,104 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
 	}
 }
 
+func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
+	const initialPrompt = `id: snapshot.test
+version: "v1"
+default_profile: snapshot-profile
+inputs:
+  - name: transcript
+    required: true
+    content_type: application/json
+messages:
+  - role: user
+    content: "Snapshot A: {{ input \"transcript\" }}"
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: adapter.schema.json
+  repair_attempts: 0
+`
+	const updatedPrompt = `id: snapshot.test
+version: "v1"
+default_profile: snapshot-profile
+inputs:
+  - name: transcript
+    required: true
+    content_type: application/json
+messages:
+  - role: user
+    content: "Snapshot B: {{ input \"transcript\" }}"
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: adapter.schema.json
+  repair_attempts: 0
+`
+	source := &switchingPromptFS{files: fstest.MapFS{
+		"snapshot.test.yaml": {Data: []byte(initialPrompt)},
+	}}
+	fake := &fakePromptKitLLM{content: `{"ok":true}`}
+	client, err := NewPromptKitClient(PromptKitClientConfig{
+		Assets: newTestPromptKitAssets(t),
+		EngineOptions: []promptkit.Option{
+			promptkit.WithPromptFS(source, "."),
+			promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}),
+			promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
+				ID:        "snapshot-profile",
+				BackendID: "snapshot-backend",
+				Model:     "snapshot-model",
+			})),
+			promptkit.WithLLMClient(fake),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
+	}
+
+	opened := source.holdNextPromptRead()
+	defer source.resumePromptRead()
+	type completion struct {
+		response contracts.StructuredCompletionResponse
+		err      error
+	}
+	completed := make(chan completion, 1)
+	go func() {
+		var out struct {
+			OK bool `json:"ok"`
+		}
+		response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+			PromptID: "snapshot.test",
+			Inputs: contracts.LLMInputSet{
+				"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+			},
+		}, &out)
+		completed <- completion{response: response, err: callErr}
+	}()
+	select {
+	case <-opened:
+	case <-time.After(time.Second):
+		t.Fatal("PromptKit did not read the prompt source")
+	}
+	source.replacePrompt([]byte(updatedPrompt))
+	source.resumePromptRead()
+
+	result := <-completed
+	if result.err != nil {
+		t.Fatalf("CompleteStructured() error = %v, want nil", result.err)
+	}
+	if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 {
+		t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug)
+	}
+	debugContent := result.response.Debug.Prompt.Messages[0].Content
+	generatedContent := fake.lastRequest().Prompt.Messages[0].Content
+	if debugContent != generatedContent {
+		t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent)
+	}
+	if !strings.Contains(debugContent, "Snapshot A") || strings.Contains(debugContent, "Snapshot B") {
+		t.Fatalf("snapshot content = %q, want the source read before it changed", debugContent)
+	}
+}
+
 func TestPromptKitClientRetainsSessionPromptVariable(t *testing.T) {
 	fake := &fakePromptKitLLM{content: `{"ok":true}`}
 	client := newTestPromptKitClient(t, fake)
@@ -674,8 +773,13 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
 		t.Fatalf("capacity error exposes PromptKit sentinel: %v", capacityErr)
 	}
 	if !strings.Contains(capacityErr.Error(), `run PromptKit prompt "adapter.direct-session"`) ||
+		!strings.Contains(capacityErr.Error(), `backend "limited-backend"`) ||
 		!strings.Contains(capacityErr.Error(), "backend capacity exceeded") {
-		t.Fatalf("capacity error = %q, want prompt context and upstream diagnostic", capacityErr)
+		t.Fatalf("capacity error = %q, want prompt, backend, and upstream diagnostic context", capacityErr)
+	}
+	var upstreamCapacityErr *promptkit.CapacityError
+	if errors.As(capacityErr, &upstreamCapacityErr) {
+		t.Fatalf("capacity error exposes PromptKit capacity type: %v", capacityErr)
 	}
 	if calls := atomic.LoadInt32(&fake.calls); calls != 1 {
 		t.Fatalf("provider calls after capacity rejection = %d, want 1", calls)
@@ -910,6 +1014,61 @@ output:
 	return registry
 }
 
+type switchingPromptFS struct {
+	mu            sync.Mutex
+	files         fstest.MapFS
+	pauseNextOpen bool
+	opened        chan struct{}
+	resume        chan struct{}
+}
+
+func (f *switchingPromptFS) Open(name string) (fs.File, error) {
+	f.mu.Lock()
+	file, err := f.files.Open(name)
+	pause := f.pauseNextOpen && name == "snapshot.test.yaml"
+	resume := f.resume
+	if pause {
+		f.pauseNextOpen = false
+		close(f.opened)
+	}
+	f.mu.Unlock()
+	if pause {
+		<-resume
+	}
+	return file, err
+}
+
+func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.files.ReadDir(name)
+}
+
+func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pauseNextOpen = true
+	f.opened = make(chan struct{})
+	f.resume = make(chan struct{})
+	return f.opened
+}
+
+func (f *switchingPromptFS) replacePrompt(content []byte) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)}
+}
+
+func (f *switchingPromptFS) resumePromptRead() {
+	f.mu.Lock()
+	resume := f.resume
+	f.resume = nil
+	f.mu.Unlock()
+	if resume != nil {
+		close(resume)
+	}
+}
+
 type fakePromptKitLLM struct {
 	content     string
 	allowEmpty  bool