diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index 2e5db94..b3057c7 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -14,8 +14,9 @@ Notarius relies on the root `promptkit` package to: - construct an `Engine` with filesystem-backed prompt, schema, and optional profile sources; -- prepare and run a `RunRequest` with named inline artifacts, variables, - a direct session ID, prompt identity, and profile selection; +- prepare one frozen execution from a `RunRequest` with named inline artifacts, + variables, a direct session ID, prompt identity, and profile selection, then + record credential-redacted details and run that exact execution; - return rendered debug material, validated structured output, selected profile, backend, effective model metadata, and token usage; - register the optional conventional `local` backend through `BackendLocal`, @@ -58,8 +59,9 @@ Notarius retains its application-wide scheduled client around the PromptKit adapter. PromptKit may apply a narrower limit for the selected backend; endpoint-only profiles have no such backend limit. The adapter translates PromptKit capacity rejection into the provider-neutral Notarius -`ErrLLMCapacityExceeded` contract and leaves retries to the calling pipeline -stage. +`ErrLLMCapacityExceeded` contract. It may include the normalized selected +backend ID in safe diagnostic context, without exposing PromptKit's capacity +error type, and leaves retries to the calling pipeline stage. ## Notarius Ownership diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 1af5b62..c21da28 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -28,13 +28,14 @@ durable schemas. Those responsibilities remain with the module and its named material to a PromptKit inline artifact while preserving its origin URI, maps the trimmed request session to PromptKit's direct per-run session field, retains the same value as the `session_id` prompt variable for maintained -prompt compatibility, forwards profile selection, then prepares and runs the -prompt. The direct field is authoritative for provider session behavior. A -session ID is a stable, non-secret correlation identifier and may be exposed -to providers and provider observability. The adapter returns PromptKit’s -validated raw bytes rather than re-encoding the decoded target. An empty -optional material is represented as one space so its named input is retained -by PromptKit. +prompt compatibility, and forwards profile selection. It then creates one +frozen prepared execution, captures its caller-owned credential-redacted +details for debug material, and executes that exact snapshot. The direct field +is authoritative for provider session behavior. A session ID is a stable, +non-secret correlation identifier and may be exposed to providers and provider +observability. The adapter returns PromptKit’s validated raw bytes rather than +re-encoding the decoded target. An empty optional material is represented as +one space so its named input is retained by PromptKit. Client construction may also receive a run-wide reasoning-effort override from the CLI factory boundary. The adapter copies the caller-owned pointer and @@ -155,10 +156,12 @@ than output-validation failures. When PromptKit rejects backend admission before generation, the adapter maps `promptkit.ErrCapacityExceeded` to `contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted -upstream diagnostic without exposing the PromptKit sentinel as a framework -contract. A canceled caller context takes precedence. The adapter does not -retry capacity failures; the pipeline's existing binding attempt policy sees -the operational error and decides whether to rerun the complete operation. +upstream diagnostic without exposing the PromptKit sentinel or capacity-error +type as a framework contract. When supplied, the normalized selected backend +ID appears only in that safe application-owned diagnostic context. A canceled +caller context takes precedence. The adapter does not retry capacity failures; +the pipeline's existing binding attempt policy sees the operational error and +decides whether to rerun the complete operation. Prompt-declared repair is executed within PromptKit’s structured-output flow. The current production D&D prompt manifests set repair attempts to zero. That diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index ba5a241..7269f11 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -139,19 +139,31 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. Vars: promptKitVars(req, sessionID), Execution: execution, } - prepared, err := c.engine.Prepare(ctx, runReq) + prepared, err := c.engine.PrepareExecution(ctx, runReq) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return contracts.StructuredCompletionResponse{}, ctxErr } return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err)) } - result, err := c.engine.Run(ctx, runReq) + defer prepared.Discard() + preparedDetails := prepared.Details() + result, err := c.engine.RunPrepared(ctx, prepared) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return contracts.StructuredCompletionResponse{}, ctxErr } if errors.Is(err, promptkit.ErrCapacityExceeded) { + var capacityErr *promptkit.CapacityError + if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" { + return contracts.StructuredCompletionResponse{}, fmt.Errorf( + "run PromptKit prompt %q on backend %q: %w: %v", + promptID, + strings.TrimSpace(capacityErr.BackendID), + contracts.ErrLLMCapacityExceeded, + redactPromptKitError(err), + ) + } return contracts.StructuredCompletionResponse{}, fmt.Errorf( "run PromptKit prompt %q: %w: %v", promptID, @@ -164,7 +176,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. if result == nil { return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput) } - response := c.responseFromResult(result, prepared) + response := c.responseFromResult(result, &preparedDetails) if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid { return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; ")) } diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 1de2d16..d39991c 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "io" + "io/fs" "net/http" "net/http/httptest" "os" @@ -114,6 +115,104 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { } } +func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) { + const initialPrompt = `id: snapshot.test +version: "v1" +default_profile: snapshot-profile +inputs: + - name: transcript + required: true + content_type: application/json +messages: + - role: user + content: "Snapshot A: {{ input \"transcript\" }}" +output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 +` + const updatedPrompt = `id: snapshot.test +version: "v1" +default_profile: snapshot-profile +inputs: + - name: transcript + required: true + content_type: application/json +messages: + - role: user + content: "Snapshot B: {{ input \"transcript\" }}" +output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 +` + source := &switchingPromptFS{files: fstest.MapFS{ + "snapshot.test.yaml": {Data: []byte(initialPrompt)}, + }} + fake := &fakePromptKitLLM{content: `{"ok":true}`} + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: newTestPromptKitAssets(t), + EngineOptions: []promptkit.Option{ + promptkit.WithPromptFS(source, "."), + promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}), + promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "snapshot-profile", + BackendID: "snapshot-backend", + Model: "snapshot-model", + })), + promptkit.WithLLMClient(fake), + }, + }) + if err != nil { + t.Fatalf("NewPromptKitClient() error = %v, want nil", err) + } + + opened := source.holdNextPromptRead() + defer source.resumePromptRead() + type completion struct { + response contracts.StructuredCompletionResponse + err error + } + completed := make(chan completion, 1) + go func() { + var out struct { + OK bool `json:"ok"` + } + response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "snapshot.test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + completed <- completion{response: response, err: callErr} + }() + select { + case <-opened: + case <-time.After(time.Second): + t.Fatal("PromptKit did not read the prompt source") + } + source.replacePrompt([]byte(updatedPrompt)) + source.resumePromptRead() + + result := <-completed + if result.err != nil { + t.Fatalf("CompleteStructured() error = %v, want nil", result.err) + } + if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 { + t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug) + } + debugContent := result.response.Debug.Prompt.Messages[0].Content + generatedContent := fake.lastRequest().Prompt.Messages[0].Content + if debugContent != generatedContent { + t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent) + } + if !strings.Contains(debugContent, "Snapshot A") || strings.Contains(debugContent, "Snapshot B") { + t.Fatalf("snapshot content = %q, want the source read before it changed", debugContent) + } +} + func TestPromptKitClientRetainsSessionPromptVariable(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) @@ -674,8 +773,13 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) { t.Fatalf("capacity error exposes PromptKit sentinel: %v", capacityErr) } if !strings.Contains(capacityErr.Error(), `run PromptKit prompt "adapter.direct-session"`) || + !strings.Contains(capacityErr.Error(), `backend "limited-backend"`) || !strings.Contains(capacityErr.Error(), "backend capacity exceeded") { - t.Fatalf("capacity error = %q, want prompt context and upstream diagnostic", capacityErr) + t.Fatalf("capacity error = %q, want prompt, backend, and upstream diagnostic context", capacityErr) + } + var upstreamCapacityErr *promptkit.CapacityError + if errors.As(capacityErr, &upstreamCapacityErr) { + t.Fatalf("capacity error exposes PromptKit capacity type: %v", capacityErr) } if calls := atomic.LoadInt32(&fake.calls); calls != 1 { t.Fatalf("provider calls after capacity rejection = %d, want 1", calls) @@ -910,6 +1014,61 @@ output: return registry } +type switchingPromptFS struct { + mu sync.Mutex + files fstest.MapFS + pauseNextOpen bool + opened chan struct{} + resume chan struct{} +} + +func (f *switchingPromptFS) Open(name string) (fs.File, error) { + f.mu.Lock() + file, err := f.files.Open(name) + pause := f.pauseNextOpen && name == "snapshot.test.yaml" + resume := f.resume + if pause { + f.pauseNextOpen = false + close(f.opened) + } + f.mu.Unlock() + if pause { + <-resume + } + return file, err +} + +func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.files.ReadDir(name) +} + +func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} { + f.mu.Lock() + defer f.mu.Unlock() + f.pauseNextOpen = true + f.opened = make(chan struct{}) + f.resume = make(chan struct{}) + return f.opened +} + +func (f *switchingPromptFS) replacePrompt(content []byte) { + f.mu.Lock() + defer f.mu.Unlock() + f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)} +} + +func (f *switchingPromptFS) resumePromptRead() { + f.mu.Lock() + resume := f.resume + f.resume = nil + f.mu.Unlock() + if resume != nil { + close(resume) + } +} + type fakePromptKitLLM struct { content string allowEmpty bool