eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Execute PromptKit requests from prepared snapshots

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 16:17:40 +00:00
parent b5c86de4d7
commit 67b315099d
4 changed files with 195 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
internal/framework/llm/promptkit_client.go
View File

@@ -139,19 +139,31 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
Vars: promptKitVars(req, sessionID),
Execution: execution,
}
prepared, err := c.engine.Prepare(ctx, runReq)
prepared, err := c.engine.PrepareExecution(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
result, err := c.engine.Run(ctx, runReq)
defer prepared.Discard()
preparedDetails := prepared.Details()
result, err := c.engine.RunPrepared(ctx, prepared)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
"run PromptKit prompt %q on backend %q: %w: %v",
promptID,
strings.TrimSpace(capacityErr.BackendID),
contracts.ErrLLMCapacityExceeded,
redactPromptKitError(err),
)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
"run PromptKit prompt %q: %w: %v",
promptID,
@@ -164,7 +176,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, prepared)
response := c.responseFromResult(result, &preparedDetails)
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}

161
internal/framework/llm/promptkit_client_test.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
@@ -114,6 +115,104 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
}
}
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
const initialPrompt = `id: snapshot.test
version: "v1"
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot A: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`
const updatedPrompt = `id: snapshot.test
version: "v1"
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot B: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`
source := &switchingPromptFS{files: fstest.MapFS{
"snapshot.test.yaml": {Data: []byte(initialPrompt)},
}}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{
promptkit.WithPromptFS(source, "."),
promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}),
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "snapshot-profile",
BackendID: "snapshot-backend",
Model: "snapshot-model",
})),
promptkit.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
opened := source.holdNextPromptRead()
defer source.resumePromptRead()
type completion struct {
response contracts.StructuredCompletionResponse
err error
}
completed := make(chan completion, 1)
go func() {
var out struct {
OK bool `json:"ok"`
}
response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "snapshot.test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
completed <- completion{response: response, err: callErr}
}()
select {
case <-opened:
case <-time.After(time.Second):
t.Fatal("PromptKit did not read the prompt source")
}
source.replacePrompt([]byte(updatedPrompt))
source.resumePromptRead()
result := <-completed
if result.err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", result.err)
}
if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 {
t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug)
}
debugContent := result.response.Debug.Prompt.Messages[0].Content
generatedContent := fake.lastRequest().Prompt.Messages[0].Content
if debugContent != generatedContent {
t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent)
}
if !strings.Contains(debugContent, "Snapshot A") || strings.Contains(debugContent, "Snapshot B") {
t.Fatalf("snapshot content = %q, want the source read before it changed", debugContent)
}
}
func TestPromptKitClientRetainsSessionPromptVariable(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
@@ -674,8 +773,13 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
t.Fatalf("capacity error exposes PromptKit sentinel: %v", capacityErr)
}
if !strings.Contains(capacityErr.Error(), `run PromptKit prompt "adapter.direct-session"`) ||
!strings.Contains(capacityErr.Error(), `backend "limited-backend"`) ||
!strings.Contains(capacityErr.Error(), "backend capacity exceeded") {
t.Fatalf("capacity error = %q, want prompt context and upstream diagnostic", capacityErr)
t.Fatalf("capacity error = %q, want prompt, backend, and upstream diagnostic context", capacityErr)
}
var upstreamCapacityErr *promptkit.CapacityError
if errors.As(capacityErr, &upstreamCapacityErr) {
t.Fatalf("capacity error exposes PromptKit capacity type: %v", capacityErr)
}
if calls := atomic.LoadInt32(&fake.calls); calls != 1 {
t.Fatalf("provider calls after capacity rejection = %d, want 1", calls)
@@ -910,6 +1014,61 @@ output:
return registry
}
type switchingPromptFS struct {
mu sync.Mutex
files fstest.MapFS
pauseNextOpen bool
opened chan struct{}
resume chan struct{}
}
func (f *switchingPromptFS) Open(name string) (fs.File, error) {
f.mu.Lock()
file, err := f.files.Open(name)
pause := f.pauseNextOpen && name == "snapshot.test.yaml"
resume := f.resume
if pause {
f.pauseNextOpen = false
close(f.opened)
}
f.mu.Unlock()
if pause {
<-resume
}
return file, err
}
func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.files.ReadDir(name)
}
func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} {
f.mu.Lock()
defer f.mu.Unlock()
f.pauseNextOpen = true
f.opened = make(chan struct{})
f.resume = make(chan struct{})
return f.opened
}
func (f *switchingPromptFS) replacePrompt(content []byte) {
f.mu.Lock()
defer f.mu.Unlock()
f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)}
}
func (f *switchingPromptFS) resumePromptRead() {
f.mu.Lock()
resume := f.resume
f.resume = nil
f.mu.Unlock()
if resume != nil {
close(resume)
}
}
type fakePromptKitLLM struct {
content string
allowEmpty bool