eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adapt corrections through PromptKit

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-26 23:50:24 +00:00
parent 85a5b52be7
commit 1a7b20c766
8 changed files with 267 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/framework/llm/promptkit_client.go
View File

@@ -119,6 +119,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
if req.StructuredOutputRepairAttempts != nil && (*req.StructuredOutputRepairAttempts < 0 || *req.StructuredOutputRepairAttempts > 3) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured output repair attempts must be between zero and three")
}
appendedMessages, err := promptKitCorrectionMessages(req.Correction)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion correction: %w", err)
}
promptID := strings.TrimSpace(req.PromptID)
if promptID == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
@@ -133,13 +137,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
}
runReq := promptkit.RunRequest{
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
AppendedMessages: appendedMessages,
}
if req.StructuredOutputRepairAttempts != nil {
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
@@ -213,6 +218,20 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
owned, err := contracts.CloneSemanticCorrection(correction)
if err != nil {
return nil, err
}
if owned == nil {
return nil, nil
}
return []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: string(owned.AssistantResponse)},
{Role: promptkit.RoleUser, Content: owned.UserGuidance},
}, nil
}
func promptKitDebugGenerationError(prepared *promptkit.PreparedRun, generationErr *promptkit.GenerationError) *contracts.LLMDebugResponse {
if generationErr == nil {
return nil

85
internal/framework/llm/promptkit_client_test.go
View File

@@ -1,6 +1,7 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -116,6 +117,74 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
}
}
func TestPromptKitClientAppendsSemanticCorrectionAfterRenderedPrompt(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
request := contracts.StructuredCompletionRequest{
PromptID: "adapter.direct-session",
ProfileID: "explicit-profile",
SessionID: "correction-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
Vars: map[string]any{"custom": "value"},
}
var ordinary map[string]any
if _, err := client.CompleteStructured(context.Background(), request, &ordinary); err != nil {
t.Fatalf("ordinary CompleteStructured() error = %v", err)
}
ordinaryMessages := append([]promptkit.RenderedMessage(nil), fake.lastRequest().Prompt.Messages...)
if len(ordinaryMessages) != 1 {
t.Fatalf("ordinary rendered messages = %#v, want only the declared prompt message", ordinaryMessages)
}
correction, err := contracts.NewSemanticCorrection([]byte(`{"previous":"response"}`), "Return the corrected JSON object.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
request.Correction = correction
var corrected map[string]any
if _, err := client.CompleteStructured(context.Background(), request, &corrected); err != nil {
t.Fatalf("corrected CompleteStructured() error = %v", err)
}
correctedMessages := fake.lastRequest().Prompt.Messages
if len(correctedMessages) != len(ordinaryMessages)+2 {
t.Fatalf("corrected message count = %d, want %d", len(correctedMessages), len(ordinaryMessages)+2)
}
if !reflect.DeepEqual(correctedMessages[:len(ordinaryMessages)], ordinaryMessages) {
t.Fatalf("ordinary rendered prefix changed: got %#v, want %#v", correctedMessages[:len(ordinaryMessages)], ordinaryMessages)
}
if got, want := correctedMessages[len(ordinaryMessages)], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"previous":"response"}`}); got != want {
t.Fatalf("assistant correction message = %#v, want %#v", got, want)
}
if got, want := correctedMessages[len(ordinaryMessages)+1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the corrected JSON object."}); got != want {
t.Fatalf("user correction message = %#v, want %#v", got, want)
}
}
func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *testing.T) {
const sensitiveResponse = "assistant-response-must-not-appear-in-errors"
for _, correction := range []*contracts.SemanticCorrection{
{AssistantResponse: []byte(sensitiveResponse), UserGuidance: " \t"},
{AssistantResponse: bytes.Repeat([]byte(sensitiveResponse), contracts.MaxAssistantResponseBytes/len(sensitiveResponse)+1), UserGuidance: "Use a smaller response."},
} {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{Correction: correction}, &out)
if err == nil || !strings.Contains(err.Error(), "correction") {
t.Fatalf("CompleteStructured() error = %v, want correction validation failure", err)
}
if strings.Contains(err.Error(), sensitiveResponse) {
t.Fatalf("correction validation error leaked response content: %v", err)
}
if got := atomic.LoadInt32(&fake.calls); got != 0 {
t.Fatalf("provider calls = %d, want no provider call after invalid correction", got)
}
}
}
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
const initialPrompt = `id: snapshot.test
version: "v1"
@@ -960,6 +1029,10 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
{Content: `{"ok":true}`, Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
}}
client := newTestPromptKitClient(t, fake)
correction, err := contracts.NewSemanticCorrection([]byte(`{"bad":true}`), "Return the required ok field.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
var out struct {
OK bool `json:"ok"`
@@ -968,6 +1041,7 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
PromptID: "adapter.test",
StructuredOutputRepairAttempts: &attempts,
SessionID: "repair-test",
Correction: correction,
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
@@ -978,6 +1052,17 @@ func TestPromptKitClientRepairsStructuredOutputAndReportsCumulativeUsage(t *test
if got := atomic.LoadInt32(&fake.calls); got != 2 {
t.Fatalf("provider calls = %d, want initial generation and one repair", got)
}
requests := fake.requestsSnapshot()
if len(requests) != 2 || len(requests[0].Prompt.Messages) < 3 {
t.Fatalf("repair requests = %#v, want correction messages on the initial prepared request", requests)
}
messages := requests[0].Prompt.Messages
if got, want := messages[len(messages)-2], (promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: `{"bad":true}`}); got != want {
t.Fatalf("repair assistant correction = %#v, want %#v", got, want)
}
if got, want := messages[len(messages)-1], (promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "Return the required ok field."}); got != want {
t.Fatalf("repair user correction = %#v, want %#v", got, want)
}
if response.RepairAttempts != 1 || response.PromptTokens != 10 || response.CompletionTokens != 16 || response.TotalTokens != 26 {
t.Fatalf("response repair and usage = %#v, want one repair and PromptKit cumulative usage", response)
}