eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the debug path to provide raw LLM prompt and response artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:37:26 -05:00
parent 451f6c0bb9
commit 68ec69f2e4
9 changed files with 389 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
internal/framework/llm/scriptorium_client.go
View File

@@ -98,6 +98,13 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
Vars: scriptoriumVars(req),
Metadata: scriptoriumMetadata(req),
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
}
result, err := c.engine.Run(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
@@ -108,7 +115,7 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
}
response := c.responseFromResult(result)
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
}
@@ -121,7 +128,7 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
return response, nil
}
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult) contracts.StructuredCompletionResponse {
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body
if len(content) == 0 {
content = []byte(result.RawOutput)
@@ -142,9 +149,111 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult) co
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
Debug: scriptoriumDebugMaterial(prepared, result),
}
}
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
material := &contracts.LLMDebugMaterial{}
if prepared != nil {
material.Prompt = scriptoriumDebugPrompt(prepared)
}
if result != nil {
material.Response = scriptoriumDebugResponse(result)
}
if material.Prompt == nil && material.Response == nil {
return nil
}
return material
}
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
if prepared == nil {
return nil
}
messages := make([]contracts.LLMDebugMessage, 0, len(prepared.Messages))
for _, message := range prepared.Messages {
messages = append(messages, contracts.LLMDebugMessage{
Role: message.Role,
Content: message.Content,
CacheControl: jsonObject(message.CacheControl),
})
}
return &contracts.LLMDebugPrompt{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: messages,
EffectiveModelParams: jsonObject(prepared.EffectiveModelParams),
OutputContract: jsonObject(prepared.OutputContract),
StructuredOutput: jsonObject(prepared.StructuredOutput),
InputHashes: cloneStringMap(prepared.InputHashes),
}
}
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
if result == nil {
return nil
}
content := result.RawOutput
if content == "" {
content = string(result.Artifact.Body)
}
return &contracts.LLMDebugResponse{
Content: content,
RunID: result.RunID,
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: jsonObject(result.EffectiveModelParams),
InputHashes: cloneStringMap(result.InputHashes),
Validation: jsonObject(result.Validation),
Usage: contracts.LLMDebugUsage{
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
CachedTokens: result.Usage.CachedTokens,
CacheWriteTokens: result.Usage.CacheWriteTokens,
},
}
}
func jsonObject(value any) map[string]any {
if value == nil {
return nil
}
data, err := json.Marshal(value)
if err != nil || string(data) == "null" {
return nil
}
var out map[string]any
if err := json.Unmarshal(data, &out); err != nil {
return nil
}
if len(out) == 0 {
return nil
}
return out
}
func cloneStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
for key, value := range values {
out[key] = value
}
return out
}
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.recorder == nil {
return nil

33
internal/framework/llm/scriptorium_client_test.go
View File

@@ -45,6 +45,25 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("usage = %#v, want mapped token counts", resp)
}
if resp.Debug == nil || resp.Debug.Prompt == nil {
t.Fatalf("debug prompt = nil, want prepared prompt material")
}
if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" {
t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt)
}
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
}
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
}
if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") {
t.Fatalf("debug material contains secret material: %s", debugJSON)
}
gotReq := fake.lastRequest()
if gotReq.Prompt.SessionID != "session-123" {
t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
@@ -100,6 +119,12 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
if got := string(resp.Content); got != `{"bad":true}` {
t.Fatalf("response content = %q, want raw failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` {
t.Fatalf("debug response = %#v, want raw failed output", resp.Debug)
}
if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 {
t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt)
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
@@ -119,13 +144,16 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
if got := string(resp.Content); got != `{"ok":true}` {
t.Fatalf("response content = %q, want raw decode-failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug)
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
@@ -141,6 +169,9 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
t.Fatalf("error = %q, want redacted bearer token", err.Error())
}
if resp.Debug != nil {
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {