eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the debug path to provide raw LLM prompt and response artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:37:26 -05:00
parent 451f6c0bb9
commit 68ec69f2e4
9 changed files with 389 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
internal/framework/pipeline/debug.go
View File

@@ -165,19 +165,26 @@ type debugStructuredCompletionResponse struct {
TotalTokens int `json:"total_tokens,omitempty"`
}
type debugStructuredLLMCall struct {
Request debugStructuredCompletionRequest `json:"request"`
Response debugStructuredCompletionResponse `json:"response,omitempty"`
Error string `json:"error,omitempty"`
type debugLLMPromptArtifact struct {
CallID string `json:"call_id"`
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
}
type debugLLMResponseArtifact struct {
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
CanonicalPath string `json:"canonical_path"`
ScopedPath string `json:"scoped_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Error bool `json:"error,omitempty"`
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
Error bool `json:"error,omitempty"`
}
type debugValidationRequest struct {
@@ -232,44 +239,65 @@ func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugReco
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.mu.Lock()
client.counter++
callID := fmt.Sprintf("call-%04d", client.counter)
callID := fmt.Sprintf("%04d", client.counter)
client.mu.Unlock()
started := time.Now().UTC()
response, err := client.inner.CompleteStructured(ctx, req, out)
completed := time.Now().UTC()
payload := debugStructuredLLMCall{
Request: debugCompletionRequest(req),
Response: debugCompletionResponse(response),
}
errorText := ""
if err != nil {
payload.Error = err.Error()
errorText = err.Error()
}
envelope := debugTimedEnvelope{
scopePrefix := cleanDebugPath(req.StageName)
if scopePrefix == "_" {
scopePrefix = "llm"
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scopePrefix = scope.prefix
}
promptPath := ""
var writeErr error
if response.Debug != nil && response.Debug.Prompt != nil {
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMPromptArtifact{
CallID: callID,
Prompt: response.Debug.Prompt,
},
}))
}
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: payload,
Error: payload.Error,
}
canonicalPath := path.Join("llm", callID+".json")
writeErr := writeDebugTimed(client.recorder, canonicalPath, envelope)
Payload: debugLLMResponseArtifact{
CallID: callID,
Response: debugResponseMaterial(response),
Fallback: debugCompletionFallback(response),
Error: errorText,
},
Error: errorText,
}))
callRef := debugLLMCallReference{
CallID: callID,
CanonicalPath: canonicalPath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Error: err != nil,
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
Error: err != nil,
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scopedPath := path.Join(scope.prefix, "llm-"+callID+".json")
callRef.ScopedPath = scopedPath
scopedWriteErr := writeDebugTimed(client.recorder, scopedPath, envelope)
if scopedWriteErr != nil {
writeErr = errors.Join(writeErr, scopedWriteErr)
}
scope.record(callRef)
}
if err != nil {
@@ -548,6 +576,37 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de
}
}
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
if response.Debug != nil && response.Debug.Response != nil {
return nil
}
fallback := debugCompletionResponse(response)
if fallback.Content == "" &&
fallback.Provider == "" &&
fallback.Model == "" &&
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 {
return nil
}
return &fallback
}
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
if response.Debug == nil {
return nil
}
return response.Debug.Response
}
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
if response.Debug == nil || response.Debug.Response == nil {
return ""
}
return response.Debug.Response.ModelName
}
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
req.Schema.JSONSchema = nil
out := debugValidationRequest{

2
internal/framework/pipeline/runner.go
View File

@@ -441,7 +441,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
var acceptedWarnings []contracts.Warning
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d-attempt-%02d", chunk.Index+1, attempt))
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := extractor.Extract(attemptCtx, contracts.ExtractionRequest{
Source: doc,

47
internal/framework/pipeline/runner_test.go
View File

@@ -1310,22 +1310,36 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.CallID != "call-0001" || call.CanonicalPath != "llm/call-0001.json" || call.ScopedPath != "chunk/attempt-01/llm-call-0001.json" {
t.Fatalf("llm call reference = %#v, want canonical and scoped paths", call)
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" {
t.Fatalf("llm call reference = %#v, want prompt and response paths", call)
}
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Error {
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Model != "debug-model" || call.Error {
t.Fatalf("llm call metadata = %#v, want prompt/profile and no call error", call)
}
scoped := recorder.envelope(t, call.ScopedPath)
scopedPayload, ok := scoped.Payload.(debugStructuredLLMCall)
prompt := recorder.envelope(t, call.PromptPath)
promptPayload, ok := prompt.Payload.(debugLLMPromptArtifact)
if !ok {
t.Fatalf("scoped payload type = %T, want debugStructuredLLMCall", scoped.Payload)
t.Fatalf("prompt payload type = %T, want debugLLMPromptArtifact", prompt.Payload)
}
if scopedPayload.Response.Content != `{"raw":true}` {
t.Fatalf("scoped response content = %q, want raw LLM response", scopedPayload.Response.Content)
if promptPayload.Prompt == nil || len(promptPayload.Prompt.Messages) != 1 || promptPayload.Prompt.Messages[0].Content != "raw prompt text" {
t.Fatalf("prompt payload = %#v, want raw prompt message", promptPayload)
}
response := recorder.envelope(t, call.ResponsePath)
responsePayload, ok := response.Payload.(debugLLMResponseArtifact)
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != `{"raw":true}` {
t.Fatalf("response payload = %#v, want raw LLM response", responsePayload)
}
if _, ok := recorder.payloads["llm/call-0001.json"]; ok {
t.Fatalf("old canonical LLM debug artifact was written")
}
if _, ok := recorder.payloads["chunk/attempt-01/llm-call-0001.json"]; ok {
t.Fatalf("old scoped LLM debug artifact was written")
}
_ = recorder.envelope(t, call.CanonicalPath)
}
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
@@ -2332,7 +2346,22 @@ func (client debugResponseLLMClient) CompleteStructured(ctx context.Context, req
}
return contracts.StructuredCompletionResponse{
Content: append([]byte(nil), client.content...),
Model: "debug-model",
ProfileID: profileID,
Debug: &contracts.LLMDebugMaterial{
Prompt: &contracts.LLMDebugPrompt{
PromptID: req.PromptID,
SelectedProfileID: profileID,
Messages: []contracts.LLMDebugMessage{
{Role: "user", Content: "raw prompt text"},
},
},
Response: &contracts.LLMDebugResponse{
Content: string(client.content),
SelectedProfileID: profileID,
ModelName: "debug-model",
},
},
}, client.err
}