Rewrite the debug path to provide raw LLM prompt and response artifacts

internal/framework/llm/scriptorium_client.go

@@ -98,6 +98,13 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
 		Vars:          scriptoriumVars(req),
 		Metadata:      scriptoriumMetadata(req),
 	}
+	prepared, err := c.engine.Prepare(ctx, runReq)
+	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return contracts.StructuredCompletionResponse{}, ctxErr
+		}
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
+	}
 	result, err := c.engine.Run(ctx, runReq)
 	if err != nil {
 		if ctxErr := ctx.Err(); ctxErr != nil {
@@ -108,7 +115,7 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
 	if result == nil {
 		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
 	}
-	response := c.responseFromResult(result)
+	response := c.responseFromResult(result, prepared)
 	if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
 		return response, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
 	}
@@ -121,7 +128,7 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
 	return response, nil
 }
 
-func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult) contracts.StructuredCompletionResponse {
+func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
 	content := result.Artifact.Body
 	if len(content) == 0 {
 		content = []byte(result.RawOutput)
@@ -142,9 +149,111 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult) co
 		PromptTokens:     result.Usage.PromptTokens,
 		CompletionTokens: result.Usage.CompletionTokens,
 		TotalTokens:      result.Usage.TotalTokens,
+		Debug:            scriptoriumDebugMaterial(prepared, result),
 	}
 }
 
+func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
+	material := &contracts.LLMDebugMaterial{}
+	if prepared != nil {
+		material.Prompt = scriptoriumDebugPrompt(prepared)
+	}
+	if result != nil {
+		material.Response = scriptoriumDebugResponse(result)
+	}
+	if material.Prompt == nil && material.Response == nil {
+		return nil
+	}
+	return material
+}
+
+func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
+	if prepared == nil {
+		return nil
+	}
+	messages := make([]contracts.LLMDebugMessage, 0, len(prepared.Messages))
+	for _, message := range prepared.Messages {
+		messages = append(messages, contracts.LLMDebugMessage{
+			Role:         message.Role,
+			Content:      message.Content,
+			CacheControl: jsonObject(message.CacheControl),
+		})
+	}
+	return &contracts.LLMDebugPrompt{
+		PromptID:             prepared.PromptID,
+		PromptVersion:        prepared.PromptVersion,
+		PromptHash:           prepared.PromptHash,
+		SelectedProfileID:    prepared.SelectedProfileID,
+		SessionID:            prepared.SessionID,
+		RenderedPromptHash:   prepared.RenderedPromptHash,
+		Messages:             messages,
+		EffectiveModelParams: jsonObject(prepared.EffectiveModelParams),
+		OutputContract:       jsonObject(prepared.OutputContract),
+		StructuredOutput:     jsonObject(prepared.StructuredOutput),
+		InputHashes:          cloneStringMap(prepared.InputHashes),
+	}
+}
+
+func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
+	if result == nil {
+		return nil
+	}
+	content := result.RawOutput
+	if content == "" {
+		content = string(result.Artifact.Body)
+	}
+	return &contracts.LLMDebugResponse{
+		Content:              content,
+		RunID:                result.RunID,
+		PromptID:             result.PromptID,
+		PromptVersion:        result.PromptVersion,
+		PromptHash:           result.PromptHash,
+		RenderedPromptHash:   result.RenderedPromptHash,
+		SelectedProfileID:    result.SelectedProfileID,
+		ModelName:            result.ModelName,
+		Endpoint:             result.Endpoint,
+		EffectiveModelParams: jsonObject(result.EffectiveModelParams),
+		InputHashes:          cloneStringMap(result.InputHashes),
+		Validation:           jsonObject(result.Validation),
+		Usage: contracts.LLMDebugUsage{
+			PromptTokens:     result.Usage.PromptTokens,
+			CompletionTokens: result.Usage.CompletionTokens,
+			TotalTokens:      result.Usage.TotalTokens,
+			CachedTokens:     result.Usage.CachedTokens,
+			CacheWriteTokens: result.Usage.CacheWriteTokens,
+		},
+	}
+}
+
+func jsonObject(value any) map[string]any {
+	if value == nil {
+		return nil
+	}
+	data, err := json.Marshal(value)
+	if err != nil || string(data) == "null" {
+		return nil
+	}
+	var out map[string]any
+	if err := json.Unmarshal(data, &out); err != nil {
+		return nil
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+func cloneStringMap(values map[string]string) map[string]string {
+	if len(values) == 0 {
+		return nil
+	}
+	out := make(map[string]string, len(values))
+	for key, value := range values {
+		out[key] = value
+	}
+	return out
+}
+
 func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
 	if c == nil || c.recorder == nil {
 		return nil