eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write workspace debug artifacts during runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 03:14:59 +00:00
parent ae9c2e1d5e
commit a5bbfea9b9
10 changed files with 1134 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
internal/framework/debug/recorder.go Normal file
View File

@@ -0,0 +1,34 @@
package debug
import (
"strings"
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type WorkspaceRecorder struct {
root string
}
func NewWorkspaceRecorder(settings coreworkspace.Settings, runID string) (pipeline.DebugRecorder, error) {
root, err := settings.DebugRunDirectory(runID)
if err != nil {
return nil, err
}
if strings.TrimSpace(root) == "" {
return pipeline.NoopDebugRecorder(), nil
}
return &WorkspaceRecorder{root: root}, nil
}
func (r *WorkspaceRecorder) Enabled() bool {
return r != nil && strings.TrimSpace(r.root) != ""
}
func (r *WorkspaceRecorder) WriteJSON(name string, payload any) error {
if !r.Enabled() {
return nil
}
return coreworkspace.WriteJSON(r.root, name, payload)
}

563
internal/framework/pipeline/debug.go Normal file
View File

@@ -0,0 +1,563 @@
package pipeline
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"path"
"regexp"
"strings"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type DebugRecorder interface {
Enabled() bool
WriteJSON(name string, payload any) error
}
type noopDebugRecorder struct{}
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
func (noopDebugRecorder) Enabled() bool { return false }
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func debugPathComponent(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "_"
}
var b strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteString(fmt.Sprintf("~%x", r))
}
}
out := b.String()
if out == "." || out == ".." || strings.Contains(out, "..") {
return "_"
}
return out
}
type debugTimedEnvelope struct {
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Attempt int `json:"attempt,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
Error string `json:"error,omitempty"`
}
type debugBinaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type debugRawPayload struct {
Content debugBinaryEnvelope `json:"content"`
}
type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw debugBinaryEnvelope `json:"raw,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceDocument struct {
ID string `json:"id"`
Kind string `json:"kind"`
Format string `json:"format,omitempty"`
Digest string `json:"digest,omitempty"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugMergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugNormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugLLMInputMaterial struct {
Name string `json:"name"`
MediaType string `json:"media_type,omitempty"`
Content string `json:"content_base64,omitempty"`
Digest string `json:"digest,omitempty"`
OriginURI string `json:"origin_uri,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
}
type debugStructuredCompletionRequest struct {
StageName string `json:"stage_name"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
}
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
}
type debugStructuredLLMCall struct {
Request debugStructuredCompletionRequest `json:"request"`
Response debugStructuredCompletionResponse `json:"response,omitempty"`
Error string `json:"error,omitempty"`
}
type debugValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *debugSourceChunk `json:"chunk,omitempty"`
Chunks []debugSourceChunk `json:"chunks,omitempty"`
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
}
type debugValidationCall struct {
ValidatorName string `json:"validator_name"`
Request debugValidationRequest `json:"request"`
Result contracts.ValidationResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMClient struct {
inner contracts.StructuredLLMClient
recorder DebugRecorder
counter int
}
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
if client == nil || recorder == nil || !recorder.Enabled() {
return client
}
return &debugLLMClient{inner: client, recorder: recorder}
}
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.counter++
started := time.Now().UTC()
response, err := client.inner.CompleteStructured(ctx, req, out)
completed := time.Now().UTC()
payload := debugStructuredLLMCall{
Request: debugCompletionRequest(req),
Response: debugCompletionResponse(response),
}
if err != nil {
payload.Error = err.Error()
}
writeErr := writeDebugTimed(client.recorder, path.Join("llm", fmt.Sprintf("call-%04d.json", client.counter)), debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: payload,
Error: payload.Error,
})
if err != nil {
return response, err
}
if writeErr != nil {
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
}
return response, err
}
func (client *debugLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
provider, ok := client.inner.(contracts.LLMProfileManifestProvider)
if !ok {
return nil
}
return provider.LLMProfileManifests()
}
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
if recorder == nil || !recorder.Enabled() {
return nil
}
if envelope.CompletedAt.IsZero() {
envelope.CompletedAt = time.Now().UTC()
}
if envelope.StartedAt.IsZero() {
envelope.StartedAt = envelope.CompletedAt
}
if envelope.DurationMS == 0 {
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
}
return recorder.WriteJSON(name, envelope)
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
content = redactSecretBytes(content)
return debugBinaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: debugContentDigest(content),
MediaType: mediaType,
Metadata: redactSensitiveMap(metadata),
Warnings: cloneWarnings(warnings),
}
}
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
if doc == nil {
return nil
}
return &debugSourceDocument{
ID: doc.ID,
Kind: doc.Kind,
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnits(doc.Units),
Metadata: redactSensitiveMap(doc.Metadata),
}
}
func debugSourceChunkEnvelope(chunk contracts.SourceChunk) debugSourceChunk {
return debugSourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
StartUnitID: chunk.StartUnitID,
EndUnitID: chunk.EndUnitID,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
}
}
func debugSourceChunkEnvelopes(chunks []contracts.SourceChunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil
}
out := make([]debugSourceChunk, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, debugSourceChunkEnvelope(chunk))
}
return out
}
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
output.Schema.JSONSchema = nil
return debugExtractOutput{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugExtractOutputEnvelope(output))
}
return out
}
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
output.Schema.JSONSchema = nil
return debugMergeOutput{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
output.Schema.JSONSchema = nil
return debugNormalizeOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugNormalizeOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugNormalizeOutputEnvelope(output))
}
return out
}
type debugOutputFile struct {
Name string `json:"name"`
ContentType string `json:"content_type,omitempty"`
Content debugBinaryEnvelope `json:"content"`
}
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
if len(files) == 0 {
return nil
}
out := make([]debugOutputFile, 0, len(files))
for _, file := range files {
out = append(out, debugOutputFile{
Name: file.Name,
ContentType: file.ContentType,
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
})
}
return out
}
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
for key, material := range req.Inputs {
inputs[key] = debugLLMInputMaterial{
Name: material.Name,
MediaType: material.MediaType,
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
Digest: material.Digest,
OriginURI: material.OriginURI,
SizeBytes: material.SizeBytes,
}
}
if len(inputs) == 0 {
inputs = nil
}
return debugStructuredCompletionRequest{
StageName: req.StageName,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
Inputs: inputs,
Vars: redactSensitiveMap(req.Vars),
}
}
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
return debugStructuredCompletionResponse{
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(response.Content)),
Provider: response.Provider,
Model: response.Model,
ProfileID: response.ProfileID,
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
}
}
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
req.Schema.JSONSchema = nil
out := debugValidationRequest{
Stage: req.Stage,
LaneID: req.LaneID,
ModuleKey: req.ModuleKey,
SourceID: req.SourceID,
SessionID: req.SessionID,
LLMProfile: req.LLMProfile,
Options: redactSensitiveMap(req.Options),
Metadata: redactSensitiveMap(req.Metadata),
Schema: req.Schema,
ChunkID: req.ChunkID,
ChunkIndex: req.ChunkIndex,
}
payload := debugPayloadEnvelope(req.Payload)
out.Payload = &payload
if req.Chunk != nil {
chunk := debugSourceChunkEnvelope(*req.Chunk)
out.Chunk = &chunk
}
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
merge := debugMergeOutputEnvelope(req.MergeOutput)
out.MergeOutput = &merge
}
return out
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message)))
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
for i := range result.Warnings {
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
}
return result
}
func debugRejectedOutputEnvelope(rejected contracts.RejectedOutput) contracts.RejectedOutput {
rejected.Message = string(redactSecretBytes([]byte(rejected.Message)))
rejected.DiagnosticArtifactPath = string(redactSecretBytes([]byte(rejected.DiagnosticArtifactPath)))
return rejected
}
func debugRejectedOutputPtr(rejected *contracts.RejectedOutput) any {
if rejected == nil {
return nil
}
out := debugRejectedOutputEnvelope(*rejected)
return out
}
func debugRejectedOutputEnvelopes(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
out := make([]contracts.RejectedOutput, 0, len(rejected))
for _, item := range rejected {
out = append(out, debugRejectedOutputEnvelope(item))
}
return out
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
var secretPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
}
func redactSecretBytes(content []byte) []byte {
if len(content) == 0 || !utf8.Valid(content) {
return append([]byte(nil), content...)
}
text := string(content)
for _, pattern := range secretPatterns {
text = pattern.ReplaceAllString(text, "[REDACTED]")
}
return []byte(text)
}
func redactSensitiveMap(values map[string]any) map[string]any {
if len(values) == 0 {
return nil
}
out := make(map[string]any, len(values))
for key, value := range values {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
continue
}
out[key] = redactSensitiveValue(value)
}
return out
}
func redactSensitiveValue(value any) any {
switch typed := value.(type) {
case string:
return string(redactSecretBytes([]byte(typed)))
case map[string]any:
return redactSensitiveMap(typed)
case map[string]string:
out := make(map[string]string, len(typed))
for key, value := range typed {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
} else {
out[key] = string(redactSecretBytes([]byte(value)))
}
}
return out
default:
return value
}
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
return strings.Contains(key, "api_key") ||
strings.Contains(key, "apikey") ||
strings.Contains(key, "authorization") ||
strings.Contains(key, "bearer") ||
strings.Contains(key, "password") ||
strings.Contains(key, "secret") ||
strings.Contains(key, "token")
}

291
internal/framework/pipeline/runner.go
View File

@@ -50,6 +50,7 @@ type RunInput struct {
Warnings []contracts.Warning
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
Debug DebugRecorder
}
type RunOutput struct {
@@ -81,10 +82,27 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if checkpointLoader == nil {
checkpointLoader = NoopCheckpointLoader()
}
debugRecorder := input.Debug
if debugRecorder == nil {
debugRecorder = NoopDebugRecorder()
}
input.Debug = debugRecorder
input.LLMClient = wrapDebugLLMClient(input.LLMClient, debugRecorder)
defer func() {
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
}()
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
Stage: "run",
StartedAt: startedTime(input.StartedAt),
Payload: map[string]any{
"pipeline_id": input.Pipeline.ID,
"pipeline_digest": input.Pipeline.Digest,
"run_id": output.Manifest.RunID,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
}
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil {
@@ -94,6 +112,21 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: debugSourceInput{
SourceID: input.SourceID,
Path: input.Path,
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
Options: redactSensitiveMap(input.Pipeline.Input.Options),
Metadata: redactSensitiveMap(input.Metadata),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
if !sourceDecision.Reused {
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
@@ -118,6 +151,18 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
}
if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: map[string]any{
"reused": sourceDecision.Reused,
"decision": sourceDecision,
"document": debugSourceDocumentEnvelope(doc),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
sessionID := resolvedSessionID(input.SessionID, doc.ID)
output.Manifest.Metadata = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
@@ -132,6 +177,22 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
var chunkWarnings []contracts.Warning
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
chunkStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: map[string]any{
"reused": chunkDecision.Reused,
"decision": chunkDecision,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.Pipeline.Chunk.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
chunksAccepted := chunkDecision.Reused
var chunkRejection *contracts.RejectedOutput
if chunkDecision.Reused {
@@ -143,6 +204,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
}
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
SourceInput: sourceInput.Clone(),
@@ -154,6 +216,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
})
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
}
if len(chunkResult.Chunks) == 0 {
@@ -163,12 +232,35 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err != nil {
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt)
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt, input.Debug)
if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": append(cloneWarnings(chunkResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
})
return false, rejection, err
}
canonicalChunks = chunks
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
if err := writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": chunkWarnings,
},
}); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
@@ -187,6 +279,23 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
}
}
chunkDebugPayload := map[string]any{
"reused": chunkDecision.Reused,
"accepted": chunksAccepted,
"chunks": debugSourceChunkEnvelopes(canonicalChunks),
"warnings": chunkWarnings,
}
if chunkRejection != nil {
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)
}
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: chunkDebugPayload,
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
if chunksAccepted {
for _, lane := range input.Pipeline.ArtifactLanes {
@@ -209,6 +318,22 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
}
attachModuleManifestMetadata(&output, "output", encoder)
outputStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
Stage: string(StageOutput),
ModuleKey: encoder.Key(),
StartedAt: outputStarted,
Payload: map[string]any{
"manifest": output.Manifest,
"normalize_outputs": debugNormalizeOutputEnvelopes(output.NormalizeOutputs),
"rejected": debugRejectedOutputEnvelopes(output.Rejected),
"warnings": output.Warnings,
"options": redactSensitiveMap(input.Pipeline.Output.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
}
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
NormalizeOutputs: cloneNormalizeOutputs(output.NormalizeOutputs),
@@ -227,6 +352,17 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
}
output.OutputFiles = files
if err := writeDebugTimed(debugRecorder, "output/output.json", debugTimedEnvelope{
Stage: string(StageOutput),
ModuleKey: encoder.Key(),
StartedAt: outputStarted,
Payload: map[string]any{
"files": debugOutputFiles(files),
"warnings": encoded.Warnings,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
}
return output, nil
}
@@ -252,6 +388,23 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
extractDependencies := digestFingerprints("chunks", joinedChunkDigest(chunks))
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
extractStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"decision": extractDecision,
"source": debugSourceDocumentEnvelope(doc),
"chunks": debugSourceChunkEnvelopes(chunks),
"options": redactSensitiveMap(lane.Extract.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if extractDecision.Reused {
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
@@ -305,6 +458,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
return false, rejection, err
@@ -330,6 +484,20 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"outputs": debugExtractOutputEnvelopes(extractOutputs),
"rejected": debugRejectedOutputEnvelopes(output.Rejected[extractRejectedStart:]),
"warnings": extractWarnings,
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if len(extractOutputs) == 0 {
return nil
@@ -340,6 +508,23 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
mergeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"decision": mergeDecision,
"source": debugSourceDocumentEnvelope(doc),
"extract_outputs": debugExtractOutputEnvelopes(extractOutputs),
"options": redactSensitiveMap(lane.Merge.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
if mergeDecision.Reused {
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
@@ -385,6 +570,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
return false, rejection, err
@@ -402,6 +588,19 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*mergeRejection),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
@@ -409,12 +608,43 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"accepted": true,
"output": debugMergeOutputEnvelope(acceptedMerge),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
var acceptedNormalize contracts.NormalizeOutput
var normalizeWarnings []contracts.Warning
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
normalizeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"decision": normalizeDecision,
"source": debugSourceDocumentEnvelope(doc),
"merge_output": debugMergeOutputEnvelope(acceptedMerge),
"options": redactSensitiveMap(lane.Normalize.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
if normalizeDecision.Reused {
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
@@ -460,6 +690,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
return false, rejection, err
@@ -477,6 +708,19 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*normalizeRejection),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
@@ -484,6 +728,20 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"accepted": true,
"output": debugNormalizeOutputEnvelope(acceptedNormalize),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
output.NormalizeOutputs = append(output.NormalizeOutputs, acceptedNormalize)
return nil
}
@@ -509,6 +767,7 @@ type rawValidationTarget struct {
metadata map[string]any
chains []ResolvedValidatorChain
attempt int
debug DebugRecorder
}
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
@@ -558,7 +817,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
return false, lastRejection, nil
}
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) ([]contracts.Warning, *contracts.RejectedOutput, error) {
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
return r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
moduleKey: moduleKey,
@@ -572,6 +831,7 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
metadata: metadata,
chains: chains,
attempt: attempt,
debug: debug,
})
}
@@ -591,7 +851,27 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
}
request := target.validationRequest(validatorBinding.Binding)
started := time.Now().UTC()
result, err := validator.Validate(ctx, request)
debugPayload := debugValidationCall{
ValidatorName: validator.Name(),
Request: debugValidationRequestEnvelope(request),
Result: debugValidationResultEnvelope(result),
}
if err != nil {
debugPayload.Error = err.Error()
}
if debugErr := writeDebugTimed(target.debug, path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d.json", len(warnings)+1, debugPathComponent(validator.Name()), target.attempt)), debugTimedEnvelope{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
Attempt: target.attempt,
StartedAt: started,
Payload: debugPayload,
Error: debugPayload.Error,
}); debugErr != nil {
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
}
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
}
@@ -733,6 +1013,13 @@ func validateRunInput(input RunInput) error {
return nil
}
func startedTime(t time.Time) time.Time {
if t.IsZero() {
return time.Now().UTC()
}
return t.UTC()
}
func manifestFromPipeline(input RunInput) artifacts.RunManifest {
startedAt := input.StartedAt
if startedAt.IsZero() {