Add durable validation provenance

2026-08-27 01:27:25 +00:00
parent 7b5f4ebd42
commit 7f01c3e79e
15 changed files with 508 additions and 83 deletions
--- a/internal/framework/pipeline/debug.go
+++ b/internal/framework/pipeline/debug.go
@@ -42,6 +42,7 @@ type debugTimedEnvelope struct {
 	LaneID      string                  `json:"lane_id,omitempty"`
 	ModuleKey   string                  `json:"module_key,omitempty"`
 	Attempt     int                     `json:"attempt,omitempty"`
+	AttemptKind string                  `json:"attempt_kind,omitempty"`
 	StartedAt   time.Time               `json:"started_at"`
 	CompletedAt time.Time               `json:"completed_at"`
 	DurationMS  int64                   `json:"duration_ms"`
@@ -143,9 +144,37 @@ type debugLLMCallReference struct {
 	PromptID            string `json:"prompt_id,omitempty"`
 	ProfileID           string `json:"profile_id,omitempty"`
 	Model               string `json:"model,omitempty"`
+	PromptTokens        int    `json:"prompt_tokens,omitempty"`
+	CompletionTokens    int    `json:"completion_tokens,omitempty"`
+	TotalTokens         int    `json:"total_tokens,omitempty"`
+	RepairAttempts      int    `json:"repair_attempts,omitempty"`
 	Error               bool   `json:"error,omitempty"`
 }

+type debugValidationOutcome struct {
+	ValidatorName string `json:"validator_name"`
+	Outcome       string `json:"outcome"`
+	AttemptCount  int    `json:"attempt_count"`
+	ReasonCode    string `json:"reason_code,omitempty"`
+}
+
+type debugProducerAttempt struct {
+	Number     int                      `json:"number"`
+	Kind       string                   `json:"kind"`
+	Outcome    string                   `json:"outcome"`
+	Validation []debugValidationOutcome `json:"validation,omitempty"`
+}
+
+type debugProducerTerminal struct {
+	ProducerAttemptCount int                         `json:"producer_attempt_count"`
+	Attempts             []debugProducerAttempt      `json:"attempts,omitempty"`
+	AggregateReasonCodes []string                    `json:"aggregate_reason_codes,omitempty"`
+	ValidationComplete   bool                        `json:"validation_complete"`
+	EffectivePolicy      ValidationPolicy            `json:"effective_policy"`
+	TerminalAction       string                      `json:"terminal_action"`
+	Summary              artifacts.ValidationSummary `json:"validation_summary"`
+}
+
 type debugValidationCall struct {
 	ValidatorName string                     `json:"validator_name"`
 	Request       any                        `json:"request"`
@@ -250,6 +279,10 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
 		PromptID:            req.PromptID,
 		ProfileID:           debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
 		Model:               debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
+		PromptTokens:        response.PromptTokens,
+		CompletionTokens:    response.CompletionTokens,
+		TotalTokens:         response.TotalTokens,
+		RepairAttempts:      response.RepairAttempts,
 		Error:               err != nil,
 	}
 	if scope := debugLLMScopeFromContext(ctx); scope != nil {
@@ -406,6 +439,50 @@ func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
 	return terminalErr
 }

+func debugValidationOutcomes(report validationReport) []debugValidationOutcome {
+	if len(report.records) == 0 {
+		return nil
+	}
+	outcomes := make([]debugValidationOutcome, 0, len(report.records))
+	for _, record := range report.records {
+		outcomes = append(outcomes, debugValidationOutcome{
+			ValidatorName: record.validatorName,
+			Outcome:       string(record.outcome),
+			AttemptCount:  record.attemptCount,
+			ReasonCode:    record.reasonCode,
+		})
+	}
+	return outcomes
+}
+
+func writeProducerTerminalDebug(recorder DebugRecorder, name string, terminal producerAttemptTerminal, policy ValidationPolicy, summary artifacts.ValidationSummary) error {
+	attempts := make([]debugProducerAttempt, 0, len(terminal.Provenance))
+	for _, item := range terminal.Provenance {
+		attempts = append(attempts, debugProducerAttempt{
+			Number:     item.Number,
+			Kind:       string(item.Kind),
+			Outcome:    string(item.Outcome),
+			Validation: debugValidationOutcomes(item.Validation),
+		})
+	}
+	return writeDebugTimed(recorder, name, debugTimedEnvelope{
+		Stage:     summary.Stage,
+		StepID:    summary.StepID,
+		LaneID:    summary.LaneID,
+		ModuleKey: summary.ModuleKey,
+		StartedAt: time.Now().UTC(),
+		Payload: debugProducerTerminal{
+			ProducerAttemptCount: summary.ProducerAttemptCount,
+			Attempts:             attempts,
+			AggregateReasonCodes: append([]string(nil), summary.ReasonCodes...),
+			ValidationComplete:   summary.Status == "complete",
+			EffectivePolicy:      policy,
+			TerminalAction:       string(terminal.Action),
+			Summary:              artifacts.CloneValidationSummary(summary),
+		},
+	})
+}
+
 func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
 	content = redactSecretBytes(content)
 	return debugBinaryEnvelope{