Collect PromptKit execution observations with logical attribution

internal/framework/runner/runner.go

@@ -82,12 +82,17 @@ type RunInput struct {
 	ProposalDiagnosticsDir   string
 	ValidationLLMClient      contracts.StructuredLLMClient
 	ValidationDiagnosticsDir string
+	timings                  *validatorTimingCollector
 }
 
 // RunOutput is the deterministic runner output.
 type RunOutput struct {
-	FinalTranscript *schema.Transcript `json:"-"`
-	ModuleResults   []ModuleResult     `json:"module_results"`
+	FinalTranscript  *schema.Transcript      `json:"-"`
+	ModuleResults    []ModuleResult          `json:"module_results"`
+	Observations     llm.ObservationSnapshot `json:"-"`
+	LLMSummary       llm.ExecutionSummary    `json:"-"`
+	RunWallTimeMS    int64                   `json:"-"`
+	ValidatorTimings []ValidatorTiming       `json:"-"`
 }
 
 func New(factory ModuleFactory) *Runner {
@@ -95,6 +100,15 @@ func New(factory ModuleFactory) *Runner {
 }
 
 func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
+	started := time.Now()
+	input.timings = newValidatorTimingCollector()
+	output, err := r.run(ctx, input)
+	output.RunWallTimeMS = time.Since(started).Milliseconds()
+	output.ValidatorTimings = input.timings.snapshot()
+	return output, err
+}
+
+func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
 	if r == nil || r.factory == nil {
 		return RunOutput{}, fmt.Errorf("runner module factory is required")
 	}
@@ -151,6 +165,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
 			DiagnosticsDir:           input.ProposalDiagnosticsDir,
 			ValidationClient:         input.ValidationLLMClient,
 			ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
+			Timings:                  input.timings,
 		})
 		if pipelineErr != nil {
 			failed := ModuleResult{
@@ -207,6 +222,7 @@ type collectSectionProposalsInput struct {
 	DiagnosticsDir           string
 	ValidationClient         contracts.StructuredLLMClient
 	ValidationDiagnosticsDir string
+	Timings                  *validatorTimingCollector
 }
 
 type sectionProposals struct {
@@ -378,6 +394,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
 					SectionEnriched:         enriched,
 					ValidationLLMClient:     input.ValidationClient,
 					DiagnosticsDir:          input.ValidationDiagnosticsDir,
+					Timings:                 input.Timings,
 				})
 				validationResults <- sectionValidationResult{
 					sectionPos: sectionPos,
@@ -477,6 +494,7 @@ type validateSectionCandidatesInput struct {
 	SectionEnriched         []proposals.EnrichedCorrectionProposal
 	ValidationLLMClient     contracts.StructuredLLMClient
 	DiagnosticsDir          string
+	Timings                 *validatorTimingCollector
 }
 
 type validateSectionCandidatesResult struct {
@@ -503,6 +521,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
 			}
 		}
 
+		validatorStarted := time.Now()
 		vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
 			WorkingTranscript: input.WorkingTranscript,
 			CandidateProposal: eligible,
@@ -514,6 +533,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
 			LLMClient:         input.ValidationLLMClient,
 			DiagnosticsWriter: diagnosticsWriter,
 		})
+		if input.Timings != nil {
+			input.Timings.record(input.Spec.ModuleKey, input.Spec.InstanceName, validator.Name(), validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked, time.Since(validatorStarted))
+		}
 
 		if err != nil {
 			return validateSectionCandidatesResult{

internal/framework/runner/runner_test.go

@@ -564,6 +564,9 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
 	if len(out.ModuleResults[0].ValidatorDecisions) != 1 {
 		t.Fatalf("expected validator decisions recorded")
 	}
+	if len(out.ValidatorTimings) != 1 || out.ValidatorTimings[0].Calls != 1 || out.ValidatorTimings[0].ValidatorName != "allow" || out.RunWallTimeMS < 0 {
+		t.Fatalf("runner timings missing: %+v", out.ValidatorTimings)
+	}
 }
 
 func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {

internal/framework/runner/timing.go

@@ -0,0 +1,56 @@
+package runner
+
+import (
+	"sort"
+	"sync"
+	"time"
+)
+
+// ValidatorTiming records domain validator work without treating a scheduler
+// permit or provider attempt as a logical LLM call.
+type ValidatorTiming struct {
+	ModuleKey      string `json:"module_key"`
+	ModuleInstance string `json:"module_instance"`
+	ValidatorName  string `json:"validator_name"`
+	LLMBacked      bool   `json:"llm_backed"`
+	Calls          int    `json:"calls"`
+	ElapsedMS      int64  `json:"elapsed_ms"`
+}
+
+type validatorTimingCollector struct {
+	mu    sync.Mutex
+	byKey map[string]*ValidatorTiming
+}
+
+func newValidatorTimingCollector() *validatorTimingCollector {
+	return &validatorTimingCollector{byKey: make(map[string]*ValidatorTiming)}
+}
+
+func (c *validatorTimingCollector) record(moduleKey, moduleInstance, validatorName string, llmBacked bool, elapsed time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	key := moduleInstance + "\x00" + validatorName
+	timing := c.byKey[key]
+	if timing == nil {
+		timing = &ValidatorTiming{ModuleKey: moduleKey, ModuleInstance: moduleInstance, ValidatorName: validatorName, LLMBacked: llmBacked}
+		c.byKey[key] = timing
+	}
+	timing.Calls++
+	timing.ElapsedMS += elapsed.Milliseconds()
+}
+
+func (c *validatorTimingCollector) snapshot() []ValidatorTiming {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	result := make([]ValidatorTiming, 0, len(c.byKey))
+	for _, timing := range c.byKey {
+		result = append(result, *timing)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].ModuleInstance != result[j].ModuleInstance {
+			return result[i].ModuleInstance < result[j].ModuleInstance
+		}
+		return result[i].ValidatorName < result[j].ValidatorName
+	})
+	return result
+}