eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Collect PromptKit execution observations with logical attribution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:23:57 +00:00
parent 4812032c9e
commit 4bfc5f3456
11 changed files with 457 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal/framework/runner/runner.go
View File

@@ -82,12 +82,17 @@ type RunInput struct {
ProposalDiagnosticsDir string
ValidationLLMClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
timings *validatorTimingCollector
}
// RunOutput is the deterministic runner output.
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Observations llm.ObservationSnapshot `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
}
func New(factory ModuleFactory) *Runner {
@@ -95,6 +100,15 @@ func New(factory ModuleFactory) *Runner {
}
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
started := time.Now()
input.timings = newValidatorTimingCollector()
output, err := r.run(ctx, input)
output.RunWallTimeMS = time.Since(started).Milliseconds()
output.ValidatorTimings = input.timings.snapshot()
return output, err
}
func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
if r == nil || r.factory == nil {
return RunOutput{}, fmt.Errorf("runner module factory is required")
}
@@ -151,6 +165,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
DiagnosticsDir: input.ProposalDiagnosticsDir,
ValidationClient: input.ValidationLLMClient,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
})
if pipelineErr != nil {
failed := ModuleResult{
@@ -207,6 +222,7 @@ type collectSectionProposalsInput struct {
DiagnosticsDir string
ValidationClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
Timings *validatorTimingCollector
}
type sectionProposals struct {
@@ -378,6 +394,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
})
validationResults <- sectionValidationResult{
sectionPos: sectionPos,
@@ -477,6 +494,7 @@ type validateSectionCandidatesInput struct {
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
}
type validateSectionCandidatesResult struct {
@@ -503,6 +521,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
}
validatorStarted := time.Now()
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
@@ -514,6 +533,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
LLMClient: input.ValidationLLMClient,
DiagnosticsWriter: diagnosticsWriter,
})
if input.Timings != nil {
input.Timings.record(input.Spec.ModuleKey, input.Spec.InstanceName, validator.Name(), validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked, time.Since(validatorStarted))
}
if err != nil {
return validateSectionCandidatesResult{