eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add utilization diagnostics and correction ledger

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 19:03:37 +00:00
parent 037121e9ce
commit ff4ed82239
15 changed files with 969 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
internal/framework/runner/runner.go
View File

@@ -77,6 +77,7 @@ type RunInput struct {
Transcript *schema.Transcript
Glossary *schema.Glossary
ModuleSpecs []contracts.ModuleRunSpec
EffectiveConcurrency EffectiveConcurrencyLimits
ProposalLLMClient contracts.StructuredLLMClient
ProposalLLMScheduler contracts.LLMScheduler
ProposalDiagnosticsDir string
@@ -89,6 +90,7 @@ type RunInput struct {
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Utilization *UtilizationDiagnostics
}
func New(factory ModuleFactory) *Runner {
@@ -100,6 +102,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return RunOutput{}, fmt.Errorf("runner module factory is required")
}
startedAt := time.Now().UTC()
instrumentation := newRunInstrumentation(startedAt)
input.ProposalLLMClient = instrumentation.wrapProposalClient(input.ProposalLLMClient)
input.ValidationLLMClient = instrumentation.wrapValidationClient(input.ValidationLLMClient)
input.ProposalLLMScheduler = instrumentation.wrapScheduler(input.ProposalLLMScheduler, true)
input.ValidationLLMScheduler = instrumentation.wrapScheduler(input.ValidationLLMScheduler, false)
working := cloneTranscript(input.Transcript)
results := make([]ModuleResult, 0, len(input.ModuleSpecs))
@@ -116,7 +126,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
return RunOutput{
FinalTranscript: working,
ModuleResults: results,
Utilization: instrumentation.finalize(&RunOutput{ModuleResults: results}, time.Now().UTC(), input.EffectiveConcurrency),
}, fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
}
policy := module.ReplacementPolicy()
@@ -132,7 +146,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q chunking failed: %w", spec.InstanceName, err)
return RunOutput{
FinalTranscript: working,
ModuleResults: results,
Utilization: instrumentation.finalize(&RunOutput{ModuleResults: results}, time.Now().UTC(), input.EffectiveConcurrency),
}, fmt.Errorf("module %q chunking failed: %w", spec.InstanceName, err)
}
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
@@ -148,6 +166,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidationClient: input.ValidationLLMClient,
ValidationScheduler: input.ValidationLLMScheduler,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Instrumentation: instrumentation,
})
if pipelineErr != nil {
failed := ModuleResult{
@@ -163,7 +182,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
return RunOutput{
FinalTranscript: working,
ModuleResults: results,
Utilization: instrumentation.finalize(&RunOutput{ModuleResults: results}, time.Now().UTC(), input.EffectiveConcurrency),
}, fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
}
applyResult := proposals.ApplyProposals(working, pipelineResult.Approved, policy)
@@ -184,7 +207,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
})
}
return RunOutput{FinalTranscript: working, ModuleResults: results}, nil
output := RunOutput{FinalTranscript: working, ModuleResults: results}
output.Utilization = instrumentation.finalize(&output, time.Now().UTC(), input.EffectiveConcurrency)
return output, nil
}
type collectSectionProposalsInput struct {
@@ -200,6 +225,7 @@ type collectSectionProposalsInput struct {
ValidationClient contracts.StructuredLLMClient
ValidationScheduler ValidationScheduler
ValidationDiagnosticsDir string
Instrumentation *runInstrumentation
}
type sectionProposals struct {
@@ -244,7 +270,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
go func() {
defer wg.Done()
meta := contracts.SectionMetadataFromSection(section)
corrected, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: input.Config,
WorkingTranscript: transcriptFromSection(section),
@@ -366,6 +392,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
ValidationLLMClient: input.ValidationClient,
ValidationScheduler: input.ValidationScheduler,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Instrumentation: input.Instrumentation,
})
validationResults <- sectionValidationResult{
sectionPos: sectionPos,
@@ -432,6 +459,7 @@ type validateSectionCandidatesInput struct {
ValidationLLMClient contracts.StructuredLLMClient
ValidationScheduler ValidationScheduler
DiagnosticsDir string
Instrumentation *runInstrumentation
}
type validateSectionCandidatesResult struct {
@@ -456,7 +484,8 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
validatorStartedAt := time.Now().UTC()
vResult, err := validator.Validate(withModuleInstanceContext(ctx, input.Spec.InstanceName), contracts.ValidationRequest{
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
ModuleKey: input.Spec.ModuleKey,
@@ -468,6 +497,15 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
Scheduler: input.ValidationScheduler,
DiagnosticsWriter: diagnosticsWriter,
})
if input.Instrumentation != nil {
input.Instrumentation.recordValidatorTiming(
input.Spec.ModuleKey,
input.Spec.InstanceName,
validator.Name(),
isLLMBackedValidator(validator),
time.Since(validatorStartedAt),
)
}
if err != nil {
return validateSectionCandidatesResult{
approved: eligible,