Add intra-module pipeline for LLM validation
This commit is contained in:
@@ -313,7 +313,13 @@ These primitives are wired into the production runner and report model. The gram
|
|||||||
- identical/no-effect rejection
|
- identical/no-effect rejection
|
||||||
- conservative protected glossary-term guard for non-glossary modules
|
- conservative protected glossary-term guard for non-glossary modules
|
||||||
|
|
||||||
`internal/framework/runner` executes validator chains in order for each module and applies only validator-approved proposals.
|
`internal/framework/runner` executes module pipelines with deterministic boundaries:
|
||||||
|
- modules still execute serially over the working transcript;
|
||||||
|
- section proposal work is launched promptly and can run concurrently;
|
||||||
|
- section-level validator-chain work starts as section proposals become available (deterministic validators before LLM-backed validators);
|
||||||
|
- proposal-generation and LLM-validator calls can overlap under composed scheduler limits;
|
||||||
|
- approved proposals are still applied once per module after section work settles.
|
||||||
|
|
||||||
Validator rejections are reported distinctly from proposal-application skips.
|
Validator rejections are reported distinctly from proposal-application skips.
|
||||||
|
|
||||||
## Implemented LLM-backed validator infrastructure
|
## Implemented LLM-backed validator infrastructure
|
||||||
|
|||||||
@@ -2,6 +2,15 @@
|
|||||||
|
|
||||||
Date: 2026-05-12
|
Date: 2026-05-12
|
||||||
|
|
||||||
|
## Implementation Status (2026-05-12 Update)
|
||||||
|
|
||||||
|
The intra-module pipelining gap identified in this audit has now been addressed:
|
||||||
|
- section proposal jobs are launched promptly for each module;
|
||||||
|
- as section proposals become available in deterministic section order, section-local validator work starts without waiting for all section proposals to finish;
|
||||||
|
- deterministic validators run before LLM-backed validators for each section;
|
||||||
|
- proposal and validation LLM calls can overlap through the existing composed scheduler path;
|
||||||
|
- module application remains a single deterministic apply barrier.
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
This audit checks whether Audita currently maximizes available LLM concurrency within each module by overlapping proposal and validation work.
|
This audit checks whether Audita currently maximizes available LLM concurrency within each module by overlapping proposal and validation work.
|
||||||
|
|||||||
@@ -27,7 +27,8 @@ The targeted concurrency gaps identified in this audit have now been addressed:
|
|||||||
- legacy `llm-concurrency` settings are preserved as compatibility aliases to total concurrency,
|
- legacy `llm-concurrency` settings are preserved as compatibility aliases to total concurrency,
|
||||||
- proposal and validation schedulers are composed with a global total-cap scheduler,
|
- proposal and validation schedulers are composed with a global total-cap scheduler,
|
||||||
- scheduler default behavior is FIFO with context-aware queued cancellation and reliable permit release,
|
- scheduler default behavior is FIFO with context-aware queued cancellation and reliable permit release,
|
||||||
- runner proposal worker fan-out is aligned with effective proposal concurrency.
|
- runner proposal worker fan-out is aligned with effective proposal concurrency,
|
||||||
|
- intra-module execution now pipelines section validation so proposal and validation LLM work can overlap within a module while retaining deterministic module-level apply ordering.
|
||||||
|
|
||||||
## Audit Findings (Questions 1-14)
|
## Audit Findings (Questions 1-14)
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,6 @@
|
|||||||
"total_skipped_changes": 1,
|
"total_skipped_changes": 1,
|
||||||
"validator_rejected_reason_codes": ["llm_rejected"],
|
"validator_rejected_reason_codes": ["llm_rejected"],
|
||||||
"expected_proposal_calls": ["grammar:proposal"],
|
"expected_proposal_calls": ["grammar:proposal"],
|
||||||
"expected_validation_calls": ["grammar:grammar_only_guard:batch-0000"]
|
"expected_validation_calls": ["grammar:section-0000:grammar_only_guard:batch-0000"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -134,125 +135,38 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q chunking failed: %w", spec.InstanceName, err)
|
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q chunking failed: %w", spec.InstanceName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
enriched, proposeErr := collectSectionProposals(ctx, collectSectionProposalsInput{
|
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
|
||||||
Module: module,
|
Module: module,
|
||||||
Spec: spec,
|
Spec: spec,
|
||||||
Policy: policy,
|
Policy: policy,
|
||||||
Config: input.Config,
|
Config: input.Config,
|
||||||
Glossary: input.Glossary,
|
Glossary: input.Glossary,
|
||||||
Sections: sections,
|
Sections: sections,
|
||||||
ProposalClient: input.ProposalLLMClient,
|
ProposalClient: input.ProposalLLMClient,
|
||||||
ProposalScheduler: input.ProposalLLMScheduler,
|
ProposalScheduler: input.ProposalLLMScheduler,
|
||||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||||
|
ValidationClient: input.ValidationLLMClient,
|
||||||
|
ValidationScheduler: input.ValidationLLMScheduler,
|
||||||
|
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||||
})
|
})
|
||||||
if proposeErr != nil {
|
if pipelineErr != nil {
|
||||||
failed := ModuleResult{
|
failed := ModuleResult{
|
||||||
ModuleKey: spec.ModuleKey,
|
ModuleKey: spec.ModuleKey,
|
||||||
ModuleInstance: spec.InstanceName,
|
ModuleInstance: spec.InstanceName,
|
||||||
ReplacementPolicy: policy,
|
ReplacementPolicy: policy,
|
||||||
Status: ModuleStatusFailed,
|
Status: ModuleStatusFailed,
|
||||||
ErrorMessage: proposeErr.Error(),
|
ProposalCount: pipelineResult.ProposalCount,
|
||||||
StartedAt: startedAt,
|
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||||
CompletedAt: time.Now().UTC(),
|
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||||
|
ErrorMessage: pipelineErr.Error(),
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: time.Now().UTC(),
|
||||||
}
|
}
|
||||||
results = append(results, failed)
|
results = append(results, failed)
|
||||||
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q failed: %w", spec.InstanceName, proposeErr)
|
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
validatorDecisions := make([]ValidatorDecisionRecord, 0)
|
applyResult := proposals.ApplyProposals(working, pipelineResult.Approved, policy)
|
||||||
validatorRejected := make([]ValidatorRejectedChange, 0)
|
|
||||||
eligible := enriched
|
|
||||||
for _, validator := range module.Validators() {
|
|
||||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
|
||||||
if input.ValidationDiagnosticsDir != "" {
|
|
||||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
|
||||||
writer: llm.NewDiagnosticsWriter(
|
|
||||||
filepath.Join(input.ValidationDiagnosticsDir, spec.InstanceName),
|
|
||||||
validatorSecrets(input.Config),
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
vResult, vErr := validator.Validate(ctx, contracts.ValidationRequest{
|
|
||||||
WorkingTranscript: working,
|
|
||||||
CandidateProposal: eligible,
|
|
||||||
ModuleKey: spec.ModuleKey,
|
|
||||||
ModuleInstance: spec.InstanceName,
|
|
||||||
ReplacementPolicy: policy,
|
|
||||||
Glossary: input.Glossary,
|
|
||||||
Config: input.Config,
|
|
||||||
LLMClient: validationLLMClientAdapter{client: input.ValidationLLMClient},
|
|
||||||
Scheduler: input.ValidationLLMScheduler,
|
|
||||||
DiagnosticsWriter: diagnosticsWriter,
|
|
||||||
})
|
|
||||||
if vErr != nil {
|
|
||||||
failed := ModuleResult{
|
|
||||||
ModuleKey: spec.ModuleKey,
|
|
||||||
ModuleInstance: spec.InstanceName,
|
|
||||||
ReplacementPolicy: policy,
|
|
||||||
Status: ModuleStatusFailed,
|
|
||||||
ProposalCount: len(enriched),
|
|
||||||
ValidatorDecisions: validatorDecisions,
|
|
||||||
ValidatorRejected: validatorRejected,
|
|
||||||
ErrorMessage: vErr.Error(),
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: time.Now().UTC(),
|
|
||||||
}
|
|
||||||
results = append(results, failed)
|
|
||||||
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q validator %q failed: %w", spec.InstanceName, validator.Name(), vErr)
|
|
||||||
}
|
|
||||||
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
|
||||||
failed := ModuleResult{
|
|
||||||
ModuleKey: spec.ModuleKey,
|
|
||||||
ModuleInstance: spec.InstanceName,
|
|
||||||
ReplacementPolicy: policy,
|
|
||||||
Status: ModuleStatusFailed,
|
|
||||||
ProposalCount: len(enriched),
|
|
||||||
ValidatorDecisions: validatorDecisions,
|
|
||||||
ValidatorRejected: validatorRejected,
|
|
||||||
ErrorMessage: err.Error(),
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: time.Now().UTC(),
|
|
||||||
}
|
|
||||||
results = append(results, failed)
|
|
||||||
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q validator %q cardinality failed: %w", spec.InstanceName, validator.Name(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
|
||||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
|
||||||
for _, p := range eligible {
|
|
||||||
byIndex[p.ProposalIndex] = p
|
|
||||||
}
|
|
||||||
for _, d := range vResult.Decisions {
|
|
||||||
validatorDecisions = append(validatorDecisions, ValidatorDecisionRecord{
|
|
||||||
ValidatorName: validator.Name(),
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
Approved: d.Approved,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
|
||||||
})
|
|
||||||
if d.Approved {
|
|
||||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p := byIndex[d.ProposalIndex]
|
|
||||||
validatorRejected = append(validatorRejected, ValidatorRejectedChange{
|
|
||||||
ValidatorName: validator.Name(),
|
|
||||||
ProposalIndex: p.ProposalIndex,
|
|
||||||
ModuleKey: p.ModuleKey,
|
|
||||||
ModuleInstance: p.ModuleInstance,
|
|
||||||
TargetSegmentID: p.TargetSegmentID,
|
|
||||||
OriginalText: p.OriginalText,
|
|
||||||
CorrectedText: p.CorrectedText,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
eligible = nextEligible
|
|
||||||
}
|
|
||||||
|
|
||||||
applyResult := proposals.ApplyProposals(working, eligible, policy)
|
|
||||||
working = applyResult.Transcript
|
working = applyResult.Transcript
|
||||||
|
|
||||||
results = append(results, ModuleResult{
|
results = append(results, ModuleResult{
|
||||||
@@ -260,9 +174,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
ModuleInstance: spec.InstanceName,
|
ModuleInstance: spec.InstanceName,
|
||||||
ReplacementPolicy: policy,
|
ReplacementPolicy: policy,
|
||||||
Status: ModuleStatusSuccess,
|
Status: ModuleStatusSuccess,
|
||||||
ProposalCount: len(enriched),
|
ProposalCount: pipelineResult.ProposalCount,
|
||||||
ValidatorDecisions: validatorDecisions,
|
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||||
ValidatorRejected: validatorRejected,
|
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||||
AppliedChanges: applyResult.Applied,
|
AppliedChanges: applyResult.Applied,
|
||||||
SkippedChanges: applyResult.Skipped,
|
SkippedChanges: applyResult.Skipped,
|
||||||
StartedAt: startedAt,
|
StartedAt: startedAt,
|
||||||
@@ -274,15 +188,18 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type collectSectionProposalsInput struct {
|
type collectSectionProposalsInput struct {
|
||||||
Module contracts.TranscriptModule
|
Module contracts.TranscriptModule
|
||||||
Spec contracts.ModuleRunSpec
|
Spec contracts.ModuleRunSpec
|
||||||
Policy proposals.ReplacementPolicy
|
Policy proposals.ReplacementPolicy
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Glossary *schema.Glossary
|
Glossary *schema.Glossary
|
||||||
Sections []chunking.Section
|
Sections []chunking.Section
|
||||||
ProposalClient contracts.StructuredLLMClient
|
ProposalClient contracts.StructuredLLMClient
|
||||||
ProposalScheduler contracts.LLMScheduler
|
ProposalScheduler contracts.LLMScheduler
|
||||||
DiagnosticsDir string
|
DiagnosticsDir string
|
||||||
|
ValidationClient contracts.StructuredLLMClient
|
||||||
|
ValidationScheduler ValidationScheduler
|
||||||
|
ValidationDiagnosticsDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
type sectionProposals struct {
|
type sectionProposals struct {
|
||||||
@@ -290,98 +207,341 @@ type sectionProposals struct {
|
|||||||
corrected []proposals.CorrectionProposal
|
corrected []proposals.CorrectionProposal
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) ([]proposals.EnrichedCorrectionProposal, error) {
|
type sectionProposalResult struct {
|
||||||
if len(input.Sections) == 0 {
|
sectionPos int
|
||||||
return []proposals.EnrichedCorrectionProposal{}, nil
|
section chunking.Section
|
||||||
}
|
corrected []proposals.CorrectionProposal
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
maxWorkers := 1
|
type sectionValidationResult struct {
|
||||||
if input.Config != nil && input.Config.EffectiveProposalLLMConcurrency() > 1 {
|
sectionPos int
|
||||||
maxWorkers = input.Config.EffectiveProposalLLMConcurrency()
|
enriched []proposals.EnrichedCorrectionProposal
|
||||||
}
|
approved []proposals.EnrichedCorrectionProposal
|
||||||
if maxWorkers > len(input.Sections) {
|
decisions []ValidatorDecisionRecord
|
||||||
maxWorkers = len(input.Sections)
|
rejected []ValidatorRejectedChange
|
||||||
}
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
sectionResults := make([]sectionProposals, len(input.Sections))
|
type modulePipelineResult struct {
|
||||||
|
ProposalCount int
|
||||||
|
Approved []proposals.EnrichedCorrectionProposal
|
||||||
|
ValidatorDecisions []ValidatorDecisionRecord
|
||||||
|
ValidatorRejected []ValidatorRejectedChange
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
|
||||||
|
results := make(chan sectionProposalResult, len(input.Sections))
|
||||||
runCtx, cancel := context.WithCancel(ctx)
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(results)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for sectionPos, section := range input.Sections {
|
||||||
|
sectionPos := sectionPos
|
||||||
|
section := section
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
meta := contracts.SectionMetadataFromSection(section)
|
||||||
|
corrected, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
||||||
|
ExecutionContext: contracts.ExecutionContext{
|
||||||
|
Config: input.Config,
|
||||||
|
WorkingTranscript: transcriptFromSection(section),
|
||||||
|
Glossary: input.Glossary,
|
||||||
|
Section: &meta,
|
||||||
|
DiagnosticsDir: input.DiagnosticsDir,
|
||||||
|
},
|
||||||
|
RunSpec: contracts.ModuleRunSpec{
|
||||||
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
|
InstanceName: input.Spec.InstanceName,
|
||||||
|
ReplacementPolicy: input.Policy,
|
||||||
|
},
|
||||||
|
LLMClient: input.ProposalClient,
|
||||||
|
LLMScheduler: input.ProposalScheduler,
|
||||||
|
})
|
||||||
|
select {
|
||||||
|
case results <- sectionProposalResult{
|
||||||
|
sectionPos: sectionPos,
|
||||||
|
section: section,
|
||||||
|
corrected: corrected,
|
||||||
|
err: err,
|
||||||
|
}:
|
||||||
|
case <-runCtx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
return runCtx, results, cancel
|
||||||
|
}
|
||||||
|
|
||||||
|
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||||
|
out := modulePipelineResult{
|
||||||
|
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
|
||||||
|
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
|
||||||
|
ValidatorRejected: make([]ValidatorRejectedChange, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(input.Sections) == 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
runCtx, proposalResults, cancel := collectSectionProposals(ctx, input)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sem := make(chan struct{}, maxWorkers)
|
pending := make(map[int]sectionProposalResult, len(input.Sections))
|
||||||
|
validationResults := make(chan sectionValidationResult, len(input.Sections))
|
||||||
|
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
|
||||||
|
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
|
||||||
|
|
||||||
|
nextSectionToProcess := 0
|
||||||
|
nextProposalIndex := 0
|
||||||
|
validationLaunches := 0
|
||||||
|
|
||||||
var (
|
var (
|
||||||
wg sync.WaitGroup
|
|
||||||
errOnce sync.Once
|
|
||||||
firstErr error
|
firstErr error
|
||||||
|
errOnce sync.Once
|
||||||
|
vwg sync.WaitGroup
|
||||||
)
|
)
|
||||||
|
|
||||||
for sectionPos, section := range input.Sections {
|
setErr := func(err error) {
|
||||||
sectionPos := sectionPos
|
if err == nil {
|
||||||
section := section
|
return
|
||||||
wg.Add(1)
|
}
|
||||||
go func() {
|
errOnce.Do(func() {
|
||||||
defer wg.Done()
|
firstErr = err
|
||||||
select {
|
cancel()
|
||||||
case sem <- struct{}{}:
|
})
|
||||||
case <-runCtx.Done():
|
}
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() { <-sem }()
|
|
||||||
|
|
||||||
meta := contracts.SectionMetadataFromSection(section)
|
for result := range proposalResults {
|
||||||
corrected, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
if result.err != nil {
|
||||||
ExecutionContext: contracts.ExecutionContext{
|
setErr(result.err)
|
||||||
Config: input.Config,
|
continue
|
||||||
WorkingTranscript: transcriptFromSection(section),
|
}
|
||||||
Glossary: input.Glossary,
|
if firstErr != nil {
|
||||||
Section: &meta,
|
continue
|
||||||
DiagnosticsDir: input.DiagnosticsDir,
|
}
|
||||||
},
|
pending[result.sectionPos] = result
|
||||||
RunSpec: contracts.ModuleRunSpec{
|
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
for {
|
||||||
InstanceName: input.Spec.InstanceName,
|
sectionResult, ok := pending[nextSectionToProcess]
|
||||||
ReplacementPolicy: input.Policy,
|
if !ok {
|
||||||
},
|
break
|
||||||
LLMClient: input.ProposalClient,
|
}
|
||||||
LLMScheduler: input.ProposalScheduler,
|
delete(pending, nextSectionToProcess)
|
||||||
})
|
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||||
if err != nil {
|
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
|
||||||
errOnce.Do(func() {
|
for i, corrected := range sectionResult.corrected {
|
||||||
firstErr = err
|
sectionIndex := sectionMeta.Index
|
||||||
cancel()
|
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
||||||
|
CorrectionProposal: corrected,
|
||||||
|
ProposalMetadata: proposals.ProposalMetadata{
|
||||||
|
ProposalIndex: nextProposalIndex + i,
|
||||||
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
|
ModuleInstance: input.Spec.InstanceName,
|
||||||
|
SectionIndex: §ionIndex,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
nextProposalIndex += len(sectionEnriched)
|
||||||
|
out.ProposalCount += len(sectionEnriched)
|
||||||
|
|
||||||
sectionResults[sectionPos] = sectionProposals{
|
validationLaunches++
|
||||||
meta: meta,
|
vwg.Add(1)
|
||||||
corrected: corrected,
|
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
||||||
}
|
defer vwg.Done()
|
||||||
}()
|
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
|
||||||
}
|
Spec: input.Spec,
|
||||||
|
Policy: input.Policy,
|
||||||
wg.Wait()
|
Glossary: input.Glossary,
|
||||||
if firstErr != nil {
|
Config: input.Config,
|
||||||
return nil, firstErr
|
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||||
}
|
ModuleInstanceForStages: fmt.Sprintf("%s:section-%04d", input.Spec.InstanceName, sectionMetadata.Index),
|
||||||
|
Validators: validatorsOrdered,
|
||||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0)
|
SectionEnriched: enriched,
|
||||||
nextProposalIndex := 0
|
ValidationLLMClient: input.ValidationClient,
|
||||||
for _, sectionResult := range sectionResults {
|
ValidationScheduler: input.ValidationScheduler,
|
||||||
for _, corrected := range sectionResult.corrected {
|
DiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||||
sectionIndex := sectionResult.meta.Index
|
})
|
||||||
enriched = append(enriched, proposals.EnrichedCorrectionProposal{
|
validationResults <- sectionValidationResult{
|
||||||
CorrectionProposal: corrected,
|
sectionPos: sectionPos,
|
||||||
ProposalMetadata: proposals.ProposalMetadata{
|
enriched: enriched,
|
||||||
ProposalIndex: nextProposalIndex,
|
approved: validated.approved,
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
decisions: validated.decisions,
|
||||||
ModuleInstance: input.Spec.InstanceName,
|
rejected: validated.rejected,
|
||||||
SectionIndex: §ionIndex,
|
err: err,
|
||||||
},
|
}
|
||||||
})
|
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
||||||
nextProposalIndex++
|
nextSectionToProcess++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return enriched, nil
|
|
||||||
|
vwg.Wait()
|
||||||
|
close(validationResults)
|
||||||
|
|
||||||
|
for v := range validationResults {
|
||||||
|
validationBySection[v.sectionPos] = v
|
||||||
|
if v.err != nil {
|
||||||
|
setErr(v.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < validationLaunches; i++ {
|
||||||
|
res, ok := validationBySection[i]
|
||||||
|
if !ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
out.Approved = append(out.Approved, res.approved...)
|
||||||
|
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||||
|
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
|
||||||
|
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
|
||||||
|
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
|
||||||
|
}
|
||||||
|
return validatorOrder[out.ValidatorDecisions[i].ValidatorName] < validatorOrder[out.ValidatorDecisions[j].ValidatorName]
|
||||||
|
})
|
||||||
|
sort.SliceStable(out.ValidatorRejected, func(i, j int) bool {
|
||||||
|
if out.ValidatorRejected[i].ProposalIndex != out.ValidatorRejected[j].ProposalIndex {
|
||||||
|
return out.ValidatorRejected[i].ProposalIndex < out.ValidatorRejected[j].ProposalIndex
|
||||||
|
}
|
||||||
|
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
|
||||||
|
})
|
||||||
|
|
||||||
|
if firstErr != nil {
|
||||||
|
return out, firstErr
|
||||||
|
}
|
||||||
|
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type validateSectionCandidatesInput struct {
|
||||||
|
Spec contracts.ModuleRunSpec
|
||||||
|
Policy proposals.ReplacementPolicy
|
||||||
|
Glossary *schema.Glossary
|
||||||
|
Config *config.Config
|
||||||
|
WorkingTranscript *schema.Transcript
|
||||||
|
ModuleInstanceForStages string
|
||||||
|
Validators []contracts.Validator
|
||||||
|
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||||
|
ValidationLLMClient contracts.StructuredLLMClient
|
||||||
|
ValidationScheduler ValidationScheduler
|
||||||
|
DiagnosticsDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
type validateSectionCandidatesResult struct {
|
||||||
|
approved []proposals.EnrichedCorrectionProposal
|
||||||
|
decisions []ValidatorDecisionRecord
|
||||||
|
rejected []ValidatorRejectedChange
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
|
||||||
|
decisions := make([]ValidatorDecisionRecord, 0)
|
||||||
|
rejected := make([]ValidatorRejectedChange, 0)
|
||||||
|
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||||
|
|
||||||
|
for _, validator := range input.Validators {
|
||||||
|
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||||
|
if input.DiagnosticsDir != "" {
|
||||||
|
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||||
|
writer: llm.NewDiagnosticsWriter(
|
||||||
|
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||||
|
validatorSecrets(input.Config),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||||
|
WorkingTranscript: input.WorkingTranscript,
|
||||||
|
CandidateProposal: eligible,
|
||||||
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
|
ModuleInstance: input.ModuleInstanceForStages,
|
||||||
|
ReplacementPolicy: input.Policy,
|
||||||
|
Glossary: input.Glossary,
|
||||||
|
Config: input.Config,
|
||||||
|
LLMClient: validationLLMClientAdapter{client: input.ValidationLLMClient},
|
||||||
|
Scheduler: input.ValidationScheduler,
|
||||||
|
DiagnosticsWriter: diagnosticsWriter,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return validateSectionCandidatesResult{
|
||||||
|
approved: eligible,
|
||||||
|
decisions: decisions,
|
||||||
|
rejected: rejected,
|
||||||
|
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
||||||
|
}
|
||||||
|
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
||||||
|
return validateSectionCandidatesResult{
|
||||||
|
approved: eligible,
|
||||||
|
decisions: decisions,
|
||||||
|
rejected: rejected,
|
||||||
|
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||||
|
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||||
|
for _, p := range eligible {
|
||||||
|
byIndex[p.ProposalIndex] = p
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range vResult.Decisions {
|
||||||
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
|
ValidatorName: validator.Name(),
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
Approved: d.Approved,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
|
})
|
||||||
|
if d.Approved {
|
||||||
|
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p := byIndex[d.ProposalIndex]
|
||||||
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
|
ValidatorName: validator.Name(),
|
||||||
|
ProposalIndex: p.ProposalIndex,
|
||||||
|
ModuleKey: p.ModuleKey,
|
||||||
|
ModuleInstance: p.ModuleInstance,
|
||||||
|
TargetSegmentID: p.TargetSegmentID,
|
||||||
|
OriginalText: p.OriginalText,
|
||||||
|
CorrectedText: p.CorrectedText,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
eligible = nextEligible
|
||||||
|
}
|
||||||
|
|
||||||
|
return validateSectionCandidatesResult{
|
||||||
|
approved: eligible,
|
||||||
|
decisions: decisions,
|
||||||
|
rejected: rejected,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
||||||
|
deterministic := make([]contracts.Validator, 0, len(in))
|
||||||
|
llmBacked := make([]contracts.Validator, 0, len(in))
|
||||||
|
for _, validator := range in {
|
||||||
|
if _, ok := validator.(*validators.LLMBackedValidator); ok {
|
||||||
|
llmBacked = append(llmBacked, validator)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deterministic = append(deterministic, validator)
|
||||||
|
}
|
||||||
|
ordered := append(deterministic, llmBacked...)
|
||||||
|
order := make(map[string]int, len(ordered))
|
||||||
|
for idx, validator := range ordered {
|
||||||
|
order[validator.Name()] = idx
|
||||||
|
}
|
||||||
|
return ordered, order
|
||||||
}
|
}
|
||||||
|
|
||||||
func chunkWorkingTranscript(cfg *config.Config, transcript *schema.Transcript) ([]chunking.Section, error) {
|
func chunkWorkingTranscript(cfg *config.Config, transcript *schema.Transcript) ([]chunking.Section, error) {
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -181,18 +183,28 @@ func TestRunnerProposalSectionConcurrencyBoundedByProposalLLMConcurrency(t *test
|
|||||||
var maxInFlight int32
|
var maxInFlight int32
|
||||||
entered := make(chan struct{}, len(transcript.Segments))
|
entered := make(chan struct{}, len(transcript.Segments))
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
scheduler, err := llm.NewScheduler(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler: %v", err)
|
||||||
|
}
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
current := atomic.AddInt32(&inFlight, 1)
|
err := req.LLMScheduler.Run(context.Background(), func(context.Context) error {
|
||||||
for {
|
current := atomic.AddInt32(&inFlight, 1)
|
||||||
prior := atomic.LoadInt32(&maxInFlight)
|
for {
|
||||||
if current <= prior || atomic.CompareAndSwapInt32(&maxInFlight, prior, current) {
|
prior := atomic.LoadInt32(&maxInFlight)
|
||||||
break
|
if current <= prior || atomic.CompareAndSwapInt32(&maxInFlight, prior, current) {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
entered <- struct{}{}
|
||||||
|
<-release
|
||||||
|
atomic.AddInt32(&inFlight, -1)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
entered <- struct{}{}
|
|
||||||
<-release
|
|
||||||
atomic.AddInt32(&inFlight, -1)
|
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}},
|
}},
|
||||||
}})
|
}})
|
||||||
@@ -200,8 +212,9 @@ func TestRunnerProposalSectionConcurrencyBoundedByProposalLLMConcurrency(t *test
|
|||||||
resultCh := make(chan error, 1)
|
resultCh := make(chan error, 1)
|
||||||
go func() {
|
go func() {
|
||||||
_, err := r.Run(context.Background(), RunInput{
|
_, err := r.Run(context.Background(), RunInput{
|
||||||
Config: &cfg,
|
Config: &cfg,
|
||||||
Transcript: transcript,
|
Transcript: transcript,
|
||||||
|
ProposalLLMScheduler: scheduler,
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{
|
ModuleSpecs: []contracts.ModuleRunSpec{
|
||||||
{ModuleKey: "m", InstanceName: "m"},
|
{ModuleKey: "m", InstanceName: "m"},
|
||||||
},
|
},
|
||||||
@@ -212,7 +225,7 @@ func TestRunnerProposalSectionConcurrencyBoundedByProposalLLMConcurrency(t *test
|
|||||||
waitForRunnerEntries(t, entered, 2, "proposal workers to enter")
|
waitForRunnerEntries(t, entered, 2, "proposal workers to enter")
|
||||||
close(release)
|
close(release)
|
||||||
|
|
||||||
err := <-resultCh
|
err = <-resultCh
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run error: %v", err)
|
t.Fatalf("Run error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -342,6 +355,436 @@ func mapKeys(values map[int]struct{}) []int {
|
|||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.MaxSectionTokens = 3
|
||||||
|
cfg.MinSectionTokens = 0
|
||||||
|
cfg.TotalLLMConcurrency = 2
|
||||||
|
cfg.ProposalLLMConcurrency = 2
|
||||||
|
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Text: "alpha one"},
|
||||||
|
{ID: 2, Text: "bravo two"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
releaseSectionOne := make(chan struct{})
|
||||||
|
validatorStarted := make(chan struct{}, 1)
|
||||||
|
|
||||||
|
validator := fakeValidator{
|
||||||
|
name: "capture",
|
||||||
|
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
|
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
|
||||||
|
select {
|
||||||
|
case validatorStarted <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
|
||||||
|
for _, p := range req.CandidateProposal {
|
||||||
|
decisions = append(decisions, validators.Decision{
|
||||||
|
ProposalIndex: p.ProposalIndex,
|
||||||
|
Approved: true,
|
||||||
|
ReasonCode: validators.ReasonApproved,
|
||||||
|
Message: "ok",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return validators.Result{ValidatorName: "capture", Decisions: decisions}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
"m": fakeModule{
|
||||||
|
key: "m",
|
||||||
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
validators: []contracts.Validator{validator},
|
||||||
|
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
|
seg := req.WorkingTranscript.Segments[0]
|
||||||
|
if req.Section != nil && req.Section.Index == 1 {
|
||||||
|
<-releaseSectionOne
|
||||||
|
}
|
||||||
|
return []proposals.CorrectionProposal{
|
||||||
|
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
|
||||||
|
resultCh := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.Run(context.Background(), RunInput{
|
||||||
|
Config: &cfg,
|
||||||
|
Transcript: transcript,
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{
|
||||||
|
{ModuleKey: "m", InstanceName: "m"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
resultCh <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-validatorStarted:
|
||||||
|
case <-time.After(350 * time.Millisecond):
|
||||||
|
t.Fatal("expected section-level validator work before all proposal sections complete")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(releaseSectionOne)
|
||||||
|
if err := <-resultCh; err != nil {
|
||||||
|
t.Fatalf("Run error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerProposalAndValidationLLMCanOverlapWithinModule(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.MaxSectionTokens = 3
|
||||||
|
cfg.MinSectionTokens = 0
|
||||||
|
cfg.TotalLLMConcurrency = 2
|
||||||
|
cfg.ProposalLLMConcurrency = 2
|
||||||
|
validationCap := 2
|
||||||
|
cfg.ValidationLLMConcurrency = &validationCap
|
||||||
|
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Text: "alpha one"},
|
||||||
|
{ID: 2, Text: "bravo two"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
global, err := llm.NewScheduler(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler: %v", err)
|
||||||
|
}
|
||||||
|
scheduler := &trackingScheduler{inner: global}
|
||||||
|
|
||||||
|
proposalSectionOneStarted := make(chan struct{}, 1)
|
||||||
|
releaseProposalSectionOne := make(chan struct{})
|
||||||
|
|
||||||
|
releaseValidation := make(chan struct{})
|
||||||
|
client := &stageAwareStructuredClient{
|
||||||
|
startedSection: make(chan int, 8),
|
||||||
|
releaseValidation: releaseValidation,
|
||||||
|
}
|
||||||
|
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
"m": fakeModule{
|
||||||
|
key: "m",
|
||||||
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
validators: []contracts.Validator{llmValidator},
|
||||||
|
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
|
seg := req.WorkingTranscript.Segments[0]
|
||||||
|
err := req.LLMScheduler.Run(context.Background(), func(context.Context) error {
|
||||||
|
if req.Section != nil && req.Section.Index == 1 {
|
||||||
|
select {
|
||||||
|
case proposalSectionOneStarted <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
<-releaseProposalSectionOne
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []proposals.CorrectionProposal{
|
||||||
|
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
|
||||||
|
resultCh := make(chan struct {
|
||||||
|
out RunOutput
|
||||||
|
err error
|
||||||
|
}, 1)
|
||||||
|
go func() {
|
||||||
|
out, err := r.Run(context.Background(), RunInput{
|
||||||
|
Config: &cfg,
|
||||||
|
Transcript: transcript,
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
|
ProposalLLMScheduler: scheduler,
|
||||||
|
ValidationLLMScheduler: scheduler,
|
||||||
|
ValidationLLMClient: client,
|
||||||
|
})
|
||||||
|
resultCh <- struct {
|
||||||
|
out RunOutput
|
||||||
|
err error
|
||||||
|
}{out: out, err: err}
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-proposalSectionOneStarted:
|
||||||
|
case <-time.After(350 * time.Millisecond):
|
||||||
|
t.Fatal("expected section-1 proposal to start")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case section := <-client.startedSection:
|
||||||
|
if section != 0 {
|
||||||
|
t.Fatalf("expected first validation to start for section 0, got section %d", section)
|
||||||
|
}
|
||||||
|
case <-time.After(350 * time.Millisecond):
|
||||||
|
t.Fatal("expected validation call for section 0 while later section proposal still running")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(releaseProposalSectionOne)
|
||||||
|
close(releaseValidation)
|
||||||
|
|
||||||
|
result := <-resultCh
|
||||||
|
if result.err != nil {
|
||||||
|
t.Fatalf("Run error: %v", result.err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&scheduler.maxInFlight); got > 2 {
|
||||||
|
t.Fatalf("expected combined proposal+validation in-flight <= 2, got %d", got)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&scheduler.maxInFlight); got < 2 {
|
||||||
|
t.Fatalf("expected observed overlap/in-flight utilization of at least 2, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerValidationLLMConcurrencyRespected(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.MaxSectionTokens = 3
|
||||||
|
cfg.MinSectionTokens = 0
|
||||||
|
cfg.TotalLLMConcurrency = 4
|
||||||
|
cfg.ProposalLLMConcurrency = 4
|
||||||
|
validationCap := 1
|
||||||
|
cfg.ValidationLLMConcurrency = &validationCap
|
||||||
|
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Text: "one one"},
|
||||||
|
{ID: 2, Text: "two two"},
|
||||||
|
{ID: 3, Text: "three three"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
validationSchedulerInner, err := llm.NewScheduler(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler(validation): %v", err)
|
||||||
|
}
|
||||||
|
validationScheduler := &trackingScheduler{inner: validationSchedulerInner}
|
||||||
|
|
||||||
|
releaseValidation := make(chan struct{})
|
||||||
|
client := &stageAwareStructuredClient{
|
||||||
|
startedSection: make(chan int, 16),
|
||||||
|
releaseValidation: releaseValidation,
|
||||||
|
}
|
||||||
|
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
"m": fakeModule{
|
||||||
|
key: "m",
|
||||||
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
validators: []contracts.Validator{llmValidator},
|
||||||
|
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
|
seg := req.WorkingTranscript.Segments[0]
|
||||||
|
return []proposals.CorrectionProposal{
|
||||||
|
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
|
||||||
|
resultCh := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.Run(context.Background(), RunInput{
|
||||||
|
Config: &cfg,
|
||||||
|
Transcript: transcript,
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
|
ValidationLLMScheduler: validationScheduler,
|
||||||
|
ValidationLLMClient: client,
|
||||||
|
})
|
||||||
|
resultCh <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-client.startedSection:
|
||||||
|
case <-time.After(350 * time.Millisecond):
|
||||||
|
t.Fatal("expected validation to start")
|
||||||
|
}
|
||||||
|
close(releaseValidation)
|
||||||
|
|
||||||
|
if err := <-resultCh; err != nil {
|
||||||
|
t.Fatalf("Run error: %v", err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&validationScheduler.maxInFlight); got > 1 {
|
||||||
|
t.Fatalf("expected validation in-flight <= 1, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.MaxSectionTokens = 3
|
||||||
|
cfg.MinSectionTokens = 0
|
||||||
|
cfg.TotalLLMConcurrency = 1
|
||||||
|
cfg.ProposalLLMConcurrency = 1
|
||||||
|
validationCap := 1
|
||||||
|
cfg.ValidationLLMConcurrency = &validationCap
|
||||||
|
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Text: "alpha one"},
|
||||||
|
{ID: 2, Text: "bravo two"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
global, err := llm.NewScheduler(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewScheduler: %v", err)
|
||||||
|
}
|
||||||
|
proposalScheduler := global
|
||||||
|
validationScheduler := global
|
||||||
|
|
||||||
|
events := make(chan string, 8)
|
||||||
|
releaseProposalSectionOne := make(chan struct{})
|
||||||
|
releaseValidation := make(chan struct{})
|
||||||
|
sectionZeroEntered := make(chan struct{})
|
||||||
|
|
||||||
|
client := &stageAwareStructuredClient{
|
||||||
|
startedSection: make(chan int, 8),
|
||||||
|
releaseValidation: releaseValidation,
|
||||||
|
eventSink: events,
|
||||||
|
}
|
||||||
|
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
"m": fakeModule{
|
||||||
|
key: "m",
|
||||||
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
validators: []contracts.Validator{llmValidator},
|
||||||
|
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
|
seg := req.WorkingTranscript.Segments[0]
|
||||||
|
if req.Section != nil && req.Section.Index == 1 {
|
||||||
|
<-sectionZeroEntered
|
||||||
|
}
|
||||||
|
err := req.LLMScheduler.Run(context.Background(), func(context.Context) error {
|
||||||
|
if req.Section != nil {
|
||||||
|
events <- "p" + strconv.Itoa(req.Section.Index)
|
||||||
|
if req.Section.Index == 0 {
|
||||||
|
select {
|
||||||
|
case sectionZeroEntered <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.Section.Index == 1 {
|
||||||
|
<-releaseProposalSectionOne
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []proposals.CorrectionProposal{
|
||||||
|
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
|
||||||
|
resultCh := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.Run(context.Background(), RunInput{
|
||||||
|
Config: &cfg,
|
||||||
|
Transcript: transcript,
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
|
ProposalLLMScheduler: proposalScheduler,
|
||||||
|
ValidationLLMScheduler: validationScheduler,
|
||||||
|
ValidationLLMClient: client,
|
||||||
|
})
|
||||||
|
resultCh <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
if got := <-events; got != "p0" {
|
||||||
|
t.Fatalf("expected first event p0, got %q", got)
|
||||||
|
}
|
||||||
|
if got := <-events; got != "p1" {
|
||||||
|
t.Fatalf("expected second event p1 (FIFO queued proposal), got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(releaseProposalSectionOne)
|
||||||
|
close(releaseValidation)
|
||||||
|
|
||||||
|
if got := <-events; got != "v0" {
|
||||||
|
t.Fatalf("expected validator event v0 after queued p1, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := <-resultCh; err != nil {
|
||||||
|
t.Fatalf("Run error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var sectionStagePattern = regexp.MustCompile(`section-(\d+)`)
|
||||||
|
|
||||||
|
type stageAwareStructuredClient struct {
|
||||||
|
startedSection chan int
|
||||||
|
releaseValidation <-chan struct{}
|
||||||
|
eventSink chan<- string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
section := parseSectionFromStage(req.StageName)
|
||||||
|
if c.startedSection != nil {
|
||||||
|
select {
|
||||||
|
case c.startedSection <- section:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.eventSink != nil {
|
||||||
|
c.eventSink <- "v" + strconv.Itoa(section)
|
||||||
|
}
|
||||||
|
if c.releaseValidation != nil {
|
||||||
|
select {
|
||||||
|
case <-c.releaseValidation:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
target := out.(*validators.LLMValidationResponse)
|
||||||
|
*target = validators.LLMValidationResponse{
|
||||||
|
Validations: []validators.LLMValidationDecision{
|
||||||
|
{CorrectionIndex: section, Approved: true, Confidence: 0.99, Reason: "ok"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSectionFromStage(stage string) int {
|
||||||
|
match := sectionStagePattern.FindStringSubmatch(stage)
|
||||||
|
if len(match) != 2 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(match[1])
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
type trackingScheduler struct {
|
||||||
|
inner contracts.LLMScheduler
|
||||||
|
inFlight int32
|
||||||
|
maxInFlight int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *trackingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||||
|
return s.inner.Run(ctx, func(callCtx context.Context) error {
|
||||||
|
current := atomic.AddInt32(&s.inFlight, 1)
|
||||||
|
for {
|
||||||
|
prior := atomic.LoadInt32(&s.maxInFlight)
|
||||||
|
if current <= prior || atomic.CompareAndSwapInt32(&s.maxInFlight, prior, current) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer atomic.AddInt32(&s.inFlight, -1)
|
||||||
|
return fn(callCtx)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerSkippedRecorded(t *testing.T) {
|
func TestRunnerSkippedRecorded(t *testing.T) {
|
||||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
|||||||
Reference in New Issue
Block a user