Hardened the proposal modules to skip malformed proposals rather than hard failing the entire run
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
This commit is contained in:
@@ -364,7 +364,7 @@ func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
func TestProcessMalformedStructuredLLMResponseViaSubprocessHookSoftSkips(t *testing.T) {
|
||||||
workDir := t.TempDir()
|
workDir := t.TempDir()
|
||||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
result := runCLISubprocessWithEnv(t,
|
result := runCLISubprocessWithEnv(t,
|
||||||
@@ -382,25 +382,34 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin
|
|||||||
"--work-dir-retention",
|
"--work-dir-retention",
|
||||||
"always",
|
"always",
|
||||||
)
|
)
|
||||||
if result.exitCode == 0 {
|
if result.exitCode != 0 {
|
||||||
t.Fatalf("expected nonzero exit code")
|
t.Fatalf("expected success exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||||
}
|
}
|
||||||
if result.stdout != "" {
|
if !json.Valid([]byte(result.stdout)) {
|
||||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(result.stderr, "runner_execution") {
|
if result.stderr != "" {
|
||||||
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
|
t.Fatalf("expected quiet stderr on success, got %q", result.stderr)
|
||||||
}
|
|
||||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
|
||||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
|
||||||
}
|
}
|
||||||
report := readFile(t, reportPath)
|
report := readFile(t, reportPath)
|
||||||
if !json.Valid(report) {
|
if !json.Valid(report) {
|
||||||
t.Fatalf("expected valid failure report JSON")
|
t.Fatalf("expected valid report JSON")
|
||||||
}
|
}
|
||||||
runDir := onlyRunDir(t, workDir)
|
var parsed struct {
|
||||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
Status string `json:"status"`
|
||||||
t.Fatalf("expected error.log, got: %v", err)
|
ModuleResults []struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
ValidatorRejected []any `json:"validator_rejected"`
|
||||||
|
} `json:"module_results"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(report, &parsed); err != nil {
|
||||||
|
t.Fatalf("unmarshal report: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Status != "success" {
|
||||||
|
t.Fatalf("expected success report status, got %+v", parsed)
|
||||||
|
}
|
||||||
|
if len(parsed.ModuleResults) == 0 || len(parsed.ModuleResults[0].ValidatorRejected) == 0 {
|
||||||
|
t.Fatalf("expected soft-skip validator rejection records, got %+v", parsed.ModuleResults)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,7 +445,7 @@ func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
func TestProcessMidPipelineMalformedCorrectionsSoftSkipSubprocess(t *testing.T) {
|
||||||
workDir := t.TempDir()
|
workDir := t.TempDir()
|
||||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
result := runCLISubprocessWithEnv(t,
|
result := runCLISubprocessWithEnv(t,
|
||||||
@@ -454,11 +463,11 @@ func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T
|
|||||||
"--work-dir-retention",
|
"--work-dir-retention",
|
||||||
"always",
|
"always",
|
||||||
)
|
)
|
||||||
if result.exitCode == 0 {
|
if result.exitCode != 0 {
|
||||||
t.Fatalf("expected nonzero exit code")
|
t.Fatalf("expected success exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||||
}
|
}
|
||||||
if result.stdout != "" {
|
if !json.Valid([]byte(result.stdout)) {
|
||||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||||
}
|
}
|
||||||
reportRaw := readFile(t, reportPath)
|
reportRaw := readFile(t, reportPath)
|
||||||
var report struct {
|
var report struct {
|
||||||
@@ -472,11 +481,11 @@ func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T
|
|||||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||||
t.Fatalf("unmarshal report: %v", err)
|
t.Fatalf("unmarshal report: %v", err)
|
||||||
}
|
}
|
||||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
if report.Status != "success" || report.ErrorPhase != "" {
|
||||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
t.Fatalf("expected successful report with no runner_execution phase, got %+v", report)
|
||||||
}
|
}
|
||||||
if len(report.ModuleResults) == 0 {
|
if len(report.ModuleResults) == 0 {
|
||||||
t.Fatalf("expected partial module results in failure report")
|
t.Fatalf("expected module results in report")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ type ProposalRequest struct {
|
|||||||
RunSpec ModuleRunSpec `json:"run_spec"`
|
RunSpec ModuleRunSpec `json:"run_spec"`
|
||||||
LLMClient StructuredLLMClient `json:"-"`
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
LLMScheduler LLMScheduler `json:"-"`
|
LLMScheduler LLMScheduler `json:"-"`
|
||||||
|
OnDroppedCandidate func(index int, targetSegmentID int, originalText string, correctedText string, confidence float64, reasonCode string, message string) `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidationRequest is the input to validator execution.
|
// ValidationRequest is the input to validator execution.
|
||||||
|
|||||||
@@ -44,6 +44,16 @@ type StructuredCorrectionSet struct {
|
|||||||
Corrections []StructuredCorrectionProposal `json:"corrections"`
|
Corrections []StructuredCorrectionProposal `json:"corrections"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type DroppedCandidate struct {
|
||||||
|
ProposalIndex int `json:"proposal_index"`
|
||||||
|
TargetSegmentID int `json:"target_segment_id"`
|
||||||
|
OriginalText string `json:"original_text"`
|
||||||
|
CorrectedText string `json:"corrected_text"`
|
||||||
|
Confidence float64 `json:"confidence"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
// Request captures reusable proposal-generation inputs for future modules.
|
// Request captures reusable proposal-generation inputs for future modules.
|
||||||
type Request struct {
|
type Request struct {
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
@@ -62,12 +72,14 @@ type Request struct {
|
|||||||
Scheduler contracts.LLMScheduler
|
Scheduler contracts.LLMScheduler
|
||||||
DiagnosticsDir string
|
DiagnosticsDir string
|
||||||
DiagnosticsWriter InteractionDiagnosticsWriter
|
DiagnosticsWriter InteractionDiagnosticsWriter
|
||||||
|
OnDroppedCandidate func(dropped DroppedCandidate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||||
type Result struct {
|
type Result struct {
|
||||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||||
|
Dropped []DroppedCandidate `json:"dropped,omitempty"`
|
||||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,7 +173,9 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
|
|
||||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||||
|
dropped := make([]DroppedCandidate, 0)
|
||||||
for i, raw := range response.Corrections {
|
for i, raw := range response.Corrections {
|
||||||
|
proposalIndex := req.StartIndex + i
|
||||||
candidate := proposals.CorrectionProposal{
|
candidate := proposals.CorrectionProposal{
|
||||||
TargetSegmentID: raw.TargetSegmentID,
|
TargetSegmentID: raw.TargetSegmentID,
|
||||||
OriginalText: raw.OriginalText,
|
OriginalText: raw.OriginalText,
|
||||||
@@ -169,14 +183,27 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
Confidence: raw.Confidence,
|
Confidence: raw.Confidence,
|
||||||
}
|
}
|
||||||
if err := candidate.Validate(); err != nil {
|
if err := candidate.Validate(); err != nil {
|
||||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
record := DroppedCandidate{
|
||||||
|
ProposalIndex: proposalIndex,
|
||||||
|
TargetSegmentID: raw.TargetSegmentID,
|
||||||
|
OriginalText: raw.OriginalText,
|
||||||
|
CorrectedText: raw.CorrectedText,
|
||||||
|
Confidence: raw.Confidence,
|
||||||
|
ReasonCode: "invalid_structured_correction",
|
||||||
|
Message: fmt.Sprintf("invalid structured correction at index %d: %v", i, err),
|
||||||
|
}
|
||||||
|
dropped = append(dropped, record)
|
||||||
|
if req.OnDroppedCandidate != nil {
|
||||||
|
req.OnDroppedCandidate(record)
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
corrections = append(corrections, candidate)
|
corrections = append(corrections, candidate)
|
||||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||||
CorrectionProposal: candidate,
|
CorrectionProposal: candidate,
|
||||||
ProposalMetadata: proposals.ProposalMetadata{
|
ProposalMetadata: proposals.ProposalMetadata{
|
||||||
ProposalIndex: req.StartIndex + i,
|
ProposalIndex: proposalIndex,
|
||||||
ModuleKey: req.ModuleKey,
|
ModuleKey: req.ModuleKey,
|
||||||
ModuleInstance: req.ModuleInstance,
|
ModuleInstance: req.ModuleInstance,
|
||||||
},
|
},
|
||||||
@@ -191,6 +218,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
return Result{
|
return Result{
|
||||||
Corrections: corrections,
|
Corrections: corrections,
|
||||||
Enriched: enriched,
|
Enriched: enriched,
|
||||||
|
Dropped: dropped,
|
||||||
Artifacts: artifacts,
|
Artifacts: artifacts,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
|||||||
responses: []StructuredCorrectionSet{
|
responses: []StructuredCorrectionSet{
|
||||||
{
|
{
|
||||||
Corrections: []StructuredCorrectionProposal{
|
Corrections: []StructuredCorrectionProposal{
|
||||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -237,9 +237,18 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
|||||||
req := defaultRequest(t)
|
req := defaultRequest(t)
|
||||||
req.LLMClient = client
|
req.LLMClient = client
|
||||||
|
|
||||||
_, err := GenerateCandidates(context.Background(), req)
|
got, err := GenerateCandidates(context.Background(), req)
|
||||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
if err != nil {
|
||||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(got.Corrections) != 0 || len(got.Enriched) != 0 {
|
||||||
|
t.Fatalf("expected invalid proposal to be dropped, got %+v", got)
|
||||||
|
}
|
||||||
|
if len(got.Dropped) != 1 {
|
||||||
|
t.Fatalf("expected one dropped candidate, got %+v", got.Dropped)
|
||||||
|
}
|
||||||
|
if got.Dropped[0].ReasonCode != "invalid_structured_correction" {
|
||||||
|
t.Fatalf("unexpected dropped reason: %+v", got.Dropped[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
|
|||||||
if strings.TrimSpace(p.OriginalText) == "" {
|
if strings.TrimSpace(p.OriginalText) == "" {
|
||||||
return fmt.Errorf("proposal original_text must not be empty")
|
return fmt.Errorf("proposal original_text must not be empty")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.CorrectedText) == "" {
|
|
||||||
return fmt.Errorf("proposal corrected_text must not be empty")
|
|
||||||
}
|
|
||||||
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
||||||
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||||
proposal := CorrectionProposal{
|
proposal := CorrectionProposal{
|
||||||
TargetSegmentID: 42,
|
TargetSegmentID: 42,
|
||||||
OriginalText: "gestures",
|
OriginalText: "gestures",
|
||||||
@@ -44,11 +44,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
err := proposal.Validate()
|
err := proposal.Validate()
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected validation error, got nil")
|
t.Fatalf("expected valid proposal, got %v", err)
|
||||||
}
|
|
||||||
if err.Error() != "proposal corrected_text must not be empty" {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -238,6 +238,8 @@ type sectionProposalResult struct {
|
|||||||
sectionPos int
|
sectionPos int
|
||||||
section chunking.Section
|
section chunking.Section
|
||||||
corrected []proposals.CorrectionProposal
|
corrected []proposals.CorrectionProposal
|
||||||
|
dropped []droppedProposalCandidate
|
||||||
|
rawCount int
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -257,6 +259,22 @@ type modulePipelineResult struct {
|
|||||||
ValidatorRejected []ValidatorRejectedChange
|
ValidatorRejected []ValidatorRejectedChange
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type droppedProposalCandidate struct {
|
||||||
|
ProposalIndex int
|
||||||
|
TargetSegmentID int
|
||||||
|
OriginalText string
|
||||||
|
CorrectedText string
|
||||||
|
ReasonCode string
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
validatorNameProposalGeneration = "proposal_generation"
|
||||||
|
reasonValidatorExecutionError = "validator_execution_error"
|
||||||
|
reasonValidatorMalformedResponse = "validator_malformed_response"
|
||||||
|
reasonValidatorMissingDecision = "validator_missing_decision"
|
||||||
|
)
|
||||||
|
|
||||||
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
|
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
|
||||||
results := make(chan sectionProposalResult, len(input.Sections))
|
results := make(chan sectionProposalResult, len(input.Sections))
|
||||||
runCtx, cancel := context.WithCancel(ctx)
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
@@ -271,6 +289,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
|||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
meta := contracts.SectionMetadataFromSection(section)
|
meta := contracts.SectionMetadataFromSection(section)
|
||||||
|
dropped := make([]droppedProposalCandidate, 0)
|
||||||
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||||
ExecutionContext: contracts.ExecutionContext{
|
ExecutionContext: contracts.ExecutionContext{
|
||||||
Config: input.Config,
|
Config: input.Config,
|
||||||
@@ -286,12 +305,25 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
|||||||
},
|
},
|
||||||
LLMClient: input.ProposalClient,
|
LLMClient: input.ProposalClient,
|
||||||
LLMScheduler: input.ProposalScheduler,
|
LLMScheduler: input.ProposalScheduler,
|
||||||
|
OnDroppedCandidate: func(index int, targetSegmentID int, originalText string, correctedText string, confidence float64, reasonCode string, message string) {
|
||||||
|
_ = confidence
|
||||||
|
dropped = append(dropped, droppedProposalCandidate{
|
||||||
|
ProposalIndex: index,
|
||||||
|
TargetSegmentID: targetSegmentID,
|
||||||
|
OriginalText: originalText,
|
||||||
|
CorrectedText: correctedText,
|
||||||
|
ReasonCode: reasonCode,
|
||||||
|
Message: message,
|
||||||
|
})
|
||||||
|
},
|
||||||
})
|
})
|
||||||
select {
|
select {
|
||||||
case results <- sectionProposalResult{
|
case results <- sectionProposalResult{
|
||||||
sectionPos: sectionPos,
|
sectionPos: sectionPos,
|
||||||
section: section,
|
section: section,
|
||||||
corrected: corrected,
|
corrected: corrected,
|
||||||
|
dropped: dropped,
|
||||||
|
rawCount: len(corrected) + len(dropped),
|
||||||
err: err,
|
err: err,
|
||||||
}:
|
}:
|
||||||
case <-runCtx.Done():
|
case <-runCtx.Done():
|
||||||
@@ -374,8 +406,21 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
nextProposalIndex += len(sectionEnriched)
|
out.ProposalCount += sectionResult.rawCount
|
||||||
out.ProposalCount += len(sectionEnriched)
|
nextProposalIndex += sectionResult.rawCount
|
||||||
|
for _, dropped := range sectionResult.dropped {
|
||||||
|
out.ValidatorRejected = append(out.ValidatorRejected, ValidatorRejectedChange{
|
||||||
|
ValidatorName: validatorNameProposalGeneration,
|
||||||
|
ProposalIndex: dropped.ProposalIndex,
|
||||||
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
|
ModuleInstance: input.Spec.InstanceName,
|
||||||
|
TargetSegmentID: dropped.TargetSegmentID,
|
||||||
|
OriginalText: dropped.OriginalText,
|
||||||
|
CorrectedText: dropped.CorrectedText,
|
||||||
|
ReasonCode: dropped.ReasonCode,
|
||||||
|
Message: dropped.Message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
validationLaunches++
|
validationLaunches++
|
||||||
vwg.Add(1)
|
vwg.Add(1)
|
||||||
@@ -508,29 +553,120 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return validateSectionCandidatesResult{
|
for _, p := range eligible {
|
||||||
approved: eligible,
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
decisions: decisions,
|
ValidatorName: validator.Name(),
|
||||||
rejected: rejected,
|
ProposalIndex: p.ProposalIndex,
|
||||||
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
Approved: false,
|
||||||
|
ReasonCode: reasonValidatorExecutionError,
|
||||||
|
Message: fmt.Sprintf("validator execution failed: %v", err),
|
||||||
|
})
|
||||||
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
|
ValidatorName: validator.Name(),
|
||||||
|
ProposalIndex: p.ProposalIndex,
|
||||||
|
ModuleKey: p.ModuleKey,
|
||||||
|
ModuleInstance: p.ModuleInstance,
|
||||||
|
TargetSegmentID: p.TargetSegmentID,
|
||||||
|
OriginalText: p.OriginalText,
|
||||||
|
CorrectedText: p.CorrectedText,
|
||||||
|
ReasonCode: reasonValidatorExecutionError,
|
||||||
|
Message: fmt.Sprintf("validator execution failed: %v", err),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
eligible = nil
|
||||||
return validateSectionCandidatesResult{
|
continue
|
||||||
approved: eligible,
|
}
|
||||||
decisions: decisions,
|
reconciled := reconcileValidatorDecisions(validator.Name(), eligible, vResult.Decisions)
|
||||||
rejected: rejected,
|
decisions = append(decisions, reconciled.decisions...)
|
||||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
|
rejected = append(rejected, reconciled.rejected...)
|
||||||
|
eligible = reconciled.nextEligible
|
||||||
}
|
}
|
||||||
|
|
||||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
return validateSectionCandidatesResult{
|
||||||
|
approved: eligible,
|
||||||
|
decisions: decisions,
|
||||||
|
rejected: rejected,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type reconciledValidatorDecisions struct {
|
||||||
|
nextEligible []proposals.EnrichedCorrectionProposal
|
||||||
|
decisions []ValidatorDecisionRecord
|
||||||
|
rejected []ValidatorRejectedChange
|
||||||
|
}
|
||||||
|
|
||||||
|
func reconcileValidatorDecisions(validatorName string, eligible []proposals.EnrichedCorrectionProposal, in []validators.Decision) reconciledValidatorDecisions {
|
||||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||||
for _, p := range eligible {
|
for _, p := range eligible {
|
||||||
byIndex[p.ProposalIndex] = p
|
byIndex[p.ProposalIndex] = p
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, d := range vResult.Decisions {
|
seen := make(map[int]struct{}, len(in))
|
||||||
|
forcedReject := make(map[int]bool)
|
||||||
|
valid := make(map[int]validators.Decision, len(in))
|
||||||
|
for _, d := range in {
|
||||||
|
if _, ok := byIndex[d.ProposalIndex]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[d.ProposalIndex]; dup {
|
||||||
|
forcedReject[d.ProposalIndex] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[d.ProposalIndex] = struct{}{}
|
||||||
|
valid[d.ProposalIndex] = d
|
||||||
|
}
|
||||||
|
|
||||||
|
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||||
|
decisions := make([]ValidatorDecisionRecord, 0, len(eligible))
|
||||||
|
rejected := make([]ValidatorRejectedChange, 0, len(eligible))
|
||||||
|
|
||||||
|
for _, p := range eligible {
|
||||||
|
idx := p.ProposalIndex
|
||||||
|
if forcedReject[idx] {
|
||||||
decisions = append(decisions, ValidatorDecisionRecord{
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
ValidatorName: validator.Name(),
|
ValidatorName: validatorName,
|
||||||
|
ProposalIndex: idx,
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: reasonValidatorMalformedResponse,
|
||||||
|
Message: "validator returned malformed response for this proposal index",
|
||||||
|
})
|
||||||
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
|
ValidatorName: validatorName,
|
||||||
|
ProposalIndex: p.ProposalIndex,
|
||||||
|
ModuleKey: p.ModuleKey,
|
||||||
|
ModuleInstance: p.ModuleInstance,
|
||||||
|
TargetSegmentID: p.TargetSegmentID,
|
||||||
|
OriginalText: p.OriginalText,
|
||||||
|
CorrectedText: p.CorrectedText,
|
||||||
|
ReasonCode: reasonValidatorMalformedResponse,
|
||||||
|
Message: "validator returned malformed response for this proposal index",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d, ok := valid[idx]
|
||||||
|
if !ok {
|
||||||
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
|
ValidatorName: validatorName,
|
||||||
|
ProposalIndex: idx,
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: reasonValidatorMissingDecision,
|
||||||
|
Message: "validator did not return a decision for this proposal index",
|
||||||
|
})
|
||||||
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
|
ValidatorName: validatorName,
|
||||||
|
ProposalIndex: p.ProposalIndex,
|
||||||
|
ModuleKey: p.ModuleKey,
|
||||||
|
ModuleInstance: p.ModuleInstance,
|
||||||
|
TargetSegmentID: p.TargetSegmentID,
|
||||||
|
OriginalText: p.OriginalText,
|
||||||
|
CorrectedText: p.CorrectedText,
|
||||||
|
ReasonCode: reasonValidatorMissingDecision,
|
||||||
|
Message: "validator did not return a decision for this proposal index",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
|
ValidatorName: validatorName,
|
||||||
ProposalIndex: d.ProposalIndex,
|
ProposalIndex: d.ProposalIndex,
|
||||||
Approved: d.Approved,
|
Approved: d.Approved,
|
||||||
ReasonCode: d.ReasonCode,
|
ReasonCode: d.ReasonCode,
|
||||||
@@ -538,12 +674,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
})
|
})
|
||||||
if d.Approved {
|
if d.Approved {
|
||||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
nextEligible = append(nextEligible, p)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
p := byIndex[d.ProposalIndex]
|
|
||||||
rejected = append(rejected, ValidatorRejectedChange{
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
ValidatorName: validator.Name(),
|
ValidatorName: validatorName,
|
||||||
ProposalIndex: p.ProposalIndex,
|
ProposalIndex: p.ProposalIndex,
|
||||||
ModuleKey: p.ModuleKey,
|
ModuleKey: p.ModuleKey,
|
||||||
ModuleInstance: p.ModuleInstance,
|
ModuleInstance: p.ModuleInstance,
|
||||||
@@ -555,14 +690,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
eligible = nextEligible
|
return reconciledValidatorDecisions{
|
||||||
}
|
nextEligible: nextEligible,
|
||||||
|
|
||||||
return validateSectionCandidatesResult{
|
|
||||||
approved: eligible,
|
|
||||||
decisions: decisions,
|
decisions: decisions,
|
||||||
rejected: rejected,
|
rejected: rejected,
|
||||||
}, nil
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
||||||
|
|||||||
@@ -1054,7 +1054,7 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
|
func TestRunnerValidatorCardinalityErrorSoftRejectsAndContinues(t *testing.T) {
|
||||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||||
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||||
@@ -1069,14 +1069,17 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
|||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
||||||
|
|
||||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected cardinality error")
|
t.Fatalf("unexpected run error: %v", err)
|
||||||
}
|
}
|
||||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||||
t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text)
|
t.Fatalf("expected second-module proposal to be rejected, got %q", out.FinalTranscript.Segments[0].Text)
|
||||||
}
|
}
|
||||||
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed {
|
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess {
|
||||||
t.Fatalf("expected second module failed")
|
t.Fatalf("expected second module success with rejection, got %+v", out.ModuleResults)
|
||||||
|
}
|
||||||
|
if len(out.ModuleResults[1].ValidatorRejected) == 0 {
|
||||||
|
t.Fatalf("expected validator rejection record, got %+v", out.ModuleResults[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1191,7 +1194,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
|
func TestRunnerLLMValidatorMalformedResponseSoftRejectsWithProgress(t *testing.T) {
|
||||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
@@ -1209,15 +1212,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
|
|||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
||||||
ValidationLLMClient: client,
|
ValidationLLMClient: client,
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected llm validator failure")
|
t.Fatalf("unexpected run error: %v", err)
|
||||||
}
|
}
|
||||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||||
t.Fatalf("expected partial progress retained")
|
t.Fatalf("expected partial progress retained")
|
||||||
}
|
}
|
||||||
|
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess {
|
||||||
|
t.Fatalf("expected second module success, got %+v", out.ModuleResults)
|
||||||
|
}
|
||||||
|
if len(out.ModuleResults[1].ValidatorRejected) == 0 {
|
||||||
|
t.Fatalf("expected llm validator rejection record")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
func TestRunnerLLMValidatorMissingDecisionSoftRejects(t *testing.T) {
|
||||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
@@ -1226,18 +1235,21 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
}})
|
}})
|
||||||
cfg := config.Default()
|
cfg := config.Default()
|
||||||
_, err := r.Run(context.Background(), RunInput{
|
out, err := r.Run(context.Background(), RunInput{
|
||||||
Config: &cfg,
|
Config: &cfg,
|
||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
ValidationLLMClient: client,
|
ValidationLLMClient: client,
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing decision failure")
|
t.Fatalf("unexpected run error: %v", err)
|
||||||
|
}
|
||||||
|
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
|
||||||
|
t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
func TestRunnerLLMValidatorDuplicateDecisionSoftRejects(t *testing.T) {
|
||||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||||
@@ -1249,14 +1261,17 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
}})
|
}})
|
||||||
cfg := config.Default()
|
cfg := config.Default()
|
||||||
_, err := r.Run(context.Background(), RunInput{
|
out, err := r.Run(context.Background(), RunInput{
|
||||||
Config: &cfg,
|
Config: &cfg,
|
||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
ValidationLLMClient: client,
|
ValidationLLMClient: client,
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected duplicate decision failure")
|
t.Fatalf("unexpected run error: %v", err)
|
||||||
|
}
|
||||||
|
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
|
||||||
|
t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ConfidenceThresholdValidator struct{}
|
type ConfidenceThresholdValidator struct{}
|
||||||
@@ -62,12 +65,27 @@ type NonEmptyCorrectionValidator struct{}
|
|||||||
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
|
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
|
||||||
|
|
||||||
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
|
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||||
|
segments := make(map[int]string)
|
||||||
|
if req.WorkingTranscript != nil {
|
||||||
|
for _, seg := range req.WorkingTranscript.Segments {
|
||||||
|
segments[seg.ID] = seg.Text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||||
for _, c := range req.CandidateProposal {
|
for _, c := range req.CandidateProposal {
|
||||||
if strings.TrimSpace(c.CorrectedText) == "" {
|
segmentText, ok := segments[c.TargetSegmentID]
|
||||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
|
if ok {
|
||||||
|
preview := proposals.PreviewProposalForSegment(
|
||||||
|
&schema.Segment{ID: c.TargetSegmentID, Text: segmentText},
|
||||||
|
c.CorrectionProposal,
|
||||||
|
req.ReplacementPolicy,
|
||||||
|
)
|
||||||
|
if preview.Applicable && strings.TrimSpace(preview.CorrectedSegmentText) == "" {
|
||||||
|
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected segment text must not be empty"))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
}
|
||||||
decisions = append(decisions, approval(c.ProposalIndex))
|
decisions = append(decisions, approval(c.ProposalIndex))
|
||||||
}
|
}
|
||||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||||
|
|||||||
@@ -143,10 +143,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
batchDecisions := mapLLMResponseToDecisions(batch.Items, response)
|
||||||
if err != nil {
|
|
||||||
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
|
|
||||||
}
|
|
||||||
for i := range batchDecisions {
|
for i := range batchDecisions {
|
||||||
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||||
}
|
}
|
||||||
@@ -259,34 +256,48 @@ func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
|
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) []Decision {
|
||||||
expected := make(map[int]LLMValidationItem, len(items))
|
expected := make(map[int]LLMValidationItem, len(items))
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
expected[item.CorrectionIndex] = item
|
expected[item.CorrectionIndex] = item
|
||||||
}
|
}
|
||||||
if len(response.Validations) == 0 {
|
|
||||||
return nil, fmt.Errorf("missing validations in structured response")
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := make(map[int]LLMValidationDecision, len(response.Validations))
|
seen := make(map[int]LLMValidationDecision, len(response.Validations))
|
||||||
|
forcedReject := make(map[int]bool)
|
||||||
for _, d := range response.Validations {
|
for _, d := range response.Validations {
|
||||||
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
|
||||||
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
|
|
||||||
}
|
|
||||||
if _, ok := expected[d.CorrectionIndex]; !ok {
|
if _, ok := expected[d.CorrectionIndex]; !ok {
|
||||||
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex)
|
continue
|
||||||
}
|
}
|
||||||
if _, exists := seen[d.CorrectionIndex]; exists {
|
if _, exists := seen[d.CorrectionIndex]; exists {
|
||||||
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
|
forcedReject[d.CorrectionIndex] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
||||||
|
forcedReject[d.CorrectionIndex] = true
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
seen[d.CorrectionIndex] = d
|
seen[d.CorrectionIndex] = d
|
||||||
}
|
}
|
||||||
|
|
||||||
decisions := make([]Decision, 0, len(items))
|
decisions := make([]Decision, 0, len(items))
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
|
if forcedReject[item.CorrectionIndex] {
|
||||||
|
decisions = append(decisions, Decision{
|
||||||
|
ProposalIndex: item.CorrectionIndex,
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonValidatorMalformed,
|
||||||
|
Message: "validator returned malformed decision payload for this proposal index",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
d, ok := seen[item.CorrectionIndex]
|
d, ok := seen[item.CorrectionIndex]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex)
|
decisions = append(decisions, Decision{
|
||||||
|
ProposalIndex: item.CorrectionIndex,
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonValidatorMissing,
|
||||||
|
Message: "validator did not return a decision for this proposal index",
|
||||||
|
})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
reasonCode := ReasonApproved
|
reasonCode := ReasonApproved
|
||||||
if !d.Approved {
|
if !d.Approved {
|
||||||
@@ -299,5 +310,5 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
|
|||||||
Message: strings.TrimSpace(d.Reason),
|
Message: strings.TrimSpace(d.Reason),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return decisions, nil
|
return decisions
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -259,18 +259,24 @@ func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
|
func TestLLMBackedValidatorMissingDecisionSoftRejects(t *testing.T) {
|
||||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||||
req.LLMClient = client
|
req.LLMClient = client
|
||||||
_, err := v.Validate(context.Background(), req)
|
res, err := v.Validate(context.Background(), req)
|
||||||
if err == nil || !strings.Contains(err.Error(), "response invalid") {
|
if err != nil {
|
||||||
t.Fatalf("expected missing decision error, got %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
|
||||||
|
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
|
||||||
|
}
|
||||||
|
if res.Decisions[0].ReasonCode != ReasonValidatorMissing {
|
||||||
|
t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
func TestLLMBackedValidatorDuplicateDecisionSoftRejects(t *testing.T) {
|
||||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||||
@@ -278,20 +284,32 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
|||||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||||
req.LLMClient = client
|
req.LLMClient = client
|
||||||
_, err := v.Validate(context.Background(), req)
|
res, err := v.Validate(context.Background(), req)
|
||||||
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
if err != nil {
|
||||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
|
||||||
|
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
|
||||||
|
}
|
||||||
|
if res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||||
|
t.Fatalf("expected %q, got %+v", ReasonValidatorMalformed, res.Decisions[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
|
func TestLLMBackedValidatorUnknownProposalIndexSoftRejectsMissing(t *testing.T) {
|
||||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
||||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||||
req.LLMClient = client
|
req.LLMClient = client
|
||||||
_, err := v.Validate(context.Background(), req)
|
res, err := v.Validate(context.Background(), req)
|
||||||
if err == nil || !strings.Contains(err.Error(), "unknown") {
|
if err != nil {
|
||||||
t.Fatalf("expected unknown index error, got %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
|
||||||
|
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
|
||||||
|
}
|
||||||
|
if res.Decisions[0].ReasonCode != ReasonValidatorMissing {
|
||||||
|
t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ const (
|
|||||||
ReasonEmptyCorrectedText = "empty_corrected_text"
|
ReasonEmptyCorrectedText = "empty_corrected_text"
|
||||||
ReasonNoEffect = "no_effect"
|
ReasonNoEffect = "no_effect"
|
||||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||||
|
ReasonValidatorMalformed = "validator_malformed_response"
|
||||||
|
ReasonValidatorMissing = "validator_missing_decision"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Request is the runtime input shared by deterministic validators.
|
// Request is the runtime input shared by deterministic validators.
|
||||||
|
|||||||
@@ -90,10 +90,14 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNonEmptyCorrectionValidator(t *testing.T) {
|
func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
req := Request{
|
||||||
|
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||||
mkCandidate(1, 1, "hello", " ", 0.9),
|
mkCandidate(1, 1, "hello", " ", 0.9),
|
||||||
}}
|
},
|
||||||
|
}
|
||||||
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
|
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Validate error: %v", err)
|
t.Fatalf("Validate error: %v", err)
|
||||||
@@ -154,6 +158,8 @@ func TestStableReasonCodes(t *testing.T) {
|
|||||||
ReasonEmptyCorrectedText,
|
ReasonEmptyCorrectedText,
|
||||||
ReasonNoEffect,
|
ReasonNoEffect,
|
||||||
ReasonProtectedGlossaryTerm,
|
ReasonProtectedGlossaryTerm,
|
||||||
|
ReasonValidatorMalformed,
|
||||||
|
ReasonValidatorMissing,
|
||||||
}
|
}
|
||||||
for _, code := range codes {
|
for _, code := range codes {
|
||||||
if strings.TrimSpace(code) == "" {
|
if strings.TrimSpace(code) == "" {
|
||||||
|
|||||||
@@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
|||||||
LLMClient: req.LLMClient,
|
LLMClient: req.LLMClient,
|
||||||
Scheduler: req.LLMScheduler,
|
Scheduler: req.LLMScheduler,
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
DiagnosticsDir: req.DiagnosticsDir,
|
||||||
|
OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) {
|
||||||
|
if req.OnDroppedCandidate == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.OnDroppedCandidate(
|
||||||
|
dropped.ProposalIndex,
|
||||||
|
dropped.TargetSegmentID,
|
||||||
|
dropped.OriginalText,
|
||||||
|
dropped.CorrectedText,
|
||||||
|
dropped.Confidence,
|
||||||
|
dropped.ReasonCode,
|
||||||
|
dropped.Message,
|
||||||
|
)
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
|||||||
LLMClient: req.LLMClient,
|
LLMClient: req.LLMClient,
|
||||||
Scheduler: req.LLMScheduler,
|
Scheduler: req.LLMScheduler,
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
DiagnosticsDir: req.DiagnosticsDir,
|
||||||
|
OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) {
|
||||||
|
if req.OnDroppedCandidate == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.OnDroppedCandidate(
|
||||||
|
dropped.ProposalIndex,
|
||||||
|
dropped.TargetSegmentID,
|
||||||
|
dropped.OriginalText,
|
||||||
|
dropped.CorrectedText,
|
||||||
|
dropped.Confidence,
|
||||||
|
dropped.ReasonCode,
|
||||||
|
dropped.Message,
|
||||||
|
)
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
|||||||
LLMClient: req.LLMClient,
|
LLMClient: req.LLMClient,
|
||||||
Scheduler: req.LLMScheduler,
|
Scheduler: req.LLMScheduler,
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
DiagnosticsDir: req.DiagnosticsDir,
|
||||||
|
OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) {
|
||||||
|
if req.OnDroppedCandidate == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.OnDroppedCandidate(
|
||||||
|
dropped.ProposalIndex,
|
||||||
|
dropped.TargetSegmentID,
|
||||||
|
dropped.OriginalText,
|
||||||
|
dropped.CorrectedText,
|
||||||
|
dropped.Confidence,
|
||||||
|
dropped.ReasonCode,
|
||||||
|
dropped.Message,
|
||||||
|
)
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
|||||||
LLMClient: req.LLMClient,
|
LLMClient: req.LLMClient,
|
||||||
Scheduler: req.LLMScheduler,
|
Scheduler: req.LLMScheduler,
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
DiagnosticsDir: req.DiagnosticsDir,
|
||||||
|
OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) {
|
||||||
|
if req.OnDroppedCandidate == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.OnDroppedCandidate(
|
||||||
|
dropped.ProposalIndex,
|
||||||
|
dropped.TargetSegmentID,
|
||||||
|
dropped.OriginalText,
|
||||||
|
dropped.CorrectedText,
|
||||||
|
dropped.Confidence,
|
||||||
|
dropped.ReasonCode,
|
||||||
|
dropped.Message,
|
||||||
|
)
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
Reference in New Issue
Block a user