Make module-stage LLM handling resilient and report warnings
This commit is contained in:
@@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, proposal := range base {
|
||||
for _, proposal := range base.Proposals {
|
||||
sectionIndex := section.Index
|
||||
out = append(out, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposal,
|
||||
@@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
|
||||
|
||||
func (m deterministicFakeModule) Validators() []Validator { return nil }
|
||||
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
|
||||
if req.WorkingTranscript == nil || req.Section == nil {
|
||||
return []proposals.CorrectionProposal{}, nil
|
||||
return ProposalResult{}, nil
|
||||
}
|
||||
|
||||
out := make([]proposals.CorrectionProposal, 0)
|
||||
@@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
return ProposalResult{Proposals: out}, nil
|
||||
}
|
||||
|
||||
func TestChunkProposalMetadataAssociation(t *testing.T) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// StructuredLLMClient provides provider-agnostic structured completion.
|
||||
@@ -28,7 +29,7 @@ type TranscriptModule interface {
|
||||
Key() string
|
||||
ReplacementPolicy() proposals.ReplacementPolicy
|
||||
Validators() []Validator
|
||||
Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
|
||||
}
|
||||
|
||||
// Validator evaluates candidate proposals and returns one decision per proposal index.
|
||||
@@ -103,6 +104,11 @@ type ProposalRequest struct {
|
||||
LLMScheduler LLMScheduler `json:"-"`
|
||||
}
|
||||
|
||||
type ProposalResult struct {
|
||||
Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationRequest is the input to validator execution.
|
||||
type ValidationRequest = validators.Request
|
||||
|
||||
|
||||
@@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator {
|
||||
return []Validator{&fakeValidator{}}
|
||||
}
|
||||
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
return ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
|
||||
t.Fatalf("unexpected module key: %q", got)
|
||||
}
|
||||
|
||||
proposalsOut, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
proposalResult, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected propose error: %v", err)
|
||||
}
|
||||
if len(proposalsOut) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalsOut))
|
||||
if len(proposalResult.Proposals) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m noopModule) Validators() []contracts.Validator { return nil }
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
|
||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
@@ -68,6 +69,7 @@ type Request struct {
|
||||
type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
@@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
if isMalformedStructuredOutputError(callErr) {
|
||||
return Result{
|
||||
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
|
||||
@@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
@@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
Enriched: enriched,
|
||||
Warnings: nil,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
@@ -240,6 +246,48 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
||||
warning := stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeProposalGeneration,
|
||||
ReasonCode: "proposal_response_malformed",
|
||||
Message: strings.TrimSpace(err.Error()),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
if section != nil {
|
||||
sectionIndex := section.Index
|
||||
warning.SectionIndex = §ionIndex
|
||||
}
|
||||
return warning
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
@@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected invalid correction to survive generation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Warnings) != 1 {
|
||||
t.Fatalf("expected one warning, got %+v", result.Warnings)
|
||||
}
|
||||
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
|
||||
return "original_text matched multiple spans under require_unique policy"
|
||||
case SkipReasonNoEffect:
|
||||
return "original_text and corrected_text are identical"
|
||||
case SkipReasonEmptyResultingText:
|
||||
return "proposal would leave the segment empty"
|
||||
case SkipReasonInvalidProposal:
|
||||
return "proposal failed structural or policy validation"
|
||||
default:
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text"
|
||||
SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text"
|
||||
SkipReasonNoEffect ProposalSkipReason = "no_effect"
|
||||
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
|
||||
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
|
||||
)
|
||||
|
||||
@@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
}
|
||||
|
||||
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
@@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
|
||||
case ReplacementPolicyReplaceAll:
|
||||
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
|
||||
@@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh hello"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh ",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if !result.Applicable {
|
||||
t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason)
|
||||
}
|
||||
if result.CorrectedSegmentText != "hello" {
|
||||
t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if result.Applicable {
|
||||
t.Fatal("expected non-applicable preview")
|
||||
}
|
||||
if result.SkipReason != SkipReasonEmptyResultingText {
|
||||
t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"}
|
||||
original := *segment
|
||||
|
||||
@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
|
||||
if strings.TrimSpace(p.OriginalText) == "" {
|
||||
return fmt.Errorf("proposal original_text must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(p.CorrectedText) == "" {
|
||||
return fmt.Errorf("proposal corrected_text must not be empty")
|
||||
}
|
||||
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
||||
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
@@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error, got nil")
|
||||
}
|
||||
if err.Error() != "proposal corrected_text must not be empty" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
if err := proposal.Validate(); err != nil {
|
||||
t.Fatalf("expected empty corrected_text to be allowed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
@@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler
|
||||
|
||||
// ModuleResult captures deterministic per-module execution output.
|
||||
type ModuleResult struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
@@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusFailed,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
ErrorMessage: pipelineErr.Error(),
|
||||
@@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusSuccess,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
AppliedChanges: applyResult.Applied,
|
||||
@@ -232,12 +236,14 @@ type collectSectionProposalsInput struct {
|
||||
type sectionProposals struct {
|
||||
meta contracts.SectionMetadata
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
type sectionProposalResult struct {
|
||||
sectionPos int
|
||||
section chunking.Section
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -247,12 +253,14 @@ type sectionValidationResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
type modulePipelineResult struct {
|
||||
ProposalCount int
|
||||
Approved []proposals.EnrichedCorrectionProposal
|
||||
Warnings []stagewarnings.StageWarning
|
||||
ValidatorDecisions []ValidatorDecisionRecord
|
||||
ValidatorRejected []ValidatorRejectedChange
|
||||
}
|
||||
@@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: input.Config,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
@@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
case results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
corrected: corrected,
|
||||
corrected: proposalResult.Proposals,
|
||||
warnings: proposalResult.Warnings,
|
||||
err: err,
|
||||
}:
|
||||
case <-runCtx.Done():
|
||||
@@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||
out := modulePipelineResult{
|
||||
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
|
||||
Warnings: make([]stagewarnings.StageWarning, 0),
|
||||
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
|
||||
ValidatorRejected: make([]ValidatorRejectedChange, 0),
|
||||
}
|
||||
@@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if firstErr != nil {
|
||||
continue
|
||||
}
|
||||
out.Warnings = append(out.Warnings, result.warnings...)
|
||||
pending[result.sectionPos] = result
|
||||
|
||||
for {
|
||||
@@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
approved: validated.approved,
|
||||
decisions: validated.decisions,
|
||||
rejected: validated.rejected,
|
||||
warnings: validated.warnings,
|
||||
err: err,
|
||||
}
|
||||
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
||||
@@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
break
|
||||
}
|
||||
out.Approved = append(out.Approved, res.approved...)
|
||||
out.Warnings = append(out.Warnings, res.warnings...)
|
||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||
}
|
||||
@@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
|
||||
})
|
||||
sort.SliceStable(out.Warnings, func(i, j int) bool {
|
||||
leftSection, rightSection := -1, -1
|
||||
if out.Warnings[i].SectionIndex != nil {
|
||||
leftSection = *out.Warnings[i].SectionIndex
|
||||
}
|
||||
if out.Warnings[j].SectionIndex != nil {
|
||||
rightSection = *out.Warnings[j].SectionIndex
|
||||
}
|
||||
if leftSection != rightSection {
|
||||
return leftSection < rightSection
|
||||
}
|
||||
leftBatch, rightBatch := -1, -1
|
||||
if out.Warnings[i].BatchIndex != nil {
|
||||
leftBatch = *out.Warnings[i].BatchIndex
|
||||
}
|
||||
if out.Warnings[j].BatchIndex != nil {
|
||||
rightBatch = *out.Warnings[j].BatchIndex
|
||||
}
|
||||
if leftBatch != rightBatch {
|
||||
return leftBatch < rightBatch
|
||||
}
|
||||
if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName {
|
||||
return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName]
|
||||
}
|
||||
if out.Warnings[i].Scope != out.Warnings[j].Scope {
|
||||
return out.Warnings[i].Scope < out.Warnings[j].Scope
|
||||
}
|
||||
if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode {
|
||||
return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode
|
||||
}
|
||||
return out.Warnings[i].Message < out.Warnings[j].Message
|
||||
})
|
||||
|
||||
if firstErr != nil {
|
||||
return out, firstErr
|
||||
@@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
|
||||
decisions := make([]ValidatorDecisionRecord, 0)
|
||||
rejected := make([]ValidatorRejectedChange, 0)
|
||||
warnings := make([]stagewarnings.StageWarning, 0)
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
|
||||
for _, validator := range input.Validators {
|
||||
@@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
||||
}
|
||||
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
||||
@@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
|
||||
}
|
||||
warnings = append(warnings, vResult.Warnings...)
|
||||
|
||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||
@@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -47,11 +47,12 @@ type fakeModule struct {
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
return m.proposeF(req)
|
||||
proposalsOut, err := m.proposeF(req)
|
||||
return contracts.ProposalResult{Proposals: proposalsOut}, err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected llm validator failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected partial progress retained")
|
||||
}
|
||||
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
|
||||
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
|
||||
}
|
||||
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1355,7 +1362,7 @@ type proposalGenerationModule struct {
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
@@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return result.Corrections, nil
|
||||
return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
|
||||
@@ -4,8 +4,35 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type ProposalShapeValidator struct{}
|
||||
|
||||
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
|
||||
|
||||
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
switch {
|
||||
case c.TargetSegmentID <= 0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
|
||||
case strings.TrimSpace(c.OriginalText) == "":
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
|
||||
case c.Confidence < 0.0 || c.Confidence > 1.0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
|
||||
default:
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type ConfidenceThresholdValidator struct{}
|
||||
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
|
||||
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
|
||||
|
||||
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
segmentsByID := make(map[int]schema.Segment)
|
||||
if req.WorkingTranscript != nil {
|
||||
for _, seg := range req.WorkingTranscript.Segments {
|
||||
segmentsByID[seg.ID] = seg
|
||||
}
|
||||
}
|
||||
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if strings.TrimSpace(c.CorrectedText) == "" {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
|
||||
segment, ok := segmentsByID[c.TargetSegmentID]
|
||||
if !ok {
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
continue
|
||||
}
|
||||
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
|
||||
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Config.ValidationMaxPromptTokens
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
|
||||
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
|
||||
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
|
||||
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
|
||||
if len(itemsForBatching) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
if isMalformedStructuredOutputError(err) {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
}
|
||||
|
||||
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
for i := range batchDecisions {
|
||||
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
all = append(all, llmDecisions...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all}, nil
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
|
||||
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
|
||||
}
|
||||
return decisions, nil
|
||||
}
|
||||
|
||||
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
|
||||
out := make([]Decision, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
|
||||
out := make([]stagewarnings.StageWarning, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
ReasonCode: ReasonValidatorInputTooLarge,
|
||||
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
|
||||
if len(decisions) == 0 {
|
||||
return append([]LLMValidationItem(nil), items...)
|
||||
}
|
||||
rejected := make(map[int]struct{}, len(decisions))
|
||||
for _, decision := range decisions {
|
||||
rejected[decision.ProposalIndex] = struct{}{}
|
||||
}
|
||||
out := make([]LLMValidationItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
if _, ok := rejected[item.CorrectionIndex]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
|
||||
out := make([]Decision, 0, len(items))
|
||||
for _, item := range items {
|
||||
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
|
||||
idx := batchIndex
|
||||
return stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
BatchIndex: &idx,
|
||||
ReasonCode: reasonCode,
|
||||
Message: strings.TrimSpace(message),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected malformed output error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed output downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "response invalid") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown") {
|
||||
t.Fatalf("expected unknown index error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown index downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: huge},
|
||||
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
|
||||
}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
ModuleKey: "homophones",
|
||||
ModuleInstance: "homophones",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
}
|
||||
req.LLMClient = client
|
||||
cfg := config.Default()
|
||||
cfg.ValidationMaxPromptTokens = 200
|
||||
req.Config = &cfg
|
||||
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected oversize downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 2 {
|
||||
t.Fatalf("expected two decisions, got %+v", res.Decisions)
|
||||
}
|
||||
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
|
||||
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
|
||||
}
|
||||
if !res.Decisions[1].Approved {
|
||||
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
|
||||
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,16 +8,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyCorrectedText = "empty_corrected_text"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyResultingText = "empty_resulting_segment"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonInvalidTargetSegment = "invalid_target_segment_id"
|
||||
ReasonEmptyOriginalText = "empty_original_text"
|
||||
ReasonInvalidConfidence = "invalid_confidence"
|
||||
ReasonValidatorMalformed = "validator_response_malformed"
|
||||
ReasonValidatorInputTooLarge = "validator_input_too_large"
|
||||
)
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
@@ -45,8 +51,9 @@ type Decision struct {
|
||||
|
||||
// Result is one validator output containing exactly one decision per proposal index.
|
||||
type Result struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationScheduler provides bounded execution for validator LLM calls.
|
||||
|
||||
@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, " ", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
|
||||
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
|
||||
}
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello", " ", 0.9),
|
||||
}}
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
}}
|
||||
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
|
||||
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
|
||||
t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyCorrectedText,
|
||||
ReasonEmptyResultingText,
|
||||
ReasonNoEffect,
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
for _, code := range codes {
|
||||
if strings.TrimSpace(code) == "" {
|
||||
|
||||
18
internal/framework/warnings/warnings.go
Normal file
18
internal/framework/warnings/warnings.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package warnings
|
||||
|
||||
type Scope string
|
||||
|
||||
const (
|
||||
ScopeProposalGeneration Scope = "proposal_generation"
|
||||
ScopeValidator Scope = "validator"
|
||||
)
|
||||
|
||||
type StageWarning struct {
|
||||
Scope Scope `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
BatchIndex *int `json:"batch_index,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
Reference in New Issue
Block a user