eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 8 deterministic validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 00:26:27 +00:00
parent 28fe899aa1
commit aeb31f1c0d
13 changed files with 940 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/framework/contracts/contracts.go
View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
// StructuredLLMClient provides provider-agnostic structured completion.
@@ -27,7 +28,7 @@ type TranscriptModule interface {
// Validator evaluates candidate proposals and returns one decision per proposal index.
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) ([]ValidationDecision, error)
Validate(ctx context.Context, req ValidationRequest) (validators.Result, error)
}
// StructuredCompletionRequest is a transport-neutral structured completion request.
@@ -90,19 +91,7 @@ type ProposalRequest struct {
}
// ValidationRequest is the input to validator execution.
type ValidationRequest struct {
ExecutionContext
RunSpec ModuleRunSpec `json:"run_spec"`
CandidateProposals []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
}
// ValidationDecision is one validator decision for one proposal index.
type ValidationDecision struct {
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Confidence *float64 `json:"confidence,omitempty"`
Reason string `json:"reason,omitempty"`
}
type ValidationRequest = validators.Request
// ResolveModuleRunSpecs deterministically resolves instance names from logical keys.
// Repeated keys are suffixed with _<n> (1-based), while singleton keys keep their raw key.

11
internal/framework/contracts/contracts_test.go
View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type fakeLLMClient struct{}
@@ -22,13 +23,13 @@ type fakeValidator struct{}
func (f *fakeValidator) Name() string { return "fake-validator" }
func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) ([]ValidationDecision, error) {
func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) {
_ = ctx
decisions := make([]ValidationDecision, len(req.CandidateProposals))
for i, proposal := range req.CandidateProposals {
decisions[i] = ValidationDecision{ProposalIndex: proposal.ProposalIndex, Approved: true}
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, proposal := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: proposal.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
}
return decisions, nil
return validators.Result{ValidatorName: f.Name(), Decisions: decisions}, nil
}
type fakeModule struct{}

144
internal/framework/runner/runner.go
View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
const (
@@ -28,16 +29,38 @@ type Runner struct {
// ModuleResult captures deterministic per-module execution output.
type ModuleResult struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
}
type ValidatorRejectedChange struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
TargetSegmentID int `json:"target_segment_id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
}
// RunInput is the deterministic runner input.
@@ -121,19 +144,100 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
})
}
applyResult := proposals.ApplyProposals(working, enriched, policy)
validatorDecisions := make([]ValidatorDecisionRecord, 0)
validatorRejected := make([]ValidatorRejectedChange, 0)
eligible := enriched
for _, validator := range module.Validators() {
vResult, vErr := validator.Validate(ctx, contracts.ValidationRequest{
WorkingTranscript: working,
CandidateProposal: eligible,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Glossary: input.Glossary,
Config: input.Config,
})
if vErr != nil {
failed := ModuleResult{
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusFailed,
ProposalCount: len(enriched),
ValidatorDecisions: validatorDecisions,
ValidatorRejected: validatorRejected,
ErrorMessage: vErr.Error(),
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q validator %q failed: %w", spec.InstanceName, validator.Name(), vErr)
}
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
failed := ModuleResult{
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusFailed,
ProposalCount: len(enriched),
ValidatorDecisions: validatorDecisions,
ValidatorRejected: validatorRejected,
ErrorMessage: err.Error(),
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return RunOutput{FinalTranscript: working, ModuleResults: results}, fmt.Errorf("module %q validator %q cardinality failed: %w", spec.InstanceName, validator.Name(), err)
}
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
for _, p := range eligible {
byIndex[p.ProposalIndex] = p
}
for _, d := range vResult.Decisions {
validatorDecisions = append(validatorDecisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
if d.Approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
}
p := byIndex[d.ProposalIndex]
validatorRejected = append(validatorRejected, ValidatorRejectedChange{
ValidatorName: validator.Name(),
ProposalIndex: p.ProposalIndex,
ModuleKey: p.ModuleKey,
ModuleInstance: p.ModuleInstance,
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
}
eligible = nextEligible
}
applyResult := proposals.ApplyProposals(working, eligible, policy)
working = applyResult.Transcript
results = append(results, ModuleResult{
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusSuccess,
ProposalCount: len(enriched),
AppliedChanges: applyResult.Applied,
SkippedChanges: applyResult.Skipped,
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusSuccess,
ProposalCount: len(enriched),
ValidatorDecisions: validatorDecisions,
ValidatorRejected: validatorRejected,
AppliedChanges: applyResult.Applied,
SkippedChanges: applyResult.Skipped,
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
})
}

165
internal/framework/runner/runner_test.go
View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type fakeFactory struct {
@@ -24,14 +25,15 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
}
type fakeModule struct {
key string
policy proposals.ReplacementPolicy
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
key string
policy proposals.ReplacementPolicy
validators []contracts.Validator
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
}
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return nil }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if m.proposeF == nil {
return nil, nil
@@ -39,6 +41,17 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
return m.proposeF(req)
}
type fakeValidator struct {
name string
validateF func(req contracts.ValidationRequest) (validators.Result, error)
}
func (v fakeValidator) Name() string { return v.name }
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
_ = ctx
return v.validateF(req)
}
func TestRunnerOneModuleAppliesProposal(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -149,4 +162,148 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
}
}
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
}
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{allowAll},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
},
},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected proposal applied, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults[0].ValidatorDecisions) != 1 {
t.Fatalf("expected validator decisions recorded")
}
}
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
}
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "teh cat" {
t.Fatalf("expected rejected proposal not applied, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("expected validator rejection recorded, got %+v", out.ModuleResults[0].ValidatorRejected)
}
}
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
}
return validators.Result{
ValidatorName: "first",
Decisions: []validators.Decision{
{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"},
{ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "reject"},
},
}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if len(req.CandidateProposal) != 1 {
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
}
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO", Confidence: 1},
}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "ONE two" {
t.Fatalf("expected only survivor applied, got %q", out.FinalTranscript.Segments[0].Text)
}
}
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
}}
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
// Missing one decision triggers cardinality error.
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
}}
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
if err == nil {
t.Fatal("expected cardinality error")
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed {
t.Fatalf("expected second module failed")
}
}
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if len(out.ModuleResults[0].SkippedChanges) != 1 {
t.Fatalf("expected application skip recorded")
}
if len(out.ModuleResults[0].ValidatorRejected) != 0 {
t.Fatalf("expected no validator rejection")
}
}
func ptrConfig(c config.Config) *config.Config { return &c }

161
internal/framework/validators/deterministic.go Normal file
View File

@@ -0,0 +1,161 @@
package validators
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
threshold := confidenceThresholdForModule(req.ModuleKey, req.Config)
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if c.Confidence < threshold {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonLowConfidence, fmt.Sprintf("confidence %.4f below threshold %.4f", c.Confidence, threshold)))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type OriginalTextPresenceValidator struct{}
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
byID := make(map[int]string)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
byID[seg.ID] = seg.Text
}
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
text, ok := byID[c.TargetSegmentID]
if !ok {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
if !strings.Contains(text, c.OriginalText) {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type NonEmptyCorrectionValidator struct{}
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_correction" }
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if strings.TrimSpace(c.CorrectedText) == "" {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type NoEffectValidator struct{}
func (v NoEffectValidator) Name() string { return "no_effect" }
func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if c.OriginalText == c.CorrectedText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonNoEffect, "original_text and corrected_text are identical"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ProtectedGlossaryTermValidator struct{}
func (v ProtectedGlossaryTermValidator) Name() string { return "protected_glossary_terms" }
func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) {
if req.ModuleKey == "glossary" {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
decisions = append(decisions, approval(c.ProposalIndex))
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
terms := glossaryTerms(req)
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if altersProtectedTerm(c.CorrectionProposal, terms) {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonProtectedGlossaryTerm, "proposal may alter protected glossary terminology"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
func glossaryTerms(req Request) []string {
if req.Glossary == nil {
return nil
}
out := make([]string, 0)
for _, e := range req.Glossary.Entries {
if t := strings.TrimSpace(strings.ToLower(e.Name)); t != "" {
out = append(out, t)
}
for _, a := range e.Aliases {
if t := strings.TrimSpace(strings.ToLower(a)); t != "" {
out = append(out, t)
}
}
if t := strings.TrimSpace(strings.ToLower(e.Plural)); t != "" {
out = append(out, t)
}
}
return out
}
func altersProtectedTerm(p proposals.CorrectionProposal, terms []string) bool {
if len(terms) == 0 {
return false
}
orig := strings.ToLower(p.OriginalText)
corr := strings.ToLower(p.CorrectedText)
for _, t := range terms {
if strings.Contains(orig, t) && !strings.Contains(corr, t) {
return true
}
}
return false
}

105
internal/framework/validators/models.go Normal file
View File

@@ -0,0 +1,105 @@
package validators
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
const (
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyCorrectedText = "empty_corrected_text"
ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term"
)
// Request is the runtime input shared by deterministic validators.
type Request struct {
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
}
// Decision is one validator decision for one proposal index.
type Decision struct {
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
}
// Result is one validator output containing exactly one decision per proposal index.
type Result struct {
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
}
// Validator is the deterministic runtime validator interface.
type Validator interface {
Name() string
Validate(ctx context.Context, req Request) (Result, error)
}
// EnforceDecisionCardinality verifies every candidate proposal index receives exactly one decision.
func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision) error {
expected := make(map[int]struct{}, len(candidate))
for _, c := range candidate {
expected[c.ProposalIndex] = struct{}{}
}
seen := make(map[int]int, len(decisions))
for _, d := range decisions {
if _, ok := expected[d.ProposalIndex]; !ok {
return fmt.Errorf("unknown decision proposal index %d", d.ProposalIndex)
}
seen[d.ProposalIndex]++
if seen[d.ProposalIndex] > 1 {
return fmt.Errorf("duplicate decision proposal index %d", d.ProposalIndex)
}
}
for idx := range expected {
if seen[idx] == 0 {
return fmt.Errorf("missing decision proposal index %d", idx)
}
}
return nil
}
func approval(index int) Decision {
return Decision{ProposalIndex: index, Approved: true, ReasonCode: ReasonApproved, Message: "approved"}
}
func rejection(index int, reasonCode string, msg string) Decision {
return Decision{ProposalIndex: index, Approved: false, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
}
func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {
if cfg == nil {
return 0.0
}
switch moduleKey {
case "glossary":
return cfg.Thresholds.Glossary
case "grammar":
return cfg.Thresholds.Grammar
case "homophones":
return cfg.Thresholds.Homophones
case "spoken_word":
return cfg.Thresholds.SpokenWord
default:
return 0.0
}
}

181
internal/framework/validators/validators_test.go Normal file
View File

@@ -0,0 +1,181 @@
package validators
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
}
}
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "missing decision") {
t.Fatalf("expected missing decision error, got %v", err)
}
}
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
t.Fatalf("expected duplicate decision error, got %v", err)
}
}
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
t.Fatalf("expected unknown decision error, got %v", err)
}
}
func TestConfidenceThresholdValidator(t *testing.T) {
cfg := config.Default()
cfg.Thresholds.Grammar = 0.8
req := Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
}
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "missing", "x", 0.9),
mkCandidate(2, 5, "hello", "hi", 0.9),
}}
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonMissingOriginalText {
t.Fatalf("expected missing_original_text, got %+v", res.Decisions[1])
}
if res.Decisions[2].ReasonCode != ReasonMissingTargetSegment {
t.Fatalf("expected missing_target_segment, got %+v", res.Decisions[2])
}
}
func TestNonEmptyCorrectionValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello", " ", 0.9),
}}
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
}
}
func TestNoEffectValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hello", 0.9),
mkCandidate(1, 1, "hello", "hi", 0.9),
}}
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if res.Decisions[0].ReasonCode != ReasonNoEffect || res.Decisions[0].Approved {
t.Fatalf("expected no_effect rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected proposal 1 approved")
}
}
func TestProtectedGlossaryTermValidator(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Aliases: []string{"Open AI"}, Plural: "OpenAIs", Category: "brand", Summary: "brand"}}}
req := Request{
Glossary: glossary,
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
},
}
res, err := (ProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if res.Decisions[0].ReasonCode != ReasonProtectedGlossaryTerm || res.Decisions[0].Approved {
t.Fatalf("expected protected glossary rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected non-glossary proposal approved")
}
}
func TestProtectedGlossaryTermValidatorAllowsGlossaryModule(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Category: "brand", Summary: "brand"}}}
req := Request{
Glossary: glossary,
ModuleKey: "glossary",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
},
}
res, err := (ProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected glossary module approval, got %+v", res.Decisions[0])
}
}
func TestStableReasonCodes(t *testing.T) {
codes := []string{
ReasonApproved,
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyCorrectedText,
ReasonNoEffect,
ReasonProtectedGlossaryTerm,
}
for _, code := range codes {
if strings.TrimSpace(code) == "" {
t.Fatalf("reason code must not be empty")
}
}
}