304 lines
11 KiB
Go
304 lines
11 KiB
Go
package validators
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
)
|
|
|
|
type fakeStructuredLLMClient struct {
|
|
responses []LLMValidationResponse
|
|
err error
|
|
calls []StructuredCompletionRequest
|
|
}
|
|
|
|
type sleepingValidationClient struct {
|
|
inFlight int32
|
|
maxInFlight int32
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
type boundedScheduler struct {
|
|
permits chan struct{}
|
|
}
|
|
|
|
func newBoundedScheduler(max int) *boundedScheduler {
|
|
return &boundedScheduler{permits: make(chan struct{}, max)}
|
|
}
|
|
|
|
func (s *boundedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
|
select {
|
|
case s.permits <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
defer func() { <-s.permits }()
|
|
return fn(ctx)
|
|
}
|
|
|
|
func (c *sleepingValidationClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
|
_ = req
|
|
current := atomic.AddInt32(&c.inFlight, 1)
|
|
for {
|
|
prior := atomic.LoadInt32(&c.maxInFlight)
|
|
if current <= prior || atomic.CompareAndSwapInt32(&c.maxInFlight, prior, current) {
|
|
break
|
|
}
|
|
}
|
|
if c.entered != nil {
|
|
c.entered <- struct{}{}
|
|
}
|
|
select {
|
|
case <-c.release:
|
|
case <-ctx.Done():
|
|
atomic.AddInt32(&c.inFlight, -1)
|
|
return StructuredCompletionResponse{}, ctx.Err()
|
|
}
|
|
atomic.AddInt32(&c.inFlight, -1)
|
|
target := out.(*LLMValidationResponse)
|
|
*target = LLMValidationResponse{
|
|
Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
},
|
|
}
|
|
return StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
|
_ = ctx
|
|
f.calls = append(f.calls, req)
|
|
if f.err != nil {
|
|
return StructuredCompletionResponse{}, f.err
|
|
}
|
|
if len(f.responses) == 0 {
|
|
return StructuredCompletionResponse{}, errors.New("unexpected call")
|
|
}
|
|
resp := f.responses[0]
|
|
f.responses = f.responses[1:]
|
|
target, ok := out.(*LLMValidationResponse)
|
|
if !ok {
|
|
return StructuredCompletionResponse{}, errors.New("unexpected output type")
|
|
}
|
|
*target = resp
|
|
return StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
|
|
cfg := config.Default()
|
|
cfg.ValidationMaxPromptTokens = 10000
|
|
return Request{
|
|
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
|
|
CandidateProposal: candidates,
|
|
ModuleKey: "homophones",
|
|
ModuleInstance: "homophones",
|
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
|
Config: &cfg,
|
|
}
|
|
}
|
|
|
|
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
|
return proposals.EnrichedCorrectionProposal{
|
|
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
}
|
|
}
|
|
|
|
func TestChunkLLMValidationItemsOneSmallBatch(t *testing.T) {
|
|
items := []LLMValidationItem{{CorrectionIndex: 0, OriginalText: "a", CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}}
|
|
batches, err := ChunkLLMValidationItems(items, 1000, chunking.NewSimpleTokenEstimator())
|
|
if err != nil {
|
|
t.Fatalf("Chunk error: %v", err)
|
|
}
|
|
if len(batches) != 1 {
|
|
t.Fatalf("expected 1 batch, got %d", len(batches))
|
|
}
|
|
}
|
|
|
|
func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
|
items := []LLMValidationItem{
|
|
{CorrectionIndex: 0, OriginalText: strings.Repeat("a", 20), CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
{CorrectionIndex: 1, OriginalText: strings.Repeat("c", 20), CorrectedText: "d", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
{CorrectionIndex: 2, OriginalText: strings.Repeat("e", 20), CorrectedText: "f", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
|
|
}
|
|
batches, err := ChunkLLMValidationItems(items, 40, chunking.NewSimpleTokenEstimator())
|
|
if err != nil {
|
|
t.Fatalf("Chunk error: %v", err)
|
|
}
|
|
if len(batches) < 2 {
|
|
t.Fatalf("expected multiple batches, got %d", len(batches))
|
|
}
|
|
seen := make([]int, 0)
|
|
for _, b := range batches {
|
|
for _, it := range b.Items {
|
|
seen = append(seen, it.CorrectionIndex)
|
|
}
|
|
}
|
|
if len(seen) != 3 || seen[0] != 0 || seen[1] != 1 || seen[2] != 2 {
|
|
t.Fatalf("unexpected ordering/drops/dups: %v", seen)
|
|
}
|
|
}
|
|
|
|
func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
|
|
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
|
|
tests := []struct {
|
|
name string
|
|
build func([]LLMValidationItem) ([]LLMMessage, error)
|
|
mustHas []string
|
|
}{
|
|
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
|
|
{"meaning_reversal", BuildMeaningReversalMessages, []string{"meaning reversals", "correction_index", "original_segment_text", "corrected_segment_text"}},
|
|
{"editorial", BuildEditorialMessages, []string{"acceptable editorial revision", "correction_index", "original_segment_text", "corrected_segment_text"}},
|
|
{"grammar_review", BuildGrammarReviewMessages, []string{"acceptable editorial revision", "correction_index"}},
|
|
{"spoken_word", BuildSpokenWordReviewMessages, []string{"acceptable editorial revision", "correction_index"}},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
msgs, err := tt.build(payload)
|
|
if err != nil {
|
|
t.Fatalf("build err: %v", err)
|
|
}
|
|
if len(msgs) != 2 {
|
|
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
|
}
|
|
combined := msgs[0].Content + "\n" + msgs[1].Content
|
|
for _, needle := range tt.mustHas {
|
|
if !strings.Contains(combined, needle) {
|
|
t.Fatalf("expected prompt to contain %q", needle)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
|
|
}}}}
|
|
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
if err != nil {
|
|
t.Fatalf("new validator error: %v", err)
|
|
}
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "Lyra")})
|
|
req.LLMClient = client
|
|
res, err := v.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("validate err: %v", err)
|
|
}
|
|
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
|
|
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
_, err := v.Validate(context.Background(), req)
|
|
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
|
t.Fatalf("expected malformed output error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
_, err := v.Validate(context.Background(), req)
|
|
if err == nil || !strings.Contains(err.Error(), "response invalid") {
|
|
t.Fatalf("expected missing decision error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
|
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
|
}}}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
_, err := v.Validate(context.Background(), req)
|
|
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
|
t.Fatalf("expected duplicate decision error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
|
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
_, err := v.Validate(context.Background(), req)
|
|
if err == nil || !strings.Contains(err.Error(), "unknown") {
|
|
t.Fatalf("expected unknown index error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
|
|
scheduler := newBoundedScheduler(2)
|
|
client := &sleepingValidationClient{
|
|
entered: make(chan struct{}, 16),
|
|
release: make(chan struct{}),
|
|
}
|
|
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
|
if err != nil {
|
|
t.Fatalf("new validator error: %v", err)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 8; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
req.Scheduler = scheduler
|
|
if _, runErr := v.Validate(context.Background(), req); runErr != nil {
|
|
t.Errorf("Validate error: %v", runErr)
|
|
}
|
|
}()
|
|
}
|
|
waitForValidationEntries(t, client.entered, 2)
|
|
close(client.release)
|
|
wg.Wait()
|
|
|
|
if got := atomic.LoadInt32(&client.maxInFlight); got > 2 {
|
|
t.Fatalf("expected scheduler cap <= 2, got %d", got)
|
|
}
|
|
if got := atomic.LoadInt32(&client.maxInFlight); got < 2 {
|
|
t.Fatalf("expected observed concurrency of at least 2, got %d", got)
|
|
}
|
|
}
|
|
|
|
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(300 * time.Millisecond)
|
|
got := 0
|
|
for got < want && time.Now().Before(deadline) {
|
|
select {
|
|
case <-entered:
|
|
got++
|
|
default:
|
|
runtime.Gosched()
|
|
}
|
|
}
|
|
if got < want {
|
|
t.Fatalf("timed out waiting for %d validator calls to enter (got %d)", want, got)
|
|
}
|
|
}
|