305 lines
14 KiB
Go
305 lines
14 KiB
Go
package semanticreconcile
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
)
|
|
|
|
func TestNewEngineValidatesConstruction(t *testing.T) {
|
|
prompt := testPromptSpec("a")
|
|
tests := []struct {
|
|
name string
|
|
client contracts.StructuredLLMClient
|
|
prompt PromptSpec
|
|
limits Limits
|
|
want string
|
|
}{
|
|
{name: "nil client", prompt: prompt, limits: DefaultLimits(), want: "client"},
|
|
{name: "empty prompt ID", client: &recordingReconciliationClient{}, prompt: PromptSpec{Version: "v1", SHA256: testDigest("a")}, limits: DefaultLimits(), want: "prompt ID"},
|
|
{name: "empty prompt version", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", SHA256: testDigest("a")}, limits: DefaultLimits(), want: "prompt version"},
|
|
{name: "missing digest", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1"}, limits: DefaultLimits(), want: "digest"},
|
|
{name: "wrong digest algorithm", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1", SHA256: "md5:" + strings.Repeat("a", 32)}, limits: DefaultLimits(), want: "sha256:"},
|
|
{name: "non canonical digest", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1", SHA256: "sha256:" + strings.Repeat("A", 64)}, limits: DefaultLimits(), want: "lowercase"},
|
|
{name: "invalid limits", client: &recordingReconciliationClient{}, prompt: prompt, limits: Limits{}, want: "limits"},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if _, err := NewEngine(test.client, test.prompt, test.limits); err == nil || !strings.Contains(err.Error(), test.want) {
|
|
t.Fatalf("NewEngine() error = %v, want %q", err, test.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
defaultPrompt, err := DefaultPromptSpec()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if defaultPrompt.ID != PromptID || defaultPrompt.Version != PromptVersion || defaultPrompt.SHA256 == "" {
|
|
t.Fatalf("DefaultPromptSpec() = %#v, want complete generic prompt identity", defaultPrompt)
|
|
}
|
|
}
|
|
|
|
func TestEnginePropagatesRequestAndAssessesResponse(t *testing.T) {
|
|
client := &recordingReconciliationClient{responses: []ProposalResponse{{DuplicateGroups: []DuplicateGroup{{
|
|
CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2,
|
|
}}}}}
|
|
engine := newTestEngine(t, client, DefaultLimits())
|
|
request := readyEngineRequest()
|
|
request.ProfileID = " profile-as-resolved "
|
|
request.SessionID = " session-as-supplied "
|
|
|
|
result, err := engine.Reconcile(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("Reconcile() error = %v, want nil", err)
|
|
}
|
|
if result.Disposition() != Complete || result.DiscardedGroupCount() != 0 || len(result.Issues()) != 0 {
|
|
t.Fatalf("result = disposition %v discarded %d issues %#v", result.Disposition(), result.DiscardedGroupCount(), result.Issues())
|
|
}
|
|
groups := result.Plan().Groups()
|
|
if len(groups) != 1 || !reflect.DeepEqual(groups[0].MemberPositions(), []int{0, 1}) || groups[0].CanonicalPosition() != 1 {
|
|
t.Fatalf("safe plan = %#v", groups)
|
|
}
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("completion calls = %d, want exactly one", len(client.requests))
|
|
}
|
|
got := client.requests[0]
|
|
if got.StageName != request.StageName || got.PromptID != engine.prompt.ID || got.PromptVersion != engine.prompt.Version || got.ProfileID != request.ProfileID || got.SessionID != request.SessionID {
|
|
t.Fatalf("structured request = %#v, want exact routing values", got)
|
|
}
|
|
if len(got.Inputs) != 2 || got.Inputs["candidates"].Name != "candidates" || got.Inputs["transcript"].Name != "transcript" || len(got.Vars) != 0 {
|
|
t.Fatalf("structured request inputs = %#v vars = %#v, want only candidate and transcript materials", got.Inputs, got.Vars)
|
|
}
|
|
}
|
|
|
|
func TestEngineClassifiesSemanticAndTransportOutcomes(t *testing.T) {
|
|
transportErr := errors.New("provider unavailable")
|
|
tests := []struct {
|
|
name string
|
|
response ProposalResponse
|
|
completion error
|
|
want ResultDisposition
|
|
wantDiscard int
|
|
wantIssues bool
|
|
wantError error
|
|
}{
|
|
{name: "empty groups complete", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{}}, want: Complete},
|
|
{name: "discarded proposal retryable", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1}}}, want: RetryableDiscardedProposalGroups, wantDiscard: 1, wantIssues: true},
|
|
{name: "invalid structured output retryable", completion: fmt.Errorf("decode response: %w", contracts.ErrInvalidStructuredOutput), want: RetryableInvalidStructuredOutput},
|
|
{name: "transport failure", completion: transportErr, wantError: transportErr},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
client := &recordingReconciliationClient{responses: []ProposalResponse{test.response}, errors: []error{test.completion}}
|
|
result, err := newTestEngine(t, client, DefaultLimits()).Reconcile(context.Background(), readyEngineRequest())
|
|
if test.wantError != nil {
|
|
if !errors.Is(err, test.wantError) || !strings.Contains(err.Error(), readyEngineRequest().StageName) {
|
|
t.Fatalf("Reconcile() error = %v, want contextual %v", err, test.wantError)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("Reconcile() error = %v, want nil", err)
|
|
}
|
|
if result.Disposition() != test.want || result.DiscardedGroupCount() != test.wantDiscard || (len(result.Issues()) > 0) != test.wantIssues {
|
|
t.Fatalf("result = disposition %v discarded %d issues %#v", result.Disposition(), result.DiscardedGroupCount(), result.Issues())
|
|
}
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("completion calls = %d, want one", len(client.requests))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEngineSkipsDeterministicOutcomesWithoutCompletion(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
request Request
|
|
limits Limits
|
|
want ResultDisposition
|
|
mappingLen int
|
|
}{
|
|
{name: "insufficient candidates", request: engineRequestWithCandidateCount(1), limits: DefaultLimits(), want: SkippedInsufficientCandidates, mappingLen: 1},
|
|
{name: "candidate limit", request: engineRequestWithCandidateCount(2), limits: Limits{ContextRadius: 0, MaximumCandidates: 1, MaximumMaterialBytes: 10000}, want: SkippedLimitExceeded, mappingLen: 2},
|
|
{name: "material limit", request: engineRequestWithCandidateCount(2), limits: Limits{ContextRadius: 0, MaximumCandidates: 2, MaximumMaterialBytes: 1}, want: SkippedLimitExceeded, mappingLen: 2},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
client := &recordingReconciliationClient{}
|
|
result, err := newTestEngine(t, client, test.limits).Reconcile(context.Background(), test.request)
|
|
if err != nil {
|
|
t.Fatalf("Reconcile() error = %v, want nil", err)
|
|
}
|
|
if result.Disposition() != test.want || len(result.CandidateMappings()) != test.mappingLen {
|
|
t.Fatalf("result disposition = %v mappings = %#v", result.Disposition(), result.CandidateMappings())
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("completion calls = %d, want zero", len(client.requests))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEngineRejectsInvalidInvocationAndHonorsCancellation(t *testing.T) {
|
|
request := readyEngineRequest()
|
|
client := &recordingReconciliationClient{}
|
|
engine := newTestEngine(t, client, DefaultLimits())
|
|
|
|
var nilEngine *Engine
|
|
if _, err := nilEngine.Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "engine must not be nil") {
|
|
t.Fatalf("nil engine error = %v", err)
|
|
}
|
|
if _, err := (&Engine{}).Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "client") {
|
|
t.Fatalf("zero engine error = %v", err)
|
|
}
|
|
invalid := newEngine(&recordingReconciliationClient{}, testPromptSpec("a"), llm.ResponseSchema{}, DefaultLimits())
|
|
if _, err := invalid.Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "invalid construction state") {
|
|
t.Fatalf("invalid construction error = %v", err)
|
|
}
|
|
if _, err := engine.Reconcile(nil, request); err == nil || !strings.Contains(err.Error(), "context") {
|
|
t.Fatalf("nil context error = %v", err)
|
|
}
|
|
withoutStage := request
|
|
withoutStage.StageName = " "
|
|
if _, err := engine.Reconcile(context.Background(), withoutStage); err == nil || !strings.Contains(err.Error(), "stage name") {
|
|
t.Fatalf("empty stage error = %v", err)
|
|
}
|
|
withoutSource := request
|
|
withoutSource.Source = nil
|
|
if _, err := engine.Reconcile(context.Background(), withoutSource); err == nil || !strings.Contains(err.Error(), "source document") {
|
|
t.Fatalf("nil source error = %v", err)
|
|
}
|
|
|
|
canceled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := engine.Reconcile(canceled, request); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "before preparation") {
|
|
t.Fatalf("preparation cancellation error = %v", err)
|
|
}
|
|
if _, err := engine.Reconcile(&cancelBeforeCompletionContext{}, request); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "before completion") {
|
|
t.Fatalf("completion cancellation error = %v", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("completion calls = %d, want zero for invalid and canceled invocations", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestEngineCallsAreIndependentAndResultsAreOwned(t *testing.T) {
|
|
client := &recordingReconciliationClient{responses: []ProposalResponse{
|
|
{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 1},
|
|
{CandidateIDs: []int{2, 99}, CanonicalCandidateID: 2},
|
|
}},
|
|
{DuplicateGroups: []DuplicateGroup{}},
|
|
}}
|
|
engine := newTestEngine(t, client, DefaultLimits())
|
|
first, err := engine.Reconcile(context.Background(), readyEngineRequest())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Plan().Groups()[0].memberPositions[0] = 99
|
|
firstIssues := first.Issues()
|
|
firstIssues[0].Category = "changed"
|
|
firstMappings := first.CandidateMappings()
|
|
firstMappings[0].CandidatePosition = 99
|
|
if first.Plan().Groups()[0].MemberPositions()[0] != 0 || first.Issues()[0].Category == "changed" || first.CandidateMappings()[0].CandidatePosition != 0 {
|
|
t.Fatal("result accessors exposed retained state")
|
|
}
|
|
|
|
second, err := engine.Reconcile(context.Background(), readyEngineRequest())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if second.Disposition() != Complete || len(second.Plan().Groups()) != 0 || len(second.Issues()) != 0 || second.DiscardedGroupCount() != 0 || len(second.CandidateMappings()) != 2 {
|
|
t.Fatalf("second result retained prior call state: disposition %v plan %#v issues %#v discarded %d mappings %#v", second.Disposition(), second.Plan().Groups(), second.Issues(), second.DiscardedGroupCount(), second.CandidateMappings())
|
|
}
|
|
}
|
|
|
|
type recordingReconciliationClient struct {
|
|
requests []contracts.StructuredCompletionRequest
|
|
responses []ProposalResponse
|
|
errors []error
|
|
}
|
|
|
|
func (client *recordingReconciliationClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
|
request.Inputs = request.Inputs.Clone()
|
|
client.requests = append(client.requests, request)
|
|
index := len(client.requests) - 1
|
|
if index < len(client.errors) && client.errors[index] != nil {
|
|
return contracts.StructuredCompletionResponse{}, client.errors[index]
|
|
}
|
|
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{}}
|
|
if index < len(client.responses) {
|
|
response = cloneProposalResponse(client.responses[index])
|
|
}
|
|
target, ok := output.(*ProposalResponse)
|
|
if !ok {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("output type = %T", output)
|
|
}
|
|
*target = response
|
|
return contracts.StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
func cloneProposalResponse(response ProposalResponse) ProposalResponse {
|
|
cloned := ProposalResponse{DuplicateGroups: make([]DuplicateGroup, len(response.DuplicateGroups))}
|
|
for index, group := range response.DuplicateGroups {
|
|
cloned.DuplicateGroups[index] = DuplicateGroup{
|
|
CandidateIDs: append([]int(nil), group.CandidateIDs...), CanonicalCandidateID: group.CanonicalCandidateID,
|
|
}
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func newTestEngine(t *testing.T, client contracts.StructuredLLMClient, limits Limits) *Engine {
|
|
t.Helper()
|
|
engine, err := NewEngine(client, testPromptSpec("a"), limits)
|
|
if err != nil {
|
|
t.Fatalf("NewEngine() error = %v", err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func testPromptSpec(digestCharacter string) PromptSpec {
|
|
return PromptSpec{ID: "test.semantic_reconciliation", Version: "v1", SHA256: testDigest(digestCharacter)}
|
|
}
|
|
|
|
func testDigest(character string) string { return "sha256:" + strings.Repeat(character, 64) }
|
|
|
|
func readyEngineRequest() Request { return engineRequestWithCandidateCount(2) }
|
|
|
|
func engineRequestWithCandidateCount(count int) Request {
|
|
document := &source.SourceDocument{ID: "source", Units: []source.SourceUnit{
|
|
{ID: 1, Kind: "speech", Text: "Mira arrived."},
|
|
{ID: 2, Kind: "speech", Text: "The captain spoke."},
|
|
}}
|
|
candidates := make([]Candidate, count)
|
|
for index := range candidates {
|
|
unitID := index%len(document.Units) + 1
|
|
candidates[index] = Candidate{
|
|
Label: fmt.Sprintf("candidate-%d", index+1),
|
|
SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: unitID, EndUnitID: unitID}},
|
|
}
|
|
}
|
|
return Request{StageName: "test/normalize", Source: document, Candidates: candidates, ProfileID: "profile", SessionID: "session"}
|
|
}
|
|
|
|
type cancelBeforeCompletionContext struct{ calls int }
|
|
|
|
func (ctx *cancelBeforeCompletionContext) Deadline() (time.Time, bool) { return time.Time{}, false }
|
|
func (ctx *cancelBeforeCompletionContext) Done() <-chan struct{} { return nil }
|
|
func (ctx *cancelBeforeCompletionContext) Value(any) any { return nil }
|
|
func (ctx *cancelBeforeCompletionContext) Err() error {
|
|
ctx.calls++
|
|
if ctx.calls > 1 {
|
|
return context.Canceled
|
|
}
|
|
return nil
|
|
}
|