268 lines
11 KiB
Go
268 lines
11 KiB
Go
package semanticreconcile
|
|
|
|
import (
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestAssessProducesAStableOriginalPositionPlan(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{5, 4}, CanonicalCandidateID: 5},
|
|
{CandidateIDs: []int{2, 1}, CanonicalCandidateID: 2},
|
|
}}
|
|
assessment := preparation.Assess(response)
|
|
want := []planGroupSnapshot{
|
|
{members: []int{0, 2}, canonical: 2},
|
|
{members: []int{5, 6}, canonical: 6},
|
|
}
|
|
if got := snapshotPlan(assessment.Plan()); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("plan = %#v, want %#v", got, want)
|
|
}
|
|
if assessment.DiscardedGroupCount() != 0 || assessment.RetryRequired() || len(assessment.Issues()) != 0 {
|
|
t.Fatalf("assessment diagnostics = discarded %d, retry %t, issues %#v", assessment.DiscardedGroupCount(), assessment.RetryRequired(), assessment.Issues())
|
|
}
|
|
|
|
reordered := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
|
{CandidateIDs: []int{4, 5}, CanonicalCandidateID: 5},
|
|
}})
|
|
if got := snapshotPlan(reordered.Plan()); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("reordered plan = %#v, want %#v", got, want)
|
|
}
|
|
|
|
empty := preparation.Assess(ProposalResponse{})
|
|
if len(empty.Plan().Groups()) != 0 || len(empty.Issues()) != 0 || empty.DiscardedGroupCount() != 0 || empty.RetryRequired() {
|
|
t.Fatalf("empty assessment = plan %#v, issues %#v, discarded %d, retry %t", empty.Plan().Groups(), empty.Issues(), empty.DiscardedGroupCount(), empty.RetryRequired())
|
|
}
|
|
}
|
|
|
|
func TestIssueDetailsPreservesIssueOrder(t *testing.T) {
|
|
issues := []Issue{
|
|
{GroupIndex: 3, Category: IssueCanonicalUnknown},
|
|
{GroupIndex: 1, Category: IssueMemberUnknown},
|
|
}
|
|
want := []string{"group 3: canonical_unknown", "group 1: member_unknown"}
|
|
if got := IssueDetails(issues); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("IssueDetails() = %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestCorrectionGuidanceCoversEveryIssueCategoryWithoutExposingInternalLabels(t *testing.T) {
|
|
if len(issueCorrectionProse) != len(allIssueCategories) {
|
|
t.Fatalf("correction prose entries = %d, categories = %d", len(issueCorrectionProse), len(allIssueCategories))
|
|
}
|
|
issues := make([]Issue, len(allIssueCategories))
|
|
seen := make(map[IssueCategory]struct{}, len(allIssueCategories))
|
|
for index, category := range allIssueCategories {
|
|
if _, duplicate := seen[category]; duplicate {
|
|
t.Fatalf("duplicate authoritative issue category %q", category)
|
|
}
|
|
seen[category] = struct{}{}
|
|
if strings.TrimSpace(issueCorrectionProse[category]) == "" {
|
|
t.Fatalf("issue category %q has no model-facing prose", category)
|
|
}
|
|
issues[index] = Issue{GroupIndex: index, Category: category}
|
|
}
|
|
|
|
details, err := CorrectionDetails(issues)
|
|
if err != nil {
|
|
t.Fatalf("CorrectionDetails() error = %v", err)
|
|
}
|
|
guidance, err := CorrectionGuidance(issues)
|
|
if err != nil {
|
|
t.Fatalf("CorrectionGuidance() error = %v", err)
|
|
}
|
|
if len(details) != len(allIssueCategories) || !strings.Contains(guidance, "Duplicate group 1") || !strings.Contains(guidance, "complete corrected JSON response") || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
|
t.Fatalf("correction details = %#v guidance = %q", details, guidance)
|
|
}
|
|
for _, category := range allIssueCategories {
|
|
if strings.Contains(guidance, string(category)) {
|
|
t.Fatalf("model guidance exposed internal category %q: %q", category, guidance)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCorrectionDetailsDeduplicatesAndBoundsAffectedGroupLists(t *testing.T) {
|
|
issues := make([]Issue, 0, 257)
|
|
for group := 0; group < 256; group++ {
|
|
issues = append(issues, Issue{GroupIndex: group, Category: IssueMemberUnknown})
|
|
}
|
|
issues = append(issues, Issue{GroupIndex: 0, Category: IssueMemberUnknown})
|
|
|
|
details, err := CorrectionDetails(issues)
|
|
if err != nil {
|
|
t.Fatalf("CorrectionDetails() error = %v", err)
|
|
}
|
|
if len(details) != 1 || !strings.Contains(details[0], "additional affected group") || strings.Count(details[0], "Duplicate groups") != 1 {
|
|
t.Fatalf("CorrectionDetails() = %#v, want one bounded grouped detail", details)
|
|
}
|
|
guidance, err := CorrectionGuidance(issues)
|
|
if err != nil || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
|
t.Fatalf("CorrectionGuidance() = %q, %v", guidance, err)
|
|
}
|
|
}
|
|
|
|
func TestCorrectionDetailsRejectsUnknownOrInvalidIssues(t *testing.T) {
|
|
for _, issues := range [][]Issue{
|
|
{{GroupIndex: 0, Category: "future_unmapped_category"}},
|
|
{{GroupIndex: -1, Category: IssueMemberUnknown}},
|
|
} {
|
|
if details, err := CorrectionDetails(issues); err == nil || details != nil {
|
|
t.Fatalf("CorrectionDetails(%#v) = %#v, %v; want fail-closed error", issues, details, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
tests := []struct {
|
|
name string
|
|
group DuplicateGroup
|
|
category IssueCategory
|
|
}{
|
|
{name: "zero member", group: DuplicateGroup{CandidateIDs: []int{0, 2}, CanonicalCandidateID: 2}, category: IssueMemberNonPositive},
|
|
{name: "negative member", group: DuplicateGroup{CandidateIDs: []int{-1, 2}, CanonicalCandidateID: 2}, category: IssueMemberNonPositive},
|
|
{name: "unknown member", group: DuplicateGroup{CandidateIDs: []int{99, 2}, CanonicalCandidateID: 2}, category: IssueMemberUnknown},
|
|
{name: "repeated member", group: DuplicateGroup{CandidateIDs: []int{1, 1}, CanonicalCandidateID: 1}, category: IssueRepeatedMember},
|
|
{name: "too small", group: DuplicateGroup{CandidateIDs: []int{1}, CanonicalCandidateID: 1}, category: IssueFewerThanTwoMembers},
|
|
{name: "zero canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 0}, category: IssueCanonicalNonPositive},
|
|
{name: "negative canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: -1}, category: IssueCanonicalNonPositive},
|
|
{name: "unknown canonical", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 99}, category: IssueCanonicalUnknown},
|
|
{name: "canonical not member", group: DuplicateGroup{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 3}, category: IssueCanonicalNotMember},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
assessment := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{test.group}})
|
|
if len(assessment.Plan().Groups()) != 0 || assessment.DiscardedGroupCount() != 1 || !assessment.RetryRequired() {
|
|
t.Fatalf("assessment = plan %#v, discarded %d, retry %t", assessment.Plan().Groups(), assessment.DiscardedGroupCount(), assessment.RetryRequired())
|
|
}
|
|
if !hasIssue(assessment.Issues(), 0, test.category) {
|
|
t.Fatalf("issues = %#v, want category %q", assessment.Issues(), test.category)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAssessDiscardsEveryOverlappingGroupAndRetainsIndependentGroups(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
assessment := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 1},
|
|
{CandidateIDs: []int{2, 3}, CanonicalCandidateID: 2},
|
|
{CandidateIDs: []int{4, 5}, CanonicalCandidateID: 5},
|
|
}})
|
|
wantPlan := []planGroupSnapshot{{members: []int{5, 6}, canonical: 6}}
|
|
if got := snapshotPlan(assessment.Plan()); !reflect.DeepEqual(got, wantPlan) {
|
|
t.Fatalf("plan = %#v, want %#v", got, wantPlan)
|
|
}
|
|
if assessment.DiscardedGroupCount() != 2 || !assessment.RetryRequired() {
|
|
t.Fatalf("discarded = %d, retry = %t", assessment.DiscardedGroupCount(), assessment.RetryRequired())
|
|
}
|
|
issues := assessment.Issues()
|
|
if len(issues) != 2 || !hasIssue(issues, 0, IssueOverlappingMember) || !hasIssue(issues, 1, IssueOverlappingMember) {
|
|
t.Fatalf("issues = %#v, want both conflicting group indexes", issues)
|
|
}
|
|
|
|
invalidAndSafe := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1},
|
|
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
|
}})
|
|
wantPlan = []planGroupSnapshot{{members: []int{0, 2}, canonical: 2}}
|
|
if got := snapshotPlan(invalidAndSafe.Plan()); !reflect.DeepEqual(got, wantPlan) {
|
|
t.Fatalf("plan with independent invalid group = %#v, want %#v", got, wantPlan)
|
|
}
|
|
if invalidAndSafe.DiscardedGroupCount() != 1 || !invalidAndSafe.RetryRequired() || !hasIssue(invalidAndSafe.Issues(), 0, IssueMemberUnknown) {
|
|
t.Fatalf("invalid-and-safe assessment = discarded %d, retry %t, issues %#v", invalidAndSafe.DiscardedGroupCount(), invalidAndSafe.RetryRequired(), invalidAndSafe.Issues())
|
|
}
|
|
}
|
|
|
|
func TestAssessmentAccessorsAndInputsDoNotShareRetainedState(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{
|
|
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2},
|
|
}}
|
|
assessment := preparation.Assess(response)
|
|
want := snapshotPlan(assessment.Plan())
|
|
|
|
response.DuplicateGroups[0].CandidateIDs[0] = 99
|
|
response.DuplicateGroups[0].CanonicalCandidateID = 99
|
|
plan := assessment.Plan()
|
|
groups := plan.Groups()
|
|
members := groups[0].MemberPositions()
|
|
members[0] = 99
|
|
groups[0] = PlanGroup{}
|
|
if got := snapshotPlan(assessment.Plan()); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("assessment plan changed through returned or input data: %#v", got)
|
|
}
|
|
|
|
invalid := preparation.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{{
|
|
CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1,
|
|
}}})
|
|
issues := invalid.Issues()
|
|
issues[0].GroupIndex = 99
|
|
issues[0].Category = IssueOverlappingMember
|
|
if retained := invalid.Issues(); retained[0].GroupIndex == 99 || retained[0].Category == IssueOverlappingMember {
|
|
t.Fatalf("Issues() exposed retained state: %#v", retained)
|
|
}
|
|
}
|
|
|
|
type planGroupSnapshot struct {
|
|
members []int
|
|
canonical int
|
|
}
|
|
|
|
func snapshotPlan(plan Plan) []planGroupSnapshot {
|
|
groups := plan.Groups()
|
|
snapshot := make([]planGroupSnapshot, len(groups))
|
|
for index, group := range groups {
|
|
snapshot[index] = planGroupSnapshot{
|
|
members: group.MemberPositions(),
|
|
canonical: group.CanonicalPosition(),
|
|
}
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func hasIssue(issues []Issue, groupIndex int, category IssueCategory) bool {
|
|
for _, issue := range issues {
|
|
if issue.GroupIndex == groupIndex && issue.Category == category {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func proposalPreparation(t *testing.T) Preparation {
|
|
t.Helper()
|
|
document := &source.SourceDocument{ID: "session", Units: make([]source.SourceUnit, 7)}
|
|
candidates := make([]Candidate, len(document.Units))
|
|
for position := range document.Units {
|
|
unitID := position + 1
|
|
document.Units[position] = source.SourceUnit{ID: unitID, Text: "unit"}
|
|
candidates[position] = Candidate{
|
|
Label: "candidate",
|
|
SourceRefs: []source.SourceRef{{
|
|
SourceID: document.ID,
|
|
StartUnitID: unitID,
|
|
EndUnitID: unitID,
|
|
}},
|
|
}
|
|
}
|
|
for _, position := range []int{1, 4} {
|
|
candidates[position].SourceRefs[0].SourceID = "other"
|
|
}
|
|
preparation, err := Prepare(document, candidates, Limits{
|
|
ContextRadius: 0,
|
|
MaximumCandidates: len(candidates),
|
|
MaximumMaterialBytes: 10000,
|
|
})
|
|
if err != nil || preparation.Disposition() != Ready {
|
|
t.Fatalf("Prepare() disposition = %v, error = %v", preparation.Disposition(), err)
|
|
}
|
|
return preparation
|
|
}
|