From ee71dc4937b246ee48f7803f38ce05065f2eedc0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 9 Aug 2026 15:52:55 +0000 Subject: [PATCH] Add bounded semantic candidate preparation --- internal/framework/semanticreconcile/doc.go | 3 + .../semanticreconcile/preparation.go | 338 ++++++++++++++++++ .../semanticreconcile/preparation_test.go | 332 +++++++++++++++++ 3 files changed, 673 insertions(+) create mode 100644 internal/framework/semanticreconcile/doc.go create mode 100644 internal/framework/semanticreconcile/preparation.go create mode 100644 internal/framework/semanticreconcile/preparation_test.go diff --git a/internal/framework/semanticreconcile/doc.go b/internal/framework/semanticreconcile/doc.go new file mode 100644 index 0000000..d23b6a8 --- /dev/null +++ b/internal/framework/semanticreconcile/doc.go @@ -0,0 +1,3 @@ +// Package semanticreconcile prepares and validates bounded materials for +// domain-neutral semantic reconciliation. +package semanticreconcile diff --git a/internal/framework/semanticreconcile/preparation.go b/internal/framework/semanticreconcile/preparation.go new file mode 100644 index 0000000..09070e9 --- /dev/null +++ b/internal/framework/semanticreconcile/preparation.go @@ -0,0 +1,338 @@ +package semanticreconcile + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +const ( + candidateInputName = "candidates" + transcriptInputName = "transcript" + jsonMediaType = "application/json" +) + +var defaultLimits = Limits{ + ContextRadius: 2, + MaximumCandidates: 128, + MaximumMaterialBytes: 262144, +} + +// Candidate is contextual source-backed input supplied by a typed consumer. +// Prepare does not retain or mutate Label or SourceRefs. +type Candidate struct { + Label string + SourceRefs []source.SourceRef +} + +// Limits bounds source context and serialized model input. +type Limits struct { + ContextRadius int + MaximumCandidates int + MaximumMaterialBytes int +} + +// DefaultLimits returns the core-owned production limits. +func DefaultLimits() Limits { + return defaultLimits +} + +// Validate rejects limits that cannot safely bound preparation. +func (limits Limits) Validate() error { + if limits.ContextRadius < 0 { + return fmt.Errorf("semantic reconciliation limits: context radius must not be negative") + } + if limits.MaximumCandidates <= 0 { + return fmt.Errorf("semantic reconciliation limits: maximum candidates must be positive") + } + if limits.MaximumMaterialBytes <= 0 { + return fmt.Errorf("semantic reconciliation limits: maximum bytes must be positive") + } + return nil +} + +// Disposition describes whether prepared materials may be sent to a model. +type Disposition uint8 + +const ( + // Ready indicates that the result contains complete bounded materials. + Ready Disposition = iota + 1 + // InsufficientCandidates indicates that fewer than two candidates were + // eligible after source-reference validation. + InsufficientCandidates + // LimitExceeded indicates that a candidate or serialized-material bound was + // exceeded and no request should be split or sent. + LimitExceeded +) + +// CandidateMapping relates one model-visible request-local ID to the +// corresponding zero-based position in the caller's candidate slice. +type CandidateMapping struct { + CandidateID int + CandidatePosition int +} + +// Preparation owns the visible candidate mapping and prompt materials. +type Preparation struct { + disposition Disposition + mappings []CandidateMapping + materials contracts.LLMInputSet +} + +// Disposition returns the preparation outcome. +func (preparation Preparation) Disposition() Disposition { + return preparation.disposition +} + +// CandidateMappings returns an owned copy in model-visible candidate order. +func (preparation Preparation) CandidateMappings() []CandidateMapping { + return append([]CandidateMapping(nil), preparation.mappings...) +} + +// Materials returns independently owned candidate and transcript materials. +// It is empty unless Disposition returns Ready. +func (preparation Preparation) Materials() contracts.LLMInputSet { + return preparation.materials.Clone() +} + +type sourceRange struct { + StartUnitID int `json:"start_unit_id"` + EndUnitID int `json:"end_unit_id"` +} + +type visibleCandidate struct { + CandidateID int `json:"candidate_id"` + Label string `json:"label"` + SourceRefs []sourceRange `json:"source_refs"` +} + +type candidateInput struct { + Candidates []visibleCandidate `json:"candidates"` +} + +type transcriptInput struct { + Windows []transcriptWindow `json:"windows"` +} + +type transcriptWindow struct { + Units []transcriptUnit `json:"units"` +} + +type transcriptUnit struct { + ID int `json:"id"` + Kind string `json:"kind"` + Text string `json:"text"` + Metadata map[string]any `json:"metadata,omitempty"` + Cited bool `json:"cited"` +} + +type sourceInterval struct { + start int + end int +} + +type preparedCandidate struct { + position int + references []sourceRange + intervals []sourceInterval +} + +// Prepare validates candidates and constructs bounded, source-ordered model +// inputs. Deterministic skip conditions are represented by the returned +// disposition rather than an error. +func Prepare(document *source.SourceDocument, candidates []Candidate, limits Limits) (Preparation, error) { + if err := limits.Validate(); err != nil { + return Preparation{}, err + } + + documentIndex := source.NewDocumentIndex(document) + prepared := make([]preparedCandidate, 0, len(candidates)) + for candidatePosition, candidate := range candidates { + references, intervals, valid := prepareReferences(documentIndex, candidate.SourceRefs) + if !valid { + continue + } + prepared = append(prepared, preparedCandidate{ + position: candidatePosition, + references: references, + intervals: intervals, + }) + } + + result := Preparation{ + disposition: InsufficientCandidates, + mappings: make([]CandidateMapping, len(prepared)), + } + views := make([]visibleCandidate, len(prepared)) + for index, candidate := range prepared { + candidateID := index + 1 + result.mappings[index] = CandidateMapping{ + CandidateID: candidateID, + CandidatePosition: candidate.position, + } + views[index] = visibleCandidate{ + CandidateID: candidateID, + Label: candidates[candidate.position].Label, + SourceRefs: cloneSourceRanges(candidate.references), + } + } + if len(prepared) < 2 { + return result, nil + } + if len(prepared) > limits.MaximumCandidates { + result.disposition = LimitExceeded + return result, nil + } + + candidateContent, err := json.Marshal(candidateInput{Candidates: views}) + if err != nil { + return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode candidate material: %w", err) + } + if len(candidateContent) > limits.MaximumMaterialBytes { + result.disposition = LimitExceeded + return result, nil + } + + intervals := make([]sourceInterval, 0) + cited := make([]bool, len(document.Units)) + for _, candidate := range prepared { + for _, interval := range candidate.intervals { + for position := interval.start; position <= interval.end; position++ { + cited[position] = true + } + intervals = append(intervals, sourceInterval{ + start: max(0, interval.start-limits.ContextRadius), + end: min(len(document.Units)-1, interval.end+limits.ContextRadius), + }) + } + } + windows, err := buildContextWindows(document.Units, coalesceIntervals(intervals), cited) + if err != nil { + return Preparation{}, fmt.Errorf("prepare semantic reconciliation: build transcript material: %w", err) + } + transcriptContent, err := json.Marshal(transcriptInput{Windows: windows}) + if err != nil { + return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode transcript material: %w", err) + } + if len(transcriptContent) > limits.MaximumMaterialBytes-len(candidateContent) { + result.disposition = LimitExceeded + return result, nil + } + + result.disposition = Ready + result.materials = contracts.LLMInputSet{ + candidateInputName: newInputMaterial(candidateInputName, candidateContent), + transcriptInputName: newInputMaterial(transcriptInputName, transcriptContent), + } + return result, nil +} + +func prepareReferences(index source.DocumentIndex, references []source.SourceRef) ([]sourceRange, []sourceInterval, bool) { + if len(references) == 0 { + return nil, nil, false + } + type referencedInterval struct { + reference sourceRange + interval sourceInterval + } + prepared := make([]referencedInterval, 0, len(references)) + for _, reference := range references { + if err := index.ValidateRef(reference); err != nil { + return nil, nil, false + } + start, _ := index.Position(reference.StartUnitID) + end, _ := index.Position(reference.EndUnitID) + prepared = append(prepared, referencedInterval{ + reference: sourceRange{StartUnitID: reference.StartUnitID, EndUnitID: reference.EndUnitID}, + interval: sourceInterval{start: start, end: end}, + }) + } + sort.Slice(prepared, func(left, right int) bool { + if prepared[left].interval.start != prepared[right].interval.start { + return prepared[left].interval.start < prepared[right].interval.start + } + return prepared[left].interval.end < prepared[right].interval.end + }) + + canonicalReferences := make([]sourceRange, 0, len(prepared)) + intervals := make([]sourceInterval, 0, len(prepared)) + for _, item := range prepared { + if len(canonicalReferences) > 0 && canonicalReferences[len(canonicalReferences)-1] == item.reference { + continue + } + canonicalReferences = append(canonicalReferences, item.reference) + intervals = append(intervals, item.interval) + } + return canonicalReferences, intervals, true +} + +func cloneSourceRanges(ranges []sourceRange) []sourceRange { + if len(ranges) == 0 { + return []sourceRange{} + } + return append([]sourceRange(nil), ranges...) +} + +func coalesceIntervals(intervals []sourceInterval) []sourceInterval { + if len(intervals) == 0 { + return nil + } + ordered := append([]sourceInterval(nil), intervals...) + sort.Slice(ordered, func(left, right int) bool { + if ordered[left].start != ordered[right].start { + return ordered[left].start < ordered[right].start + } + return ordered[left].end < ordered[right].end + }) + + coalesced := make([]sourceInterval, 0, len(ordered)) + for _, interval := range ordered { + if len(coalesced) == 0 || interval.start > coalesced[len(coalesced)-1].end+1 { + coalesced = append(coalesced, interval) + continue + } + if interval.end > coalesced[len(coalesced)-1].end { + coalesced[len(coalesced)-1].end = interval.end + } + } + return coalesced +} + +func buildContextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]transcriptWindow, error) { + windows := make([]transcriptWindow, 0, len(intervals)) + for _, interval := range intervals { + window := transcriptWindow{Units: make([]transcriptUnit, 0, interval.end-interval.start+1)} + for position := interval.start; position <= interval.end; position++ { + unit := units[position] + metadata, err := source.CloneMetadata(unit.Metadata) + if err != nil { + return nil, err + } + window.Units = append(window.Units, transcriptUnit{ + ID: unit.ID, + Kind: unit.Kind, + Text: unit.Text, + Metadata: metadata, + Cited: cited[position], + }) + } + windows = append(windows, window) + } + return windows, nil +} + +func newInputMaterial(name string, content []byte) contracts.LLMInputMaterial { + digest := sha256.Sum256(content) + return contracts.NewLLMInputMaterial( + name, + jsonMediaType, + content, + "sha256:"+hex.EncodeToString(digest[:]), + "", + ) +} diff --git a/internal/framework/semanticreconcile/preparation_test.go b/internal/framework/semanticreconcile/preparation_test.go new file mode 100644 index 0000000..eb0dd74 --- /dev/null +++ b/internal/framework/semanticreconcile/preparation_test.go @@ -0,0 +1,332 @@ +package semanticreconcile + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" +) + +func TestPrepareBuildsContiguousCandidatesAndOwnedSourceContext(t *testing.T) { + document := &source.SourceDocument{ID: "private-source-id", Units: []source.SourceUnit{ + {ID: 40, Kind: "narration", Text: "zero"}, + {ID: 10, Kind: "speech", Text: "one", Metadata: map[string]any{"speaker": map[string]any{"name": "Mira"}}}, + {ID: 70, Kind: "speech", Text: "two"}, + {ID: 20, Kind: "narration", Text: "three"}, + {ID: 90, Kind: "speech", Text: "four"}, + }} + references := []source.SourceRef{ + {SourceID: document.ID, StartUnitID: 90, EndUnitID: 90}, + {SourceID: document.ID, StartUnitID: 10, EndUnitID: 20}, + {SourceID: document.ID, StartUnitID: 10, EndUnitID: 20}, + } + candidates := []Candidate{ + {Label: "The Tavern", SourceRefs: append([]source.SourceRef(nil), references...)}, + {Label: "The Tavern", SourceRefs: append([]source.SourceRef(nil), references...)}, + {Label: "Broken", SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: 20, EndUnitID: 10}}}, + } + before := cloneCandidates(candidates) + + preparation, err := Prepare(document, candidates, Limits{ + ContextRadius: 1, + MaximumCandidates: len(candidates), + MaximumMaterialBytes: 10000, + }) + if err != nil { + t.Fatal(err) + } + if preparation.Disposition() != Ready { + t.Fatalf("Disposition() = %v, want Ready", preparation.Disposition()) + } + if !reflect.DeepEqual(candidates, before) { + t.Fatalf("Prepare() mutated candidates: %#v", candidates) + } + if got, want := preparation.CandidateMappings(), []CandidateMapping{ + {CandidateID: 1, CandidatePosition: 0}, + {CandidateID: 2, CandidatePosition: 1}, + }; !reflect.DeepEqual(got, want) { + t.Fatalf("CandidateMappings() = %#v, want %#v", got, want) + } + + materials := preparation.Materials() + if len(materials) != 2 { + t.Fatalf("materials = %#v, want candidates and transcript", materials) + } + if _, ok := materials[candidateInputName]; !ok { + t.Fatal("candidate material is missing") + } + if _, ok := materials[transcriptInputName]; !ok { + t.Fatal("transcript material is missing") + } + for name, material := range materials { + if material.Name != name || material.MediaType != "application/json" || material.OriginURI != "" || material.SizeBytes != int64(len(material.Content)) { + t.Fatalf("material %q metadata = %#v", name, material) + } + digest := sha256.Sum256(material.Content) + if want := "sha256:" + hex.EncodeToString(digest[:]); material.Digest != want { + t.Fatalf("material %q digest = %q, want %q", name, material.Digest, want) + } + } + + candidateContent := append([]byte(nil), materials[candidateInputName].Content...) + var candidatePayload candidateInput + if err := json.Unmarshal(candidateContent, &candidatePayload); err != nil { + t.Fatal(err) + } + wantCandidates := []visibleCandidate{ + {CandidateID: 1, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}}, + {CandidateID: 2, Label: "The Tavern", SourceRefs: []sourceRange{{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 90, EndUnitID: 90}}}, + } + if !reflect.DeepEqual(candidatePayload.Candidates, wantCandidates) { + t.Fatalf("candidate payload = %#v, want %#v", candidatePayload.Candidates, wantCandidates) + } + var candidateObjects struct { + Candidates []map[string]json.RawMessage `json:"candidates"` + } + if err := json.Unmarshal(candidateContent, &candidateObjects); err != nil { + t.Fatal(err) + } + for _, candidate := range candidateObjects.Candidates { + if len(candidate) != 3 || candidate["candidate_id"] == nil || candidate["label"] == nil || candidate["source_refs"] == nil { + t.Fatalf("model-facing candidate fields = %#v", candidate) + } + } + combined := string(materials[candidateInputName].Content) + string(materials[transcriptInputName].Content) + for _, forbidden := range []string{document.ID, "application_entity_id", "private-entity-id"} { + if strings.Contains(combined, forbidden) { + t.Fatalf("model material leaked %q: %s", forbidden, combined) + } + } + + var transcript transcriptInput + if err := json.Unmarshal(materials[transcriptInputName].Content, &transcript); err != nil { + t.Fatal(err) + } + if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != len(document.Units) { + t.Fatalf("windows = %#v, want one coalesced source window", transcript.Windows) + } + for index, wantID := range []int{40, 10, 70, 20, 90} { + if transcript.Windows[0].Units[index].ID != wantID { + t.Fatalf("unit %d id = %d, want %d", index, transcript.Windows[0].Units[index].ID, wantID) + } + } + if transcript.Windows[0].Units[0].Cited { + t.Fatal("radius-only unit marked cited") + } + for index := 1; index < len(transcript.Windows[0].Units); index++ { + if !transcript.Windows[0].Units[index].Cited { + t.Fatalf("evidence unit %d was not marked cited", index) + } + } + + candidates[0].SourceRefs[0].StartUnitID = 40 + document.Units[1].Metadata["speaker"].(map[string]any)["name"] = "changed" + if got := preparation.Materials()[candidateInputName].Content; !reflect.DeepEqual(got, candidateContent) { + t.Fatalf("candidate material changed through caller input: %s", got) + } + var retained transcriptInput + if err := json.Unmarshal(preparation.Materials()[transcriptInputName].Content, &retained); err != nil { + t.Fatal(err) + } + if got := retained.Windows[0].Units[1].Metadata["speaker"].(map[string]any)["name"]; got != "Mira" { + t.Fatalf("retained metadata = %v, want Mira", got) + } + + returnedMappings := preparation.CandidateMappings() + returnedMappings[0].CandidatePosition = 99 + returnedMaterials := preparation.Materials() + candidateMaterial := returnedMaterials[candidateInputName] + candidateMaterial.Content[0] = '[' + returnedMaterials[candidateInputName] = candidateMaterial + delete(returnedMaterials, transcriptInputName) + if preparation.CandidateMappings()[0].CandidatePosition != 0 || !json.Valid(preparation.Materials()[candidateInputName].Content) || len(preparation.Materials()) != 2 { + t.Fatal("preparation accessors exposed retained data") + } +} + +func TestPrepareFiltersUnsafeCandidatesAndCoalescesAdjacentWindows(t *testing.T) { + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 9}, {ID: 3}, {ID: 8}, {ID: 1}, {ID: 7}, + }} + candidates := []Candidate{ + {Label: "One", SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: 3, EndUnitID: 3}}}, + {Label: "Two", SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: 8, EndUnitID: 8}}}, + {Label: "Missing", SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: 99, EndUnitID: 99}}}, + {Label: "Foreign", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}}}, + {Label: "No references"}, + {Label: "Partly invalid", SourceRefs: []source.SourceRef{ + {SourceID: document.ID, StartUnitID: 1, EndUnitID: 1}, + {SourceID: document.ID, StartUnitID: 100, EndUnitID: 100}, + }}, + } + limits := Limits{ContextRadius: 0, MaximumCandidates: len(candidates), MaximumMaterialBytes: 10000} + + preparation, err := Prepare(document, candidates, limits) + if err != nil || preparation.Disposition() != Ready { + t.Fatalf("Prepare() disposition = %v, error = %v", preparation.Disposition(), err) + } + if got, want := preparation.CandidateMappings(), []CandidateMapping{ + {CandidateID: 1, CandidatePosition: 0}, + {CandidateID: 2, CandidatePosition: 1}, + }; !reflect.DeepEqual(got, want) { + t.Fatalf("CandidateMappings() = %#v, want %#v", got, want) + } + var transcript transcriptInput + if err := json.Unmarshal(preparation.Materials()[transcriptInputName].Content, &transcript); err != nil { + t.Fatal(err) + } + if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 2 || transcript.Windows[0].Units[0].ID != 3 || transcript.Windows[0].Units[1].ID != 8 { + t.Fatalf("windows = %#v, want adjacent source-order units coalesced", transcript.Windows) + } + oneCandidate, err := Prepare(document, candidates[:1], limits) + if err != nil { + t.Fatal(err) + } + if got, want := oneCandidate.CandidateMappings(), []CandidateMapping{{CandidateID: 1, CandidatePosition: 0}}; oneCandidate.Disposition() != InsufficientCandidates || !reflect.DeepEqual(got, want) || len(oneCandidate.Materials()) != 0 { + t.Fatalf("Prepare(one candidate) = disposition %v, mappings %#v, materials %#v", oneCandidate.Disposition(), got, oneCandidate.Materials()) + } + + nilPreparation, err := Prepare(nil, candidates, limits) + if err != nil { + t.Fatal(err) + } + if nilPreparation.Disposition() != InsufficientCandidates || len(nilPreparation.CandidateMappings()) != 0 || len(nilPreparation.Materials()) != 0 { + t.Fatalf("Prepare(nil) = disposition %v, mappings %#v, materials %#v", nilPreparation.Disposition(), nilPreparation.CandidateMappings(), nilPreparation.Materials()) + } +} + +func TestPrepareValidatesLimitsBeforeBuildingMaterials(t *testing.T) { + if err := DefaultLimits().Validate(); err != nil { + t.Fatalf("DefaultLimits().Validate() error = %v", err) + } + cycle := map[string]any{} + cycle["self"] = cycle + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 1, Metadata: cycle}, {ID: 2}, + }} + candidates := candidatesForEveryUnit(document) + tests := []struct { + name string + limits Limits + want string + }{ + {name: "negative radius", limits: Limits{ContextRadius: -1, MaximumCandidates: 2, MaximumMaterialBytes: 100}, want: "radius"}, + {name: "zero candidates", limits: Limits{ContextRadius: 0, MaximumCandidates: 0, MaximumMaterialBytes: 100}, want: "candidates"}, + {name: "negative candidates", limits: Limits{ContextRadius: 0, MaximumCandidates: -1, MaximumMaterialBytes: 100}, want: "candidates"}, + {name: "zero bytes", limits: Limits{ContextRadius: 0, MaximumCandidates: 2, MaximumMaterialBytes: 0}, want: "bytes"}, + {name: "negative bytes", limits: Limits{ContextRadius: 0, MaximumCandidates: 2, MaximumMaterialBytes: -1}, want: "bytes"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := Prepare(document, candidates, test.limits); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Prepare() error = %v, want %q validation", err, test.want) + } + }) + } +} + +func TestPrepareEnforcesCandidateLimitBeforeRenderingContext(t *testing.T) { + cycle := map[string]any{} + cycle["self"] = cycle + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 1, Text: "one", Metadata: cycle}, + {ID: 2, Text: "two"}, + {ID: 3, Text: "three"}, + }} + candidates := candidatesForEveryUnit(document) + limits := Limits{ContextRadius: 0, MaximumCandidates: 2, MaximumMaterialBytes: 10000} + + exceeded, err := Prepare(document, candidates, limits) + if err != nil { + t.Fatalf("Prepare(over limit) error = %v; context should not be rendered", err) + } + if exceeded.Disposition() != LimitExceeded || len(exceeded.CandidateMappings()) != 3 || len(exceeded.Materials()) != 0 { + t.Fatalf("Prepare(over limit) = disposition %v, mappings %#v, materials %#v", exceeded.Disposition(), exceeded.CandidateMappings(), exceeded.Materials()) + } + + document.Units[0].Metadata = nil + exact, err := Prepare(document, candidates[:2], limits) + if err != nil || exact.Disposition() != Ready { + t.Fatalf("Prepare(at limit) = disposition %v, error %v", exact.Disposition(), err) + } +} + +func TestPrepareAcceptsExactCombinedByteLimitAndSkipsOneOver(t *testing.T) { + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 1, Text: "one"}, + {ID: 2, Text: "two"}, + }} + candidates := candidatesForEveryUnit(document) + baseLimits := Limits{ContextRadius: 0, MaximumCandidates: len(candidates), MaximumMaterialBytes: 10000} + base, err := Prepare(document, candidates, baseLimits) + if err != nil || base.Disposition() != Ready { + t.Fatalf("Prepare(base) = disposition %v, error %v", base.Disposition(), err) + } + materials := base.Materials() + totalBytes := len(materials[candidateInputName].Content) + len(materials[transcriptInputName].Content) + + exactLimits := baseLimits + exactLimits.MaximumMaterialBytes = totalBytes + exact, err := Prepare(document, candidates, exactLimits) + if err != nil || exact.Disposition() != Ready { + t.Fatalf("Prepare(exact bytes) = disposition %v, error %v", exact.Disposition(), err) + } + oneOverLimits := exactLimits + oneOverLimits.MaximumMaterialBytes-- + oneOver, err := Prepare(document, candidates, oneOverLimits) + if err != nil || oneOver.Disposition() != LimitExceeded || len(oneOver.Materials()) != 0 { + t.Fatalf("Prepare(one over) = disposition %v, materials %#v, error %v", oneOver.Disposition(), oneOver.Materials(), err) + } +} + +func TestPrepareSerializationIsDeterministic(t *testing.T) { + document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 5, Text: "five", Metadata: map[string]any{"z": 1, "a": []any{"first", "second"}}}, + {ID: 2, Text: "two", Metadata: map[string]any{"nested": map[string]any{"b": true, "a": false}}}, + }} + candidates := candidatesForEveryUnit(document) + limits := Limits{ContextRadius: 0, MaximumCandidates: len(candidates), MaximumMaterialBytes: 10000} + + first, err := Prepare(document, candidates, limits) + if err != nil { + t.Fatal(err) + } + second, err := Prepare(document, cloneCandidates(candidates), limits) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{candidateInputName, transcriptInputName} { + firstMaterial := first.Materials()[name] + secondMaterial := second.Materials()[name] + if !reflect.DeepEqual(firstMaterial, secondMaterial) { + t.Fatalf("material %q is not deterministic:\n%#v\n%#v", name, firstMaterial, secondMaterial) + } + } +} + +func candidatesForEveryUnit(document *source.SourceDocument) []Candidate { + candidates := make([]Candidate, len(document.Units)) + for index, unit := range document.Units { + candidates[index] = Candidate{ + Label: "candidate", + SourceRefs: []source.SourceRef{{ + SourceID: document.ID, + StartUnitID: unit.ID, + EndUnitID: unit.ID, + }}, + } + } + return candidates +} + +func cloneCandidates(candidates []Candidate) []Candidate { + cloned := append([]Candidate(nil), candidates...) + for index := range cloned { + cloned[index].SourceRefs = append([]source.SourceRef(nil), candidates[index].SourceRefs...) + } + return cloned +}