69 lines
2.1 KiB
Go
69 lines
2.1 KiB
Go
package validators
|
|
|
|
import (
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
)
|
|
|
|
// validationContext supplies adjacent source segments within the current chunk
|
|
// and glossary definitions mentioned in the candidate or that local context.
|
|
// These fields travel in each item so batching accounts for their size.
|
|
func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) {
|
|
var neighbors []schema.Segment
|
|
var source string
|
|
if req.WorkingTranscript != nil {
|
|
segments := req.WorkingTranscript.Segments
|
|
for i, segment := range segments {
|
|
if segment.ID != segmentID {
|
|
continue
|
|
}
|
|
source = segment.Text
|
|
if i > 0 {
|
|
neighbors = append(neighbors, segments[i-1])
|
|
}
|
|
if i+1 < len(segments) {
|
|
neighbors = append(neighbors, segments[i+1])
|
|
}
|
|
break
|
|
}
|
|
}
|
|
texts := []string{source, corrected}
|
|
for _, neighbor := range neighbors {
|
|
texts = append(texts, neighbor.Text)
|
|
}
|
|
var entries []schema.GlossaryEntry
|
|
if req.Glossary != nil {
|
|
vocabulary := NewProtectedVocabulary(req.Glossary)
|
|
mentioned := make(map[int]bool)
|
|
for _, text := range texts {
|
|
for identity := range vocabulary.occurrencesByIdentity(text) {
|
|
mentioned[identity] = true
|
|
}
|
|
}
|
|
for i, entry := range req.Glossary.Entries {
|
|
if mentioned[i] {
|
|
entries = append(entries, entry)
|
|
}
|
|
}
|
|
}
|
|
return neighbors, entries
|
|
}
|
|
|
|
// fitValidationContext drops optional context before treating required candidate
|
|
// text as oversized. Glossary definitions take priority over adjacent segments.
|
|
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem {
|
|
for {
|
|
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
|
if err != nil || tokens <= budget {
|
|
return item
|
|
}
|
|
if len(item.NeighboringSegments) > 0 {
|
|
item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1]
|
|
} else if len(item.GlossaryEntries) > 0 {
|
|
item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1]
|
|
} else {
|
|
return item
|
|
}
|
|
}
|
|
}
|