eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete the semantic reconciliation roadmap

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-09 18:30:32 +00:00
parent e95e2f2220
commit 67338798aa
14 changed files with 187 additions and 1283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
internal/framework/semanticreconcile/preparation.go
View File

@@ -189,37 +189,36 @@ func Prepare(document *source.SourceDocument, candidates []Candidate, limits Lim
return result, nil
}
candidateContent, err := json.Marshal(candidateInput{Candidates: views})
candidateContent, withinLimit, err := marshalCandidateInput(views, limits.MaximumMaterialBytes)
if err != nil {
return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode candidate material: %w", err)
}
if len(candidateContent) > limits.MaximumMaterialBytes {
if !withinLimit {
result.disposition = LimitExceeded
return result, nil
}
intervals := make([]sourceInterval, 0)
cited := make([]bool, len(document.Units))
contextIntervals := make([]sourceInterval, 0)
citedIntervals := make([]sourceInterval, 0)
for _, candidate := range prepared {
for _, interval := range candidate.intervals {
for position := interval.start; position <= interval.end; position++ {
cited[position] = true
}
intervals = append(intervals, sourceInterval{
citedIntervals = append(citedIntervals, interval)
contextIntervals = append(contextIntervals, sourceInterval{
start: max(0, interval.start-limits.ContextRadius),
end: min(len(document.Units)-1, interval.end+limits.ContextRadius),
})
}
}
windows, err := buildContextWindows(document.Units, coalesceIntervals(intervals), cited)
transcriptContent, withinLimit, err := marshalTranscriptInput(
document.Units,
coalesceIntervals(contextIntervals),
coalesceIntervals(citedIntervals),
limits.MaximumMaterialBytes-len(candidateContent),
)
if err != nil {
return Preparation{}, fmt.Errorf("prepare semantic reconciliation: build transcript material: invalid source metadata")
}
transcriptContent, err := json.Marshal(transcriptInput{Windows: windows})
if err != nil {
return Preparation{}, fmt.Errorf("prepare semantic reconciliation: encode transcript material: %w", err)
}
if len(transcriptContent) > limits.MaximumMaterialBytes-len(candidateContent) {
if !withinLimit {
result.disposition = LimitExceeded
return result, nil
}
@@ -303,27 +302,93 @@ func coalesceIntervals(intervals []sourceInterval) []sourceInterval {
return coalesced
}
func buildContextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]transcriptWindow, error) {
windows := make([]transcriptWindow, 0, len(intervals))
for _, interval := range intervals {
window := transcriptWindow{Units: make([]transcriptUnit, 0, interval.end-interval.start+1)}
func marshalCandidateInput(candidates []visibleCandidate, maximumBytes int) ([]byte, bool, error) {
content := make([]byte, 0, min(maximumBytes, 4096))
var withinLimit bool
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte(`{"candidates":[`))
if !withinLimit {
return nil, false, nil
}
for index, candidate := range candidates {
encoded, err := json.Marshal(candidate)
if err != nil {
return nil, false, err
}
separator := []byte(nil)
if index > 0 {
separator = []byte(",")
}
content, withinLimit = appendWithinLimit(content, maximumBytes, separator, encoded)
if !withinLimit {
return nil, false, nil
}
}
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}"))
return content, withinLimit, nil
}
// marshalTranscriptInput retains at most maximumBytes while visiting source
// units in order. It deliberately serializes one unit at a time so an oversized
// request does not require a document-sized transcript copy before rejection.
func marshalTranscriptInput(units []source.SourceUnit, contextIntervals, citedIntervals []sourceInterval, maximumBytes int) ([]byte, bool, error) {
content := make([]byte, 0, min(maximumBytes, 4096))
content, withinLimit := appendWithinLimit(content, maximumBytes, []byte(`{"windows":[`))
if !withinLimit {
return nil, false, nil
}
citedIndex := 0
for windowIndex, interval := range contextIntervals {
separator := []byte(nil)
if windowIndex > 0 {
separator = []byte(",")
}
content, withinLimit = appendWithinLimit(content, maximumBytes, separator, []byte(`{"units":[`))
if !withinLimit {
return nil, false, nil
}
for position := interval.start; position <= interval.end; position++ {
for citedIndex < len(citedIntervals) && citedIntervals[citedIndex].end < position {
citedIndex++
}
cited := citedIndex < len(citedIntervals) && citedIntervals[citedIndex].start <= position
unit := units[position]
metadata, err := source.CloneMetadata(unit.Metadata)
if err != nil {
return nil, err
return nil, false, err
}
window.Units = append(window.Units, transcriptUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: metadata,
Cited: cited[position],
encoded, err := json.Marshal(transcriptUnit{
ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: metadata, Cited: cited,
})
if err != nil {
return nil, false, err
}
separator = nil
if position > interval.start {
separator = []byte(",")
}
content, withinLimit = appendWithinLimit(content, maximumBytes, separator, encoded)
if !withinLimit {
return nil, false, nil
}
}
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}"))
if !withinLimit {
return nil, false, nil
}
windows = append(windows, window)
}
return windows, nil
content, withinLimit = appendWithinLimit(content, maximumBytes, []byte("]}"))
return content, withinLimit, nil
}
func appendWithinLimit(content []byte, maximumBytes int, parts ...[]byte) ([]byte, bool) {
for _, part := range parts {
if len(content) > maximumBytes || len(part) > maximumBytes-len(content) {
return content, false
}
content = append(content, part...)
}
return content, true
}
func newInputMaterial(name string, content []byte) contracts.LLMInputMaterial {

32
internal/framework/semanticreconcile/preparation_test.go
View File

@@ -277,6 +277,38 @@ func TestPrepareEnforcesCandidateLimitBeforeRenderingContext(t *testing.T) {
}
}
func TestPrepareStopsRenderingContextWhenMaterialLimitIsExceeded(t *testing.T) {
cycle := map[string]any{}
cycle["self"] = cycle
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
{ID: 1, Text: strings.Repeat("oversized", 100)},
{ID: 2, Text: "must not be inspected"},
}}
candidates := candidatesForEveryUnit(document)
base, err := Prepare(document, candidates, Limits{
ContextRadius: 0,
MaximumCandidates: len(candidates),
MaximumMaterialBytes: 10000,
})
if err != nil || base.Disposition() != Ready {
t.Fatalf("Prepare(base) = disposition %v, error %v", base.Disposition(), err)
}
candidateBytes := len(base.Materials()[candidateInputName].Content)
document.Units[1].Metadata = cycle
limited, err := Prepare(document, candidates, Limits{
ContextRadius: 0,
MaximumCandidates: len(candidates),
MaximumMaterialBytes: candidateBytes + 64,
})
if err != nil {
t.Fatalf("Prepare(limited) error = %v; rendering should stop at the material bound", err)
}
if limited.Disposition() != LimitExceeded || len(limited.Materials()) != 0 {
t.Fatalf("Prepare(limited) = disposition %v, materials %#v, want bounded skip", limited.Disposition(), limited.Materials())
}
}
func TestPrepareAcceptsExactCombinedByteLimitAndSkipsOneOver(t *testing.T) {
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
{ID: 1, Text: "one"},

5
internal/modules/dnd/normalize/itemregistry/normalizer.go
View File

@@ -102,11 +102,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err := ctx.Err(); err != nil {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context error before normalize: %w", err)
}
if req.Source == nil {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil")
}
order := shared.NewSourceRefOrder(req.Source)
records, warnings := preprocessRecords(req.MergeOutput.Value, order)
deterministic := recordList(records)
if len(records) < 2 || req.Source == nil {
if len(records) < 2 {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil
}

15
internal/modules/dnd/normalize/itemregistry/normalizer_test.go
View File

@@ -47,6 +47,16 @@ func TestModuleContractAndMetadata(t *testing.T) {
}
}
func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
_, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(
context.Background(),
contracts.TypedNormalizeRequest[dnd.ItemRegistry]{},
)
if err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
t.Fatalf("Normalize() error = %v, want nil source rejection", err)
}
}
func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
{ID: 1, Text: "The rope is secured."},
@@ -380,7 +390,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz
return normalizer
}
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}}
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{
Source: &source.SourceDocument{},
MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value},
}
}
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
request := normalizeRequest(value)

5
internal/modules/dnd/normalize/locationregistry/normalizer.go
View File

@@ -103,11 +103,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err := ctx.Err(); err != nil {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context error before normalize: %w", err)
}
if req.Source == nil {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("source document must not be nil")
}
order := shared.NewSourceRefOrder(req.Source)
records, warnings := preprocessRecords(req.MergeOutput.Value, order)
deterministic := recordList(records)
if len(records) < 2 || req.Source == nil {
if len(records) < 2 {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil
}

10
internal/modules/dnd/normalize/locationregistry/normalizer_test.go
View File

@@ -44,6 +44,16 @@ func TestModuleContractAndMetadata(t *testing.T) {
}
}
func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
_, err := newNormalizer(t, &recordingLocationNormalizerClient{}).Normalize(
context.Background(),
contracts.TypedNormalizeRequest[dnd.LocationRegistry]{},
)
if err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
t.Fatalf("Normalize() error = %v, want nil source rejection", err)
}
}
func TestNormalizePreparesOnlyExactDuplicatesAndRetainsSameNameAndNestedPlaces(t *testing.T) {
input := dnd.LocationRegistry{Locations: []dnd.Location{
{Name: " The Tavern ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},

5
internal/modules/dnd/normalize/locationregistry/test_helpers_test.go
View File

@@ -41,7 +41,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz
return normalizer
}
func normalizeRequest(value dnd.LocationRegistry) contracts.TypedNormalizeRequest[dnd.LocationRegistry] {
return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value}}
return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{
Source: &source.SourceDocument{},
MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value},
}
}
func normalizeRequestWithSource(value dnd.LocationRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.LocationRegistry] {
request := normalizeRequest(value)

5
internal/modules/dnd/normalize/npcregistry/normalizer.go
View File

@@ -102,11 +102,14 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err := ctx.Err(); err != nil {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("context error before normalize: %w", err)
}
if req.Source == nil {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("source document must not be nil")
}
order := shared.NewSourceRefOrder(req.Source)
records, warnings := preprocessRecords(req.MergeOutput.Value, order)
deterministic := recordList(records)
if len(records) < 2 || req.Source == nil {
if len(records) < 2 {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil
}

8
internal/modules/dnd/normalize/npcregistry/normalizer_test.go
View File

@@ -126,6 +126,9 @@ func TestNormalizePreservesInvalidCandidatesAndDoesNotAliasInput(t *testing.T) {
func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) {
normalizer := newNormalizer(t, &recordingNPCNormalizerClient{})
if _, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[dnd.NPCRegistry]{}); err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
t.Fatalf("nil source Normalize() error = %v", err)
}
result, err := normalizer.Normalize(context.Background(), normalizeRequest(dnd.NPCRegistry{NPCs: nil}))
if err != nil || result.Value.NPCs != nil {
t.Fatalf("nil list result = %#v, error = %v", result.Value, err)
@@ -177,7 +180,10 @@ func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normaliz
}
func normalizeRequest(value dnd.NPCRegistry) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {
return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value}}
return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{
Source: &source.SourceDocument{},
MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value},
}
}
func normalizeRequestWithSource(value dnd.NPCRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {