eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement integer source units and chunk payloads

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 18:34:23 +00:00
parent 4f057b99ac
commit 9e3f8809b3
45 changed files with 618 additions and 451 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/modules/chunk/dnd/scenes/assets/prompts/instructions.md
View File

@@ -22,8 +22,7 @@ dnd/scenes boundary policy:
- return sequential scenes with no gaps;
- do not overlap scenes;
- preserve source-unit order;
- use 1-based integer source-unit numbers from the transcript, where 1 is the
first provided source unit;
- use integer source-unit IDs from the transcript;
- each scene must have start_unit_id and end_unit_id;
- do not include final chunk IDs or chunk indexes.

52
internal/modules/chunk/dnd/scenes/chunker.go
View File

@@ -2,6 +2,7 @@ package scenes
import (
"context"
"encoding/json"
"fmt"
"strings"
@@ -142,7 +143,7 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
return nil, fmt.Errorf("scenes must not be empty")
}
unitIndexes := make(map[string]int, len(doc.Units))
unitIndexes := make(map[int]int, len(doc.Units))
for i, unit := range doc.Units {
unitIndexes[unit.ID] = i
}
@@ -157,18 +158,18 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
startIndex, ok := unitIndexes[normalized.StartUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
return nil, fmt.Errorf("scene[%d] start_unit_id %d was not found", i, normalized.StartUnitID)
}
endIndex, ok := unitIndexes[normalized.EndUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
return nil, fmt.Errorf("scene[%d] end_unit_id %d was not found", i, normalized.EndUnitID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
return nil, fmt.Errorf("scene[%d] start_unit_id %d appears after end_unit_id %d", i, normalized.StartUnitID, normalized.EndUnitID)
}
if i == 0 && startIndex != 0 {
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
return nil, fmt.Errorf("first scene must start at first source unit %d", doc.Units[0].ID)
}
if i > 0 {
if startIndex <= previousEnd {
@@ -181,11 +182,19 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
previousEnd = endIndex
units := cloneUnits(doc.Units[startIndex : endIndex+1])
content, err := chunkContent(units)
if err != nil {
return nil, err
}
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
Units: units,
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
Content: content,
MediaType: "application/json",
Units: units,
Metadata: map[string]any{
"scene_title": normalized.ShortTitle,
"primary_mode": normalized.PrimaryMode,
@@ -201,11 +210,23 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
}
if previousEnd != len(doc.Units)-1 {
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
return nil, fmt.Errorf("final scene must end at final source unit %d", doc.Units[len(doc.Units)-1].ID)
}
return chunks, nil
}
func chunkContent(units []source.SourceUnit) ([]byte, error) {
content, err := json.Marshal(struct {
Units []source.SourceUnit `json:"units"`
}{
Units: units,
})
if err != nil {
return nil, fmt.Errorf("encode chunk content: %w", err)
}
return content, nil
}
func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) {
startUnitID, err := dnd.ResolveUnitID(doc, "start_unit_id", scene.StartUnitID)
if err != nil {
@@ -226,9 +247,16 @@ func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse)
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
}
requiredInts := map[string]int{
"start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID,
}
for field, value := range requiredInts {
if value <= 0 {
return normalizedScene{}, fmt.Errorf("scene[%d] %s must be positive", index, field)
}
}
required := map[string]string{
"start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID,
"short_title": out.ShortTitle,
"primary_mode": out.PrimaryMode,
"summary": out.Summary,

64
internal/modules/chunk/dnd/scenes/chunker_test.go
View File

@@ -170,8 +170,8 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
}
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]int{{1, 2}, {3, 4}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
}
@@ -179,13 +179,19 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.StartUnitID != 1 || first.EndUnitID != 2 {
t.Fatalf("first boundaries = %d-%d, want 1-2", first.StartUnitID, first.EndUnitID)
}
if first.MediaType != "application/json" || len(first.Content) == 0 {
t.Fatalf("first payload = media type %q length %d, want JSON content", first.MediaType, len(first.Content))
}
if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != "seg-001" ||
first.Metadata["end_unit_id"] != "seg-002" ||
first.Metadata["start_unit_id"] != 1 ||
first.Metadata["end_unit_id"] != 2 ||
first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
}
@@ -311,11 +317,11 @@ func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
t.Fatalf("Chunk() error = %v, want nil", err)
}
doc.Units[0].ID = "mutated"
doc.Units[0].ID = 99
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != "seg-001" {
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
@@ -362,7 +368,7 @@ func TestChunkRejectsInvalidRequests(t *testing.T) {
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = ""
invalidDoc.Units[0].ID = 0
emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil
@@ -407,37 +413,37 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
{
name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-999"),
scene(1, 999),
}),
want: "was not found",
},
{
name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-003", "seg-002"),
scene(3, 2),
}),
want: "appears after",
},
{
name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-001"),
scene("seg-003", "seg-004"),
scene(1, 1),
scene(3, 4),
}),
want: "gap",
},
{
name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-002"),
scene("seg-002", "seg-004"),
scene(1, 2),
scene(2, 4),
}),
want: "overlap",
},
{
name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-003"),
scene(1, 3),
}),
want: "final scene",
},
@@ -445,8 +451,8 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: dnd.UnitRefFromString("seg-001"),
EndUnitID: dnd.UnitRefFromString("seg-004"),
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: " ",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
@@ -461,8 +467,8 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: dnd.UnitRefFromString("seg-001"),
EndUnitID: dnd.UnitRefFromString("seg-004"),
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: "Title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "},
@@ -511,7 +517,7 @@ func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.Chun
}
}
const sceneTranscriptJSON = `{"id":"session-alpha","segments":[{"id":"seg-001","text":"Aria asks whether the goblin will parley."}]}`
const sceneTranscriptJSON = `{"id":"session-alpha","segments":[{"id":1,"text":"Aria asks whether the goblin will parley."}]}`
func sceneSourceInput() contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
@@ -529,10 +535,10 @@ func sceneSourceDocument() *source.SourceDocument {
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."},
},
}
}
@@ -540,7 +546,7 @@ func sceneSourceDocument() *source.SourceDocument {
func validSceneResponse() chunkResponse {
return chunkResponse{
Scenes: []sceneResponse{
scene("seg-001", "seg-004"),
scene(1, 4),
},
BoundaryCaveats: []string{},
}
@@ -551,10 +557,10 @@ func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse
return response
}
func scene(startUnitID string, endUnitID string) sceneResponse {
func scene(startUnitID int, endUnitID int) sceneResponse {
return sceneResponse{
StartUnitID: dnd.UnitRefFromString(startUnitID),
EndUnitID: dnd.UnitRefFromString(endUnitID),
StartUnitID: dnd.UnitRefFromInt(startUnitID),
EndUnitID: dnd.UnitRefFromInt(endUnitID),
ShortTitle: "Scene title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
@@ -572,8 +578,8 @@ func chunkIDs(chunks []contracts.SourceChunk) []string {
return ids
}
func unitIDs(units []source.SourceUnit) []string {
ids := make([]string, 0, len(units))
func unitIDs(units []source.SourceUnit) []int {
ids := make([]int, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}

4
internal/modules/chunk/dnd/scenes/model.go
View File

@@ -19,8 +19,8 @@ type sceneResponse struct {
}
type normalizedScene struct {
StartUnitID string
EndUnitID string
StartUnitID int
EndUnitID int
ShortTitle string
PrimaryMode string
MainParticipants []string