Files
notarius/internal/modules/chunk/dnd/scenes/chunker_test.go

629 lines
21 KiB
Go

package scenes
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
)
func TestNewModuleSpecAndRegister(t *testing.T) {
chunker := New()
if chunker == nil {
t.Fatal("New() = nil, want chunker")
}
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
ReferenceSlots: wantReferenceSlots(),
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got := ModuleSpec()
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
registry := pipeline.NewChunkerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
registered, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(registered, want) {
t.Fatalf("registered spec = %#v, want %#v", registered, want)
}
built, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
if slots := built.ReferenceSlots(); !reflect.DeepEqual(slots, want.ReferenceSlots) {
t.Fatalf("ReferenceSlots() = %#v, want %#v", slots, want.ReferenceSlots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {
t.Fatal("Register(nil) error = nil, want error")
}
if !strings.Contains(err.Error(), "chunker registry") {
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
}
}
func wantReferenceSlots() []contracts.ReferenceSlot {
accepted := []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}
return []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "party",
Description: "Optional party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "players",
Description: "Optional player list reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "roster",
Description: "Deprecated alias for party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
}
}
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(2),
ShortTitle: " Goblin parley ",
PrimaryMode: "Discussion",
MainParticipants: []string{" Aria ", "Goblin scout"},
Summary: " The party negotiates with a scout. ",
BoundaryNote: " The scene covers the discussion before fighting starts. ",
BoundaryConfidence: "High",
},
{
StartUnitID: dnd.UnitRefFromInt(3),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: "Ambush at the gate",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria", "Goblin ambushers"},
Summary: "The goblins attack at the gate.",
BoundaryNote: "Combat begins and resolves the immediate threat.",
BoundaryConfidence: "Medium",
},
},
BoundaryCaveats: []string{" The transition into combat is gradual. "},
},
}
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
req := client.requests[0]
if req.StageName != Key {
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
}
if req.PromptID != PromptID || req.PromptVersion != ResponseSchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", req.PromptID, req.PromptVersion, PromptID, ResponseSchemaVersion)
}
if req.SessionID != "session-123" || req.ProfileID != "profile-scenes" {
t.Fatalf("session/profile = %q/%q, want session-123/profile-scenes", req.SessionID, req.ProfileID)
}
transcript, ok := req.Inputs["transcript"]
if !ok {
t.Fatalf("transcript input missing from %#v", req.Inputs)
}
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" {
t.Fatalf("transcript metadata = %#v", transcript)
}
if got := string(transcript.Content); got != sceneTranscriptJSON {
t.Fatalf("transcript content = %q, want original source input", got)
}
if got := string(req.Inputs["players"].Content); got != " " {
t.Fatalf("players input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["party"].Content); got != " " {
t.Fatalf("party input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["glossary"].Content); got != " " {
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
}
gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]int{{1, 2}, {3, 4}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
}
first := result.Chunks[0]
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.StartUnitID != 1 || first.EndUnitID != 2 {
t.Fatalf("first boundaries = %d-%d, want 1-2", first.StartUnitID, first.EndUnitID)
}
if first.MediaType != "application/json" || len(first.Content) == 0 {
t.Fatalf("first payload = media type %q length %d, want JSON content", first.MediaType, len(first.Content))
}
if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != 1 ||
first.Metadata["end_unit_id"] != 2 ||
first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
}
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
}
if got := result.Warnings; len(got) != 1 ||
got[0].Scope != Key ||
got[0].ReasonCode != "scene_boundary_caveat" ||
got[0].Message != "The transition into combat is gradual." {
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
}
}
func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
client := &fakeScenesLLMClient{response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: "Ambush",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria"},
Summary: "The party is ambushed.",
BoundaryNote: "One scene covers the short fixture.",
BoundaryConfidence: "High",
},
},
}}
req := chunkRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"players": {
Slot: contracts.ReferenceSlot{Name: "players"},
Items: []contracts.ReferenceItem{
{SlotName: "players", Content: []byte("Alice: Aria")},
},
},
"party": {
Slot: contracts.ReferenceSlot{Name: "party"},
Items: []contracts.ReferenceItem{
{SlotName: "party", Content: []byte("Aria: cleric")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Brightmantle: local temple")},
},
},
},
}
if _, err := New().Chunk(context.Background(), req); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
request := client.requests[0]
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
t.Fatalf("players input = %q, want reference content", got)
}
if got := string(request.Inputs["party"].Content); got != "Aria: cleric" {
t.Fatalf("party input = %q, want reference content", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" {
t.Fatalf("glossary input = %q, want reference content", got)
}
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria: cleric") {
t.Fatalf("transcript input contains reference content")
}
}
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
inputs := dnd.PromptInputs(sceneSourceInput(), contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Legacy roster text")},
},
},
},
})
if got := string(inputs["party"].Content); got != "Legacy roster text" {
t.Fatalf("party input = %q, want legacy roster content", got)
}
if _, ok := inputs["roster"]; ok {
t.Fatalf("roster prompt input was present; want only party input")
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: validSceneResponse().Scenes,
BoundaryCaveats: []string{
" ",
},
},
}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want malformed structured output error")
}
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
}
}
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
doc := sceneSourceDocument()
client := &fakeScenesLLMClient{response: validSceneResponse()}
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
Source: doc,
SourceInput: sceneSourceInput(),
SessionID: "session-123",
LLMProfile: "profile-scenes",
LLMClient: client,
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
doc.Units[0].ID = 99
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
if !ok || participants[0] != "Aria" {
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
}
}
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := New().ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": ResponseSchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
if _, ok := metadata[forbidden]; ok {
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
}
}
}
func TestChunkRejectsInvalidRequests(t *testing.T) {
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
validReq := chunkRequestWithClient(validClient)
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = 0
emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil
tests := []struct {
name string
chunker *Chunker
ctx context.Context
req contracts.ChunkRequest
want string
}{
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
tests := []struct {
name string
response chunkResponse
want string
}{
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
{
name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene(1, 999),
}),
want: "was not found",
},
{
name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene(3, 2),
}),
want: "appears after",
},
{
name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene(1, 1),
scene(3, 4),
}),
want: "gap",
},
{
name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene(1, 2),
scene(2, 4),
}),
want: "overlap",
},
{
name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene(1, 3),
}),
want: "final scene",
},
{
name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: " ",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "short_title",
},
{
name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: "Title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "main_participants",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &fakeScenesLLMClient{response: tt.response}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkWrapsLLMClientError(t *testing.T) {
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want LLM error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
}
}
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: sceneSourceDocument(),
SourceInput: sceneSourceInput(),
SessionID: "session-123",
LLMProfile: "profile-scenes",
LLMClient: client,
}
}
const sceneTranscriptJSON = `{"id":"session-alpha","segments":[{"id":1,"text":"Aria asks whether the goblin will parley."}]}`
func sceneSourceInput() contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
}
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
req.Options = map[string]any{"max_units": 2}
return req
}
func sceneSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."},
},
}
}
func validSceneResponse() chunkResponse {
return chunkResponse{
Scenes: []sceneResponse{
scene(1, 4),
},
BoundaryCaveats: []string{},
}
}
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
response.Scenes = scenes
return response
}
func scene(startUnitID int, endUnitID int) sceneResponse {
return sceneResponse{
StartUnitID: dnd.UnitRefFromInt(startUnitID),
EndUnitID: dnd.UnitRefFromInt(endUnitID),
ShortTitle: "Scene title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "A compact summary.",
BoundaryNote: "The source units form one coherent scene.",
BoundaryConfidence: "High",
}
}
func chunkIDs(chunks []contracts.SourceChunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)
}
return ids
}
func unitIDs(units []source.SourceUnit) []int {
ids := make([]int, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
type fakeScenesLLMClient struct {
response chunkResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*chunkResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
req.Vars = cloneVars(req.Vars)
return req
}
func cloneVars(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
}
out := make(map[string]any, len(in))
for key, value := range in {
out[key] = value
}
return out
}