488 lines
16 KiB
Go
488 lines
16 KiB
Go
package scenes
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
func TestNewModuleSpecAndRegister(t *testing.T) {
|
|
chunker := New()
|
|
if chunker == nil {
|
|
t.Fatal("New() = nil, want chunker")
|
|
}
|
|
if chunker.Key() != Key {
|
|
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
|
|
}
|
|
|
|
want := pipeline.ModuleSpec{
|
|
Key: Key,
|
|
Stage: pipeline.StageChunk,
|
|
Requires: []string{"source.transcript"},
|
|
Provides: []string{"chunks", "chunks.scenes"},
|
|
}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
got := ModuleSpec()
|
|
got.Requires[0] = "changed"
|
|
got.Provides[0] = "changed"
|
|
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
|
|
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
|
|
}
|
|
|
|
registry := pipeline.NewChunkerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatalf("Register() error = %v, want nil", err)
|
|
}
|
|
registered, ok := registry.Spec(Key)
|
|
if !ok {
|
|
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
|
}
|
|
if !reflect.DeepEqual(registered, want) {
|
|
t.Fatalf("registered spec = %#v, want %#v", registered, want)
|
|
}
|
|
built, err := registry.Build(Key)
|
|
if err != nil {
|
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
|
}
|
|
if built.Key() != Key {
|
|
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
|
|
}
|
|
if slots := built.ReferenceSlots(); len(slots) != 0 {
|
|
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
|
}
|
|
}
|
|
|
|
func TestRegisterNilRegistryReturnsError(t *testing.T) {
|
|
err := Register(nil)
|
|
if err == nil {
|
|
t.Fatal("Register(nil) error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "chunker registry") {
|
|
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|
client := &fakeScenesLLMClient{
|
|
response: chunkResponse{
|
|
Scenes: []sceneResponse{
|
|
{
|
|
StartUnitID: "seg-001",
|
|
EndUnitID: "seg-002",
|
|
ShortTitle: " Goblin parley ",
|
|
PrimaryMode: "Discussion",
|
|
MainParticipants: []string{" Aria ", "Goblin scout"},
|
|
Summary: " The party negotiates with a scout. ",
|
|
BoundaryNote: " The scene covers the discussion before fighting starts. ",
|
|
BoundaryConfidence: "High",
|
|
},
|
|
{
|
|
StartUnitID: "seg-003",
|
|
EndUnitID: "seg-004",
|
|
ShortTitle: "Ambush at the gate",
|
|
PrimaryMode: "Combat",
|
|
MainParticipants: []string{"Aria", "Goblin ambushers"},
|
|
Summary: "The goblins attack at the gate.",
|
|
BoundaryNote: "Combat begins and resolves the immediate threat.",
|
|
BoundaryConfidence: "Medium",
|
|
},
|
|
},
|
|
BoundaryCaveats: []string{" The transition into combat is gradual. "},
|
|
},
|
|
}
|
|
|
|
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
|
if err != nil {
|
|
t.Fatalf("Chunk() error = %v, want nil", err)
|
|
}
|
|
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
|
}
|
|
req := client.requests[0]
|
|
if req.StageName != Key {
|
|
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
|
|
}
|
|
schema, err := loadResponseSchema()
|
|
if err != nil {
|
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
|
}
|
|
if req.ResponseSchemaName != schema.Name {
|
|
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
|
|
}
|
|
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
|
|
t.Fatal("ResponseSchema does not match D&D scenes schema")
|
|
}
|
|
if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
|
|
t.Fatalf("Messages = %#v, want system then user", req.Messages)
|
|
}
|
|
for _, want := range []string{"session-alpha", "seg-001", "seg-004", "start_unit_id", "boundary_confidence"} {
|
|
if !strings.Contains(req.Messages[1].Content, want) {
|
|
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
|
|
}
|
|
}
|
|
|
|
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
|
|
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
|
|
}
|
|
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
|
|
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
|
|
if !reflect.DeepEqual(gotUnits, wantUnits) {
|
|
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
|
|
}
|
|
first := result.Chunks[0]
|
|
if first.SourceID != "session-alpha" || first.Index != 0 {
|
|
t.Fatalf("first chunk = %#v, want source and index fields", first)
|
|
}
|
|
if first.Metadata["scene_title"] != "Goblin parley" ||
|
|
first.Metadata["primary_mode"] != "Discussion" ||
|
|
first.Metadata["summary"] != "The party negotiates with a scout." ||
|
|
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
|
|
first.Metadata["boundary_confidence"] != "High" ||
|
|
first.Metadata["start_unit_id"] != "seg-001" ||
|
|
first.Metadata["end_unit_id"] != "seg-002" ||
|
|
first.Metadata["unit_count"] != 2 {
|
|
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
|
|
}
|
|
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
|
|
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
|
|
}
|
|
if got := result.Warnings; len(got) != 1 ||
|
|
got[0].Scope != Key ||
|
|
got[0].ReasonCode != "scene_boundary_caveat" ||
|
|
got[0].Message != "The transition into combat is gradual." {
|
|
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
|
|
}
|
|
}
|
|
|
|
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
|
client := &fakeScenesLLMClient{
|
|
response: chunkResponse{
|
|
Scenes: validSceneResponse().Scenes,
|
|
BoundaryCaveats: []string{
|
|
" ",
|
|
},
|
|
},
|
|
}
|
|
|
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
|
if err == nil {
|
|
t.Fatal("Chunk() error = nil, want malformed structured output error")
|
|
}
|
|
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
|
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
|
doc := sceneSourceDocument()
|
|
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
|
|
|
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
|
|
Source: doc,
|
|
LLMClient: client,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Chunk() error = %v, want nil", err)
|
|
}
|
|
|
|
doc.Units[0].ID = "mutated"
|
|
doc.Units[0].Metadata["speaker"] = "mutated"
|
|
client.response.Scenes[0].MainParticipants[0] = "mutated"
|
|
|
|
if result.Chunks[0].Units[0].ID != "seg-001" {
|
|
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
|
|
}
|
|
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
|
|
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
|
|
}
|
|
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
|
|
if !ok || participants[0] != "Aria" {
|
|
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
|
|
}
|
|
}
|
|
|
|
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
|
|
metadata := New().ManifestMetadata()
|
|
|
|
tests := map[string]string{
|
|
"prompt_id": PromptID,
|
|
"prompt_version": ResponseSchemaVersion,
|
|
"response_schema_key": string(ResponseSchemaKey),
|
|
"response_schema_id": ResponseSchemaID,
|
|
"response_schema_name": ResponseSchemaName,
|
|
"response_schema_version": ResponseSchemaVersion,
|
|
}
|
|
for key, want := range tests {
|
|
if metadata[key] != want {
|
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
|
}
|
|
}
|
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
|
value, ok := metadata[key].(string)
|
|
if !ok || !strings.HasPrefix(value, "sha256:") {
|
|
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
|
|
}
|
|
}
|
|
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
|
|
if _, ok := metadata[forbidden]; ok {
|
|
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestChunkRejectsInvalidRequests(t *testing.T) {
|
|
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
|
|
validReq := chunkRequestWithClient(validClient)
|
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
invalidDoc := sceneSourceDocument()
|
|
invalidDoc.Units[0].ID = ""
|
|
emptyDoc := sceneSourceDocument()
|
|
emptyDoc.Units = nil
|
|
|
|
tests := []struct {
|
|
name string
|
|
chunker *Chunker
|
|
ctx context.Context
|
|
req contracts.ChunkRequest
|
|
want string
|
|
}{
|
|
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
|
|
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
|
|
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
|
|
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
|
|
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
|
|
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
|
|
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
|
|
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
|
|
if err == nil {
|
|
t.Fatal("Chunk() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
|
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
response chunkResponse
|
|
want string
|
|
}{
|
|
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
|
|
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
|
|
{
|
|
name: "unknown boundary id",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
scene("seg-001", "seg-999"),
|
|
}),
|
|
want: "was not found",
|
|
},
|
|
{
|
|
name: "out of order boundaries",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
scene("seg-003", "seg-002"),
|
|
}),
|
|
want: "appears after",
|
|
},
|
|
{
|
|
name: "gap",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
scene("seg-001", "seg-001"),
|
|
scene("seg-003", "seg-004"),
|
|
}),
|
|
want: "gap",
|
|
},
|
|
{
|
|
name: "overlap",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
scene("seg-001", "seg-002"),
|
|
scene("seg-002", "seg-004"),
|
|
}),
|
|
want: "overlap",
|
|
},
|
|
{
|
|
name: "incomplete coverage",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
scene("seg-001", "seg-003"),
|
|
}),
|
|
want: "final scene",
|
|
},
|
|
{
|
|
name: "empty metadata field",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
{
|
|
StartUnitID: "seg-001",
|
|
EndUnitID: "seg-004",
|
|
ShortTitle: " ",
|
|
PrimaryMode: "Narrative",
|
|
MainParticipants: []string{"Aria"},
|
|
Summary: "Summary.",
|
|
BoundaryNote: "Note.",
|
|
BoundaryConfidence: "High",
|
|
},
|
|
}),
|
|
want: "short_title",
|
|
},
|
|
{
|
|
name: "empty participant",
|
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
|
{
|
|
StartUnitID: "seg-001",
|
|
EndUnitID: "seg-004",
|
|
ShortTitle: "Title",
|
|
PrimaryMode: "Narrative",
|
|
MainParticipants: []string{"Aria", " "},
|
|
Summary: "Summary.",
|
|
BoundaryNote: "Note.",
|
|
BoundaryConfidence: "High",
|
|
},
|
|
}),
|
|
want: "main_participants",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
client := &fakeScenesLLMClient{response: tt.response}
|
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
|
if err == nil {
|
|
t.Fatal("Chunk() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
|
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkWrapsLLMClientError(t *testing.T) {
|
|
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
|
|
|
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
|
if err == nil {
|
|
t.Fatal("Chunk() error = nil, want LLM error")
|
|
}
|
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
|
|
}
|
|
}
|
|
|
|
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
|
|
return contracts.ChunkRequest{
|
|
Source: sceneSourceDocument(),
|
|
LLMClient: client,
|
|
}
|
|
}
|
|
|
|
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
|
|
req.Options = map[string]any{"max_units": 2}
|
|
return req
|
|
}
|
|
|
|
func sceneSourceDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{
|
|
ID: "session-alpha",
|
|
Kind: "transcript",
|
|
Format: "application/vnd.seriatim.minimal+json",
|
|
Digest: "sha256:source",
|
|
Units: []source.SourceUnit{
|
|
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
|
|
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
|
|
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
|
|
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
|
|
},
|
|
}
|
|
}
|
|
|
|
func validSceneResponse() chunkResponse {
|
|
return chunkResponse{
|
|
Scenes: []sceneResponse{
|
|
scene("seg-001", "seg-004"),
|
|
},
|
|
BoundaryCaveats: []string{},
|
|
}
|
|
}
|
|
|
|
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
|
|
response.Scenes = scenes
|
|
return response
|
|
}
|
|
|
|
func scene(startUnitID string, endUnitID string) sceneResponse {
|
|
return sceneResponse{
|
|
StartUnitID: startUnitID,
|
|
EndUnitID: endUnitID,
|
|
ShortTitle: "Scene title",
|
|
PrimaryMode: "Narrative",
|
|
MainParticipants: []string{"Aria"},
|
|
Summary: "A compact summary.",
|
|
BoundaryNote: "The source units form one coherent scene.",
|
|
BoundaryConfidence: "High",
|
|
}
|
|
}
|
|
|
|
func chunkIDs(chunks []contracts.SourceChunk) []string {
|
|
ids := make([]string, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
ids = append(ids, chunk.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func unitIDs(units []source.SourceUnit) []string {
|
|
ids := make([]string, 0, len(units))
|
|
for _, unit := range units {
|
|
ids = append(ids, unit.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
type fakeScenesLLMClient struct {
|
|
response chunkResponse
|
|
err error
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
|
|
StageName: req.StageName,
|
|
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
|
|
Model: req.Model,
|
|
ResponseSchemaName: req.ResponseSchemaName,
|
|
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
|
|
})
|
|
if client.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, client.err
|
|
}
|
|
|
|
target, ok := out.(*chunkResponse)
|
|
if !ok {
|
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
|
}
|
|
*target = client.response
|
|
content, err := json.Marshal(client.response)
|
|
if err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
|
}
|