Add D&D scene description extractor
This commit is contained in:
6
internal/modules/dnd/extract/scenedescriptions/assets.go
Normal file
6
internal/modules/dnd/extract/scenedescriptions/assets.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package scenedescriptions
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/schemas/dnd_scene_descriptions_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md
|
||||
var embeddedAssets embed.FS
|
||||
@@ -0,0 +1,40 @@
|
||||
id: dnd.scene_descriptions
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./task.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||
repair_attempts: 0
|
||||
@@ -0,0 +1,9 @@
|
||||
Choose exactly one kind: combat, narrative, recap, or meta. When a chunk is
|
||||
mixed, use this precedence: combat, then recap, then meta, then narrative.
|
||||
|
||||
The title should identify the central event or subject. The summary should
|
||||
describe only what the accepted chunk establishes. Campaign references may
|
||||
disambiguate names but never add events or lore.
|
||||
|
||||
Do not return identifiers, source identifiers, source ranges, unit identifiers,
|
||||
participants, confidence, or any fields besides kind, title, and summary.
|
||||
@@ -0,0 +1,5 @@
|
||||
Describe exactly one accepted Dungeons & Dragons scene from the supplied
|
||||
transcript chunk. The complete chunk is the evidence boundary: do not split it
|
||||
into multiple scenes or use facts that are not supported by it.
|
||||
|
||||
Return one kind, one concise title, and one concise summary.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.scene_descriptions.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "title", "summary"],
|
||||
"properties": {
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": ["combat", "narrative", "recap", "meta"]
|
||||
},
|
||||
"title": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"summary": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
195
internal/modules/dnd/extract/scenedescriptions/extractor.go
Normal file
195
internal/modules/dnd/extract/scenedescriptions/extractor.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const Key = "dnd/scene-descriptions"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
"source.transcript",
|
||||
}
|
||||
|
||||
var providedCapabilities = []string{
|
||||
"dnd.scene_descriptions",
|
||||
}
|
||||
|
||||
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||
Glossary: "Optional campaign glossary reference material used only to disambiguate scene descriptions.",
|
||||
Party: "Optional party roster reference material used only to disambiguate scene descriptions.",
|
||||
Players: "Optional player list reference material used only to disambiguate scene descriptions.",
|
||||
}
|
||||
|
||||
func referenceSlots() []contracts.ReferenceSlot {
|
||||
all := shared.ReferenceSlots(referenceSlotDescriptions)
|
||||
slots := make([]contracts.ReferenceSlot, 0, 3)
|
||||
for _, slot := range all {
|
||||
if slot.Name != "roster" {
|
||||
slots = append(slots, slot)
|
||||
}
|
||||
}
|
||||
return slots
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.SceneDescriptionList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("context error before extraction: %w", err)
|
||||
}
|
||||
if req.Source == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("chunk must not be nil")
|
||||
}
|
||||
if len(req.Chunk.Units) == 0 {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("chunk %q units must not be empty", req.Chunk.ID)
|
||||
}
|
||||
sourceInput, err := shared.ChunkPromptMaterial(req)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("%w", err)
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: mapResponse(response, req.Chunk)}, nil
|
||||
}
|
||||
|
||||
func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDescriptionList {
|
||||
return dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
|
||||
ID: chunk.ID,
|
||||
SourceRef: chunk.Ref,
|
||||
Kind: response.Kind,
|
||||
Title: strings.TrimSpace(response.Title),
|
||||
Summary: strings.TrimSpace(response.Summary),
|
||||
}}}
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SceneDescriptionList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error {
|
||||
_, err := DecodeOptions(options)
|
||||
return err
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, extractorErrorf("%w", err)
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func extractorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("dnd scene descriptions extractor: "+format, args...)
|
||||
}
|
||||
126
internal/modules/dnd/extract/scenedescriptions/extractor_test.go
Normal file
126
internal/modules/dnd/extract/scenedescriptions/extractor_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestExtractMapsExactlyOneSceneToTheChunk(t *testing.T) {
|
||||
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
|
||||
Kind: dnd.SceneKindCombat, Title: " Fight at the Watchtower ", Summary: " Bandits attack the party. ",
|
||||
}}
|
||||
request := extractionRequest()
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
|
||||
ID: request.Chunk.ID, SourceRef: request.Chunk.Ref, Kind: dnd.SceneKindCombat,
|
||||
Title: "Fight at the Watchtower", Summary: "Bandits attack the party.",
|
||||
}}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
completed := client.requests[0]
|
||||
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != "profile-scene-descriptions" || completed.SessionID != "session-123" {
|
||||
t.Fatalf("LLM request identity = %#v", completed)
|
||||
}
|
||||
transcript := completed.Inputs["transcript"]
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(request.Chunk.Content) {
|
||||
t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) {
|
||||
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."}}
|
||||
req := extractionRequest()
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
for _, slot := range []string{"players", "party", "glossary"} {
|
||||
if got := string(client.requests[0].Inputs[slot].Content); got != " " {
|
||||
t.Fatalf("empty %s input = %q, want explicit placeholder", slot, got)
|
||||
}
|
||||
}
|
||||
|
||||
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}},
|
||||
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}},
|
||||
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Greencloak: local title")}}},
|
||||
}}
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
inputs := client.requests[1].Inputs
|
||||
if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" {
|
||||
t.Fatalf("reference inputs = %#v", inputs)
|
||||
}
|
||||
if strings.Contains(string(inputs["transcript"].Content), "local title") {
|
||||
t.Fatal("transcript input contains reference content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPreservesTheModelKindWithoutRepair(t *testing.T) {
|
||||
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
|
||||
Kind: dnd.SceneKind("unrecognized"), Title: " Untitled scene ", Summary: " Summary ",
|
||||
}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
scene := result.Value.Scenes[0]
|
||||
if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "Untitled scene" || scene.Summary != "Summary" {
|
||||
t.Fatalf("scene = %#v, want model kind preserved and textual fields trimmed", scene)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractValidatesRequestsAndSurfacesProviderFailures(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
extractor := newExtractor(t, &fakeSceneDescriptionsLLMClient{response: extractionResponse{Kind: dnd.SceneKindMeta, Title: "Table talk", Summary: "The group discusses rules."}})
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
req contracts.TypedExtractionRequest
|
||||
}{
|
||||
{name: "nil source", req: func() contracts.TypedExtractionRequest { r := request; r.Source = nil; return r }()},
|
||||
{name: "nil chunk", req: func() contracts.TypedExtractionRequest { r := request; r.Chunk = nil; return r }()},
|
||||
{name: "empty chunk", req: emptyChunkRequest(request)},
|
||||
{name: "mismatched source input", req: mismatchedSourceInputRequest(request)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd scene descriptions") {
|
||||
t.Fatalf("Extract() error = %v, want contextual validation error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := extractor.Extract(canceled, request); err == nil || !strings.Contains(err.Error(), "context") {
|
||||
t.Fatalf("canceled Extract() error = %v, want context error", err)
|
||||
}
|
||||
_, err := newExtractor(t, &fakeSceneDescriptionsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), request)
|
||||
if err == nil || !strings.Contains(err.Error(), "dnd scene descriptions") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider Extract() error = %v, want contextual provider error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapResponseUsesOnlyChunkIdentityAndRange(t *testing.T) {
|
||||
chunk := &source.Chunk{ID: "source:chunk:7", Ref: source.SourceRef{SourceID: "source", StartUnitID: 12, EndUnitID: 14}}
|
||||
got := mapResponse(extractionResponse{Kind: dnd.SceneKindRecap, Title: " Recap ", Summary: " Summary "}, chunk)
|
||||
want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
|
||||
ID: "source:chunk:7", SourceRef: source.SourceRef{SourceID: "source", StartUnitID: 12, EndUnitID: 14},
|
||||
Kind: dnd.SceneKindRecap, Title: "Recap", Summary: "Summary",
|
||||
}}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("mapResponse() = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
9
internal/modules/dnd/extract/scenedescriptions/model.go
Normal file
9
internal/modules/dnd/extract/scenedescriptions/model.go
Normal file
@@ -0,0 +1,9 @@
|
||||
package scenedescriptions
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
|
||||
type extractionResponse struct {
|
||||
Kind dnd.SceneKind `json:"kind"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
t.Fatalf("New(nil) error = %v, want dependency error", err)
|
||||
}
|
||||
if _, err := New(&fakeSceneDescriptionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v, want reference-set error", err)
|
||||
}
|
||||
if got := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).Key(); got != Key {
|
||||
t.Fatalf("Key() = %q, want %q", got, Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleSpecAndReferenceSlots(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key, Stage: pipeline.StageExtract, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "glossary", Description: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "party", Description: "Optional party roster reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "players", Description: "Optional player list reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
},
|
||||
}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
got := ModuleSpec()
|
||||
got.Requires[0] = "changed"
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("ModuleSpec() returned mutable shared slices")
|
||||
}
|
||||
if slots := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ReferenceSlots(); !reflect.DeepEqual(slots, want.ReferenceSlots) {
|
||||
t.Fatalf("ReferenceSlots() = %#v, want %#v", slots, want.ReferenceSlots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterStoresTypedModuleSpecAndOptionsAreStrict(t *testing.T) {
|
||||
registry := pipeline.NewExtractorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, ModuleSpec()) {
|
||||
t.Fatalf("registry.Spec(%q) = %#v, present = %t", Key, got, ok)
|
||||
}
|
||||
if err := Register(nil); err == nil || !strings.Contains(err.Error(), "extractor registry") {
|
||||
t.Fatalf("Register(nil) error = %v, want registry error", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("DecodeOptions() error = %v, want strict options error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) {
|
||||
metadata := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"mapping_policy": mappingPolicy,
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key])
|
||||
}
|
||||
}
|
||||
want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "mapping_policy": mappingPolicy}
|
||||
fingerprints := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).CheckpointFingerprints()
|
||||
if len(fingerprints) != len(want) {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want))
|
||||
}
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Value != want[fingerprint.Name] {
|
||||
t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
22
internal/modules/dnd/extract/scenedescriptions/schema.go
Normal file
22
internal/modules/dnd/extract/scenedescriptions/schema.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package scenedescriptions
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.scene_descriptions"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_descriptions_llm")
|
||||
ResponseSchemaID = "notarius.dnd.scene_descriptions.llm"
|
||||
ResponseSchemaName = "notarius_dnd_scene_descriptions_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
mappingPolicy = "dnd.scene_descriptions.mapping.v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "assets/schemas/dnd_scene_descriptions_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private scene-description schema identity", schema)
|
||||
}
|
||||
valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."}
|
||||
if err := validateJSONSchema(t, valid, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid private response rejected: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
valid bool
|
||||
}{
|
||||
{name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}},
|
||||
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
|
||||
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
|
||||
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
|
||||
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "wrong field type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}},
|
||||
{name: "blank title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}},
|
||||
{name: "blank summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := validateJSONSchema(t, test.response, schema.JSONSchema)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := validateJSONSchemaContent([]byte(`{"kind":`), schema.JSONSchema); err == nil {
|
||||
t.Fatal("validateJSONSchemaContent() error = nil, want malformed JSON rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return validateJSONSchemaContent(content, schemaContent)
|
||||
}
|
||||
|
||||
func validateJSONSchemaContent(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(instance)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const scriptoriumPromptRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.scene_descriptions",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "dnd.scene_descriptions.yaml", Path: "assets/prompts/dnd.scene_descriptions.yaml"},
|
||||
{Name: "task.md", Path: "assets/prompts/task.md"},
|
||||
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-transcript.md",
|
||||
},
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
promptFS, err := promptAssetManifest.PromptFS(embeddedAssets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare scene-description prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets)
|
||||
})
|
||||
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
scriptoriumPromptHashOnce sync.Once
|
||||
scriptoriumPromptHash string
|
||||
scriptoriumPromptHashErr error
|
||||
)
|
||||
@@ -0,0 +1,106 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||
}
|
||||
if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("message roles = %#v, want %#v", got, want)
|
||||
}
|
||||
for _, index := range []int{1, 2, 4} {
|
||||
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
|
||||
}
|
||||
}
|
||||
for _, index := range []int{0, 3, 5} {
|
||||
if prepared.Messages[index].CacheControl != nil {
|
||||
t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") {
|
||||
t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content)
|
||||
}
|
||||
for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} {
|
||||
if !strings.Contains(prepared.Messages[2].Content, want) {
|
||||
t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content)
|
||||
}
|
||||
}
|
||||
if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) {
|
||||
t.Fatal("transcript rendered before its final message")
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) {
|
||||
t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content)
|
||||
}
|
||||
transcriptMessages := 0
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, `{"units":[1]}`) {
|
||||
transcriptMessages++
|
||||
}
|
||||
}
|
||||
if transcriptMessages != 1 {
|
||||
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
hash, err := scriptoriumPromptMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("scriptoriumPromptMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func messageRoles(messages []scriptorium.RenderedMessage) []string {
|
||||
roles := make([]string, len(messages))
|
||||
for i, message := range messages {
|
||||
roles[i] = string(message.Role)
|
||||
}
|
||||
return roles
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
SourceID: doc.ID,
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 3},
|
||||
Content: []byte(`{"units":[1,2,3]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-scene-descriptions",
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha",
|
||||
Kind: "transcript",
|
||||
Format: "application/vnd.seriatim.minimal+json",
|
||||
Digest: "sha256:test",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "transcript_segment", Text: "Mira Thorn enters the ruined watchtower."},
|
||||
{ID: 2, Kind: "transcript_segment", Text: "Bandits attack from the upper floor."},
|
||||
{ID: 3, Kind: "transcript_segment", Text: "The party drives them back."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func emptyChunkRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.Chunk = &source.Chunk{ID: req.Chunk.ID, SourceID: req.Chunk.SourceID, Index: req.Chunk.Index}
|
||||
return req
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeSceneDescriptionsLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeSceneDescriptionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user