diff --git a/internal/modules/dnd/extract/scenedescriptions/assets.go b/internal/modules/dnd/extract/scenedescriptions/assets.go new file mode 100644 index 0000000..d298053 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/assets.go @@ -0,0 +1,6 @@ +package scenedescriptions + +import "embed" + +//go:embed assets/schemas/dnd_scene_descriptions_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md +var embeddedAssets embed.FS diff --git a/internal/modules/dnd/extract/scenedescriptions/assets/prompts/dnd.scene_descriptions.yaml b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/dnd.scene_descriptions.yaml new file mode 100644 index 0000000..397f934 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/dnd.scene_descriptions.yaml @@ -0,0 +1,40 @@ +id: dnd.scene_descriptions +version: "v1" +default_profile: gemini-2-flash +inputs: + - name: transcript + required: true + content_type: application/json + - name: players + required: false + content_type: text/plain + - name: party + required: false + content_type: text/plain + - name: glossary + required: false + content_type: text/plain +messages: + - role: system + content_file: ./sharedassets/common-dnd-system.md + - role: user + content_file: ./sharedassets/common-dnd-identity.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-references.md + cache_control: + type: ephemeral + - role: user + content_file: ./task.md + - role: user + content_file: ./instructions.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-transcript.md +output: + format: json + validation_mode: json_schema + schema_path: dnd_scene_descriptions_llm.v1.json + repair_attempts: 0 diff --git a/internal/modules/dnd/extract/scenedescriptions/assets/prompts/instructions.md b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/instructions.md new file mode 100644 index 0000000..6e9efc9 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/instructions.md @@ -0,0 +1,9 @@ +Choose exactly one kind: combat, narrative, recap, or meta. When a chunk is +mixed, use this precedence: combat, then recap, then meta, then narrative. + +The title should identify the central event or subject. The summary should +describe only what the accepted chunk establishes. Campaign references may +disambiguate names but never add events or lore. + +Do not return identifiers, source identifiers, source ranges, unit identifiers, +participants, confidence, or any fields besides kind, title, and summary. diff --git a/internal/modules/dnd/extract/scenedescriptions/assets/prompts/task.md b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/task.md new file mode 100644 index 0000000..36ed668 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/assets/prompts/task.md @@ -0,0 +1,5 @@ +Describe exactly one accepted Dungeons & Dragons scene from the supplied +transcript chunk. The complete chunk is the evidence boundary: do not split it +into multiple scenes or use facts that are not supported by it. + +Return one kind, one concise title, and one concise summary. diff --git a/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json b/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json new file mode 100644 index 0000000..64beae5 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.scene_descriptions.llm", + "type": "object", + "additionalProperties": false, + "required": ["kind", "title", "summary"], + "properties": { + "kind": { + "type": "string", + "enum": ["combat", "narrative", "recap", "meta"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string", + "minLength": 1 + } + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor.go b/internal/modules/dnd/extract/scenedescriptions/extractor.go new file mode 100644 index 0000000..c593e47 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/extractor.go @@ -0,0 +1,195 @@ +package scenedescriptions + +import ( + "context" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const Key = "dnd/scene-descriptions" + +var requiredCapabilities = []string{ + "chunks", + "source.transcript", +} + +var providedCapabilities = []string{ + "dnd.scene_descriptions", +} + +var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{ + Glossary: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", + Party: "Optional party roster reference material used only to disambiguate scene descriptions.", + Players: "Optional player list reference material used only to disambiguate scene descriptions.", +} + +func referenceSlots() []contracts.ReferenceSlot { + all := shared.ReferenceSlots(referenceSlotDescriptions) + slots := make([]contracts.ReferenceSlot, 0, 3) + for _, slot := range all { + if slot.Name != "roster" { + slots = append(slots, slot) + } + } + return slots +} + +var _ contracts.Extractor[dnd.SceneDescriptionList] = (*Extractor)(nil) +var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) + +type Options struct{} + +type Extractor struct { + llm contracts.StructuredLLMClient + promptSHA string + responseSchemaSHA string +} + +func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if llmClient == nil { + return nil, extractorErrorf("LLM client must not be nil") + } + if len(references) > 1 { + return nil, extractorErrorf("at most one reference set may be supplied") + } + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + return nil, extractorErrorf("load prompt metadata: %w", err) + } + responseSchema, err := loadResponseSchema() + if err != nil { + return nil, extractorErrorf("load response schema: %w", err) + } + return &Extractor{ + llm: llmClient, + promptSHA: promptSHA, + responseSchemaSHA: responseSchema.SHA256, + }, nil +} + +func (e *Extractor) Key() string { return Key } + +func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } + +func (e *Extractor) ManifestMetadata() map[string]any { + if e == nil { + return nil + } + return map[string]any{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "prompt_sha256": e.promptSHA, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + "response_schema_sha256": e.responseSchemaSHA, + "mapping_policy": mappingPolicy, + } +} + +func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if e == nil { + return nil + } + return []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: e.promptSHA}, + {Name: "response_schema", Value: e.responseSchemaSHA}, + {Name: "mapping_policy", Value: mappingPolicy}, + } +} + +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) { + if e == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("extractor must not be nil") + } + if e.llm == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("LLM client must not be nil") + } + if ctx == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("context must not be nil") + } + if err := ctx.Err(); err != nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("context error before extraction: %w", err) + } + if req.Source == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("source must not be nil") + } + if req.Chunk == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("chunk must not be nil") + } + if len(req.Chunk.Units) == 0 { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("chunk %q units must not be empty", req.Chunk.ID) + } + sourceInput, err := shared.ChunkPromptMaterial(req) + if err != nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("%w", err) + } + + var response extractionResponse + if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: Key, + PromptID: PromptID, + PromptVersion: SchemaVersion, + ProfileID: req.LLMProfile, + SessionID: req.SessionID, + Inputs: shared.PromptInputs(sourceInput, req.References), + }, &response); err != nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err) + } + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: mapResponse(response, req.Chunk)}, nil +} + +func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDescriptionList { + return dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ + ID: chunk.ID, + SourceRef: chunk.Ref, + Kind: response.Kind, + Title: strings.TrimSpace(response.Title), + Summary: strings.TrimSpace(response.Summary), + }}} +} + +func ModuleSpec() pipeline.ModuleSpec { + return pipeline.ModuleSpec{ + Key: Key, + Stage: pipeline.StageExtract, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.SceneDescriptionListKind, + ReferenceSlots: referenceSlots(), + } +} + +func Register(registry *pipeline.ExtractorRegistry) error { + return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SceneDescriptionList], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(request.Dependencies.LLM, options, request.References) + }) +} + +func validateOptions(options map[string]any) error { + _, err := DecodeOptions(options) + return err +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, extractorErrorf("%w", err) + } + return Options{}, nil +} + +func extractorErrorf(format string, args ...any) error { + return fmt.Errorf("dnd scene descriptions extractor: "+format, args...) +} diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor_test.go b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go new file mode 100644 index 0000000..10fec7f --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go @@ -0,0 +1,126 @@ +package scenedescriptions + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestExtractMapsExactlyOneSceneToTheChunk(t *testing.T) { + client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{ + Kind: dnd.SceneKindCombat, Title: " Fight at the Watchtower ", Summary: " Bandits attack the party. ", + }} + request := extractionRequest() + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ + ID: request.Chunk.ID, SourceRef: request.Chunk.Ref, Kind: dnd.SceneKindCombat, + Title: "Fight at the Watchtower", Summary: "Bandits attack the party.", + }}} + if !reflect.DeepEqual(result.Value, want) { + t.Fatalf("Value = %#v, want %#v", result.Value, want) + } + if len(client.requests) != 1 { + t.Fatalf("LLM calls = %d, want 1", len(client.requests)) + } + completed := client.requests[0] + if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != "profile-scene-descriptions" || completed.SessionID != "session-123" { + t.Fatalf("LLM request identity = %#v", completed) + } + transcript := completed.Inputs["transcript"] + if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(request.Chunk.Content) { + t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript) + } +} + +func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) { + client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."}} + req := extractionRequest() + if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + for _, slot := range []string{"players", "party", "glossary"} { + if got := string(client.requests[0].Inputs[slot].Content); got != " " { + t.Fatalf("empty %s input = %q, want explicit placeholder", slot, got) + } + } + + req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}}, + "party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}}, + "glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Greencloak: local title")}}}, + }} + if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + inputs := client.requests[1].Inputs + if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" { + t.Fatalf("reference inputs = %#v", inputs) + } + if strings.Contains(string(inputs["transcript"].Content), "local title") { + t.Fatal("transcript input contains reference content") + } +} + +func TestExtractPreservesTheModelKindWithoutRepair(t *testing.T) { + client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{ + Kind: dnd.SceneKind("unrecognized"), Title: " Untitled scene ", Summary: " Summary ", + }} + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + scene := result.Value.Scenes[0] + if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "Untitled scene" || scene.Summary != "Summary" { + t.Fatalf("scene = %#v, want model kind preserved and textual fields trimmed", scene) + } +} + +func TestExtractValidatesRequestsAndSurfacesProviderFailures(t *testing.T) { + request := extractionRequest() + extractor := newExtractor(t, &fakeSceneDescriptionsLLMClient{response: extractionResponse{Kind: dnd.SceneKindMeta, Title: "Table talk", Summary: "The group discusses rules."}}) + for _, test := range []struct { + name string + req contracts.TypedExtractionRequest + }{ + {name: "nil source", req: func() contracts.TypedExtractionRequest { r := request; r.Source = nil; return r }()}, + {name: "nil chunk", req: func() contracts.TypedExtractionRequest { r := request; r.Chunk = nil; return r }()}, + {name: "empty chunk", req: emptyChunkRequest(request)}, + {name: "mismatched source input", req: mismatchedSourceInputRequest(request)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd scene descriptions") { + t.Fatalf("Extract() error = %v, want contextual validation error", err) + } + }) + } + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := extractor.Extract(canceled, request); err == nil || !strings.Contains(err.Error(), "context") { + t.Fatalf("canceled Extract() error = %v, want context error", err) + } + _, err := newExtractor(t, &fakeSceneDescriptionsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), request) + if err == nil || !strings.Contains(err.Error(), "dnd scene descriptions") || !strings.Contains(err.Error(), "provider unavailable") { + t.Fatalf("provider Extract() error = %v, want contextual provider error", err) + } +} + +func TestMapResponseUsesOnlyChunkIdentityAndRange(t *testing.T) { + chunk := &source.Chunk{ID: "source:chunk:7", Ref: source.SourceRef{SourceID: "source", StartUnitID: 12, EndUnitID: 14}} + got := mapResponse(extractionResponse{Kind: dnd.SceneKindRecap, Title: " Recap ", Summary: " Summary "}, chunk) + want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ + ID: "source:chunk:7", SourceRef: source.SourceRef{SourceID: "source", StartUnitID: 12, EndUnitID: 14}, + Kind: dnd.SceneKindRecap, Title: "Recap", Summary: "Summary", + }}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("mapResponse() = %#v, want %#v", got, want) + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/model.go b/internal/modules/dnd/extract/scenedescriptions/model.go new file mode 100644 index 0000000..afac668 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/model.go @@ -0,0 +1,9 @@ +package scenedescriptions + +import "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + +type extractionResponse struct { + Kind dnd.SceneKind `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` +} diff --git a/internal/modules/dnd/extract/scenedescriptions/registry_test.go b/internal/modules/dnd/extract/scenedescriptions/registry_test.go new file mode 100644 index 0000000..547c599 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/registry_test.go @@ -0,0 +1,92 @@ +package scenedescriptions + +import ( + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + t.Fatalf("New(nil) error = %v, want dependency error", err) + } + if _, err := New(&fakeSceneDescriptionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + t.Fatalf("New() error = %v, want reference-set error", err) + } + if got := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).Key(); got != Key { + t.Fatalf("Key() = %q, want %q", got, Key) + } +} + +func TestModuleSpecAndReferenceSlots(t *testing.T) { + want := pipeline.ModuleSpec{ + Key: Key, Stage: pipeline.StageExtract, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind, + ReferenceSlots: []contracts.ReferenceSlot{ + {Name: "glossary", Description: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + {Name: "party", Description: "Optional party roster reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + {Name: "players", Description: "Optional player list reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + }, + } + if got := ModuleSpec(); !reflect.DeepEqual(got, want) { + t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) + } + got := ModuleSpec() + got.Requires[0] = "changed" + got.Provides[0] = "changed" + got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" + if !reflect.DeepEqual(ModuleSpec(), want) { + t.Fatal("ModuleSpec() returned mutable shared slices") + } + if slots := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ReferenceSlots(); !reflect.DeepEqual(slots, want.ReferenceSlots) { + t.Fatalf("ReferenceSlots() = %#v, want %#v", slots, want.ReferenceSlots) + } +} + +func TestRegisterStoresTypedModuleSpecAndOptionsAreStrict(t *testing.T) { + registry := pipeline.NewExtractorRegistry() + if err := Register(registry); err != nil { + t.Fatalf("Register() error = %v, want nil", err) + } + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, ModuleSpec()) { + t.Fatalf("registry.Spec(%q) = %#v, present = %t", Key, got, ok) + } + if err := Register(nil); err == nil || !strings.Contains(err.Error(), "extractor registry") { + t.Fatalf("Register(nil) error = %v, want registry error", err) + } + if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") { + t.Fatalf("DecodeOptions() error = %v, want strict options error", err) + } +} + +func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) { + metadata := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata() + for key, want := range map[string]string{ + "prompt_id": PromptID, "prompt_version": SchemaVersion, + "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, + "mapping_policy": mappingPolicy, + } { + if metadata[key] != want { + t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) + } + } + for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { + if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { + t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key]) + } + } + want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "mapping_policy": mappingPolicy} + fingerprints := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).CheckpointFingerprints() + if len(fingerprints) != len(want) { + t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want)) + } + for _, fingerprint := range fingerprints { + if fingerprint.Value != want[fingerprint.Name] { + t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want) + } + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/schema.go b/internal/modules/dnd/extract/scenedescriptions/schema.go new file mode 100644 index 0000000..86a8351 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/schema.go @@ -0,0 +1,22 @@ +package scenedescriptions + +import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + +const ( + PromptID = "dnd.scene_descriptions" + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_descriptions_llm") + ResponseSchemaID = "notarius.dnd.scene_descriptions.llm" + ResponseSchemaName = "notarius_dnd_scene_descriptions_llm_v1" + SchemaVersion = "v1" + mappingPolicy = "dnd.scene_descriptions.mapping.v1" +) + +func loadResponseSchema() (llm.ResponseSchema, error) { + return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: "assets/schemas/dnd_scene_descriptions_llm.v1.json", + }) +} diff --git a/internal/modules/dnd/extract/scenedescriptions/schema_test.go b/internal/modules/dnd/extract/scenedescriptions/schema_test.go new file mode 100644 index 0000000..42b5130 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/schema_test.go @@ -0,0 +1,92 @@ +package scenedescriptions + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatalf("loadResponseSchema() error = %v, want nil", err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v, want private scene-description schema identity", schema) + } + valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."} + if err := validateJSONSchema(t, valid, schema.JSONSchema); err != nil { + t.Fatalf("valid private response rejected: %v", err) + } + for _, test := range []struct { + name string + response map[string]any + valid bool + }{ + {name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}}, + {name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}}, + {name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}}, + {name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}}, + {name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}}, + {name: "wrong field type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}}, + {name: "blank title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}}, + {name: "blank summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}}, + } { + t.Run(test.name, func(t *testing.T) { + err := validateJSONSchema(t, test.response, schema.JSONSchema) + if (err == nil) != test.valid { + t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid) + } + }) + } + if err := validateJSONSchemaContent([]byte(`{"kind":`), schema.JSONSchema); err == nil { + t.Fatal("validateJSONSchemaContent() error = nil, want malformed JSON rejected") + } +} + +func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { + first, err := loadResponseSchema() + if err != nil { + t.Fatal(err) + } + first.JSONSchema[0] = '[' + second, err := loadResponseSchema() + if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { + t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) + } + if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil { + t.Fatalf("schema diagnostics included raw content: %#v", diagnostics) + } +} + +func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error { + t.Helper() + content, err := json.Marshal(instance) + if err != nil { + t.Fatal(err) + } + return validateJSONSchemaContent(content, schemaContent) +} + +func validateJSONSchemaContent(instanceContent, schemaContent []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return err + } + schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", schemaDocument); err != nil { + return err + } + compiled, err := compiler.Compile("schema.json") + if err != nil { + return err + } + return compiled.Validate(instance) +} diff --git a/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets.go b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets.go new file mode 100644 index 0000000..2e918f1 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets.go @@ -0,0 +1,51 @@ +package scenedescriptions + +import ( + "fmt" + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const scriptoriumPromptRoot = "assets/prompts" + +var promptAssetManifest = shared.PromptAssetManifest{ + ModuleDir: "dnd.scene_descriptions", + ModuleFiles: []promptfs.ModulePromptFile{ + {Name: "dnd.scene_descriptions.yaml", Path: "assets/prompts/dnd.scene_descriptions.yaml"}, + {Name: "task.md", Path: "assets/prompts/task.md"}, + {Name: "instructions.md", Path: "assets/prompts/instructions.md"}, + }, + SharedFiles: []string{ + "common-dnd-system.md", + "common-dnd-identity.md", + "common-dnd-references.md", + "common-dnd-transcript.md", + }, +} + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + promptFS, err := promptAssetManifest.PromptFS(embeddedAssets) + if err != nil { + return fmt.Errorf("prepare scene-description prompt assets: %w", err) + } + if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil { + return err + } + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +} + +func scriptoriumPromptMetadata() (string, error) { + scriptoriumPromptHashOnce.Do(func() { + scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets) + }) + return scriptoriumPromptHash, scriptoriumPromptHashErr +} + +var ( + scriptoriumPromptHashOnce sync.Once + scriptoriumPromptHash string + scriptoriumPromptHashErr error +) diff --git a/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go new file mode 100644 index 0000000..c833bc4 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go @@ -0,0 +1,106 @@ +package scenedescriptions + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) + } + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`), + "players": scriptorium.Inline("Dana: Mira"), + "party": scriptorium.Inline("Mira: ranger"), + "glossary": scriptorium.Inline("Greencloak: title"), + }, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" { + t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared) + } + if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) { + t.Fatalf("message roles = %#v, want %#v", got, want) + } + for _, index := range []int{1, 2, 4} { + if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral { + t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl) + } + } + for _, index := range []int{0, 3, 5} { + if prepared.Messages[index].CacheControl != nil { + t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl) + } + } + if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") { + t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content) + } + for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} { + if !strings.Contains(prepared.Messages[2].Content, want) { + t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content) + } + } + if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) { + t.Fatal("transcript rendered before its final message") + } + if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) { + t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content) + } + transcriptMessages := 0 + for _, message := range prepared.Messages { + if strings.Contains(message.Content, `{"units":[1]}`) { + transcriptMessages++ + } + } + if transcriptMessages != 1 { + t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) + } +} + +func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { + hash, err := scriptoriumPromptMetadata() + if err != nil || !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("scriptoriumPromptMetadata() = %q, %v; want hash", hash, err) + } + payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata()) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} { + if strings.Contains(string(payload), forbidden) { + t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload) + } + } +} + +func messageRoles(messages []scriptorium.RenderedMessage) []string { + roles := make([]string, len(messages)) + for i, message := range messages { + roles[i] = string(message.Role) + } + return roles +} diff --git a/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go b/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go new file mode 100644 index 0000000..1a992e0 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go @@ -0,0 +1,102 @@ +package scenedescriptions + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func extractionRequest() contracts.TypedExtractionRequest { + doc := sourceDocument() + chunk := &source.Chunk{ + ID: "session-alpha:chunk:0", + SourceID: doc.ID, + Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 3}, + Content: []byte(`{"units":[1,2,3]}`), + MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + Metadata: map[string]any{"ignored": "chunk metadata"}, + } + return contracts.TypedExtractionRequest{ + Source: doc, + Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), + SessionID: "session-123", + LLMProfile: "profile-scene-descriptions", + } +} + +func sourceDocument() *source.SourceDocument { + return &source.SourceDocument{ + ID: "session-alpha", + Kind: "transcript", + Format: "application/vnd.seriatim.minimal+json", + Digest: "sha256:test", + Units: []source.SourceUnit{ + {ID: 1, Kind: "transcript_segment", Text: "Mira Thorn enters the ruined watchtower."}, + {ID: 2, Kind: "transcript_segment", Text: "Bandits attack from the upper floor."}, + {ID: 3, Kind: "transcript_segment", Text: "The party drives them back."}, + }, + } +} + +func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + extractor, err := New(client, Options{}, references...) + if err != nil { + t.Fatalf("New() error = %v, want nil", err) + } + return extractor +} + +func emptyChunkRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { + req.Chunk = &source.Chunk{ID: req.Chunk.ID, SourceID: req.Chunk.SourceID, Index: req.Chunk.Index} + return req +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { + req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") + return req +} + +func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { + req.Inputs = req.Inputs.Clone() + if len(req.Vars) == 0 { + req.Vars = nil + return req + } + vars := make(map[string]any, len(req.Vars)) + for key, value := range req.Vars { + vars[key] = value + } + req.Vars = vars + return req +} + +type fakeSceneDescriptionsLLMClient struct { + response extractionResponse + err error + requests []contracts.StructuredCompletionRequest +} + +func (client *fakeSceneDescriptionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) + if client.err != nil { + return contracts.StructuredCompletionResponse{}, client.err + } + target, ok := out.(*extractionResponse) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + } + *target = client.response + content, err := json.Marshal(client.response) + if err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: content}, nil +}