diff --git a/docs/integrations/dnd-spell-artifacts.md b/docs/integrations/dnd-spell-artifacts.md index 7bdc573..7f38610 100644 --- a/docs/integrations/dnd-spell-artifacts.md +++ b/docs/integrations/dnd-spell-artifacts.md @@ -14,8 +14,10 @@ This document is the durable raw output contract for the implemented - Media type: `application/json` The extractor requires source chunks and transcript source capability. It -returns the structured LLM response as raw JSON. The default `appendorder` -merger passes a single chunk output through and concatenates multiple +returns canonical spell-cast JSON derived from the structured LLM response. The +extractor assigns source IDs deterministically and keeps source-unit ranges as +model-authored evidence locations. The default `appendorder` merger passes a +single chunk output through and concatenates multiple `spell_casts` arrays in chunk order. The default `noop` normalizer passes the merge output through unchanged. @@ -76,7 +78,8 @@ Each spell cast contains: - `spell`: spell name; - `effect`: concise spell effect in the scene; - `narrative_description`: short description of the spell cast in context; -- `source_refs`: transcript source references supplied by the model. +- `source_refs`: transcript source references with extractor-assigned source + IDs and model-supplied unit ranges. `caster` is the in-world caster, not the transcript speaker. @@ -88,8 +91,10 @@ Each source reference uses the generic source-reference shape: - `start_unit_id` - `end_unit_id` -The extractor prompt and schema use integer `start_unit_id` and `end_unit_id` -values matching source-unit IDs. +The LLM-facing prompt schema asks only for integer `start_unit_id` and +`end_unit_id` values matching source-unit IDs. `source_id` is assigned by the +extractor from the source document ID before validation and output, and is +required in this durable output contract. ## References diff --git a/internal/modules/extract/dnd/spells/assets/prompts/dnd.spells.yaml b/internal/modules/extract/dnd/spells/assets/prompts/dnd.spells.yaml index e52871a..f631f09 100644 --- a/internal/modules/extract/dnd/spells/assets/prompts/dnd.spells.yaml +++ b/internal/modules/extract/dnd/spells/assets/prompts/dnd.spells.yaml @@ -32,5 +32,5 @@ messages: output: format: json validation_mode: json_schema - schema_path: dnd_spells.v1.json + schema_path: dnd_spells_llm.v1.json repair_attempts: 0 diff --git a/internal/modules/extract/dnd/spells/assets/prompts/instructions.md b/internal/modules/extract/dnd/spells/assets/prompts/instructions.md index c534b75..32e909c 100644 --- a/internal/modules/extract/dnd/spells/assets/prompts/instructions.md +++ b/internal/modules/extract/dnd/spells/assets/prompts/instructions.md @@ -1,8 +1,9 @@ -Source references must use integer source-unit IDs from the transcript. +Source references must use integer source-unit IDs from the transcript. Provide +start_unit_id and end_unit_id for each source reference; the extractor assigns +source_id automatically. Return only D&D spell-cast artifacts. For each spell cast, identify the in-world -caster, spell name, effect, narrative description, and source references using -source_id, start_unit_id, and end_unit_id. +caster, spell name, effect, narrative description, and source references. Use player, party, and glossary reference material only to clarify source text. Do not return spells, casters, or effects that are mentioned only in reference diff --git a/internal/modules/extract/dnd/spells/assets/schemas/dnd_spells_llm.v1.json b/internal/modules/extract/dnd/spells/assets/schemas/dnd_spells_llm.v1.json new file mode 100644 index 0000000..fcbf873 --- /dev/null +++ b/internal/modules/extract/dnd/spells/assets/schemas/dnd_spells_llm.v1.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.spells.llm", + "type": "object", + "additionalProperties": false, + "required": ["spell_casts"], + "properties": { + "spell_casts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "caster", + "spell", + "effect", + "narrative_description", + "source_refs" + ], + "properties": { + "caster": { + "type": "string", + "minLength": 1 + }, + "spell": { + "type": "string", + "minLength": 1 + }, + "effect": { + "type": "string", + "minLength": 1 + }, + "narrative_description": { + "type": "string", + "minLength": 1 + }, + "source_refs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["start_unit_id", "end_unit_id"], + "properties": { + "start_unit_id": { + "type": "integer", + "minimum": 1 + }, + "end_unit_id": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + } + } +} diff --git a/internal/modules/extract/dnd/spells/canonicalize.go b/internal/modules/extract/dnd/spells/canonicalize.go new file mode 100644 index 0000000..8e74973 --- /dev/null +++ b/internal/modules/extract/dnd/spells/canonicalize.go @@ -0,0 +1,92 @@ +package spells + +import ( + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd" +) + +func canonicalizeResponse(response *extractionResponse, sourceID string) { + if response == nil { + return + } + for index := range response.SpellCasts { + canonicalizeSpellCast(&response.SpellCasts[index], sourceID) + } + sort.SliceStable(response.SpellCasts, func(i, j int) bool { + left, leftOK := earliestSourceUnit(response.SpellCasts[i]) + right, rightOK := earliestSourceUnit(response.SpellCasts[j]) + if leftOK != rightOK { + return leftOK + } + if !leftOK { + return false + } + return left < right + }) +} + +func canonicalizeSpellCast(spell *spellCastResponse, sourceID string) { + for index := range spell.SourceRefs { + spell.SourceRefs[index].SourceID = sourceID + spell.SourceRefs[index].StartUnitID = canonicalUnitRef(spell.SourceRefs[index].StartUnitID) + spell.SourceRefs[index].EndUnitID = canonicalUnitRef(spell.SourceRefs[index].EndUnitID) + } + sort.SliceStable(spell.SourceRefs, func(i, j int) bool { + left := spell.SourceRefs[i] + right := spell.SourceRefs[j] + if left.StartUnitID.Int() != right.StartUnitID.Int() { + return unitSortValue(left.StartUnitID) < unitSortValue(right.StartUnitID) + } + return unitSortValue(left.EndUnitID) < unitSortValue(right.EndUnitID) + }) + spell.SourceRefs = dedupeSourceRefs(spell.SourceRefs) +} + +func canonicalUnitRef(ref dnd.UnitRef) dnd.UnitRef { + value := ref.Int() + if value <= 0 { + return ref + } + return dnd.UnitRefFromInt(value) +} + +func dedupeSourceRefs(refs []dnd.SourceRefResponse) []dnd.SourceRefResponse { + if len(refs) < 2 { + return refs + } + out := refs[:0] + var previous dnd.SourceRefResponse + for index, ref := range refs { + if index > 0 && sameSourceRef(previous, ref) { + continue + } + out = append(out, ref) + previous = ref + } + return out +} + +func sameSourceRef(left dnd.SourceRefResponse, right dnd.SourceRefResponse) bool { + return left.SourceID == right.SourceID && + left.StartUnitID.Int() == right.StartUnitID.Int() && + left.EndUnitID.Int() == right.EndUnitID.Int() +} + +func earliestSourceUnit(spell spellCastResponse) (int, bool) { + for _, ref := range spell.SourceRefs { + start := ref.StartUnitID.Int() + if start > 0 { + return start, true + } + } + return 0, false +} + +func unitSortValue(ref dnd.UnitRef) int { + value := ref.Int() + if value <= 0 { + return int(^uint(0) >> 1) + } + return value +} diff --git a/internal/modules/extract/dnd/spells/extractor.go b/internal/modules/extract/dnd/spells/extractor.go index 3c317aa..373c47a 100644 --- a/internal/modules/extract/dnd/spells/extractor.go +++ b/internal/modules/extract/dnd/spells/extractor.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -96,24 +95,20 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest } var response extractionResponse - completion, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: dnd.PromptInputs(sourceInput, req.References), - }, &response) - if err != nil { + }, &response); err != nil { return contracts.ExtractionResult{}, extractorErrorf("complete structured output: %w", err) } - content := append([]byte(nil), completion.Content...) - if len(strings.TrimSpace(string(content))) == 0 { - var err error - content, err = json.Marshal(response) - if err != nil { - return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err) - } + canonicalizeResponse(&response, req.Source.ID) + content, err := json.Marshal(response) + if err != nil { + return contracts.ExtractionResult{}, extractorErrorf("marshal canonical output: %w", err) } schema, err := loadResponseSchema() if err != nil { diff --git a/internal/modules/extract/dnd/spells/extractor_test.go b/internal/modules/extract/dnd/spells/extractor_test.go index 0cfb553..2371959 100644 --- a/internal/modules/extract/dnd/spells/extractor_test.go +++ b/internal/modules/extract/dnd/spells/extractor_test.go @@ -11,7 +11,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd" ) -func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) { +func TestExtractReturnsCanonicalOutputFromStructuredResponse(t *testing.T) { client := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ @@ -20,7 +20,7 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) { Spell: " Cure Wounds ", Effect: " Heals an injured ally. ", NarrativeDescription: " Aria restores the fighter after the fight. ", - SourceRefs: responseSourceRefsInt("session-alpha", 1, 2), + SourceRefs: responseSourceRefsInt("transcript", 1, 2), }, }, }, @@ -63,8 +63,8 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) { if !json.Valid(result.Output.Schema.JSONSchema) { t.Fatalf("schema JSON is invalid or missing: %s", result.Output.Schema.JSONSchema) } - if got := string(result.Output.Payload.Content); got != string(client.content) { - t.Fatalf("content = %q, want exact raw completion content", got) + if strings.Contains(string(result.Output.Payload.Content), "raw_marker") { + t.Fatalf("content = %q, want canonical payload without raw completion marker", result.Output.Payload.Content) } var payload extractionResponse @@ -72,7 +72,10 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) { t.Fatalf("Unmarshal(Content) error = %v, want nil", err) } if len(payload.SpellCasts) != 1 || payload.SpellCasts[0].Spell != " Cure Wounds " { - t.Fatalf("payload = %#v, want raw structured response", payload) + t.Fatalf("payload = %#v, want structured response fields", payload) + } + if got := payload.SpellCasts[0].SourceRefs[0].SourceID; got != "session-alpha" { + t.Fatalf("source_id = %q, want canonical source document ID", got) } } @@ -187,7 +190,7 @@ func TestExtractReturnsRawOutputForEmptyResponse(t *testing.T) { } } -func TestExtractCarriesMalformedStructuredContentAsRawOutput(t *testing.T) { +func TestExtractReturnsCanonicalOutputForMalformedStructuredResponse(t *testing.T) { client := &fakeSpellsLLMClient{response: extractionResponse{}} result, err := New().Extract(context.Background(), extractionRequestWithClient(client)) @@ -195,7 +198,7 @@ func TestExtractCarriesMalformedStructuredContentAsRawOutput(t *testing.T) { t.Fatalf("Extract() error = %v, want nil", err) } if string(result.Output.Payload.Content) != `{"spell_casts":null}` { - t.Fatalf("content = %s, want raw structured output", result.Output.Payload.Content) + t.Fatalf("content = %s, want canonical structured output", result.Output.Payload.Content) } } @@ -247,10 +250,17 @@ func TestExtractRejectsInvalidRequests(t *testing.T) { } } -func TestExtractPreservesResponseOrder(t *testing.T) { +func TestExtractOrdersSpellCastsByEarliestSourceUnit(t *testing.T) { client := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ + { + Caster: "Bandit Shaman", + Spell: "Fire Bolt", + Effect: "Burns.", + NarrativeDescription: "Second spell.", + SourceRefs: responseSourceRefs("session-alpha", 2, 2), + }, { Caster: "Aria", Spell: "Cure Wounds", @@ -259,11 +269,10 @@ func TestExtractPreservesResponseOrder(t *testing.T) { SourceRefs: responseSourceRefs("session-alpha", 1, 1), }, { - Caster: "Bandit Shaman", - Spell: "Fire Bolt", - Effect: "Burns.", - NarrativeDescription: "Second spell.", - SourceRefs: responseSourceRefs("session-alpha", 2, 2), + Caster: "Narrator", + Spell: "Unknown Spell", + Effect: "No cited range.", + NarrativeDescription: "This should sort after cited spell casts.", }, }, }, @@ -277,8 +286,84 @@ func TestExtractPreservesResponseOrder(t *testing.T) { if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil { t.Fatalf("Unmarshal(Content) error = %v, want nil", err) } - if len(payload.SpellCasts) != 2 || payload.SpellCasts[0].Spell != "Cure Wounds" || payload.SpellCasts[1].Spell != "Fire Bolt" { - t.Fatalf("spell order = %#v, want response order", payload.SpellCasts) + if len(payload.SpellCasts) != 3 || + payload.SpellCasts[0].Spell != "Cure Wounds" || + payload.SpellCasts[1].Spell != "Fire Bolt" || + payload.SpellCasts[2].Spell != "Unknown Spell" { + t.Fatalf("spell order = %#v, want earliest source-unit order with uncited spell last", payload.SpellCasts) + } +} + +func TestExtractCanonicalizesSourceRefs(t *testing.T) { + client := &fakeSpellsLLMClient{ + response: extractionResponse{ + SpellCasts: []spellCastResponse{ + { + Caster: "Aria", + Spell: "Cure Wounds", + Effect: "Heals.", + NarrativeDescription: "Aria heals.", + SourceRefs: []dnd.SourceRefResponse{ + {SourceID: "gameplay_transcript", StartUnitID: dnd.UnitRefFromInt(2), EndUnitID: dnd.UnitRefFromInt(2)}, + {SourceID: "", StartUnitID: dnd.UnitRefFromInt(1), EndUnitID: dnd.UnitRefFromInt(2)}, + {SourceID: "transcript", StartUnitID: dnd.UnitRefFromInt(1), EndUnitID: dnd.UnitRefFromInt(2)}, + }, + }, + }, + }, + } + + result, err := New().Extract(context.Background(), extractionRequestWithClient(client)) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + var payload extractionResponse + if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil { + t.Fatalf("Unmarshal(Content) error = %v, want nil", err) + } + refs := payload.SpellCasts[0].SourceRefs + if len(refs) != 2 { + t.Fatalf("source refs = %#v, want duplicate collapsed", refs) + } + for _, ref := range refs { + if ref.SourceID != "session-alpha" { + t.Fatalf("source ref = %#v, want canonical source_id", ref) + } + } + if refs[0].StartUnitID.Int() != 1 || refs[0].EndUnitID.Int() != 2 || + refs[1].StartUnitID.Int() != 2 || refs[1].EndUnitID.Int() != 2 { + t.Fatalf("source refs = %#v, want sorted unit ranges", refs) + } +} + +func TestExtractPreservesInvalidSourceRefsForValidators(t *testing.T) { + client := &fakeSpellsLLMClient{ + response: extractionResponse{ + SpellCasts: []spellCastResponse{ + { + Caster: "Aria", + Spell: "Cure Wounds", + Effect: "Heals.", + NarrativeDescription: "Aria heals.", + SourceRefs: []dnd.SourceRefResponse{ + {SourceID: "transcript", StartUnitID: dnd.UnitRefFromInt(99), EndUnitID: dnd.UnitRefFromString("missing")}, + }, + }, + }, + }, + } + + result, err := New().Extract(context.Background(), extractionRequestWithClient(client)) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + var payload map[string][]map[string]any + if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil { + t.Fatalf("Unmarshal(Content) error = %v, want nil", err) + } + ref := payload["spell_casts"][0]["source_refs"].([]any)[0].(map[string]any) + if ref["source_id"] != "session-alpha" || ref["start_unit_id"] != float64(99) || ref["end_unit_id"] != "" { + t.Fatalf("source ref = %#v, want source_id canonicalized without unit repair", ref) } } diff --git a/internal/modules/extract/dnd/spells/runner_test.go b/internal/modules/extract/dnd/spells/runner_test.go index fb9d1a2..ba6aef1 100644 --- a/internal/modules/extract/dnd/spells/runner_test.go +++ b/internal/modules/extract/dnd/spells/runner_test.go @@ -61,7 +61,7 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { } first, second := response.SpellCasts[0], response.SpellCasts[1] if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" { - t.Fatalf("spell order = %q, %q; want response order", first.Spell, second.Spell) + t.Fatalf("spell order = %q, %q; want source-unit order", first.Spell, second.Spell) } if first.Caster != "Aria" || second.Caster != "Borin" { t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster) @@ -296,7 +296,7 @@ func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) { t.Fatalf("len(NormalizeOutputs) = %d, want raw output", len(output.NormalizeOutputs)) } if string(output.NormalizeOutputs[0].Payload.Content) != `{"spell_casts":null}` { - t.Fatalf("content = %s, want raw structured output", output.NormalizeOutputs[0].Payload.Content) + t.Fatalf("content = %s, want canonical structured output", output.NormalizeOutputs[0].Payload.Content) } if output.Manifest.ValidationStatus != "approved" { t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus) diff --git a/internal/modules/extract/dnd/spells/schema_test.go b/internal/modules/extract/dnd/spells/schema_test.go index 6579609..0b2b2a9 100644 --- a/internal/modules/extract/dnd/spells/schema_test.go +++ b/internal/modules/extract/dnd/spells/schema_test.go @@ -37,6 +37,10 @@ func TestLoadResponseSchemaForSpells(t *testing.T) { properties := decoded["properties"].(map[string]any) spellCastProperties := properties["spell_casts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) sourceRefProperties := spellCastProperties["source_refs"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) + sourceRefRequired := spellCastProperties["source_refs"].(map[string]any)["items"].(map[string]any)["required"].([]any) + if !containsJSONField(sourceRefRequired, "source_id") { + t.Fatalf("canonical source refs required = %#v, want source_id", sourceRefRequired) + } for _, field := range []string{"start_unit_id", "end_unit_id"} { property := sourceRefProperties[field].(map[string]any) if property["type"] != "integer" { @@ -48,6 +52,32 @@ func TestLoadResponseSchemaForSpells(t *testing.T) { } } +func TestLLMResponseSchemaOmitsSourceID(t *testing.T) { + raw, err := embeddedAssets.ReadFile("assets/schemas/dnd_spells_llm.v1.json") + if err != nil { + t.Fatalf("ReadFile(LLM schema) error = %v, want nil", err) + } + if !json.Valid(raw) { + t.Fatalf("LLM schema is invalid JSON: %s", raw) + } + + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatalf("Unmarshal(LLM schema) error = %v, want nil", err) + } + properties := decoded["properties"].(map[string]any) + spellCastProperties := properties["spell_casts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) + sourceRefItems := spellCastProperties["source_refs"].(map[string]any)["items"].(map[string]any) + sourceRefProperties := sourceRefItems["properties"].(map[string]any) + sourceRefRequired := sourceRefItems["required"].([]any) + if _, ok := sourceRefProperties["source_id"]; ok { + t.Fatalf("LLM source ref schema contains source_id property: %#v", sourceRefProperties) + } + if containsJSONField(sourceRefRequired, "source_id") { + t.Fatalf("LLM source refs required = %#v, want no source_id", sourceRefRequired) + } +} + func TestResponseSchemaJSONIsMutationSafe(t *testing.T) { first, err := loadResponseSchema() if err != nil { @@ -89,3 +119,12 @@ func TestResponseSchemaDiagnosticsOmitRawSchema(t *testing.T) { t.Fatalf("diagnostics should omit raw schema content: %#v", diagnostics) } } + +func containsJSONField(fields []any, want string) bool { + for _, field := range fields { + if field == want { + return true + } + } + return false +} diff --git a/internal/modules/extract/dnd/spells/scriptorium_assets_test.go b/internal/modules/extract/dnd/spells/scriptorium_assets_test.go index cfe251d..631a5c8 100644 --- a/internal/modules/extract/dnd/spells/scriptorium_assets_test.go +++ b/internal/modules/extract/dnd/spells/scriptorium_assets_test.go @@ -18,6 +18,9 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing if prepared.PromptID != PromptID { t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) } + if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" { + t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath) + } if got := len(prepared.Messages); got != 5 { t.Fatalf("message count = %d, want 5", got) } diff --git a/internal/modules/sharedassets/dnd/unit_refs.go b/internal/modules/sharedassets/dnd/unit_refs.go index 66f7701..96058b9 100644 --- a/internal/modules/sharedassets/dnd/unit_refs.go +++ b/internal/modules/sharedassets/dnd/unit_refs.go @@ -40,6 +40,10 @@ func (ref UnitRef) String() string { return strconv.Itoa(ref.value) } +func (ref UnitRef) Int() int { + return ref.value +} + func (ref *UnitRef) UnmarshalJSON(raw []byte) error { raw = bytes.TrimSpace(raw) if len(raw) == 0 { diff --git a/internal/modules/sharedassets/dnd/unit_refs_test.go b/internal/modules/sharedassets/dnd/unit_refs_test.go index 0023402..a07b046 100644 --- a/internal/modules/sharedassets/dnd/unit_refs_test.go +++ b/internal/modules/sharedassets/dnd/unit_refs_test.go @@ -16,6 +16,9 @@ func TestUnitRefUnmarshalAcceptsIntegerAndNumericString(t *testing.T) { if got := integerRef.String(); got != "12" { t.Fatalf("integer ref = %q, want 12", got) } + if got := integerRef.Int(); got != 12 { + t.Fatalf("integer ref value = %d, want 12", got) + } var stringRef UnitRef if err := json.Unmarshal([]byte(`"12"`), &stringRef); err != nil {