diff --git a/docs/config.md b/docs/config.md index 94231c8b..032473b5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -504,7 +504,24 @@ Available validator keys are: | Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** | | Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** | | NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** | -| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** | +| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** | + +`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene: + +~~~yaml +extract: + module: dnd/scene-descriptions + retries: 1 + validators: + - generic/valid_json + - extract/dnd/scene-descriptions/shape + - extract/dnd/scene-descriptions/source_refs + - generic/valid_json_schema + - extract/dnd/scene-descriptions/source_relatedness + - extract/dnd/scene-descriptions/combat_semantics +~~~ + +An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules. | Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** | | Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** | | Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** | diff --git a/docs/internal/dnd.md b/docs/internal/dnd.md index 97aa9cfc..abf88f8e 100644 --- a/docs/internal/dnd.md +++ b/docs/internal/dnd.md @@ -134,6 +134,19 @@ relatedness validators report advisory evidence concerns. The configured order is documented in [Configuration](../config.md#production-validator-keys-and-default-chains). +The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the +D&D family's LLM-backed review of only combat versus non-combat classification. +It selects the shared combat-policy prompt fragment, receives the proposed kind +and current chunk, and maps its verdict deterministically into producer +guidance. It does not assess titles, summaries, non-combat subtype, or scene +boundaries; deferred boundary-coherence review remains separate. It is opt-in; +[Configuration](../config.md) owns selection and retry/failure behavior. + +Before proposing default-chain inclusion, run a deliberate provider evaluation +against the synthetic corpus and record false acceptance, false rejection, +producer-correction success, added calls, latency, and token use. Default tests +remain offline and do not measure provider quality. + Every D&D rejection describes the correction in transcript-grounded domain terms, using contextual names, artifact fields, and source segment ranges when useful. The guidance must not ask the model to reproduce durable entity IDs, diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 203b87c8..8fe48b81 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -374,6 +374,8 @@ This stage is small enough for one implementation prompt. ## Stage 6: Add Evaluation Material And Documentation +✅ Complete + ### Goal Provide accurate opt-in configuration guidance and a bounded human-reviewed diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json new file mode 100644 index 00000000..3b95d0e9 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json @@ -0,0 +1,10 @@ +[ + {"name":"active encounter","transcript_units":[{"id":1,"text":"Roll initiative; the goblins attack."},{"id":2,"text":"The ranger hits and deals damage."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"Initiative and hostile actions organize the chunk."}, + {"name":"combat after setup","transcript_units":[{"id":1,"text":"They open the crypt door."},{"id":2,"text":"Skeletons attack and turns begin."}],"proposed_kind":"narrative","expected_verdict":"combat_should_be_added","reviewer_rationale":"Brief setup does not displace substantive active combat."}, + {"name":"combat aftermath","transcript_units":[{"id":1,"text":"The last enemy falls."},{"id":2,"text":"They search bodies and heal."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Looting and healing after a completed fight are not active combat."}, + {"name":"multi phase encounter","transcript_units":[{"id":1,"text":"The dragon attacks."},{"id":2,"text":"After a rules clarification, its next turn begins."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"A brief rules interruption does not end the encounter."}, + {"name":"planning","transcript_units":[{"id":1,"text":"They plan how to ambush the guard."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Planning a possible fight is not active encounter play."}, + {"name":"hostile dialogue","transcript_units":[{"id":1,"text":"The captain threatens them and they argue."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Threats and hostile dialogue alone are insufficient."}, + {"name":"recap recollection","transcript_units":[{"id":1,"text":"They recap last session's battle with the lich."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Recounting earlier combat is not current active combat."}, + {"name":"rules discussion","transcript_units":[{"id":1,"text":"The table discusses how concentration works."}],"proposed_kind":"meta","expected_verdict":"approved","reviewer_rationale":"Sustained out-of-character rules discussion has no active encounter."} +] diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go index f12dbf28..4fabdd8e 100644 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "os" "reflect" "strings" "testing" @@ -167,6 +168,54 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) { } } +func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) { + content, err := os.ReadFile("testdata/evaluation_cases.json") + if err != nil { + t.Fatal(err) + } + var cases []struct { + Name string `json:"name"` + TranscriptUnits []struct { + ID int `json:"id"` + Text string `json:"text"` + } `json:"transcript_units"` + ProposedKind string `json:"proposed_kind"` + ExpectedVerdict string `json:"expected_verdict"` + ReviewerRationale string `json:"reviewer_rationale"` + } + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cases); err != nil { + t.Fatalf("decode evaluation cases: %v", err) + } + seenNames, seenVerdicts := map[string]bool{}, map[string]bool{} + for _, value := range cases { + if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" { + t.Fatalf("invalid evaluation case: %#v", value) + } + seenNames[value.Name] = true + switch value.ProposedKind { + case "combat", "narrative", "recap", "meta": + default: + t.Fatalf("unsupported proposed kind %q", value.ProposedKind) + } + switch value.ExpectedVerdict { + case "approved", "combat_should_be_added", "combat_should_be_removed": + seenVerdicts[value.ExpectedVerdict] = true + default: + t.Fatalf("unsupported verdict %q", value.ExpectedVerdict) + } + for _, unit := range value.TranscriptUnits { + if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" { + t.Fatalf("invalid transcript unit: %#v", unit) + } + } + } + if len(seenVerdicts) != 3 { + t.Fatalf("evaluation verdicts = %#v, want all classes", seenVerdicts) + } +} + func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator { t.Helper() validator, err := New(client, Options{})