Compare commits
9 Commits
v0.5.0
...
a2610757cd
| Author | SHA1 | Date | |
|---|---|---|---|
| a2610757cd | |||
| 3ad34956c5 | |||
| 22e6caa2a0 | |||
| 9cb7462800 | |||
| 87385b7e14 | |||
| 0ae5ea7637 | |||
| 6bc883dfb6 | |||
| fcfff3ad15 | |||
| 0f2b23dce1 |
@@ -5,9 +5,7 @@ into multiple scenes or use facts that are not supported by it.
|
||||
Return one kind, one concise title, and one concise summary. Choose exactly one
|
||||
kind:
|
||||
|
||||
- combat: active combat materially organizes the scene, including
|
||||
initiative-like exchanges or sustained hostile action. Planning a fight or
|
||||
discussing a completed fight is not combat by itself.
|
||||
- combat: a scene classified as combat under the shared combat policy.
|
||||
- narrative: current-session in-world play that is not principally active
|
||||
combat, a prior-session recap, or sustained out-of-character session
|
||||
discussion. This includes exploration, travel, dialogue, investigation,
|
||||
@@ -20,15 +18,13 @@ kind:
|
||||
play.
|
||||
|
||||
Narrative is the default for actual current-session gameplay that does not meet
|
||||
another definition. When the accepted chunk is mixed:
|
||||
another definition. When the accepted chunk has no substantive active combat:
|
||||
|
||||
1. use combat when active combat is a substantive central activity, even with
|
||||
brief setup, rules clarification, or immediate aftermath;
|
||||
2. otherwise use recap when recounting a previous session is the chunk's
|
||||
1. use recap when recounting a previous session is the chunk's
|
||||
primary table purpose;
|
||||
3. otherwise use meta when sustained out-of-character session discussion is
|
||||
2. otherwise use meta when sustained out-of-character session discussion is
|
||||
primary and in-world progression is no more than incidental; and
|
||||
4. use narrative for all remaining current-session in-world play.
|
||||
3. use narrative for all remaining current-session in-world play.
|
||||
|
||||
Brief table talk, dice resolution, rules clarification, jokes, or
|
||||
administrative comments do not make a gameplay scene meta. A short recollection
|
||||
|
||||
@@ -27,6 +27,8 @@ messages:
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
Review only whether the proposed scene kind correctly represents substantive
|
||||
active combat in the supplied transcript chunk under the shared combat policy.
|
||||
Do not judge the scene title, summary, non-combat subtype, scene boundary, or
|
||||
any other aspect of a scene description.
|
||||
|
||||
Return `approved` only when the proposed kind's combat status is supported;
|
||||
approval does not endorse any other part of the scene description. Return
|
||||
`combat_should_be_added` when a proposed non-combat kind omits substantive
|
||||
active combat. Return `combat_should_be_removed` when a proposed `combat` kind
|
||||
is unsupported. Give a concise, transcript-grounded explanation for every
|
||||
verdict.
|
||||
@@ -0,0 +1,30 @@
|
||||
id: dnd.scene_descriptions.validate_combat
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: proposed_kind
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./proposed-kind.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||
repair_attempts: 1
|
||||
@@ -0,0 +1,3 @@
|
||||
The proposed scene kind is:
|
||||
|
||||
{{ input "proposed_kind" }}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.scene_descriptions.combat_semantics.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["verdict", "explanation"],
|
||||
"properties": {
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"enum": ["approved", "combat_should_be_added", "combat_should_be_removed"]
|
||||
},
|
||||
"explanation": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 512
|
||||
}
|
||||
}
|
||||
}
|
||||
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
@@ -0,0 +1,13 @@
|
||||
Use `combat` only when substantive active combat materially organizes the
|
||||
scene. Active combat includes initiative or turn exchanges, attacks, combat
|
||||
spells, damage, saves, movement, or similarly sustained hostile action.
|
||||
|
||||
Do not use `combat` for planning or preparing for a possible fight; threats,
|
||||
hostile dialogue, or a tense confrontation; immediate aftermath, looting,
|
||||
healing, or discussion of a completed fight; a recap or in-world recollection
|
||||
of earlier combat; or out-of-character rules discussion without active
|
||||
encounter play.
|
||||
|
||||
When a chunk contains substantive active combat alongside brief setup, rules
|
||||
clarification, interruption, phase transition, or immediate aftermath, classify
|
||||
it as `combat`.
|
||||
@@ -504,11 +504,28 @@ Available validator keys are:
|
||||
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
|
||||
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
|
||||
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** |
|
||||
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
|
||||
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
|
||||
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
|
||||
|
||||
`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene:
|
||||
|
||||
~~~yaml
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 1
|
||||
validators:
|
||||
- generic/valid_json
|
||||
- extract/dnd/scene-descriptions/shape
|
||||
- extract/dnd/scene-descriptions/source_refs
|
||||
- generic/valid_json_schema
|
||||
- extract/dnd/scene-descriptions/source_relatedness
|
||||
- extract/dnd/scene-descriptions/combat_semantics
|
||||
~~~
|
||||
|
||||
An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules.
|
||||
|
||||
When no override is configured, production D&D bindings use the following
|
||||
ordered chains. Each row lists extract then normalize; spell chains are the
|
||||
same at both stages.
|
||||
|
||||
@@ -134,6 +134,64 @@ relatedness validators report advisory evidence concerns. The configured order
|
||||
is documented in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
|
||||
The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the
|
||||
D&D family's LLM-backed review of only combat versus non-combat classification.
|
||||
It selects the shared combat-policy prompt fragment, receives the proposed kind
|
||||
and current chunk, and maps its verdict deterministically into producer
|
||||
guidance. It does not assess titles, summaries, non-combat subtype, or scene
|
||||
boundaries; deferred boundary-coherence review remains separate. It is opt-in;
|
||||
[Configuration](../config.md) owns selection and retry/failure behavior.
|
||||
|
||||
### Combat-semantics provider evaluation
|
||||
|
||||
The human-reviewed corpus at
|
||||
`internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json`
|
||||
owns the proposed kind, expected verdict, and reviewer rationale for each
|
||||
synthetic case. Its package test validates the fixture contract only. Provider
|
||||
evaluation remains an explicit maintainer operation and must not be added to
|
||||
the default offline test suite.
|
||||
|
||||
Use the following protocol before proposing default-chain inclusion:
|
||||
|
||||
1. Record the Notarius commit, prompt and schema fingerprints, provider, model,
|
||||
profile settings, reasoning effort, structural-repair setting, number of
|
||||
repetitions, and evaluation date before collecting results. Do not revise
|
||||
expected verdicts merely to agree with provider output; a substantive corpus
|
||||
correction requires independent human review.
|
||||
2. Exercise the production validator construction and prompt assets from an
|
||||
explicitly invoked, disposable evaluation driver or test in the validator
|
||||
package. For each corpus case, construct transcript source units from the
|
||||
listed IDs and text, assign matching per-unit source references, and use
|
||||
`source.MaterializeChunkPlan` with one range spanning those units. Construct
|
||||
exactly one scene whose ID and source range match that chunk and whose kind
|
||||
is the case's `proposed_kind`; title and summary may use fixed placeholders
|
||||
because the validator neither receives nor evaluates them. Invoke the typed
|
||||
validator at extract stage through the production registry and scheduled LLM
|
||||
client. Do not commit provider credentials, generated source material, or an
|
||||
always-on live test.
|
||||
3. Compare the resulting approval or reason code with `expected_verdict`.
|
||||
Record an unexpected approval of an expected rejection as a false
|
||||
acceptance, an unexpected rejection of an expected approval as a false
|
||||
rejection, the opposite rejection direction separately, and any validator
|
||||
execution failure separately from semantic accuracy. Retain per-case results
|
||||
so repeated trials and systematic failure modes remain visible.
|
||||
4. Evaluate producer correction separately with representative complete
|
||||
scene-description runs configured as shown in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
For every initial semantic rejection, record whether the next producer
|
||||
attempt returns the requested combat status and is approved. Do not count a
|
||||
PromptKit structural repair as a producer-correction attempt.
|
||||
5. Run the correction evaluation with debug capture enabled and without reused
|
||||
extraction checkpoints. Record added validator and producer calls, elapsed
|
||||
latency, and cumulative prompt, completion, cached, and total token usage
|
||||
from the debug attempt records. Compare these values with an otherwise
|
||||
identical run whose scene-description chain omits the semantic validator.
|
||||
|
||||
The default-chain review must consider false acceptance, false rejection,
|
||||
wrong-direction rejection, execution failure, producer-correction success,
|
||||
added calls, latency, and token use together. A structurally successful
|
||||
provider run alone is not evidence that the validator should become a default.
|
||||
|
||||
Every D&D rejection describes the correction in transcript-grounded domain
|
||||
terms, using contextual names, artifact fields, and source segment ranges when
|
||||
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
||||
|
||||
@@ -14,47 +14,8 @@ eligibility, and the separation of actionable process warnings from quality
|
||||
diagnostics. The remaining near-term work applies those completed foundations
|
||||
to domain review and empirical evaluation.
|
||||
|
||||
### D&D Combat Scene Semantic Validation
|
||||
|
||||
- Add an optional production LLM-backed D&D validator that determines whether
|
||||
proposed scene boundaries and classifications represent substantive active
|
||||
combat correctly. Its central quality goal is that active combat is kept in
|
||||
coherent scenes classified as `combat`, rather than split incorrectly or
|
||||
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
||||
- Resolve the validator's exact target before implementation. The current
|
||||
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
||||
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
||||
`recap`, and `meta` classification. The preferred initial placement is
|
||||
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
||||
can compare one proposed kind with the corresponding transcript chunk.
|
||||
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
||||
question that can be answered from the complete transcript and proposed
|
||||
range map, such as whether one continuous combat was fragmented across
|
||||
inappropriate scene boundaries. Do not duplicate the same classification
|
||||
judgment at both stages. Moving classification into chunk-plan annotations
|
||||
would change the deliberately minimal, annotation-free chunk contract and
|
||||
requires an explicit architecture review before it is selected.
|
||||
- Validate both false negatives and false positives: a non-combat kind must not
|
||||
omit substantive active combat, and a combat kind must be supported by such
|
||||
combat. Keep the existing deterministic downstream rule that combat-turn
|
||||
extraction runs only for an exact `combat` scene classification; semantic
|
||||
review improves the upstream classification but does not replace that gate.
|
||||
- Run the semantic validator through PromptKit, use a minimal required-field
|
||||
structured response schema, and let PromptKit repair structural validator
|
||||
output within its bounded budget. A contract-invalid final validator response
|
||||
is a validator execution failure, not a semantic rejection and not a reason
|
||||
to recursively validate the validator.
|
||||
- Evaluate the prompt and decision policy against a small human-reviewed set
|
||||
containing combat setup, active turns, interruptions, multi-phase encounters,
|
||||
brief rules discussion, aftermath, recalled combat, and false-positive
|
||||
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
||||
added calls, latency, and token cost before placing it in the production
|
||||
default chain.
|
||||
- An ADR is not required if classification remains owned by
|
||||
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
||||
Create or supersede an ADR if the work transfers scene classification into
|
||||
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
||||
contract.
|
||||
The active first production application is defined by
|
||||
[D&D Combat Scene Semantic Validation](combat-scene-validation.md).
|
||||
|
||||
## Near-Term D&D Pipeline
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ var promptAssetManifest = shared.PromptAssetManifest{
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-scene-combat-policy.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,23 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||
}
|
||||
const combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
transcriptIndex := -1
|
||||
policyIndex := -1
|
||||
instructionsIndex := len(prepared.Messages) - 1
|
||||
policyOccurrences := 0
|
||||
for index, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, "scene-description-transcript") {
|
||||
transcriptIndex = index
|
||||
}
|
||||
if count := strings.Count(message.Content, combatPolicySentinel); count > 0 {
|
||||
policyIndex = index
|
||||
policyOccurrences += count
|
||||
}
|
||||
}
|
||||
if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex {
|
||||
t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
)
|
||||
@@ -157,6 +158,7 @@ func registerPromptAssets(assets *llm.AssetRegistry) error {
|
||||
{name: "item registry normalization prompt assets", register: func() error { return itemregistrynormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "npc occurrences prompt assets", register: func() error { return occurrenceextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene descriptions prompt assets", register: func() error { return scenedescriptionextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene combat-semantics validator prompt assets", register: func() error { return combatsemantics.RegisterPromptAssets(assets) }},
|
||||
{name: "location registry prompt assets", register: func() error { return locationextract.RegisterPromptAssets(assets) }},
|
||||
{name: "location normalization prompt assets", register: func() error { return locationnormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "location occurrences prompt assets", register: func() error { return locationoccurrenceextract.RegisterPromptAssets(assets) }},
|
||||
|
||||
@@ -23,13 +23,14 @@ import (
|
||||
|
||||
func TestExtractionPromptComposition(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
)
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := registerPromptAssets(registry); err != nil {
|
||||
@@ -43,11 +44,12 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
middleSentinel string
|
||||
}{
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
@@ -55,7 +57,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
||||
}), suffixGroups: [][]string{{evidenceSentinel}, {"item-registry-sentinel"}}},
|
||||
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs, middleSentinel: combatPolicySentinel},
|
||||
{
|
||||
name: "combat turns",
|
||||
promptID: combatextract.PromptID,
|
||||
@@ -155,6 +157,12 @@ func TestExtractionPromptComposition(t *testing.T) {
|
||||
previousIndex = groupIndex
|
||||
}
|
||||
instructionIndex := len(prepared.Messages) - 1
|
||||
if testCase.middleSentinel != "" {
|
||||
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
|
||||
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
|
||||
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
|
||||
}
|
||||
}
|
||||
if instructionIndex <= previousIndex {
|
||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
)
|
||||
|
||||
func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
@@ -57,6 +58,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
"dnd.item_registry.normalize/prompt.yaml",
|
||||
"dnd.npc_occurrences/prompt.yaml",
|
||||
"dnd.scene_descriptions/prompt.yaml",
|
||||
"dnd.scene_descriptions.validate_combat/prompt.yaml",
|
||||
"dnd.npc_registry.normalize/prompt.yaml",
|
||||
"dnd.location_registry/prompt.yaml",
|
||||
"dnd.location_registry.normalize/prompt.yaml",
|
||||
@@ -124,10 +126,14 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
"extract/dnd/scene-descriptions/shape",
|
||||
"extract/dnd/scene-descriptions/source_refs",
|
||||
"extract/dnd/scene-descriptions/source_relatedness",
|
||||
combatsemantics.Key,
|
||||
"normalize/dnd/scene-descriptions/invariants",
|
||||
"generic/always_accept",
|
||||
"generic/always_reject",
|
||||
})
|
||||
if spec, ok := registries.Validators.Spec(combatsemantics.Key); !ok || spec.Key != combatsemantics.Key || spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||
t.Fatalf("combat semantics validator spec = %#v, present=%t; want registered LLM-backed scene validator", spec, ok)
|
||||
}
|
||||
locationExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
||||
locationNormalizeChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("normalize/dnd/location-registry/identity"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
||||
occurrenceExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-occurrences/shape"), pipeline.Binding("extract/dnd/location-occurrences/registry"), pipeline.Binding("extract/dnd/location-occurrences/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-occurrences/source_relatedness")}
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
||||
npcsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
||||
npcrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_relatedness"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
sceneinvariants "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/invariants"
|
||||
sceneshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
scenerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
|
||||
@@ -87,6 +88,7 @@ func registerValidators(registries pipeline.Registries) error {
|
||||
{name: "scene description shape validator", register: func() error { return sceneshape.Register(registries.Validators) }},
|
||||
{name: "scene description source references validator", register: func() error { return scenerefs.Register(registries.Validators) }},
|
||||
{name: "scene description source relatedness validator", register: func() error { return scenerelatedness.Register(registries.Validators) }},
|
||||
{name: "scene description combat semantics validator", register: func() error { return combatsemantics.Register(registries.Validators) }},
|
||||
{name: "scene description normalized invariants validator", register: func() error { return sceneinvariants.Register(registries.Validators) }},
|
||||
{name: "location shape validator", register: func() error { return locationshape.Register(registries.Validators) }},
|
||||
{name: "location identity validator", register: func() error { return locationidentity.Register(registries.Validators) }},
|
||||
|
||||
@@ -27,6 +27,7 @@ var sharedPromptPaths = map[string]string{
|
||||
"common-dnd-identity.md": "prompts/common-dnd-identity.md",
|
||||
"common-dnd-transcript-full.md": "prompts/common-dnd-transcript-full.md",
|
||||
"common-dnd-transcript-chunk.md": "prompts/common-dnd-transcript-chunk.md",
|
||||
"common-dnd-scene-combat-policy.md": "prompts/common-dnd-scene-combat-policy.md",
|
||||
"common-dnd-references.md": "prompts/common-dnd-references.md",
|
||||
"common-dnd-npc-registry.md": "prompts/common-dnd-npc-registry.md",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
{Name: "proposed-kind.md", Path: "prompts/proposed-kind.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-scene-combat-policy.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -0,0 +1,100 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
engine := newCombatSemanticsPromptEngine(t, registry)
|
||||
inputs := map[string]promptkit.ArtifactRef{
|
||||
"proposed_kind": promptkit.Inline("narrative"),
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
|
||||
}
|
||||
if len(prepared.Messages) != 5 {
|
||||
t.Fatalf("prepared message count = %d, want 5", len(prepared.Messages))
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
wantEphemeral := index == 3 || index == 4
|
||||
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
|
||||
if gotEphemeral != wantEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
|
||||
}
|
||||
}
|
||||
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
|
||||
assertRenderedExactlyOnce(t, prepared.Messages, "narrative", 3)
|
||||
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 4)
|
||||
for name := range inputs {
|
||||
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
|
||||
for inputName, input := range inputs {
|
||||
if inputName != name {
|
||||
missing[inputName] = input
|
||||
}
|
||||
}
|
||||
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing,
|
||||
}); err == nil {
|
||||
t.Fatalf("Prepare() without required %q input = nil error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
if strings.Contains(hash, "combat_should_be_added") {
|
||||
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
|
||||
}
|
||||
}
|
||||
|
||||
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "combat-semantics-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-semantics-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
for messageIndex, message := range messages {
|
||||
count := strings.Count(message.Content, sentinel)
|
||||
if count > 0 {
|
||||
index = messageIndex
|
||||
occurrences += count
|
||||
}
|
||||
}
|
||||
if occurrences != 1 || index != wantIndex {
|
||||
t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package combat_semantics
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.scene_descriptions.validate_combat"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm")
|
||||
ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm"
|
||||
ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
|
||||
}
|
||||
for _, verdict := range []string{"approved", "combat_should_be_added", "combat_should_be_removed"} {
|
||||
if err := validateCombatSemanticsSchema(map[string]any{"verdict": verdict, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid %q verdict rejected: %v", verdict, err)
|
||||
}
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
}{
|
||||
{name: "missing explanation", response: map[string]any{"verdict": "approved"}},
|
||||
{name: "missing verdict", response: map[string]any{"explanation": "The proposed kind is supported."}},
|
||||
{name: "unknown property", response: map[string]any{"verdict": "approved", "explanation": "The proposed kind is supported.", "confidence": 1}},
|
||||
{name: "wrong verdict type", response: map[string]any{"verdict": 1, "explanation": "The proposed kind is supported."}},
|
||||
{name: "wrong explanation type", response: map[string]any{"verdict": "approved", "explanation": 1}},
|
||||
{name: "unsupported verdict", response: map[string]any{"verdict": "uncertain", "explanation": "The proposed kind is supported."}},
|
||||
{name: "blank explanation", response: map[string]any{"verdict": "approved", "explanation": ""}},
|
||||
{name: "oversized explanation", response: map[string]any{"verdict": "approved", "explanation": strings.Repeat("a", 513)}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {
|
||||
t.Fatal("invalid private response accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) {
|
||||
t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
||||
instanceContent, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(parsedInstance)
|
||||
}
|
||||
10
internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json
vendored
Normal file
10
internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{"name":"active encounter","transcript_units":[{"id":1,"text":"Roll initiative; the goblins attack."},{"id":2,"text":"The ranger hits and deals damage."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"Initiative and hostile actions organize the chunk."},
|
||||
{"name":"combat after setup","transcript_units":[{"id":1,"text":"They open the crypt door."},{"id":2,"text":"Skeletons attack and turns begin."}],"proposed_kind":"narrative","expected_verdict":"combat_should_be_added","reviewer_rationale":"Brief setup does not displace substantive active combat."},
|
||||
{"name":"combat aftermath","transcript_units":[{"id":1,"text":"The last enemy falls."},{"id":2,"text":"They search bodies and heal."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Looting and healing after a completed fight are not active combat."},
|
||||
{"name":"multi phase encounter","transcript_units":[{"id":1,"text":"The dragon attacks."},{"id":2,"text":"After a rules clarification, its next turn begins."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"A brief rules interruption does not end the encounter."},
|
||||
{"name":"planning","transcript_units":[{"id":1,"text":"They plan how to ambush the guard."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Planning a possible fight is not active encounter play."},
|
||||
{"name":"hostile dialogue","transcript_units":[{"id":1,"text":"The captain threatens them and they argue."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Threats and hostile dialogue alone are insufficient."},
|
||||
{"name":"recap recollection","transcript_units":[{"id":1,"text":"They recap last session's battle with the lich."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Recounting earlier combat is not current active combat."},
|
||||
{"name":"rules discussion","transcript_units":[{"id":1,"text":"The table discusses how concentration works."}],"proposed_kind":"meta","expected_verdict":"approved","reviewer_rationale":"Sustained out-of-character rules discussion has no active encounter."}
|
||||
]
|
||||
@@ -0,0 +1,230 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
|
||||
maximumExplanationRunes = 512
|
||||
)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type completionResponse struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Explanation string `json:"explanation"`
|
||||
}
|
||||
|
||||
type Validator struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
|
||||
if llmClient == nil {
|
||||
return nil, validatorErrorf("LLM client must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
}
|
||||
|
||||
func (v *Validator) Name() string { return Key }
|
||||
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (v *Validator) ManifestMetadata() map[string]any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": v.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": v.responseSchemaSHA,
|
||||
"combat_policy": combatPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: v.promptSHA},
|
||||
{Name: "response_schema", Value: v.responseSchemaSHA},
|
||||
{Name: "combat_policy", Value: combatPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||
if v == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
||||
}
|
||||
if v.llm == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
|
||||
}
|
||||
sourceInput, scene, err := validateRequest(ctx, req)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
|
||||
var response completionResponse
|
||||
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": shared.TranscriptPromptMaterial(sourceInput),
|
||||
"proposed_kind": contracts.NewLLMInputMaterial("proposed_kind", "text/plain", []byte(scene.Kind), "", ""),
|
||||
},
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
return interpretResponse(response, scene.Kind)
|
||||
}
|
||||
|
||||
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
||||
if ctx == nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context error before validation: %w", err)
|
||||
}
|
||||
if req.Stage != string(pipeline.StageExtract) {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||
if err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(req.Chunk.ID) == "" || req.Chunk.ID != strings.TrimSpace(req.Chunk.ID) {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk ID must be nonblank and trimmed")
|
||||
}
|
||||
if req.Chunk.SourceID != req.Source.ID {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk source ID %q does not match source %q", req.Chunk.SourceID, req.Source.ID)
|
||||
}
|
||||
if err := source.NewDocumentIndex(req.Source).ValidateRef(req.Chunk.Ref); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate current chunk source range: %w", err)
|
||||
}
|
||||
if err := shape.ValidateForStage(req.Value, req.Stage); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate scene description shape: %w", err)
|
||||
}
|
||||
scene := req.Value.Scenes[0]
|
||||
if scene.ID != req.Chunk.ID {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene ID must equal current chunk ID")
|
||||
}
|
||||
if scene.SourceRef != req.Chunk.Ref {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene source range must equal current chunk range")
|
||||
}
|
||||
return sourceInput, scene, nil
|
||||
}
|
||||
|
||||
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||
explanation, err := validateExplanation(response.Explanation)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
switch response.Verdict {
|
||||
case "approved":
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
case "combat_should_be_added":
|
||||
if proposedKind == dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_added verdict is inconsistent with proposed combat kind")
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||
}, nil
|
||||
case "combat_should_be_removed":
|
||||
if proposedKind != dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_removed verdict is inconsistent with proposed non-combat kind")
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||
Message: "The current chunk does not support a combat classification.",
|
||||
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||
}, nil
|
||||
default:
|
||||
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics verdict %q", response.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func validateExplanation(value string) (string, error) {
|
||||
if !utf8.ValidString(value) {
|
||||
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", validatorErrorf("validator explanation must not be blank")
|
||||
}
|
||||
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
||||
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||
}
|
||||
return trimmed, nil
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func validatorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
kind dnd.SceneKind
|
||||
response completionResponse
|
||||
approved bool
|
||||
reasonCode string
|
||||
guidance string
|
||||
correction string
|
||||
wantError string
|
||||
}{
|
||||
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}, approved: true},
|
||||
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "approved", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
||||
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
||||
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
||||
{name: "inconsistent added", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks."}, wantError: "inconsistent"},
|
||||
{name: "inconsistent removed", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans."}, wantError: "inconsistent"},
|
||||
{name: "unknown verdict", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
||||
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: " "}, wantError: "blank"},
|
||||
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
||||
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request.Value.Scenes[0].Kind = test.kind
|
||||
client := &fakeCombatSemanticsClient{response: test.response}
|
||||
validator := newValidator(t, client)
|
||||
result, err := validator.Validate(context.Background(), request)
|
||||
if test.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
||||
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
||||
}
|
||||
if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) {
|
||||
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
||||
}
|
||||
if err := contracts.ValidateValidationResult(result); err != nil {
|
||||
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorUsesOnlyChunkTranscriptAndProposedKind(t *testing.T) {
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
repairAttempts := 2
|
||||
request.LLMProfile = "validator-profile"
|
||||
request.SessionID = "validator-session"
|
||||
request.StructuredOutputRepairAttempts = &repairAttempts
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
|
||||
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
|
||||
validator := newValidator(t, client)
|
||||
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("completion count = %d, want 1", len(client.requests))
|
||||
}
|
||||
completed := client.requests[0]
|
||||
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
|
||||
t.Fatalf("completion request = %#v", completed)
|
||||
}
|
||||
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"proposed_kind", "transcript"}) || string(completed.Inputs["proposed_kind"].Content) != "narrative" || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
|
||||
t.Fatalf("completion inputs = %#v", completed.Inputs)
|
||||
}
|
||||
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
|
||||
for name, input := range completed.Inputs {
|
||||
if strings.Contains(string(input.Content), forbidden) {
|
||||
t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||
base := validRequest(dnd.SceneKindNarrative)
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
mutate func(*contracts.TypedValidationRequest[dnd.SceneDescriptionList])
|
||||
wantErr string
|
||||
}{
|
||||
{name: "nil context", ctx: nil, wantErr: "context must not be nil"},
|
||||
{name: "canceled context", ctx: canceled, wantErr: "context error"},
|
||||
{name: "wrong stage", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||
req.Stage = string(pipeline.StageNormalize)
|
||||
}, wantErr: "stage must be extract"},
|
||||
{name: "missing source", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) { req.Source = nil }, wantErr: "source must not be nil"},
|
||||
{name: "missing chunk", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) { req.Chunk = nil }, wantErr: "chunk must not be nil"},
|
||||
{name: "mismatched source input", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||
req.SourceInput.Content = []byte(`{"other":true}`)
|
||||
}, wantErr: "must match chunk"},
|
||||
{name: "multiple scenes", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||
req.Value.Scenes = append(req.Value.Scenes, req.Value.Scenes[0])
|
||||
}, wantErr: "exactly one"},
|
||||
{name: "wrong scene ID", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||
req.Value.Scenes[0].ID = "other"
|
||||
}, wantErr: "scene ID"},
|
||||
{name: "wrong scene range", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||
req.Value.Scenes[0].SourceRef.EndUnitID = 1
|
||||
}, wantErr: "scene source range"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := cloneValidationRequest(base)
|
||||
if test.mutate != nil {
|
||||
test.mutate(&request)
|
||||
}
|
||||
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
|
||||
_, err := newValidator(t, client).Validate(test.ctx, request)
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
|
||||
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
|
||||
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") {
|
||||
t.Fatalf("completion failure error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown options")
|
||||
}
|
||||
validator := newValidator(t, &fakeCombatSemanticsClient{})
|
||||
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" {
|
||||
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
|
||||
}
|
||||
metadata, err := json.Marshal(validator.ManifestMetadata())
|
||||
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "combat_should_be_added") {
|
||||
t.Fatalf("manifest metadata = %s, %v", metadata, err)
|
||||
}
|
||||
fingerprints := validator.CheckpointFingerprints()
|
||||
if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) {
|
||||
t.Fatalf("fingerprints = %#v", fingerprints)
|
||||
}
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if spec, ok := registry.Spec(Key); !ok || spec.Key != Key || spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||
t.Fatalf("registered spec = %#v, present=%t", spec, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
|
||||
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cases []struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptUnits []struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
} `json:"transcript_units"`
|
||||
ProposedKind string `json:"proposed_kind"`
|
||||
ExpectedVerdict string `json:"expected_verdict"`
|
||||
ReviewerRationale string `json:"reviewer_rationale"`
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&cases); err != nil {
|
||||
t.Fatalf("decode evaluation cases: %v", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
||||
}
|
||||
seenNames, seenVerdicts := map[string]bool{}, map[string]bool{}
|
||||
for _, value := range cases {
|
||||
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
||||
t.Fatalf("invalid evaluation case: %#v", value)
|
||||
}
|
||||
seenNames[value.Name] = true
|
||||
switch value.ProposedKind {
|
||||
case "combat", "narrative", "recap", "meta":
|
||||
default:
|
||||
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
||||
}
|
||||
switch value.ExpectedVerdict {
|
||||
case "approved", "combat_should_be_added", "combat_should_be_removed":
|
||||
seenVerdicts[value.ExpectedVerdict] = true
|
||||
default:
|
||||
t.Fatalf("unsupported verdict %q", value.ExpectedVerdict)
|
||||
}
|
||||
for _, unit := range value.TranscriptUnits {
|
||||
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
||||
t.Fatalf("invalid transcript unit: %#v", unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seenVerdicts) != 3 {
|
||||
t.Fatalf("evaluation verdicts = %#v, want all classes", seenVerdicts)
|
||||
}
|
||||
}
|
||||
|
||||
func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator {
|
||||
t.Helper()
|
||||
validator, err := New(client, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
return validator
|
||||
}
|
||||
|
||||
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
|
||||
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
|
||||
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
||||
}
|
||||
|
||||
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||
request.SourceInput = request.SourceInput.Clone()
|
||||
chunk := *request.Chunk
|
||||
chunk.Content = append([]byte(nil), chunk.Content...)
|
||||
chunk.Units = append([]source.SourceUnit(nil), chunk.Units...)
|
||||
request.Chunk = &chunk
|
||||
request.Value.Scenes = append([]dnd.SceneDescription(nil), request.Value.Scenes...)
|
||||
return request
|
||||
}
|
||||
|
||||
func sortedInputNames(inputs contracts.LLMInputSet) []string {
|
||||
names := make([]string, 0, len(inputs))
|
||||
for name := range inputs {
|
||||
names = append(names, name)
|
||||
}
|
||||
if len(names) == 2 && names[0] == "transcript" {
|
||||
names[0], names[1] = names[1], names[0]
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
type fakeCombatSemanticsClient struct {
|
||||
response completionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
cloned, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
client.requests = append(client.requests, cloned)
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
response, ok := out.(*completionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target")
|
||||
}
|
||||
*response = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
)
|
||||
|
||||
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
||||
@@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
||||
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
||||
}
|
||||
|
||||
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
||||
sceneLane.Extract.Retries = 1
|
||||
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
|
||||
pipeline.Binding(combatsemantics.Key),
|
||||
}}
|
||||
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
|
||||
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||
|
||||
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
|
||||
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
|
||||
}
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
if group.Origin.ValidatorKey == combatsemantics.Key {
|
||||
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
||||
}
|
||||
}
|
||||
var scenes dnd.SceneDescriptionList
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.LaneID == "scene-descriptions" {
|
||||
var err error
|
||||
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
||||
}
|
||||
requests := client.requestsSnapshot()
|
||||
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
|
||||
for _, request := range requests {
|
||||
switch request.PromptID {
|
||||
case sceneextract.PromptID:
|
||||
sceneRequests = append(sceneRequests, request)
|
||||
case combatsemantics.PromptID:
|
||||
validatorRequests = append(validatorRequests, request)
|
||||
}
|
||||
}
|
||||
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
|
||||
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||
}
|
||||
correction := sceneRequests[1].Correction
|
||||
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
|
||||
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
|
||||
}
|
||||
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
||||
if strings.Contains(correction.UserGuidance, forbidden) {
|
||||
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
@@ -508,12 +573,16 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str
|
||||
}
|
||||
|
||||
type groundedDNDLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
sceneKind dnd.SceneKind
|
||||
sceneTitle string
|
||||
firstUnitID int
|
||||
thirdUnitID int
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
sceneKind dnd.SceneKind
|
||||
sceneTitle string
|
||||
firstUnitID int
|
||||
thirdUnitID int
|
||||
sceneKinds []dnd.SceneKind
|
||||
combatSemanticsVerdicts []string
|
||||
sceneCalls int
|
||||
combatSemanticsCalls int
|
||||
}
|
||||
|
||||
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
@@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
payload = map[string]any{"duplicate_groups": []any{}}
|
||||
case sceneextract.PromptID:
|
||||
kind := client.sceneKind
|
||||
if client.sceneCalls < len(client.sceneKinds) {
|
||||
kind = client.sceneKinds[client.sceneCalls]
|
||||
}
|
||||
client.sceneCalls++
|
||||
if kind == "" {
|
||||
kind = dnd.SceneKindCombat
|
||||
}
|
||||
@@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
title = "A combat encounter"
|
||||
}
|
||||
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
||||
case combatsemantics.PromptID:
|
||||
verdict := "approved"
|
||||
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
|
||||
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
|
||||
}
|
||||
client.combatSemanticsCalls++
|
||||
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
|
||||
case spells.PromptID:
|
||||
payload = map[string]any{"spell_casts": []any{map[string]any{
|
||||
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
||||
|
||||
Reference in New Issue
Block a user