Compare commits
9 Commits
v0.5.0
...
a2610757cd
| Author | SHA1 | Date | |
|---|---|---|---|
| a2610757cd | |||
| 3ad34956c5 | |||
| 22e6caa2a0 | |||
| 9cb7462800 | |||
| 87385b7e14 | |||
| 0ae5ea7637 | |||
| 6bc883dfb6 | |||
| fcfff3ad15 | |||
| 0f2b23dce1 |
@@ -5,9 +5,7 @@ into multiple scenes or use facts that are not supported by it.
|
|||||||
Return one kind, one concise title, and one concise summary. Choose exactly one
|
Return one kind, one concise title, and one concise summary. Choose exactly one
|
||||||
kind:
|
kind:
|
||||||
|
|
||||||
- combat: active combat materially organizes the scene, including
|
- combat: a scene classified as combat under the shared combat policy.
|
||||||
initiative-like exchanges or sustained hostile action. Planning a fight or
|
|
||||||
discussing a completed fight is not combat by itself.
|
|
||||||
- narrative: current-session in-world play that is not principally active
|
- narrative: current-session in-world play that is not principally active
|
||||||
combat, a prior-session recap, or sustained out-of-character session
|
combat, a prior-session recap, or sustained out-of-character session
|
||||||
discussion. This includes exploration, travel, dialogue, investigation,
|
discussion. This includes exploration, travel, dialogue, investigation,
|
||||||
@@ -20,15 +18,13 @@ kind:
|
|||||||
play.
|
play.
|
||||||
|
|
||||||
Narrative is the default for actual current-session gameplay that does not meet
|
Narrative is the default for actual current-session gameplay that does not meet
|
||||||
another definition. When the accepted chunk is mixed:
|
another definition. When the accepted chunk has no substantive active combat:
|
||||||
|
|
||||||
1. use combat when active combat is a substantive central activity, even with
|
1. use recap when recounting a previous session is the chunk's
|
||||||
brief setup, rules clarification, or immediate aftermath;
|
|
||||||
2. otherwise use recap when recounting a previous session is the chunk's
|
|
||||||
primary table purpose;
|
primary table purpose;
|
||||||
3. otherwise use meta when sustained out-of-character session discussion is
|
2. otherwise use meta when sustained out-of-character session discussion is
|
||||||
primary and in-world progression is no more than incidental; and
|
primary and in-world progression is no more than incidental; and
|
||||||
4. use narrative for all remaining current-session in-world play.
|
3. use narrative for all remaining current-session in-world play.
|
||||||
|
|
||||||
Brief table talk, dice resolution, rules clarification, jokes, or
|
Brief table talk, dice resolution, rules clarification, jokes, or
|
||||||
administrative comments do not make a gameplay scene meta. A short recollection
|
administrative comments do not make a gameplay scene meta. A short recollection
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ messages:
|
|||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./instructions.md
|
content_file: ./instructions.md
|
||||||
cache_control:
|
cache_control:
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
Review only whether the proposed scene kind correctly represents substantive
|
||||||
|
active combat in the supplied transcript chunk under the shared combat policy.
|
||||||
|
Do not judge the scene title, summary, non-combat subtype, scene boundary, or
|
||||||
|
any other aspect of a scene description.
|
||||||
|
|
||||||
|
Return `approved` only when the proposed kind's combat status is supported;
|
||||||
|
approval does not endorse any other part of the scene description. Return
|
||||||
|
`combat_should_be_added` when a proposed non-combat kind omits substantive
|
||||||
|
active combat. Return `combat_should_be_removed` when a proposed `combat` kind
|
||||||
|
is unsupported. Give a concise, transcript-grounded explanation for every
|
||||||
|
verdict.
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
id: dnd.scene_descriptions.validate_combat
|
||||||
|
version: "v1"
|
||||||
|
default_profile: dnd-extraction
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
- name: proposed_kind
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./instructions.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./proposed-kind.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
The proposed scene kind is:
|
||||||
|
|
||||||
|
{{ input "proposed_kind" }}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "notarius.dnd.scene_descriptions.combat_semantics.llm",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["verdict", "explanation"],
|
||||||
|
"properties": {
|
||||||
|
"verdict": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["approved", "combat_should_be_added", "combat_should_be_removed"]
|
||||||
|
},
|
||||||
|
"explanation": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 512
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
13
assets/dnd/shared/prompts/common-dnd-scene-combat-policy.md
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
Use `combat` only when substantive active combat materially organizes the
|
||||||
|
scene. Active combat includes initiative or turn exchanges, attacks, combat
|
||||||
|
spells, damage, saves, movement, or similarly sustained hostile action.
|
||||||
|
|
||||||
|
Do not use `combat` for planning or preparing for a possible fight; threats,
|
||||||
|
hostile dialogue, or a tense confrontation; immediate aftermath, looting,
|
||||||
|
healing, or discussion of a completed fight; a recap or in-world recollection
|
||||||
|
of earlier combat; or out-of-character rules discussion without active
|
||||||
|
encounter play.
|
||||||
|
|
||||||
|
When a chunk contains substantive active combat alongside brief setup, rules
|
||||||
|
clarification, interruption, phase transition, or immediate aftermath, classify
|
||||||
|
it as `combat`.
|
||||||
@@ -504,11 +504,28 @@ Available validator keys are:
|
|||||||
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
|
| Item occurrences | **extract/dnd/item-occurrences/shape**, **extract/dnd/item-occurrences/registry**, **extract/dnd/item-occurrences/source_refs**, **extract/dnd/item-occurrences/source_relatedness**, **normalize/dnd/item-occurrences/invariants** |
|
||||||
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
|
| Item registry | **extract/dnd/item-registry/shape**, **extract/dnd/item-registry/source_refs**, **extract/dnd/item-registry/source_relatedness**, **normalize/dnd/item-registry/identity** |
|
||||||
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
|
| NPC occurrences | **extract/dnd/npc-occurrences/shape**, **extract/dnd/npc-occurrences/registry**, **extract/dnd/npc-occurrences/source_refs**, **extract/dnd/npc-occurrences/source_relatedness**, **normalize/dnd/npc-occurrences/invariants** |
|
||||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** |
|
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **extract/dnd/scene-descriptions/combat_semantics** (LLM-backed, opt-in), **normalize/dnd/scene-descriptions/invariants** |
|
||||||
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
|
| Enemy events | **extract/dnd/enemy-events/shape**, **extract/dnd/enemy-events/engagements**, **extract/dnd/enemy-events/source_refs**, **extract/dnd/enemy-events/source_relatedness**, **normalize/dnd/enemy-events/invariants** |
|
||||||
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
|
| Location registry | **extract/dnd/location-registry/shape**, **extract/dnd/location-registry/source_refs**, **extract/dnd/location-registry/source_relatedness**, **normalize/dnd/location-registry/identity** |
|
||||||
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
|
| Location occurrences | **extract/dnd/location-occurrences/shape**, **extract/dnd/location-occurrences/registry**, **extract/dnd/location-occurrences/source_refs**, **extract/dnd/location-occurrences/source_relatedness**, **normalize/dnd/location-occurrences/invariants** |
|
||||||
|
|
||||||
|
`extract/dnd/scene-descriptions/combat_semantics` is not in a production default chain. To opt in, replace the scene extractor validator chain with the current ordered chain plus the semantic validator last, and set a positive producer retry budget if a rejection should request a corrected scene:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
extract:
|
||||||
|
module: dnd/scene-descriptions
|
||||||
|
retries: 1
|
||||||
|
validators:
|
||||||
|
- generic/valid_json
|
||||||
|
- extract/dnd/scene-descriptions/shape
|
||||||
|
- extract/dnd/scene-descriptions/source_refs
|
||||||
|
- generic/valid_json_schema
|
||||||
|
- extract/dnd/scene-descriptions/source_relatedness
|
||||||
|
- extract/dnd/scene-descriptions/combat_semantics
|
||||||
|
~~~
|
||||||
|
|
||||||
|
An override replaces, rather than extends, the default chain. See [Module Bindings And Validators](#module-bindings-and-validators) for binding, profile, repair, retry, and failure-policy rules.
|
||||||
|
|
||||||
When no override is configured, production D&D bindings use the following
|
When no override is configured, production D&D bindings use the following
|
||||||
ordered chains. Each row lists extract then normalize; spell chains are the
|
ordered chains. Each row lists extract then normalize; spell chains are the
|
||||||
same at both stages.
|
same at both stages.
|
||||||
|
|||||||
@@ -134,6 +134,64 @@ relatedness validators report advisory evidence concerns. The configured order
|
|||||||
is documented in
|
is documented in
|
||||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||||
|
|
||||||
|
The optional `extract/dnd/scene-descriptions/combat_semantics` validator is the
|
||||||
|
D&D family's LLM-backed review of only combat versus non-combat classification.
|
||||||
|
It selects the shared combat-policy prompt fragment, receives the proposed kind
|
||||||
|
and current chunk, and maps its verdict deterministically into producer
|
||||||
|
guidance. It does not assess titles, summaries, non-combat subtype, or scene
|
||||||
|
boundaries; deferred boundary-coherence review remains separate. It is opt-in;
|
||||||
|
[Configuration](../config.md) owns selection and retry/failure behavior.
|
||||||
|
|
||||||
|
### Combat-semantics provider evaluation
|
||||||
|
|
||||||
|
The human-reviewed corpus at
|
||||||
|
`internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json`
|
||||||
|
owns the proposed kind, expected verdict, and reviewer rationale for each
|
||||||
|
synthetic case. Its package test validates the fixture contract only. Provider
|
||||||
|
evaluation remains an explicit maintainer operation and must not be added to
|
||||||
|
the default offline test suite.
|
||||||
|
|
||||||
|
Use the following protocol before proposing default-chain inclusion:
|
||||||
|
|
||||||
|
1. Record the Notarius commit, prompt and schema fingerprints, provider, model,
|
||||||
|
profile settings, reasoning effort, structural-repair setting, number of
|
||||||
|
repetitions, and evaluation date before collecting results. Do not revise
|
||||||
|
expected verdicts merely to agree with provider output; a substantive corpus
|
||||||
|
correction requires independent human review.
|
||||||
|
2. Exercise the production validator construction and prompt assets from an
|
||||||
|
explicitly invoked, disposable evaluation driver or test in the validator
|
||||||
|
package. For each corpus case, construct transcript source units from the
|
||||||
|
listed IDs and text, assign matching per-unit source references, and use
|
||||||
|
`source.MaterializeChunkPlan` with one range spanning those units. Construct
|
||||||
|
exactly one scene whose ID and source range match that chunk and whose kind
|
||||||
|
is the case's `proposed_kind`; title and summary may use fixed placeholders
|
||||||
|
because the validator neither receives nor evaluates them. Invoke the typed
|
||||||
|
validator at extract stage through the production registry and scheduled LLM
|
||||||
|
client. Do not commit provider credentials, generated source material, or an
|
||||||
|
always-on live test.
|
||||||
|
3. Compare the resulting approval or reason code with `expected_verdict`.
|
||||||
|
Record an unexpected approval of an expected rejection as a false
|
||||||
|
acceptance, an unexpected rejection of an expected approval as a false
|
||||||
|
rejection, the opposite rejection direction separately, and any validator
|
||||||
|
execution failure separately from semantic accuracy. Retain per-case results
|
||||||
|
so repeated trials and systematic failure modes remain visible.
|
||||||
|
4. Evaluate producer correction separately with representative complete
|
||||||
|
scene-description runs configured as shown in
|
||||||
|
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||||
|
For every initial semantic rejection, record whether the next producer
|
||||||
|
attempt returns the requested combat status and is approved. Do not count a
|
||||||
|
PromptKit structural repair as a producer-correction attempt.
|
||||||
|
5. Run the correction evaluation with debug capture enabled and without reused
|
||||||
|
extraction checkpoints. Record added validator and producer calls, elapsed
|
||||||
|
latency, and cumulative prompt, completion, cached, and total token usage
|
||||||
|
from the debug attempt records. Compare these values with an otherwise
|
||||||
|
identical run whose scene-description chain omits the semantic validator.
|
||||||
|
|
||||||
|
The default-chain review must consider false acceptance, false rejection,
|
||||||
|
wrong-direction rejection, execution failure, producer-correction success,
|
||||||
|
added calls, latency, and token use together. A structurally successful
|
||||||
|
provider run alone is not evidence that the validator should become a default.
|
||||||
|
|
||||||
Every D&D rejection describes the correction in transcript-grounded domain
|
Every D&D rejection describes the correction in transcript-grounded domain
|
||||||
terms, using contextual names, artifact fields, and source segment ranges when
|
terms, using contextual names, artifact fields, and source segment ranges when
|
||||||
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
useful. The guidance must not ask the model to reproduce durable entity IDs,
|
||||||
|
|||||||
@@ -14,47 +14,8 @@ eligibility, and the separation of actionable process warnings from quality
|
|||||||
diagnostics. The remaining near-term work applies those completed foundations
|
diagnostics. The remaining near-term work applies those completed foundations
|
||||||
to domain review and empirical evaluation.
|
to domain review and empirical evaluation.
|
||||||
|
|
||||||
### D&D Combat Scene Semantic Validation
|
The active first production application is defined by
|
||||||
|
[D&D Combat Scene Semantic Validation](combat-scene-validation.md).
|
||||||
- Add an optional production LLM-backed D&D validator that determines whether
|
|
||||||
proposed scene boundaries and classifications represent substantive active
|
|
||||||
combat correctly. Its central quality goal is that active combat is kept in
|
|
||||||
coherent scenes classified as `combat`, rather than split incorrectly or
|
|
||||||
hidden inside scenes classified as `narrative`, `recap`, or `meta`.
|
|
||||||
- Resolve the validator's exact target before implementation. The current
|
|
||||||
`dnd/scenes` chunker owns only complete, gap-free source ranges, while the
|
|
||||||
per-chunk `dnd/scene-descriptions` extractor owns the `combat`, `narrative`,
|
|
||||||
`recap`, and `meta` classification. The preferred initial placement is
|
|
||||||
therefore an extract-stage validator for `dnd/scene-descriptions`, where it
|
|
||||||
can compare one proposed kind with the corresponding transcript chunk.
|
|
||||||
- Consider a chunk-stage LLM validator only for a distinct boundary-coherence
|
|
||||||
question that can be answered from the complete transcript and proposed
|
|
||||||
range map, such as whether one continuous combat was fragmented across
|
|
||||||
inappropriate scene boundaries. Do not duplicate the same classification
|
|
||||||
judgment at both stages. Moving classification into chunk-plan annotations
|
|
||||||
would change the deliberately minimal, annotation-free chunk contract and
|
|
||||||
requires an explicit architecture review before it is selected.
|
|
||||||
- Validate both false negatives and false positives: a non-combat kind must not
|
|
||||||
omit substantive active combat, and a combat kind must be supported by such
|
|
||||||
combat. Keep the existing deterministic downstream rule that combat-turn
|
|
||||||
extraction runs only for an exact `combat` scene classification; semantic
|
|
||||||
review improves the upstream classification but does not replace that gate.
|
|
||||||
- Run the semantic validator through PromptKit, use a minimal required-field
|
|
||||||
structured response schema, and let PromptKit repair structural validator
|
|
||||||
output within its bounded budget. A contract-invalid final validator response
|
|
||||||
is a validator execution failure, not a semantic rejection and not a reason
|
|
||||||
to recursively validate the validator.
|
|
||||||
- Evaluate the prompt and decision policy against a small human-reviewed set
|
|
||||||
containing combat setup, active turns, interruptions, multi-phase encounters,
|
|
||||||
brief rules discussion, aftermath, recalled combat, and false-positive
|
|
||||||
hostile dialogue. Measure false acceptance, false rejection, retry success,
|
|
||||||
added calls, latency, and token cost before placing it in the production
|
|
||||||
default chain.
|
|
||||||
- An ADR is not required if classification remains owned by
|
|
||||||
`dnd/scene-descriptions` and the validator follows the generic validation ADR.
|
|
||||||
Create or supersede an ADR if the work transfers scene classification into
|
|
||||||
the chunker or otherwise changes stage ownership or the durable chunk-plan
|
|
||||||
contract.
|
|
||||||
|
|
||||||
## Near-Term D&D Pipeline
|
## Near-Term D&D Pipeline
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ var promptAssetManifest = shared.PromptAssetManifest{
|
|||||||
"common-dnd-identity.md",
|
"common-dnd-identity.md",
|
||||||
"common-dnd-references.md",
|
"common-dnd-references.md",
|
||||||
"common-dnd-transcript-chunk.md",
|
"common-dnd-transcript-chunk.md",
|
||||||
|
"common-dnd-scene-combat-policy.md",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,23 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
|||||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||||
}
|
}
|
||||||
|
const combatPolicySentinel = "substantive active combat materially organizes the"
|
||||||
|
transcriptIndex := -1
|
||||||
|
policyIndex := -1
|
||||||
|
instructionsIndex := len(prepared.Messages) - 1
|
||||||
|
policyOccurrences := 0
|
||||||
|
for index, message := range prepared.Messages {
|
||||||
|
if strings.Contains(message.Content, "scene-description-transcript") {
|
||||||
|
transcriptIndex = index
|
||||||
|
}
|
||||||
|
if count := strings.Count(message.Content, combatPolicySentinel); count > 0 {
|
||||||
|
policyIndex = index
|
||||||
|
policyOccurrences += count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex {
|
||||||
|
t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ import (
|
|||||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||||
)
|
)
|
||||||
@@ -157,6 +158,7 @@ func registerPromptAssets(assets *llm.AssetRegistry) error {
|
|||||||
{name: "item registry normalization prompt assets", register: func() error { return itemregistrynormalize.RegisterPromptAssets(assets) }},
|
{name: "item registry normalization prompt assets", register: func() error { return itemregistrynormalize.RegisterPromptAssets(assets) }},
|
||||||
{name: "npc occurrences prompt assets", register: func() error { return occurrenceextract.RegisterPromptAssets(assets) }},
|
{name: "npc occurrences prompt assets", register: func() error { return occurrenceextract.RegisterPromptAssets(assets) }},
|
||||||
{name: "scene descriptions prompt assets", register: func() error { return scenedescriptionextract.RegisterPromptAssets(assets) }},
|
{name: "scene descriptions prompt assets", register: func() error { return scenedescriptionextract.RegisterPromptAssets(assets) }},
|
||||||
|
{name: "scene combat-semantics validator prompt assets", register: func() error { return combatsemantics.RegisterPromptAssets(assets) }},
|
||||||
{name: "location registry prompt assets", register: func() error { return locationextract.RegisterPromptAssets(assets) }},
|
{name: "location registry prompt assets", register: func() error { return locationextract.RegisterPromptAssets(assets) }},
|
||||||
{name: "location normalization prompt assets", register: func() error { return locationnormalize.RegisterPromptAssets(assets) }},
|
{name: "location normalization prompt assets", register: func() error { return locationnormalize.RegisterPromptAssets(assets) }},
|
||||||
{name: "location occurrences prompt assets", register: func() error { return locationoccurrenceextract.RegisterPromptAssets(assets) }},
|
{name: "location occurrences prompt assets", register: func() error { return locationoccurrenceextract.RegisterPromptAssets(assets) }},
|
||||||
|
|||||||
@@ -23,13 +23,14 @@ import (
|
|||||||
|
|
||||||
func TestExtractionPromptComposition(t *testing.T) {
|
func TestExtractionPromptComposition(t *testing.T) {
|
||||||
const (
|
const (
|
||||||
transcriptSentinel = "shared-transcript-sentinel"
|
transcriptSentinel = "shared-transcript-sentinel"
|
||||||
playersSentinel = "shared-players-sentinel"
|
playersSentinel = "shared-players-sentinel"
|
||||||
partySentinel = "shared-party-sentinel"
|
partySentinel = "shared-party-sentinel"
|
||||||
glossarySentinel = "shared-glossary-sentinel"
|
glossarySentinel = "shared-glossary-sentinel"
|
||||||
npcSentinel = "npc-registry-sentinel"
|
npcSentinel = "npc-registry-sentinel"
|
||||||
catalogSentinel = "spell-catalog-sentinel"
|
catalogSentinel = "spell-catalog-sentinel"
|
||||||
evidenceSentinel = "Transcript units are the only evidence"
|
evidenceSentinel = "Transcript units are the only evidence"
|
||||||
|
combatPolicySentinel = "substantive active combat materially organizes the"
|
||||||
)
|
)
|
||||||
registry := llm.NewAssetRegistry()
|
registry := llm.NewAssetRegistry()
|
||||||
if err := registerPromptAssets(registry); err != nil {
|
if err := registerPromptAssets(registry); err != nil {
|
||||||
@@ -43,11 +44,12 @@ func TestExtractionPromptComposition(t *testing.T) {
|
|||||||
"glossary": promptkit.Inline(glossarySentinel),
|
"glossary": promptkit.Inline(glossarySentinel),
|
||||||
}
|
}
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
promptID string
|
promptID string
|
||||||
promptVersion string
|
promptVersion string
|
||||||
inputs map[string]promptkit.ArtifactRef
|
inputs map[string]promptkit.ArtifactRef
|
||||||
suffixGroups [][]string
|
suffixGroups [][]string
|
||||||
|
middleSentinel string
|
||||||
}{
|
}{
|
||||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||||
@@ -55,7 +57,7 @@ func TestExtractionPromptComposition(t *testing.T) {
|
|||||||
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
||||||
}), suffixGroups: [][]string{{evidenceSentinel}, {"item-registry-sentinel"}}},
|
}), suffixGroups: [][]string{{evidenceSentinel}, {"item-registry-sentinel"}}},
|
||||||
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
|
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs, middleSentinel: combatPolicySentinel},
|
||||||
{
|
{
|
||||||
name: "combat turns",
|
name: "combat turns",
|
||||||
promptID: combatextract.PromptID,
|
promptID: combatextract.PromptID,
|
||||||
@@ -155,6 +157,12 @@ func TestExtractionPromptComposition(t *testing.T) {
|
|||||||
previousIndex = groupIndex
|
previousIndex = groupIndex
|
||||||
}
|
}
|
||||||
instructionIndex := len(prepared.Messages) - 1
|
instructionIndex := len(prepared.Messages) - 1
|
||||||
|
if testCase.middleSentinel != "" {
|
||||||
|
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
|
||||||
|
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
|
||||||
|
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
|
||||||
|
}
|
||||||
|
}
|
||||||
if instructionIndex <= previousIndex {
|
if instructionIndex <= previousIndex {
|
||||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import (
|
|||||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterAddsDNDFamily(t *testing.T) {
|
func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||||
@@ -57,6 +58,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
"dnd.item_registry.normalize/prompt.yaml",
|
"dnd.item_registry.normalize/prompt.yaml",
|
||||||
"dnd.npc_occurrences/prompt.yaml",
|
"dnd.npc_occurrences/prompt.yaml",
|
||||||
"dnd.scene_descriptions/prompt.yaml",
|
"dnd.scene_descriptions/prompt.yaml",
|
||||||
|
"dnd.scene_descriptions.validate_combat/prompt.yaml",
|
||||||
"dnd.npc_registry.normalize/prompt.yaml",
|
"dnd.npc_registry.normalize/prompt.yaml",
|
||||||
"dnd.location_registry/prompt.yaml",
|
"dnd.location_registry/prompt.yaml",
|
||||||
"dnd.location_registry.normalize/prompt.yaml",
|
"dnd.location_registry.normalize/prompt.yaml",
|
||||||
@@ -124,10 +126,14 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
|||||||
"extract/dnd/scene-descriptions/shape",
|
"extract/dnd/scene-descriptions/shape",
|
||||||
"extract/dnd/scene-descriptions/source_refs",
|
"extract/dnd/scene-descriptions/source_refs",
|
||||||
"extract/dnd/scene-descriptions/source_relatedness",
|
"extract/dnd/scene-descriptions/source_relatedness",
|
||||||
|
combatsemantics.Key,
|
||||||
"normalize/dnd/scene-descriptions/invariants",
|
"normalize/dnd/scene-descriptions/invariants",
|
||||||
"generic/always_accept",
|
"generic/always_accept",
|
||||||
"generic/always_reject",
|
"generic/always_reject",
|
||||||
})
|
})
|
||||||
|
if spec, ok := registries.Validators.Spec(combatsemantics.Key); !ok || spec.Key != combatsemantics.Key || spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
t.Fatalf("combat semantics validator spec = %#v, present=%t; want registered LLM-backed scene validator", spec, ok)
|
||||||
|
}
|
||||||
locationExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
locationExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
||||||
locationNormalizeChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("normalize/dnd/location-registry/identity"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
locationNormalizeChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-registry/shape"), pipeline.Binding("normalize/dnd/location-registry/identity"), pipeline.Binding("extract/dnd/location-registry/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-registry/source_relatedness")}
|
||||||
occurrenceExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-occurrences/shape"), pipeline.Binding("extract/dnd/location-occurrences/registry"), pipeline.Binding("extract/dnd/location-occurrences/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-occurrences/source_relatedness")}
|
occurrenceExtractChain := []pipeline.ModuleBinding{pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/location-occurrences/shape"), pipeline.Binding("extract/dnd/location-occurrences/registry"), pipeline.Binding("extract/dnd/location-occurrences/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/location-occurrences/source_relatedness")}
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ import (
|
|||||||
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
||||||
npcsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
npcsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
||||||
npcrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_relatedness"
|
npcrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_relatedness"
|
||||||
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||||
sceneinvariants "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/invariants"
|
sceneinvariants "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/invariants"
|
||||||
sceneshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
sceneshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||||
scenerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
|
scenerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
|
||||||
@@ -87,6 +88,7 @@ func registerValidators(registries pipeline.Registries) error {
|
|||||||
{name: "scene description shape validator", register: func() error { return sceneshape.Register(registries.Validators) }},
|
{name: "scene description shape validator", register: func() error { return sceneshape.Register(registries.Validators) }},
|
||||||
{name: "scene description source references validator", register: func() error { return scenerefs.Register(registries.Validators) }},
|
{name: "scene description source references validator", register: func() error { return scenerefs.Register(registries.Validators) }},
|
||||||
{name: "scene description source relatedness validator", register: func() error { return scenerelatedness.Register(registries.Validators) }},
|
{name: "scene description source relatedness validator", register: func() error { return scenerelatedness.Register(registries.Validators) }},
|
||||||
|
{name: "scene description combat semantics validator", register: func() error { return combatsemantics.Register(registries.Validators) }},
|
||||||
{name: "scene description normalized invariants validator", register: func() error { return sceneinvariants.Register(registries.Validators) }},
|
{name: "scene description normalized invariants validator", register: func() error { return sceneinvariants.Register(registries.Validators) }},
|
||||||
{name: "location shape validator", register: func() error { return locationshape.Register(registries.Validators) }},
|
{name: "location shape validator", register: func() error { return locationshape.Register(registries.Validators) }},
|
||||||
{name: "location identity validator", register: func() error { return locationidentity.Register(registries.Validators) }},
|
{name: "location identity validator", register: func() error { return locationidentity.Register(registries.Validators) }},
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ var sharedPromptPaths = map[string]string{
|
|||||||
"common-dnd-identity.md": "prompts/common-dnd-identity.md",
|
"common-dnd-identity.md": "prompts/common-dnd-identity.md",
|
||||||
"common-dnd-transcript-full.md": "prompts/common-dnd-transcript-full.md",
|
"common-dnd-transcript-full.md": "prompts/common-dnd-transcript-full.md",
|
||||||
"common-dnd-transcript-chunk.md": "prompts/common-dnd-transcript-chunk.md",
|
"common-dnd-transcript-chunk.md": "prompts/common-dnd-transcript-chunk.md",
|
||||||
|
"common-dnd-scene-combat-policy.md": "prompts/common-dnd-scene-combat-policy.md",
|
||||||
"common-dnd-references.md": "prompts/common-dnd-references.md",
|
"common-dnd-references.md": "prompts/common-dnd-references.md",
|
||||||
"common-dnd-npc-registry.md": "prompts/common-dnd-npc-registry.md",
|
"common-dnd-npc-registry.md": "prompts/common-dnd-npc-registry.md",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
)
|
||||||
|
|
||||||
|
const promptAssetRoot = "assets/prompts"
|
||||||
|
|
||||||
|
var promptAssetManifest = shared.PromptAssetManifest{
|
||||||
|
ModuleDir: PromptID,
|
||||||
|
ModuleFiles: []promptfs.ModulePromptFile{
|
||||||
|
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||||
|
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||||
|
{Name: "proposed-kind.md", Path: "prompts/proposed-kind.md"},
|
||||||
|
},
|
||||||
|
SharedFiles: []string{
|
||||||
|
"common-dnd-system.md",
|
||||||
|
"common-dnd-scene-combat-policy.md",
|
||||||
|
"common-dnd-transcript-chunk.md",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleAssetFS() (fs.FS, error) {
|
||||||
|
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err)
|
||||||
|
}
|
||||||
|
return assets, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||||
|
assets, err := moduleAssetFS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err)
|
||||||
|
}
|
||||||
|
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
schemas, err := fs.Sub(assets, "schemas")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err)
|
||||||
|
}
|
||||||
|
return registry.RegisterSchemaFS(schemas, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptAssetMetadata() (string, error) {
|
||||||
|
promptAssetHashOnce.Do(func() {
|
||||||
|
assets, err := moduleAssetFS()
|
||||||
|
if err != nil {
|
||||||
|
promptAssetHashErr = err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||||
|
})
|
||||||
|
return promptAssetHash, promptAssetHashErr
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
promptAssetHashOnce sync.Once
|
||||||
|
promptAssetHash string
|
||||||
|
promptAssetHashErr error
|
||||||
|
)
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
|
||||||
|
registry := llm.NewAssetRegistry()
|
||||||
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
|
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
engine := newCombatSemanticsPromptEngine(t, registry)
|
||||||
|
inputs := map[string]promptkit.ArtifactRef{
|
||||||
|
"proposed_kind": promptkit.Inline("narrative"),
|
||||||
|
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
|
||||||
|
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 5 {
|
||||||
|
t.Fatalf("prepared message count = %d, want 5", len(prepared.Messages))
|
||||||
|
}
|
||||||
|
for index, message := range prepared.Messages {
|
||||||
|
wantEphemeral := index == 3 || index == 4
|
||||||
|
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
|
||||||
|
if gotEphemeral != wantEphemeral {
|
||||||
|
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
|
||||||
|
assertRenderedExactlyOnce(t, prepared.Messages, "narrative", 3)
|
||||||
|
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 4)
|
||||||
|
for name := range inputs {
|
||||||
|
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
|
||||||
|
for inputName, input := range inputs {
|
||||||
|
if inputName != name {
|
||||||
|
missing[inputName] = input
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing,
|
||||||
|
}); err == nil {
|
||||||
|
t.Fatalf("Prepare() without required %q input = nil error", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
|
||||||
|
hash, err := promptAssetMetadata()
|
||||||
|
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||||
|
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(hash, "combat_should_be_added") {
|
||||||
|
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
options, err := registry.PromptKitOptions()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "combat-semantics-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-semantics-test-model",
|
||||||
|
})))
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {
|
||||||
|
t.Helper()
|
||||||
|
index := -1
|
||||||
|
occurrences := 0
|
||||||
|
for messageIndex, message := range messages {
|
||||||
|
count := strings.Count(message.Content, sentinel)
|
||||||
|
if count > 0 {
|
||||||
|
index = messageIndex
|
||||||
|
occurrences += count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if occurrences != 1 || index != wantIndex {
|
||||||
|
t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
|
||||||
|
const (
|
||||||
|
PromptID = "dnd.scene_descriptions.validate_combat"
|
||||||
|
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm")
|
||||||
|
ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm"
|
||||||
|
ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1"
|
||||||
|
SchemaVersion = "v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||||
|
assets, err := moduleAssetFS()
|
||||||
|
if err != nil {
|
||||||
|
return llm.ResponseSchema{}, err
|
||||||
|
}
|
||||||
|
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||||
|
Key: ResponseSchemaKey,
|
||||||
|
ID: ResponseSchemaID,
|
||||||
|
Version: SchemaVersion,
|
||||||
|
Name: ResponseSchemaName,
|
||||||
|
AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json",
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||||
|
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
|
||||||
|
}
|
||||||
|
for _, verdict := range []string{"approved", "combat_should_be_added", "combat_should_be_removed"} {
|
||||||
|
if err := validateCombatSemanticsSchema(map[string]any{"verdict": verdict, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
|
||||||
|
t.Fatalf("valid %q verdict rejected: %v", verdict, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
response map[string]any
|
||||||
|
}{
|
||||||
|
{name: "missing explanation", response: map[string]any{"verdict": "approved"}},
|
||||||
|
{name: "missing verdict", response: map[string]any{"explanation": "The proposed kind is supported."}},
|
||||||
|
{name: "unknown property", response: map[string]any{"verdict": "approved", "explanation": "The proposed kind is supported.", "confidence": 1}},
|
||||||
|
{name: "wrong verdict type", response: map[string]any{"verdict": 1, "explanation": "The proposed kind is supported."}},
|
||||||
|
{name: "wrong explanation type", response: map[string]any{"verdict": "approved", "explanation": 1}},
|
||||||
|
{name: "unsupported verdict", response: map[string]any{"verdict": "uncertain", "explanation": "The proposed kind is supported."}},
|
||||||
|
{name: "blank explanation", response: map[string]any{"verdict": "approved", "explanation": ""}},
|
||||||
|
{name: "oversized explanation", response: map[string]any{"verdict": "approved", "explanation": strings.Repeat("a", 513)}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {
|
||||||
|
t.Fatal("invalid private response accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) {
|
||||||
|
t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||||
|
first, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first.JSONSchema[0] = '['
|
||||||
|
second, err := loadResponseSchema()
|
||||||
|
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||||
|
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||||
|
}
|
||||||
|
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
||||||
|
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
||||||
|
instanceContent, err := json.Marshal(instance)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compiled, err := compiler.Compile("schema.json")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return compiled.Validate(parsedInstance)
|
||||||
|
}
|
||||||
10
internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json
vendored
Normal file
10
internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[
|
||||||
|
{"name":"active encounter","transcript_units":[{"id":1,"text":"Roll initiative; the goblins attack."},{"id":2,"text":"The ranger hits and deals damage."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"Initiative and hostile actions organize the chunk."},
|
||||||
|
{"name":"combat after setup","transcript_units":[{"id":1,"text":"They open the crypt door."},{"id":2,"text":"Skeletons attack and turns begin."}],"proposed_kind":"narrative","expected_verdict":"combat_should_be_added","reviewer_rationale":"Brief setup does not displace substantive active combat."},
|
||||||
|
{"name":"combat aftermath","transcript_units":[{"id":1,"text":"The last enemy falls."},{"id":2,"text":"They search bodies and heal."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Looting and healing after a completed fight are not active combat."},
|
||||||
|
{"name":"multi phase encounter","transcript_units":[{"id":1,"text":"The dragon attacks."},{"id":2,"text":"After a rules clarification, its next turn begins."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"A brief rules interruption does not end the encounter."},
|
||||||
|
{"name":"planning","transcript_units":[{"id":1,"text":"They plan how to ambush the guard."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Planning a possible fight is not active encounter play."},
|
||||||
|
{"name":"hostile dialogue","transcript_units":[{"id":1,"text":"The captain threatens them and they argue."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Threats and hostile dialogue alone are insufficient."},
|
||||||
|
{"name":"recap recollection","transcript_units":[{"id":1,"text":"They recap last session's battle with the lich."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Recounting earlier combat is not current active combat."},
|
||||||
|
{"name":"rules discussion","transcript_units":[{"id":1,"text":"The table discusses how concentration works."}],"proposed_kind":"meta","expected_verdict":"approved","reviewer_rationale":"Sustained out-of-character rules discussion has no active encounter."}
|
||||||
|
]
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||||
|
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||||
|
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||||
|
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
|
||||||
|
maximumExplanationRunes = 512
|
||||||
|
)
|
||||||
|
|
||||||
|
type Options struct{}
|
||||||
|
|
||||||
|
type completionResponse struct {
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Explanation string `json:"explanation"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Validator struct {
|
||||||
|
llm contracts.StructuredLLMClient
|
||||||
|
promptSHA string
|
||||||
|
responseSchemaSHA string
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||||
|
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
|
||||||
|
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||||
|
|
||||||
|
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
|
||||||
|
if llmClient == nil {
|
||||||
|
return nil, validatorErrorf("LLM client must not be nil")
|
||||||
|
}
|
||||||
|
promptSHA, err := promptAssetMetadata()
|
||||||
|
if err != nil {
|
||||||
|
return nil, validatorErrorf("load prompt metadata: %w", err)
|
||||||
|
}
|
||||||
|
responseSchema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
return nil, validatorErrorf("load response schema: %w", err)
|
||||||
|
}
|
||||||
|
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Name() string { return Key }
|
||||||
|
|
||||||
|
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) ManifestMetadata() map[string]any {
|
||||||
|
if v == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": SchemaVersion,
|
||||||
|
"prompt_sha256": v.promptSHA,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
"response_schema_version": SchemaVersion,
|
||||||
|
"response_schema_sha256": v.responseSchemaSHA,
|
||||||
|
"combat_policy": combatPolicy,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||||
|
if v == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []pipeline.CheckpointFingerprint{
|
||||||
|
{Name: "prompt", Value: v.promptSHA},
|
||||||
|
{Name: "response_schema", Value: v.responseSchemaSHA},
|
||||||
|
{Name: "combat_policy", Value: combatPolicy},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||||
|
if v == nil {
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
||||||
|
}
|
||||||
|
if v.llm == nil {
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
|
||||||
|
}
|
||||||
|
sourceInput, scene, err := validateRequest(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var response completionResponse
|
||||||
|
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
|
StageName: Key,
|
||||||
|
PromptID: PromptID,
|
||||||
|
PromptVersion: SchemaVersion,
|
||||||
|
ProfileID: req.LLMProfile,
|
||||||
|
SessionID: req.SessionID,
|
||||||
|
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": shared.TranscriptPromptMaterial(sourceInput),
|
||||||
|
"proposed_kind": contracts.NewLLMInputMaterial("proposed_kind", "text/plain", []byte(scene.Kind), "", ""),
|
||||||
|
},
|
||||||
|
}, &response)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
|
||||||
|
}
|
||||||
|
return interpretResponse(response, scene.Kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context error before validation: %w", err)
|
||||||
|
}
|
||||||
|
if req.Stage != string(pipeline.StageExtract) {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
||||||
|
}
|
||||||
|
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||||
|
if err != nil {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.Chunk.ID) == "" || req.Chunk.ID != strings.TrimSpace(req.Chunk.ID) {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk ID must be nonblank and trimmed")
|
||||||
|
}
|
||||||
|
if req.Chunk.SourceID != req.Source.ID {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk source ID %q does not match source %q", req.Chunk.SourceID, req.Source.ID)
|
||||||
|
}
|
||||||
|
if err := source.NewDocumentIndex(req.Source).ValidateRef(req.Chunk.Ref); err != nil {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate current chunk source range: %w", err)
|
||||||
|
}
|
||||||
|
if err := shape.ValidateForStage(req.Value, req.Stage); err != nil {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate scene description shape: %w", err)
|
||||||
|
}
|
||||||
|
scene := req.Value.Scenes[0]
|
||||||
|
if scene.ID != req.Chunk.ID {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene ID must equal current chunk ID")
|
||||||
|
}
|
||||||
|
if scene.SourceRef != req.Chunk.Ref {
|
||||||
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene source range must equal current chunk range")
|
||||||
|
}
|
||||||
|
return sourceInput, scene, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||||
|
explanation, err := validateExplanation(response.Explanation)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
switch response.Verdict {
|
||||||
|
case "approved":
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
case "combat_should_be_added":
|
||||||
|
if proposedKind == dnd.SceneKindCombat {
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_added verdict is inconsistent with proposed combat kind")
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||||
|
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||||
|
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||||
|
}, nil
|
||||||
|
case "combat_should_be_removed":
|
||||||
|
if proposedKind != dnd.SceneKindCombat {
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_removed verdict is inconsistent with proposed non-combat kind")
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||||
|
Message: "The current chunk does not support a combat classification.",
|
||||||
|
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics verdict %q", response.Verdict)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateExplanation(value string) (string, error) {
|
||||||
|
if !utf8.ValidString(value) {
|
||||||
|
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", validatorErrorf("validator explanation must not be blank")
|
||||||
|
}
|
||||||
|
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
||||||
|
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||||
|
}
|
||||||
|
return trimmed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Spec() pipeline.ValidatorSpec {
|
||||||
|
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
|
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) {
|
||||||
|
options, err := DecodeOptions(request.Options)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return New(request.Dependencies.LLM, options)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func DecodeOptions(options map[string]any) (Options, error) {
|
||||||
|
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||||
|
return Options{}, err
|
||||||
|
}
|
||||||
|
return Options{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||||
|
|
||||||
|
func validatorErrorf(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
||||||
|
request := validRequest(dnd.SceneKindNarrative)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
kind dnd.SceneKind
|
||||||
|
response completionResponse
|
||||||
|
approved bool
|
||||||
|
reasonCode string
|
||||||
|
guidance string
|
||||||
|
correction string
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}, approved: true},
|
||||||
|
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "approved", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
||||||
|
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
||||||
|
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
||||||
|
{name: "inconsistent added", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks."}, wantError: "inconsistent"},
|
||||||
|
{name: "inconsistent removed", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans."}, wantError: "inconsistent"},
|
||||||
|
{name: "unknown verdict", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
||||||
|
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: " "}, wantError: "blank"},
|
||||||
|
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
||||||
|
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
request.Value.Scenes[0].Kind = test.kind
|
||||||
|
client := &fakeCombatSemanticsClient{response: test.response}
|
||||||
|
validator := newValidator(t, client)
|
||||||
|
result, err := validator.Validate(context.Background(), request)
|
||||||
|
if test.wantError != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
||||||
|
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
||||||
|
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
||||||
|
}
|
||||||
|
if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) {
|
||||||
|
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
||||||
|
}
|
||||||
|
if err := contracts.ValidateValidationResult(result); err != nil {
|
||||||
|
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorUsesOnlyChunkTranscriptAndProposedKind(t *testing.T) {
|
||||||
|
request := validRequest(dnd.SceneKindNarrative)
|
||||||
|
repairAttempts := 2
|
||||||
|
request.LLMProfile = "validator-profile"
|
||||||
|
request.SessionID = "validator-session"
|
||||||
|
request.StructuredOutputRepairAttempts = &repairAttempts
|
||||||
|
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
|
||||||
|
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
|
||||||
|
validator := newValidator(t, client)
|
||||||
|
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
|
||||||
|
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("completion count = %d, want 1", len(client.requests))
|
||||||
|
}
|
||||||
|
completed := client.requests[0]
|
||||||
|
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
|
||||||
|
t.Fatalf("completion request = %#v", completed)
|
||||||
|
}
|
||||||
|
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"proposed_kind", "transcript"}) || string(completed.Inputs["proposed_kind"].Content) != "narrative" || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
|
||||||
|
t.Fatalf("completion inputs = %#v", completed.Inputs)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
|
||||||
|
for name, input := range completed.Inputs {
|
||||||
|
if strings.Contains(string(input.Content), forbidden) {
|
||||||
|
t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||||
|
base := validRequest(dnd.SceneKindNarrative)
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
ctx context.Context
|
||||||
|
mutate func(*contracts.TypedValidationRequest[dnd.SceneDescriptionList])
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "nil context", ctx: nil, wantErr: "context must not be nil"},
|
||||||
|
{name: "canceled context", ctx: canceled, wantErr: "context error"},
|
||||||
|
{name: "wrong stage", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||||
|
req.Stage = string(pipeline.StageNormalize)
|
||||||
|
}, wantErr: "stage must be extract"},
|
||||||
|
{name: "missing source", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) { req.Source = nil }, wantErr: "source must not be nil"},
|
||||||
|
{name: "missing chunk", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) { req.Chunk = nil }, wantErr: "chunk must not be nil"},
|
||||||
|
{name: "mismatched source input", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||||
|
req.SourceInput.Content = []byte(`{"other":true}`)
|
||||||
|
}, wantErr: "must match chunk"},
|
||||||
|
{name: "multiple scenes", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||||
|
req.Value.Scenes = append(req.Value.Scenes, req.Value.Scenes[0])
|
||||||
|
}, wantErr: "exactly one"},
|
||||||
|
{name: "wrong scene ID", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||||
|
req.Value.Scenes[0].ID = "other"
|
||||||
|
}, wantErr: "scene ID"},
|
||||||
|
{name: "wrong scene range", ctx: context.Background(), mutate: func(req *contracts.TypedValidationRequest[dnd.SceneDescriptionList]) {
|
||||||
|
req.Value.Scenes[0].SourceRef.EndUnitID = 1
|
||||||
|
}, wantErr: "scene source range"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
request := cloneValidationRequest(base)
|
||||||
|
if test.mutate != nil {
|
||||||
|
test.mutate(&request)
|
||||||
|
}
|
||||||
|
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
|
||||||
|
_, err := newValidator(t, client).Validate(test.ctx, request)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
|
||||||
|
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
|
||||||
|
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") {
|
||||||
|
t.Fatalf("completion failure error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
||||||
|
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||||
|
t.Fatalf("New(nil) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||||
|
t.Fatal("DecodeOptions() accepted unknown options")
|
||||||
|
}
|
||||||
|
validator := newValidator(t, &fakeCombatSemanticsClient{})
|
||||||
|
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" {
|
||||||
|
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
|
||||||
|
}
|
||||||
|
metadata, err := json.Marshal(validator.ManifestMetadata())
|
||||||
|
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "combat_should_be_added") {
|
||||||
|
t.Fatalf("manifest metadata = %s, %v", metadata, err)
|
||||||
|
}
|
||||||
|
fingerprints := validator.CheckpointFingerprints()
|
||||||
|
if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) {
|
||||||
|
t.Fatalf("fingerprints = %#v", fingerprints)
|
||||||
|
}
|
||||||
|
registry := pipeline.NewValidatorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v", err)
|
||||||
|
}
|
||||||
|
if spec, ok := registry.Spec(Key); !ok || spec.Key != Key || spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
t.Fatalf("registered spec = %#v, present=%t", spec, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
|
||||||
|
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var cases []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
TranscriptUnits []struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"transcript_units"`
|
||||||
|
ProposedKind string `json:"proposed_kind"`
|
||||||
|
ExpectedVerdict string `json:"expected_verdict"`
|
||||||
|
ReviewerRationale string `json:"reviewer_rationale"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(&cases); err != nil {
|
||||||
|
t.Fatalf("decode evaluation cases: %v", err)
|
||||||
|
}
|
||||||
|
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||||
|
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
||||||
|
}
|
||||||
|
seenNames, seenVerdicts := map[string]bool{}, map[string]bool{}
|
||||||
|
for _, value := range cases {
|
||||||
|
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
||||||
|
t.Fatalf("invalid evaluation case: %#v", value)
|
||||||
|
}
|
||||||
|
seenNames[value.Name] = true
|
||||||
|
switch value.ProposedKind {
|
||||||
|
case "combat", "narrative", "recap", "meta":
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
||||||
|
}
|
||||||
|
switch value.ExpectedVerdict {
|
||||||
|
case "approved", "combat_should_be_added", "combat_should_be_removed":
|
||||||
|
seenVerdicts[value.ExpectedVerdict] = true
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported verdict %q", value.ExpectedVerdict)
|
||||||
|
}
|
||||||
|
for _, unit := range value.TranscriptUnits {
|
||||||
|
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
||||||
|
t.Fatalf("invalid transcript unit: %#v", unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(seenVerdicts) != 3 {
|
||||||
|
t.Fatalf("evaluation verdicts = %#v, want all classes", seenVerdicts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator {
|
||||||
|
t.Helper()
|
||||||
|
validator, err := New(client, Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
return validator
|
||||||
|
}
|
||||||
|
|
||||||
|
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||||
|
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
|
||||||
|
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
|
||||||
|
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||||
|
request.SourceInput = request.SourceInput.Clone()
|
||||||
|
chunk := *request.Chunk
|
||||||
|
chunk.Content = append([]byte(nil), chunk.Content...)
|
||||||
|
chunk.Units = append([]source.SourceUnit(nil), chunk.Units...)
|
||||||
|
request.Chunk = &chunk
|
||||||
|
request.Value.Scenes = append([]dnd.SceneDescription(nil), request.Value.Scenes...)
|
||||||
|
return request
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedInputNames(inputs contracts.LLMInputSet) []string {
|
||||||
|
names := make([]string, 0, len(inputs))
|
||||||
|
for name := range inputs {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
if len(names) == 2 && names[0] == "transcript" {
|
||||||
|
names[0], names[1] = names[1], names[0]
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeCombatSemanticsClient struct {
|
||||||
|
response completionResponse
|
||||||
|
err error
|
||||||
|
requests []contracts.StructuredCompletionRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
cloned, err := contracts.CloneStructuredCompletionRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
client.requests = append(client.requests, cloned)
|
||||||
|
if client.err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, client.err
|
||||||
|
}
|
||||||
|
response, ok := out.(*completionResponse)
|
||||||
|
if !ok {
|
||||||
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target")
|
||||||
|
}
|
||||||
|
*response = client.response
|
||||||
|
content, err := json.Marshal(client.response)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||||
|
}
|
||||||
@@ -28,6 +28,7 @@ import (
|
|||||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||||
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
||||||
@@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
|||||||
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||||
|
registries := productionNPCRegistries(t)
|
||||||
|
configValue := loadGroundedPipelineConfig(t)
|
||||||
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||||
|
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
||||||
|
sceneLane.Extract.Retries = 1
|
||||||
|
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding("generic/valid_json"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
|
||||||
|
pipeline.Binding("generic/valid_json_schema"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
|
||||||
|
pipeline.Binding(combatsemantics.Key),
|
||||||
|
}}
|
||||||
|
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
|
||||||
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||||
|
|
||||||
|
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
|
||||||
|
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||||
|
if len(output.Rejected) != 0 {
|
||||||
|
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
|
||||||
|
}
|
||||||
|
for _, group := range output.Diagnostics.Groups {
|
||||||
|
if group.Origin.ValidatorKey == combatsemantics.Key {
|
||||||
|
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var scenes dnd.SceneDescriptionList
|
||||||
|
for _, serialized := range output.NormalizeOutputs {
|
||||||
|
if serialized.LaneID == "scene-descriptions" {
|
||||||
|
var err error
|
||||||
|
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||||
|
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
||||||
|
}
|
||||||
|
requests := client.requestsSnapshot()
|
||||||
|
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
|
||||||
|
for _, request := range requests {
|
||||||
|
switch request.PromptID {
|
||||||
|
case sceneextract.PromptID:
|
||||||
|
sceneRequests = append(sceneRequests, request)
|
||||||
|
case combatsemantics.PromptID:
|
||||||
|
validatorRequests = append(validatorRequests, request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
|
||||||
|
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||||
|
}
|
||||||
|
correction := sceneRequests[1].Correction
|
||||||
|
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
|
||||||
|
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
||||||
|
if strings.Contains(correction.UserGuidance, forbidden) {
|
||||||
|
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
||||||
registries := productionNPCRegistries(t)
|
registries := productionNPCRegistries(t)
|
||||||
configValue := loadGroundedPipelineConfig(t)
|
configValue := loadGroundedPipelineConfig(t)
|
||||||
@@ -508,12 +573,16 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str
|
|||||||
}
|
}
|
||||||
|
|
||||||
type groundedDNDLLMClient struct {
|
type groundedDNDLLMClient struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
requests []contracts.StructuredCompletionRequest
|
requests []contracts.StructuredCompletionRequest
|
||||||
sceneKind dnd.SceneKind
|
sceneKind dnd.SceneKind
|
||||||
sceneTitle string
|
sceneTitle string
|
||||||
firstUnitID int
|
firstUnitID int
|
||||||
thirdUnitID int
|
thirdUnitID int
|
||||||
|
sceneKinds []dnd.SceneKind
|
||||||
|
combatSemanticsVerdicts []string
|
||||||
|
sceneCalls int
|
||||||
|
combatSemanticsCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
@@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
|||||||
payload = map[string]any{"duplicate_groups": []any{}}
|
payload = map[string]any{"duplicate_groups": []any{}}
|
||||||
case sceneextract.PromptID:
|
case sceneextract.PromptID:
|
||||||
kind := client.sceneKind
|
kind := client.sceneKind
|
||||||
|
if client.sceneCalls < len(client.sceneKinds) {
|
||||||
|
kind = client.sceneKinds[client.sceneCalls]
|
||||||
|
}
|
||||||
|
client.sceneCalls++
|
||||||
if kind == "" {
|
if kind == "" {
|
||||||
kind = dnd.SceneKindCombat
|
kind = dnd.SceneKindCombat
|
||||||
}
|
}
|
||||||
@@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
|||||||
title = "A combat encounter"
|
title = "A combat encounter"
|
||||||
}
|
}
|
||||||
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
||||||
|
case combatsemantics.PromptID:
|
||||||
|
verdict := "approved"
|
||||||
|
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
|
||||||
|
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
|
||||||
|
}
|
||||||
|
client.combatSemanticsCalls++
|
||||||
|
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
|
||||||
case spells.PromptID:
|
case spells.PromptID:
|
||||||
payload = map[string]any{"spell_casts": []any{map[string]any{
|
payload = map[string]any{"spell_casts": []any{map[string]any{
|
||||||
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
||||||
|
|||||||
Reference in New Issue
Block a user