eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make combat scene validation more reliable

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 11:39:02 +00:00
parent 7e626753bf
commit f208dbe954
18 changed files with 146 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go
View File

@@ -18,7 +18,6 @@ var promptAssetManifest = shared.PromptAssetManifest{
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
{Name: "proposed-kind.md", Path: "prompts/proposed-kind.md"},
},
SharedFiles: []string{
"common-dnd-system.md",

14
internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go
View File

@@ -17,8 +17,7 @@ func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
}
engine := newCombatSemanticsPromptEngine(t, registry)
inputs := map[string]promptkit.ArtifactRef{
"proposed_kind": promptkit.Inline("narrative"),
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
@@ -29,19 +28,18 @@ func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
}
if len(prepared.Messages) != 5 {
t.Fatalf("prepared message count = %d, want 5", len(prepared.Messages))
if len(prepared.Messages) != 4 {
t.Fatalf("prepared message count = %d, want 4", len(prepared.Messages))
}
for index, message := range prepared.Messages {
wantEphemeral := index == 3 || index == 4
wantEphemeral := index == 3
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
if gotEphemeral != wantEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
}
}
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
assertRenderedExactlyOnce(t, prepared.Messages, "narrative", 3)
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 4)
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 3)
for name := range inputs {
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
for inputName, input := range inputs {
@@ -62,7 +60,7 @@ func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
}
if strings.Contains(hash, "combat_should_be_added") {
if strings.Contains(hash, "non_combat") {
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
}
}

22
internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go
View File

@@ -17,23 +17,23 @@ func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
}
for _, verdict := range []string{"approved", "combat_should_be_added", "combat_should_be_removed"} {
if err := validateCombatSemanticsSchema(map[string]any{"verdict": verdict, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
t.Fatalf("valid %q verdict rejected: %v", verdict, err)
for _, classification := range []string{"combat", "non_combat"} {
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
t.Fatalf("valid %q classification rejected: %v", classification, err)
}
}
for _, test := range []struct {
name string
response map[string]any
}{
{name: "missing explanation", response: map[string]any{"verdict": "approved"}},
{name: "missing verdict", response: map[string]any{"explanation": "The proposed kind is supported."}},
{name: "unknown property", response: map[string]any{"verdict": "approved", "explanation": "The proposed kind is supported.", "confidence": 1}},
{name: "wrong verdict type", response: map[string]any{"verdict": 1, "explanation": "The proposed kind is supported."}},
{name: "wrong explanation type", response: map[string]any{"verdict": "approved", "explanation": 1}},
{name: "unsupported verdict", response: map[string]any{"verdict": "uncertain", "explanation": "The proposed kind is supported."}},
{name: "blank explanation", response: map[string]any{"verdict": "approved", "explanation": ""}},
{name: "oversized explanation", response: map[string]any{"verdict": "approved", "explanation": strings.Repeat("a", 513)}},
{name: "missing explanation", response: map[string]any{"classification": "combat"}},
{name: "missing classification", response: map[string]any{"explanation": "The chunk contains active combat."}},
{name: "unknown property", response: map[string]any{"classification": "combat", "explanation": "The chunk contains active combat.", "confidence": 1}},
{name: "wrong classification type", response: map[string]any{"classification": 1, "explanation": "The chunk contains active combat."}},
{name: "wrong explanation type", response: map[string]any{"classification": "combat", "explanation": 1}},
{name: "unsupported classification", response: map[string]any{"classification": "uncertain", "explanation": "The classification is uncertain."}},
{name: "blank explanation", response: map[string]any{"classification": "combat", "explanation": ""}},
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
} {
t.Run(test.name, func(t *testing.T) {
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {

16
internal/modules/dnd/validate/scenedescriptions/combat_semantics/testdata/evaluation_cases.json vendored
View File

@@ -1,10 +1,10 @@
[
{"name":"active encounter","transcript_units":[{"id":1,"text":"Roll initiative; the goblins attack."},{"id":2,"text":"The ranger hits and deals damage."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"Initiative and hostile actions organize the chunk."},
{"name":"combat after setup","transcript_units":[{"id":1,"text":"They open the crypt door."},{"id":2,"text":"Skeletons attack and turns begin."}],"proposed_kind":"narrative","expected_verdict":"combat_should_be_added","reviewer_rationale":"Brief setup does not displace substantive active combat."},
{"name":"combat aftermath","transcript_units":[{"id":1,"text":"The last enemy falls."},{"id":2,"text":"They search bodies and heal."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Looting and healing after a completed fight are not active combat."},
{"name":"multi phase encounter","transcript_units":[{"id":1,"text":"The dragon attacks."},{"id":2,"text":"After a rules clarification, its next turn begins."}],"proposed_kind":"combat","expected_verdict":"approved","reviewer_rationale":"A brief rules interruption does not end the encounter."},
{"name":"planning","transcript_units":[{"id":1,"text":"They plan how to ambush the guard."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Planning a possible fight is not active encounter play."},
{"name":"hostile dialogue","transcript_units":[{"id":1,"text":"The captain threatens them and they argue."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Threats and hostile dialogue alone are insufficient."},
{"name":"recap recollection","transcript_units":[{"id":1,"text":"They recap last session's battle with the lich."}],"proposed_kind":"combat","expected_verdict":"combat_should_be_removed","reviewer_rationale":"Recounting earlier combat is not current active combat."},
{"name":"rules discussion","transcript_units":[{"id":1,"text":"The table discusses how concentration works."}],"proposed_kind":"meta","expected_verdict":"approved","reviewer_rationale":"Sustained out-of-character rules discussion has no active encounter."}
{"name":"active encounter","transcript_units":[{"id":1,"text":"Roll initiative; the goblins attack."},{"id":2,"text":"The ranger hits and deals damage."}],"proposed_kind":"combat","expected_classification":"combat","reviewer_rationale":"Initiative and hostile actions organize the chunk."},
{"name":"combat after setup","transcript_units":[{"id":1,"text":"They open the crypt door."},{"id":2,"text":"Skeletons attack and turns begin."}],"proposed_kind":"narrative","expected_classification":"combat","reviewer_rationale":"Brief setup does not displace substantive active combat."},
{"name":"combat aftermath","transcript_units":[{"id":1,"text":"The last enemy falls."},{"id":2,"text":"They search bodies and heal."}],"proposed_kind":"combat","expected_classification":"non_combat","reviewer_rationale":"Looting and healing after a completed fight are not active combat."},
{"name":"multi phase encounter","transcript_units":[{"id":1,"text":"The dragon attacks."},{"id":2,"text":"After a rules clarification, its next turn begins."}],"proposed_kind":"combat","expected_classification":"combat","reviewer_rationale":"A brief rules interruption does not end the encounter."},
{"name":"planning","transcript_units":[{"id":1,"text":"They plan how to ambush the guard."}],"proposed_kind":"combat","expected_classification":"non_combat","reviewer_rationale":"Planning a possible fight is not active encounter play."},
{"name":"hostile dialogue","transcript_units":[{"id":1,"text":"The captain threatens them and they argue."}],"proposed_kind":"combat","expected_classification":"non_combat","reviewer_rationale":"Threats and hostile dialogue alone are insufficient."},
{"name":"recap recollection","transcript_units":[{"id":1,"text":"They recap last session's battle with the lich."}],"proposed_kind":"combat","expected_classification":"non_combat","reviewer_rationale":"Recounting earlier combat is not current active combat."},
{"name":"rules discussion","transcript_units":[{"id":1,"text":"The table discusses how concentration works."}],"proposed_kind":"meta","expected_classification":"non_combat","reviewer_rationale":"Sustained out-of-character rules discussion has no active encounter."}
]

21
internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go
View File

@@ -25,8 +25,8 @@ const (
type Options struct{}
type completionResponse struct {
Verdict string `json:"verdict"`
Explanation string `json:"explanation"`
Classification string `json:"classification"`
Explanation string `json:"explanation"`
}
type Validator struct {
@@ -109,8 +109,7 @@ func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationR
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Inputs: contracts.LLMInputSet{
"transcript": shared.TranscriptPromptMaterial(sourceInput),
"proposed_kind": contracts.NewLLMInputMaterial("proposed_kind", "text/plain", []byte(scene.Kind), "", ""),
"transcript": shared.TranscriptPromptMaterial(sourceInput),
},
}, &response)
if err != nil {
@@ -160,12 +159,10 @@ func interpretResponse(response completionResponse, proposedKind dnd.SceneKind)
if err != nil {
return contracts.ValidationResult{}, err
}
switch response.Verdict {
case "approved":
return contracts.ValidationResult{Approved: true}, nil
case "combat_should_be_added":
switch response.Classification {
case "combat":
if proposedKind == dnd.SceneKindCombat {
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_added verdict is inconsistent with proposed combat kind")
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
@@ -173,9 +170,9 @@ func interpretResponse(response completionResponse, proposedKind dnd.SceneKind)
Message: "The current chunk contains substantive active combat that is not classified as combat.",
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
}, nil
case "combat_should_be_removed":
case "non_combat":
if proposedKind != dnd.SceneKindCombat {
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_removed verdict is inconsistent with proposed non-combat kind")
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
@@ -184,7 +181,7 @@ func interpretResponse(response completionResponse, proposedKind dnd.SceneKind)
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
}, nil
default:
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics verdict %q", response.Verdict)
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
}
}

50
internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go
View File

@@ -28,16 +28,14 @@ func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
correction string
wantError string
}{
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}, approved: true},
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "approved", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
{name: "inconsistent added", kind: dnd.SceneKindCombat, response: completionResponse{Verdict: "combat_should_be_added", Explanation: "The combatants exchange attacks."}, wantError: "inconsistent"},
{name: "inconsistent removed", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_removed", Explanation: "The group only plans."}, wantError: "inconsistent"},
{name: "unknown verdict", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: " "}, wantError: "blank"},
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "combat_should_be_added", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Verdict: "approved", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
} {
t.Run(test.name, func(t *testing.T) {
request.Value.Scenes[0].Kind = test.kind
@@ -66,14 +64,14 @@ func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
}
}
func TestValidatorUsesOnlyChunkTranscriptAndProposedKind(t *testing.T) {
func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
request := validRequest(dnd.SceneKindNarrative)
repairAttempts := 2
request.LLMProfile = "validator-profile"
request.SessionID = "validator-session"
request.StructuredOutputRepairAttempts = &repairAttempts
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
validator := newValidator(t, client)
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
t.Fatalf("Validate() = %#v, %v", result, err)
@@ -85,7 +83,7 @@ func TestValidatorUsesOnlyChunkTranscriptAndProposedKind(t *testing.T) {
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
t.Fatalf("completion request = %#v", completed)
}
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"proposed_kind", "transcript"}) || string(completed.Inputs["proposed_kind"].Content) != "narrative" || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"transcript"}) || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
t.Fatalf("completion inputs = %#v", completed.Inputs)
}
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
@@ -132,7 +130,7 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
if test.mutate != nil {
test.mutate(&request)
}
client := &fakeCombatSemanticsClient{response: completionResponse{Verdict: "approved", Explanation: "No active encounter occurs."}}
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
_, err := newValidator(t, client).Validate(test.ctx, request)
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
@@ -157,7 +155,7 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
}
metadata, err := json.Marshal(validator.ManifestMetadata())
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "combat_should_be_added") {
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "non_combat") {
t.Fatalf("manifest metadata = %s, %v", metadata, err)
}
fingerprints := validator.CheckpointFingerprints()
@@ -173,7 +171,7 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
}
}
func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
content, err := os.ReadFile("testdata/evaluation_cases.json")
if err != nil {
t.Fatal(err)
@@ -184,9 +182,9 @@ func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
ID int `json:"id"`
Text string `json:"text"`
} `json:"transcript_units"`
ProposedKind string `json:"proposed_kind"`
ExpectedVerdict string `json:"expected_verdict"`
ReviewerRationale string `json:"reviewer_rationale"`
ProposedKind string `json:"proposed_kind"`
ExpectedClassification string `json:"expected_classification"`
ReviewerRationale string `json:"reviewer_rationale"`
}
decoder := json.NewDecoder(strings.NewReader(string(content)))
decoder.DisallowUnknownFields()
@@ -196,7 +194,7 @@ func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
}
seenNames, seenVerdicts := map[string]bool{}, map[string]bool{}
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
for _, value := range cases {
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
t.Fatalf("invalid evaluation case: %#v", value)
@@ -207,11 +205,11 @@ func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
default:
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
}
switch value.ExpectedVerdict {
case "approved", "combat_should_be_added", "combat_should_be_removed":
seenVerdicts[value.ExpectedVerdict] = true
switch value.ExpectedClassification {
case "combat", "non_combat":
seenClassifications[value.ExpectedClassification] = true
default:
t.Fatalf("unsupported verdict %q", value.ExpectedVerdict)
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
}
for _, unit := range value.TranscriptUnits {
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
@@ -219,8 +217,8 @@ func TestEvaluationCasesAreStrictAndCoverVerdicts(t *testing.T) {
}
}
}
if len(seenVerdicts) != 3 {
t.Fatalf("evaluation verdicts = %#v, want all classes", seenVerdicts)
if len(seenClassifications) != 2 {
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
}
}

30
internal/modules/integration/dnd_npc_grounded_test.go
View File

@@ -276,7 +276,7 @@ func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
configValue.Pipelines["dnd-npc-grounded"] = profile
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}}
output := runGroundedPipeline(t, configValue, registries, client, nil)
if len(output.Rejected) != 0 {
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
@@ -573,16 +573,16 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str
}
type groundedDNDLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
sceneKind dnd.SceneKind
sceneTitle string
firstUnitID int
thirdUnitID int
sceneKinds []dnd.SceneKind
combatSemanticsVerdicts []string
sceneCalls int
combatSemanticsCalls int
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
sceneKind dnd.SceneKind
sceneTitle string
firstUnitID int
thirdUnitID int
sceneKinds []dnd.SceneKind
combatSemanticsClassifications []string
sceneCalls int
combatSemanticsCalls int
}
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -629,12 +629,12 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
}
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
case combatsemantics.PromptID:
verdict := "approved"
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
classification := "combat"
if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) {
classification = client.combatSemanticsClassifications[client.combatSemanticsCalls]
}
client.combatSemanticsCalls++
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."}
case spells.PromptID:
payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds",