Harden checkpoint reuse and combat validation
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
Return the combat_turns array even when no combat turn is established. Return
|
||||
one or more actions for every turn. Use one of the supported turn_kind and
|
||||
action category values. Set round to null when the transcript does not state an
|
||||
one or more actions for every turn. For turn_kind, use exactly one of: turn,
|
||||
reaction, legendary_action, lair_action, or other. For each action category,
|
||||
use exactly one of: attack, spell, movement, item, ability_check, saving_throw,
|
||||
condition, or other. Set round to null when the transcript does not state an
|
||||
explicit or unambiguous positive round number. Set resolution to null when the
|
||||
transcript establishes the declaration but not an immediate resolution.
|
||||
|
||||
@@ -87,9 +87,9 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
|
||||
Module: combatextract.Key,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding(validjson.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(combatshape.Key),
|
||||
pipeline.Binding(combatsourcerefs.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(combatrelatedness.Key),
|
||||
},
|
||||
})
|
||||
@@ -100,10 +100,10 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
|
||||
Module: combatnormalize.Key,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding(validjson.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(combatshape.Key),
|
||||
pipeline.Binding(combatinvariants.Key),
|
||||
pipeline.Binding(combatsourcerefs.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(combatrelatedness.Key),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -87,9 +87,9 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
}
|
||||
combatExtractChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/shape"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
|
||||
}
|
||||
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, combatextract.Key); !reflect.DeepEqual(got, combatExtractChain) {
|
||||
@@ -97,10 +97,10 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
}
|
||||
combatNormalizeChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/shape"),
|
||||
pipeline.Binding("normalize/dnd/combat-turns/invariants"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
|
||||
}
|
||||
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, combatNormalizeChain) {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
|
||||
combatshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
@@ -47,7 +48,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
}
|
||||
|
||||
client := &fakeCombatLLMClient{responses: []string{
|
||||
combatTestInvalidResponse(),
|
||||
combatTestInvalidEnumResponse("unsupported", "attack"),
|
||||
combatTestTurnResponse("The Greencloak", "turn", "watches", "The Greencloak", 1, 1),
|
||||
combatTestTurnResponse("Mira Thorn", "reaction", "asks", "Hooded Guard", 2, 2),
|
||||
combatTestTurnResponse("Hooded Guard", "turn", "attacks", "The Greencloak", 3, 3),
|
||||
@@ -125,6 +126,48 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidation(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := combatOnlyConfig()
|
||||
profile := configValue.Pipelines["dnd-combat-fixture"]
|
||||
profile.Chunk.Options["max_units"] = 100
|
||||
configValue.Pipelines["dnd-combat-fixture"] = profile
|
||||
effective, err := configValue.Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-combat-fixture",
|
||||
Catalog: moduleCatalog(registries),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
client := &fakeCombatLLMClient{responses: []string{
|
||||
combatTestInvalidEnumResponse("unsupported", "attack"),
|
||||
combatTestInvalidEnumResponse("turn", "unsupported"),
|
||||
combatTestInvalidEnumResponse("unsupported", "unsupported"),
|
||||
}}
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
ExtractWorkers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
||||
}
|
||||
if len(client.requests) != 3 || len(output.Rejected) != 1 {
|
||||
t.Fatalf("LLM requests = %d rejected = %#v, want exhausted retry and one rejection", len(client.requests), output.Rejected)
|
||||
}
|
||||
rejected := output.Rejected[0]
|
||||
if rejected.ReasonCode != combatshape.ReasonCode || rejected.ValidatorName != combatshape.Key || rejected.AttemptCount != 3 {
|
||||
t.Fatalf("rejected output = %#v, want exhausted combat shape rejection", rejected)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("output = %#v, want non-fatal rejected combat result without normalized artifacts", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
catalog := moduleCatalog(registries)
|
||||
@@ -269,8 +312,8 @@ func (client *fakeCombatLLMClient) CompleteStructured(ctx context.Context, req c
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "combat-fake"}, nil
|
||||
}
|
||||
|
||||
func combatTestInvalidResponse() string {
|
||||
return `{"combat_turns":[{"actor":"","turn_kind":"turn","round":1,"actions":[{"category":"attack","declaration":"watches","targets":[],"resolution":null}],"summary":"invalid candidate","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
func combatTestInvalidEnumResponse(turnKind, category string) string {
|
||||
return fmt.Sprintf(`{"combat_turns":[{"actor":"Aria","turn_kind":%q,"round":1,"actions":[{"category":%q,"declaration":"watches","targets":["Mira"],"resolution":null}],"summary":"invalid candidate","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, turnKind, category)
|
||||
}
|
||||
|
||||
func combatTestTurnResponse(actor, turnKind, declaration, target string, round, unit int) string {
|
||||
|
||||
Reference in New Issue
Block a user