Align D&D validator ordering

This commit is contained in:
2026-07-22 14:30:20 +00:00
parent 748e02db80
commit 7b2fb0880d
10 changed files with 115 additions and 19 deletions

View File

@@ -29,6 +29,12 @@ variants. The D&D production registrar registers the canonical typed spell,
NPC, and combat implementations, including their kind-specific merge and NPC, and combat implementations, including their kind-specific merge and
normalize behavior. normalize behavior.
For D&D artifact defaults, generic JSON syntax validation runs first. Rejecting
domain validators then own semantic diagnostics before generic JSON Schema
validation provides the final rejecting representation backstop; warning-only
relatedness validators run last. This default composition does not reorder an
explicitly configured validator chain.
Prepared extractors, extract validators, and codecs may be reused concurrently Prepared extractors, extract validators, and codecs may be reused concurrently
by the run-wide extract pool. Production implementations are immutable after by the run-wide extract pool. Production implementations are immutable after
construction: they retain only typed options, immutable assets, or the shared construction: they retain only typed options, immutable assets, or the shared
@@ -410,7 +416,8 @@ validator owns display normalization, comparison-unique targets, canonical
source-reference order, chronology, and exact duplicate identity; it defers source-reference order, chronology, and exact duplicate identity; it defers
shape and source-reference failures. All four validators are deterministic and shape and source-reference failures. All four validators are deterministic and
expose local policy fingerprints. The D&D registrar orders them after generic expose local policy fingerprints. The D&D registrar orders them after generic
JSON and response-schema validation at extraction and normalization. JSON validation and before response-schema validation at extraction and
normalization.
## Production Registration ## Production Registration

View File

@@ -298,6 +298,13 @@ canonical chunk JSON or artifact codec bytes. Validators execute in resolved
order and stop at the first error or rejection. An empty chain approves the order and stop at the first error or rejection. An empty chain approves the
result. result.
Production D&D artifact chains keep generic JSON syntax validation first, then
run every rejecting domain validator before generic JSON Schema validation. The
domain validator therefore owns expected semantic diagnostics; the generic
schema validator remains the final rejecting representation backstop, before
warning-only relatedness validation. Explicitly configured validator chains
retain their configured order.
`runWithRetry` applies the effective retry policy around module execution and `runWithRetry` applies the effective retry policy around module execution and
its complete validation chain. A module or validator error becomes a framework its complete validation chain. A module or validator error becomes a framework
error when attempts are exhausted. A rejection becomes a recorded error when attempts are exhausted. A rejection becomes a recorded

View File

@@ -50,17 +50,17 @@ func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
wantExtractChain := []pipeline.ModuleBinding{ wantExtractChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/shape"), pipeline.Binding("extract/dnd/npcs/shape"),
pipeline.Binding("extract/dnd/npcs/source_refs"), pipeline.Binding("extract/dnd/npcs/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/source_relatedness"), pipeline.Binding("extract/dnd/npcs/source_relatedness"),
} }
wantNormalizeChain := []pipeline.ModuleBinding{ wantNormalizeChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/shape"), pipeline.Binding("extract/dnd/npcs/shape"),
pipeline.Binding("normalize/dnd/npcs/identity"), pipeline.Binding("normalize/dnd/npcs/identity"),
pipeline.Binding("extract/dnd/npcs/source_refs"), pipeline.Binding("extract/dnd/npcs/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/source_relatedness"), pipeline.Binding("extract/dnd/npcs/source_relatedness"),
} }
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageExtract, npcextract.Key); !reflect.DeepEqual(got, wantExtractChain) { if got := validatorChain(effective.ResolvedPipeline, pipeline.StageExtract, npcextract.Key); !reflect.DeepEqual(got, wantExtractChain) {

View File

@@ -63,10 +63,10 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
wantChain := []pipeline.ModuleBinding{ wantChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/spells/shape"), pipeline.Binding("extract/dnd/spells/shape"),
pipeline.Binding("extract/dnd/spells/catalog"), pipeline.Binding("extract/dnd/spells/catalog"),
pipeline.Binding("extract/dnd/spells/source_refs"), pipeline.Binding("extract/dnd/spells/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/spells/source_relatedness"), pipeline.Binding("extract/dnd/spells/source_relatedness"),
} }
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
@@ -438,7 +438,7 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) { func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml")) base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n") content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
path := writeProductionContractConfig(t, content) path := writeProductionContractConfig(t, content)
components := productionTestComponents(t) components := productionTestComponents(t)
effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
@@ -449,8 +449,8 @@ func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T)
if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key { if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key {
continue continue
} }
if len(chain.Validators) != 1 || chain.Validators[0].Binding.Module != "generic/always_accept" { if len(chain.Validators) != 2 || chain.Validators[0].Binding.Module != "generic/always_accept" || chain.Validators[1].Binding.Module != "generic/valid_json" {
t.Fatalf("normalize validator chain = %#v, want explicit always-accept override", chain) t.Fatalf("normalize validator chain = %#v, want explicit validator order", chain)
} }
return return
} }

View File

@@ -154,9 +154,9 @@ func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenc
fingerprints := prepared.CheckpointFingerprints() fingerprints := prepared.CheckpointFingerprints()
wantNames := map[string]struct{}{ wantNames := map[string]struct{}{
"extract:spells:" + spells.Key + ":effective_catalog": {}, "extract:spells:" + spells.Key + ":effective_catalog": {},
"extract:spells:" + spells.Key + ":validator:3:extract/dnd/spells/catalog:effective_catalog": {}, "extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {}, "normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
"normalize:spells:" + spellnormalize.Key + ":validator:3:extract/dnd/spells/catalog:effective_catalog": {}, "normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
} }
seen := make(map[string]string, len(fingerprints)) seen := make(map[string]string, len(fingerprints))
for _, fingerprint := range fingerprints { for _, fingerprint := range fingerprints {

View File

@@ -32,10 +32,10 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
Module: spellextract.Key, Module: spellextract.Key,
Validators: []pipeline.ModuleBinding{ Validators: []pipeline.ModuleBinding{
pipeline.Binding(validjson.Key), pipeline.Binding(validjson.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(spellshape.Key), pipeline.Binding(spellshape.Key),
pipeline.Binding(spellcatalog.Key), pipeline.Binding(spellcatalog.Key),
pipeline.Binding(spellsourcerefs.Key), pipeline.Binding(spellsourcerefs.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(spellrelatedness.Key), pipeline.Binding(spellrelatedness.Key),
}, },
}) })
@@ -46,10 +46,10 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
Module: spellnormalize.Key, Module: spellnormalize.Key,
Validators: []pipeline.ModuleBinding{ Validators: []pipeline.ModuleBinding{
pipeline.Binding(validjson.Key), pipeline.Binding(validjson.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(spellshape.Key), pipeline.Binding(spellshape.Key),
pipeline.Binding(spellcatalog.Key), pipeline.Binding(spellcatalog.Key),
pipeline.Binding(spellsourcerefs.Key), pipeline.Binding(spellsourcerefs.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(spellrelatedness.Key), pipeline.Binding(spellrelatedness.Key),
}, },
}) })
@@ -60,9 +60,9 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
Module: npcextract.Key, Module: npcextract.Key,
Validators: []pipeline.ModuleBinding{ Validators: []pipeline.ModuleBinding{
pipeline.Binding(validjson.Key), pipeline.Binding(validjson.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(npcshape.Key), pipeline.Binding(npcshape.Key),
pipeline.Binding(npcsourcerefs.Key), pipeline.Binding(npcsourcerefs.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(npcrelatedness.Key), pipeline.Binding(npcrelatedness.Key),
}, },
}) })
@@ -73,10 +73,10 @@ func registerDefaultChains(registry *pipeline.ValidatorChainRegistry) error {
Module: npcnormalize.Key, Module: npcnormalize.Key,
Validators: []pipeline.ModuleBinding{ Validators: []pipeline.ModuleBinding{
pipeline.Binding(validjson.Key), pipeline.Binding(validjson.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(npcshape.Key), pipeline.Binding(npcshape.Key),
pipeline.Binding(npcidentity.Key), pipeline.Binding(npcidentity.Key),
pipeline.Binding(npcsourcerefs.Key), pipeline.Binding(npcsourcerefs.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(npcrelatedness.Key), pipeline.Binding(npcrelatedness.Key),
}, },
}) })

View File

@@ -52,10 +52,10 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
}) })
wantChain := []pipeline.ModuleBinding{ wantChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/spells/shape"), pipeline.Binding("extract/dnd/spells/shape"),
pipeline.Binding("extract/dnd/spells/catalog"), pipeline.Binding("extract/dnd/spells/catalog"),
pipeline.Binding("extract/dnd/spells/source_refs"), pipeline.Binding("extract/dnd/spells/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/spells/source_relatedness"), pipeline.Binding("extract/dnd/spells/source_relatedness"),
} }
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
@@ -66,9 +66,9 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
} }
npcExtractChain := []pipeline.ModuleBinding{ npcExtractChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/shape"), pipeline.Binding("extract/dnd/npcs/shape"),
pipeline.Binding("extract/dnd/npcs/source_refs"), pipeline.Binding("extract/dnd/npcs/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/source_relatedness"), pipeline.Binding("extract/dnd/npcs/source_relatedness"),
} }
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, npcextract.Key); !reflect.DeepEqual(got, npcExtractChain) { if got := registries.ValidatorChains.Validators(pipeline.StageExtract, npcextract.Key); !reflect.DeepEqual(got, npcExtractChain) {
@@ -76,10 +76,10 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
} }
npcNormalizeChain := []pipeline.ModuleBinding{ npcNormalizeChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/shape"), pipeline.Binding("extract/dnd/npcs/shape"),
pipeline.Binding("normalize/dnd/npcs/identity"), pipeline.Binding("normalize/dnd/npcs/identity"),
pipeline.Binding("extract/dnd/npcs/source_refs"), pipeline.Binding("extract/dnd/npcs/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/npcs/source_relatedness"), pipeline.Binding("extract/dnd/npcs/source_relatedness"),
} }
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, npcnormalize.Key); !reflect.DeepEqual(got, npcNormalizeChain) { if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, npcnormalize.Key); !reflect.DeepEqual(got, npcNormalizeChain) {

View File

@@ -17,6 +17,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register" dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/shape"
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register" genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register" seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
@@ -130,6 +131,35 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
} }
} }
func TestProductionNPCPipelineAttributesInvalidShapeBeforeSchemaValidation(t *testing.T) {
registries := productionNPCRegistries(t)
effective, err := loadNPCPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-npcs-fixture",
Catalog: moduleCatalog(registries),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
client := &fakeNPCProductionLLMClient{response: npcProductionResponse{NPCs: []npcProductionRecord{{
Name: "",
Aliases: []string{},
Description: "A participant.",
Relationships: []npcProductionRelationship{},
SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}},
}}}}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
}
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted shape rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
}
rejection := output.Rejected[0]
if rejection.ReasonCode != npcshape.ReasonCode || rejection.ValidatorName != npcshape.Key || rejection.AttemptCount != 3 {
t.Fatalf("rejection = %#v, want exhausted NPC shape rejection", rejection)
}
}
type npcProductionResponse struct { type npcProductionResponse struct {
NPCs []npcProductionRecord `json:"npcs"` NPCs []npcProductionRecord `json:"npcs"`
} }

View File

@@ -22,13 +22,22 @@ type spellCastResponse struct {
} }
type fakeSpellsLLMClient struct { type fakeSpellsLLMClient struct {
response extractionResponse response extractionResponse
requests []contracts.StructuredCompletionRequest responses []extractionResponse
requests []contracts.StructuredCompletionRequest
} }
func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
content, err := json.Marshal(client.response) response := client.response
if client.responses != nil {
index := len(client.requests) - 1
if index >= len(client.responses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
}
response = client.responses[index]
}
content, err := json.Marshal(response)
if err != nil { if err != nil {
return contracts.StructuredCompletionResponse{}, err return contracts.StructuredCompletionResponse{}, err
} }

View File

@@ -12,6 +12,8 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
) )
@@ -164,6 +166,47 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
} }
} }
func TestProductionSpellPipelineAttributesInvalidShapeBeforeSchemaValidation(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := config.Default()
configValue.Pipelines["dnd-spells-shape"] = pipeline.PipelineProfile{
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {
Extract: pipeline.ModuleBinding{Module: spells.Key, Retries: 2},
Normalize: pipeline.Binding(spellnormalize.Key),
},
},
}
effective, err := configValue.Resolve(config.ResolveInput{
PipelineID: "dnd-spells-shape",
Catalog: moduleCatalog(registries),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
response := extractionResponse{SpellCasts: []spellCastResponse{{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "",
NarrativeDescription: "Aria casts the spell.",
SourceRefs: responseSourceRefs("spell-session", 1, 1),
}}}
client := &fakeSpellsLLMClient{responses: []extractionResponse{response, response, response}}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
}
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted shape rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
}
rejection := output.Rejected[0]
if rejection.ReasonCode != spellshape.ReasonCode || rejection.ValidatorName != spellshape.Key || rejection.AttemptCount != 3 {
t.Fatalf("rejection = %#v, want exhausted spell shape rejection", rejection)
}
}
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet { func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot) slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(party) != "" { if strings.TrimSpace(party) != "" {