From 0b5cc4f251b6b2d6476c315bc8ec7049ba36180d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 9 Aug 2026 02:08:11 +0000 Subject: [PATCH] Require chunk-local extraction evidence --- docs/internal/dnd.md | 3 +- .../combatturns/source_refs/validator.go | 47 +++++++++++++++++-- .../combatturns/source_refs/validator_test.go | 33 +++++++++++++ .../npcregistry/source_refs/validator.go | 43 ++++++++++++++++- .../npcregistry/source_refs/validator_test.go | 31 +++++++++++- .../validate/spells/source_refs/validator.go | 43 ++++++++++++++++- .../spells/source_refs/validator_test.go | 28 +++++++++++ .../integration/dnd_combat_runner_test.go | 4 +- 8 files changed, 223 insertions(+), 9 deletions(-) diff --git a/docs/internal/dnd.md b/docs/internal/dnd.md index 76aa279..43a9e91 100644 --- a/docs/internal/dnd.md +++ b/docs/internal/dnd.md @@ -116,7 +116,8 @@ result. Default chains keep responsibilities separate: structural validators assess the candidate, source-reference validators resolve cited ranges against the current -source, durable-schema validation checks an approved representation, and +source and require extraction evidence to stay within the current chunk, +durable-schema validation checks an approved representation, and relatedness validators report advisory evidence concerns. The configured order is documented in [Configuration](../config.md#production-validator-keys-and-default-chains). diff --git a/internal/modules/dnd/validate/combatturns/source_refs/validator.go b/internal/modules/dnd/validate/combatturns/source_refs/validator.go index 8e10b80..d8a9d33 100644 --- a/internal/modules/dnd/validate/combatturns/source_refs/validator.go +++ b/internal/modules/dnd/validate/combatturns/source_refs/validator.go @@ -15,7 +15,7 @@ import ( const ( Key = "extract/dnd/combat-turns/source_refs" ReasonCode = "invalid_combat_turn_source_refs" - policy = "dnd.combat_turns.validator.source_refs.v1" + policy = "dnd.combat_turns.validator.source_refs.v2" ) type Options struct{} @@ -34,10 +34,17 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { } func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.CombatTurnList]) (contracts.ValidationResult, error) { + if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil { + return contracts.ValidationResult{}, fmt.Errorf("combat-turn source-reference validator requires the current extraction chunk") + } if err := combatshape.Validate(req.Value); err != nil { return contracts.ValidationResult{Approved: true}, nil } - issues := sourceRefIssues(source.NewDocumentIndex(req.Source), req.Value) + var coverage *chunkCoverage + if req.Stage == string(pipeline.StageExtract) { + coverage = newChunkCoverage(req.Chunk) + } + issues := sourceRefIssues(source.NewDocumentIndex(req.Source), req.Source, coverage, req.Value) if len(issues) == 0 { return contracts.ValidationResult{Approved: true}, nil } @@ -48,18 +55,52 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq }, nil } -func sourceRefIssues(index source.DocumentIndex, value dnd.CombatTurnList) []string { +func sourceRefIssues(index source.DocumentIndex, doc *source.SourceDocument, coverage *chunkCoverage, value dnd.CombatTurnList) []string { issues := make([]string, 0) for turnIndex, turn := range value.CombatTurns { for refIndex, ref := range turn.SourceRefs { if err := index.ValidateRef(ref); err != nil { issues = append(issues, fmt.Sprintf("combat_turns[%d].source_refs[%d]: %s", turnIndex, refIndex, diagnostics.Truncate(err.Error()))) + continue + } + if coverage != nil && !coverage.contains(doc, ref) { + issues = append(issues, fmt.Sprintf("combat_turns[%d].source_refs[%d]: source reference is outside the current extraction chunk", turnIndex, refIndex)) } } } return issues } +type chunkCoverage struct { + sourceID string + unitIDs map[int]struct{} +} + +func newChunkCoverage(chunk *source.Chunk) *chunkCoverage { + coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))} + for _, unit := range chunk.Units { + coverage.unitIDs[unit.ID] = struct{}{} + } + return coverage +} + +func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool { + if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID { + return false + } + start, startOK := source.UnitIndex(doc, ref.StartUnitID) + end, endOK := source.UnitIndex(doc, ref.EndUnitID) + if !startOK || !endOK || start > end { + return false + } + for position := start; position <= end; position++ { + if _, found := coverage.unitIDs[doc.Units[position].ID]; !found { + return false + } + } + return true +} + func Spec() pipeline.ValidatorSpec { return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} } diff --git a/internal/modules/dnd/validate/combatturns/source_refs/validator_test.go b/internal/modules/dnd/validate/combatturns/source_refs/validator_test.go index 25674c4..a6467c0 100644 --- a/internal/modules/dnd/validate/combatturns/source_refs/validator_test.go +++ b/internal/modules/dnd/validate/combatturns/source_refs/validator_test.go @@ -46,6 +46,39 @@ func TestValidatorRejectsInvalidSourceIdentityExistenceAndOrder(t *testing.T) { } } +func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) { + doc := validDocument() + value := validCombatTurnList() + value.CombatTurns[0].SourceRefs[0].EndUnitID = 2 + req := contracts.TypedValidationRequest[dnd.CombatTurnList]{ + Stage: string(pipeline.StageExtract), + Source: doc, + Chunk: &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)}, + Value: value, + } + result, err := New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err) + } + + req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}} + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") { + t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err) + } + + req.Chunk = nil + if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") { + t.Fatalf("missing extraction chunk error = %v", err) + } + + req.Stage = string(pipeline.StageNormalize) + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err) + } +} + func TestValidatorDefersMalformedShape(t *testing.T) { value := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{{Actor: "Aria"}}} result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: validDocument(), Value: value}) diff --git a/internal/modules/dnd/validate/npcregistry/source_refs/validator.go b/internal/modules/dnd/validate/npcregistry/source_refs/validator.go index 69af798..1505744 100644 --- a/internal/modules/dnd/validate/npcregistry/source_refs/validator.go +++ b/internal/modules/dnd/validate/npcregistry/source_refs/validator.go @@ -15,7 +15,7 @@ import ( const ( Key = "extract/dnd/npc-registry/source_refs" ReasonCode = "invalid_npc_source_refs" - policy = "dnd.npc_registry.validator.source_refs.v1" + policy = "dnd.npc_registry.validator.source_refs.v2" ) type Options struct{} @@ -34,15 +34,26 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { } func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCRegistry]) (contracts.ValidationResult, error) { + if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil { + return contracts.ValidationResult{}, fmt.Errorf("NPC source-reference validator requires the current extraction chunk") + } if err := npcshape.Validate(req.Value); err != nil { return contracts.ValidationResult{Approved: true}, nil } index := source.NewDocumentIndex(req.Source) + var coverage *chunkCoverage + if req.Stage == string(pipeline.StageExtract) { + coverage = newChunkCoverage(req.Chunk) + } issues := make([]string, 0) for npcIndex, npc := range req.Value.NPCs { for refIndex, ref := range npc.SourceRefs { if err := index.ValidateRef(ref); err != nil { issues = append(issues, fmt.Sprintf("npcs[%d].source_refs[%d]: %s", npcIndex, refIndex, diagnostics.Truncate(err.Error()))) + continue + } + if coverage != nil && !coverage.contains(req.Source, ref) { + issues = append(issues, fmt.Sprintf("npcs[%d].source_refs[%d]: source reference is outside the current extraction chunk", npcIndex, refIndex)) } } } @@ -52,6 +63,36 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq return rejection(diagnostics.Aggregate("invalid NPC source references", issues)), nil } +type chunkCoverage struct { + sourceID string + unitIDs map[int]struct{} +} + +func newChunkCoverage(chunk *source.Chunk) *chunkCoverage { + coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))} + for _, unit := range chunk.Units { + coverage.unitIDs[unit.ID] = struct{}{} + } + return coverage +} + +func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool { + if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID { + return false + } + start, startOK := source.UnitIndex(doc, ref.StartUnitID) + end, endOK := source.UnitIndex(doc, ref.EndUnitID) + if !startOK || !endOK || start > end { + return false + } + for position := start; position <= end; position++ { + if _, found := coverage.unitIDs[doc.Units[position].ID]; !found { + return false + } + } + return true +} + func Spec() pipeline.ValidatorSpec { return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} } diff --git a/internal/modules/dnd/validate/npcregistry/source_refs/validator_test.go b/internal/modules/dnd/validate/npcregistry/source_refs/validator_test.go index 724c68e..77e8177 100644 --- a/internal/modules/dnd/validate/npcregistry/source_refs/validator_test.go +++ b/internal/modules/dnd/validate/npcregistry/source_refs/validator_test.go @@ -33,6 +33,35 @@ func TestValidatorRejectsInvalidSourceReferences(t *testing.T) { } } +func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) { + doc := validDocument() + value := validNPCRegistry() + req := requestWithValue(doc, value) + req.Stage = string(pipeline.StageExtract) + req.Chunk = &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)} + result, err := New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err) + } + + req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}} + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") { + t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err) + } + + req.Chunk = nil + if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") { + t.Fatalf("missing extraction chunk error = %v", err) + } + + req.Stage = string(pipeline.StageNormalize) + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err) + } +} + func TestValidatorDefersMalformedShape(t *testing.T) { value := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Mira Thorn"}}} result, err := New(Options{}).Validate(context.Background(), requestWithValue(validDocument(), value)) @@ -57,7 +86,7 @@ func TestValidatorBoundsDiagnosticsAndHandlesMissingDocument(t *testing.T) { } func TestValidatorSpecCheckpointAndRegistration(t *testing.T) { - if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npc_registry.validator.source_refs.v1" { + if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npc_registry.validator.source_refs.v2" { t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got) } if Spec().ExecutionClass != contracts.ExecutionClassDeterministic { diff --git a/internal/modules/dnd/validate/spells/source_refs/validator.go b/internal/modules/dnd/validate/spells/source_refs/validator.go index e7bad43..0837c27 100644 --- a/internal/modules/dnd/validate/spells/source_refs/validator.go +++ b/internal/modules/dnd/validate/spells/source_refs/validator.go @@ -15,7 +15,7 @@ import ( const ( Key = "extract/dnd/spells/source_refs" ReasonCode = "invalid_source_refs" - policy = "dnd.spells.validator.source_refs.v1" + policy = "dnd.spells.validator.source_refs.v2" ) type Options struct{} @@ -33,15 +33,26 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}} } func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) { + if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil { + return contracts.ValidationResult{}, fmt.Errorf("spell source-reference validator requires the current extraction chunk") + } if err := spellshape.Validate(req.Value); err != nil { return contracts.ValidationResult{Approved: true}, nil } index := source.NewDocumentIndex(req.Source) + var coverage *chunkCoverage + if req.Stage == string(pipeline.StageExtract) { + coverage = newChunkCoverage(req.Chunk) + } issues := make([]string, 0) for spellIndex, spell := range req.Value.SpellCasts { for refIndex, ref := range spell.SourceRefs { if err := index.ValidateRef(ref); err != nil { issues = append(issues, fmt.Sprintf("spell_casts[%d].source_refs[%d]: %s", spellIndex, refIndex, diagnostics.Truncate(err.Error()))) + continue + } + if coverage != nil && !coverage.contains(req.Source, ref) { + issues = append(issues, fmt.Sprintf("spell_casts[%d].source_refs[%d]: source reference is outside the current extraction chunk", spellIndex, refIndex)) } } } @@ -50,6 +61,36 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq } return contracts.ValidationResult{Approved: true}, nil } + +type chunkCoverage struct { + sourceID string + unitIDs map[int]struct{} +} + +func newChunkCoverage(chunk *source.Chunk) *chunkCoverage { + coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))} + for _, unit := range chunk.Units { + coverage.unitIDs[unit.ID] = struct{}{} + } + return coverage +} + +func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool { + if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID { + return false + } + start, startOK := source.UnitIndex(doc, ref.StartUnitID) + end, endOK := source.UnitIndex(doc, ref.EndUnitID) + if !startOK || !endOK || start > end { + return false + } + for position := start; position <= end; position++ { + if _, found := coverage.unitIDs[doc.Units[position].ID]; !found { + return false + } + } + return true +} func Spec() pipeline.ValidatorSpec { return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} } diff --git a/internal/modules/dnd/validate/spells/source_refs/validator_test.go b/internal/modules/dnd/validate/spells/source_refs/validator_test.go index fb043fa..13f0fc7 100644 --- a/internal/modules/dnd/validate/spells/source_refs/validator_test.go +++ b/internal/modules/dnd/validate/spells/source_refs/validator_test.go @@ -48,6 +48,34 @@ func TestValidatorRejectsMissingSourceDocument(t *testing.T) { } } +func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) { + doc := validDocument() + req := requestWithValue(doc, source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2}) + req.Stage = string(pipeline.StageExtract) + req.Chunk = &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)} + result, err := New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err) + } + + req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}} + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") { + t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err) + } + + req.Chunk = nil + if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") { + t.Fatalf("missing extraction chunk error = %v", err) + } + + req.Stage = string(pipeline.StageNormalize) + result, err = New(Options{}).Validate(context.Background(), req) + if err != nil || !result.Approved { + t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err) + } +} + func TestValidatorDefersMalformedShape(t *testing.T) { value := dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "Cure Wounds"}}} result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.SpellList]{Source: validDocument(), Value: value}) diff --git a/internal/modules/integration/dnd_combat_runner_test.go b/internal/modules/integration/dnd_combat_runner_test.go index 1f80779..430fa33 100644 --- a/internal/modules/integration/dnd_combat_runner_test.go +++ b/internal/modules/integration/dnd_combat_runner_test.go @@ -51,8 +51,8 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing client := &fakeCombatLLMClient{responses: []string{ combatTestInvalidEnumResponse("unsupported"), combatTestTurnResponse("mira thorn", "turn", 1), - combatTestTurnResponse("Mira Thorn", "reaction", 2), - combatTestTurnResponse("Hooded Guard", "turn", 3), + combatTestTurnResponse("Mira Thorn", "reaction", 3), + combatTestTurnResponse("Hooded Guard", "turn", 5), }} prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) if err != nil {