Require chunk-local extraction evidence

docs/internal/dnd.md

@@ -116,7 +116,8 @@ result.
 
 Default chains keep responsibilities separate: structural validators assess the
 candidate, source-reference validators resolve cited ranges against the current
-source, durable-schema validation checks an approved representation, and
+source and require extraction evidence to stay within the current chunk,
+durable-schema validation checks an approved representation, and
 relatedness validators report advisory evidence concerns. The configured order
 is documented in
 [Configuration](../config.md#production-validator-keys-and-default-chains).

internal/modules/dnd/validate/combatturns/source_refs/validator.go

@@ -15,7 +15,7 @@ import (
 const (
 	Key        = "extract/dnd/combat-turns/source_refs"
 	ReasonCode = "invalid_combat_turn_source_refs"
-	policy     = "dnd.combat_turns.validator.source_refs.v1"
+	policy     = "dnd.combat_turns.validator.source_refs.v2"
 )
 
 type Options struct{}
@@ -34,10 +34,17 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
 }
 
 func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.CombatTurnList]) (contracts.ValidationResult, error) {
+	if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil {
+		return contracts.ValidationResult{}, fmt.Errorf("combat-turn source-reference validator requires the current extraction chunk")
+	}
 	if err := combatshape.Validate(req.Value); err != nil {
 		return contracts.ValidationResult{Approved: true}, nil
 	}
-	issues := sourceRefIssues(source.NewDocumentIndex(req.Source), req.Value)
+	var coverage *chunkCoverage
+	if req.Stage == string(pipeline.StageExtract) {
+		coverage = newChunkCoverage(req.Chunk)
+	}
+	issues := sourceRefIssues(source.NewDocumentIndex(req.Source), req.Source, coverage, req.Value)
 	if len(issues) == 0 {
 		return contracts.ValidationResult{Approved: true}, nil
 	}
@@ -48,18 +55,52 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
 	}, nil
 }
 
-func sourceRefIssues(index source.DocumentIndex, value dnd.CombatTurnList) []string {
+func sourceRefIssues(index source.DocumentIndex, doc *source.SourceDocument, coverage *chunkCoverage, value dnd.CombatTurnList) []string {
 	issues := make([]string, 0)
 	for turnIndex, turn := range value.CombatTurns {
 		for refIndex, ref := range turn.SourceRefs {
 			if err := index.ValidateRef(ref); err != nil {
 				issues = append(issues, fmt.Sprintf("combat_turns[%d].source_refs[%d]: %s", turnIndex, refIndex, diagnostics.Truncate(err.Error())))
+				continue
+			}
+			if coverage != nil && !coverage.contains(doc, ref) {
+				issues = append(issues, fmt.Sprintf("combat_turns[%d].source_refs[%d]: source reference is outside the current extraction chunk", turnIndex, refIndex))
 			}
 		}
 	}
 	return issues
 }
 
+type chunkCoverage struct {
+	sourceID string
+	unitIDs  map[int]struct{}
+}
+
+func newChunkCoverage(chunk *source.Chunk) *chunkCoverage {
+	coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))}
+	for _, unit := range chunk.Units {
+		coverage.unitIDs[unit.ID] = struct{}{}
+	}
+	return coverage
+}
+
+func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool {
+	if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID {
+		return false
+	}
+	start, startOK := source.UnitIndex(doc, ref.StartUnitID)
+	end, endOK := source.UnitIndex(doc, ref.EndUnitID)
+	if !startOK || !endOK || start > end {
+		return false
+	}
+	for position := start; position <= end; position++ {
+		if _, found := coverage.unitIDs[doc.Units[position].ID]; !found {
+			return false
+		}
+	}
+	return true
+}
+
 func Spec() pipeline.ValidatorSpec {
 	return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
 }

internal/modules/dnd/validate/combatturns/source_refs/validator_test.go

@@ -46,6 +46,39 @@ func TestValidatorRejectsInvalidSourceIdentityExistenceAndOrder(t *testing.T) {
 	}
 }
 
+func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) {
+	doc := validDocument()
+	value := validCombatTurnList()
+	value.CombatTurns[0].SourceRefs[0].EndUnitID = 2
+	req := contracts.TypedValidationRequest[dnd.CombatTurnList]{
+		Stage:  string(pipeline.StageExtract),
+		Source: doc,
+		Chunk:  &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)},
+		Value:  value,
+	}
+	result, err := New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err)
+	}
+
+	req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}}
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") {
+		t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err)
+	}
+
+	req.Chunk = nil
+	if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") {
+		t.Fatalf("missing extraction chunk error = %v", err)
+	}
+
+	req.Stage = string(pipeline.StageNormalize)
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err)
+	}
+}
+
 func TestValidatorDefersMalformedShape(t *testing.T) {
 	value := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{{Actor: "Aria"}}}
 	result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: validDocument(), Value: value})

internal/modules/dnd/validate/npcregistry/source_refs/validator.go

@@ -15,7 +15,7 @@ import (
 const (
 	Key        = "extract/dnd/npc-registry/source_refs"
 	ReasonCode = "invalid_npc_source_refs"
-	policy     = "dnd.npc_registry.validator.source_refs.v1"
+	policy     = "dnd.npc_registry.validator.source_refs.v2"
 )
 
 type Options struct{}
@@ -34,15 +34,26 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
 }
 
 func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCRegistry]) (contracts.ValidationResult, error) {
+	if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil {
+		return contracts.ValidationResult{}, fmt.Errorf("NPC source-reference validator requires the current extraction chunk")
+	}
 	if err := npcshape.Validate(req.Value); err != nil {
 		return contracts.ValidationResult{Approved: true}, nil
 	}
 	index := source.NewDocumentIndex(req.Source)
+	var coverage *chunkCoverage
+	if req.Stage == string(pipeline.StageExtract) {
+		coverage = newChunkCoverage(req.Chunk)
+	}
 	issues := make([]string, 0)
 	for npcIndex, npc := range req.Value.NPCs {
 		for refIndex, ref := range npc.SourceRefs {
 			if err := index.ValidateRef(ref); err != nil {
 				issues = append(issues, fmt.Sprintf("npcs[%d].source_refs[%d]: %s", npcIndex, refIndex, diagnostics.Truncate(err.Error())))
+				continue
+			}
+			if coverage != nil && !coverage.contains(req.Source, ref) {
+				issues = append(issues, fmt.Sprintf("npcs[%d].source_refs[%d]: source reference is outside the current extraction chunk", npcIndex, refIndex))
 			}
 		}
 	}
@@ -52,6 +63,36 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
 	return rejection(diagnostics.Aggregate("invalid NPC source references", issues)), nil
 }
 
+type chunkCoverage struct {
+	sourceID string
+	unitIDs  map[int]struct{}
+}
+
+func newChunkCoverage(chunk *source.Chunk) *chunkCoverage {
+	coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))}
+	for _, unit := range chunk.Units {
+		coverage.unitIDs[unit.ID] = struct{}{}
+	}
+	return coverage
+}
+
+func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool {
+	if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID {
+		return false
+	}
+	start, startOK := source.UnitIndex(doc, ref.StartUnitID)
+	end, endOK := source.UnitIndex(doc, ref.EndUnitID)
+	if !startOK || !endOK || start > end {
+		return false
+	}
+	for position := start; position <= end; position++ {
+		if _, found := coverage.unitIDs[doc.Units[position].ID]; !found {
+			return false
+		}
+	}
+	return true
+}
+
 func Spec() pipeline.ValidatorSpec {
 	return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
 }

internal/modules/dnd/validate/npcregistry/source_refs/validator_test.go

@@ -33,6 +33,35 @@ func TestValidatorRejectsInvalidSourceReferences(t *testing.T) {
 	}
 }
 
+func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) {
+	doc := validDocument()
+	value := validNPCRegistry()
+	req := requestWithValue(doc, value)
+	req.Stage = string(pipeline.StageExtract)
+	req.Chunk = &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)}
+	result, err := New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err)
+	}
+
+	req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}}
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") {
+		t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err)
+	}
+
+	req.Chunk = nil
+	if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") {
+		t.Fatalf("missing extraction chunk error = %v", err)
+	}
+
+	req.Stage = string(pipeline.StageNormalize)
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err)
+	}
+}
+
 func TestValidatorDefersMalformedShape(t *testing.T) {
 	value := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Mira Thorn"}}}
 	result, err := New(Options{}).Validate(context.Background(), requestWithValue(validDocument(), value))
@@ -57,7 +86,7 @@ func TestValidatorBoundsDiagnosticsAndHandlesMissingDocument(t *testing.T) {
 }
 
 func TestValidatorSpecCheckpointAndRegistration(t *testing.T) {
-	if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npc_registry.validator.source_refs.v1" {
+	if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npc_registry.validator.source_refs.v2" {
 		t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got)
 	}
 	if Spec().ExecutionClass != contracts.ExecutionClassDeterministic {

internal/modules/dnd/validate/spells/source_refs/validator.go

@@ -15,7 +15,7 @@ import (
 const (
 	Key        = "extract/dnd/spells/source_refs"
 	ReasonCode = "invalid_source_refs"
-	policy     = "dnd.spells.validator.source_refs.v1"
+	policy     = "dnd.spells.validator.source_refs.v2"
 )
 
 type Options struct{}
@@ -33,15 +33,26 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
 	return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
 }
 func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
+	if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil {
+		return contracts.ValidationResult{}, fmt.Errorf("spell source-reference validator requires the current extraction chunk")
+	}
 	if err := spellshape.Validate(req.Value); err != nil {
 		return contracts.ValidationResult{Approved: true}, nil
 	}
 	index := source.NewDocumentIndex(req.Source)
+	var coverage *chunkCoverage
+	if req.Stage == string(pipeline.StageExtract) {
+		coverage = newChunkCoverage(req.Chunk)
+	}
 	issues := make([]string, 0)
 	for spellIndex, spell := range req.Value.SpellCasts {
 		for refIndex, ref := range spell.SourceRefs {
 			if err := index.ValidateRef(ref); err != nil {
 				issues = append(issues, fmt.Sprintf("spell_casts[%d].source_refs[%d]: %s", spellIndex, refIndex, diagnostics.Truncate(err.Error())))
+				continue
+			}
+			if coverage != nil && !coverage.contains(req.Source, ref) {
+				issues = append(issues, fmt.Sprintf("spell_casts[%d].source_refs[%d]: source reference is outside the current extraction chunk", spellIndex, refIndex))
 			}
 		}
 	}
@@ -50,6 +61,36 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
 	}
 	return contracts.ValidationResult{Approved: true}, nil
 }
+
+type chunkCoverage struct {
+	sourceID string
+	unitIDs  map[int]struct{}
+}
+
+func newChunkCoverage(chunk *source.Chunk) *chunkCoverage {
+	coverage := &chunkCoverage{sourceID: chunk.SourceID, unitIDs: make(map[int]struct{}, len(chunk.Units))}
+	for _, unit := range chunk.Units {
+		coverage.unitIDs[unit.ID] = struct{}{}
+	}
+	return coverage
+}
+
+func (coverage *chunkCoverage) contains(doc *source.SourceDocument, ref source.SourceRef) bool {
+	if coverage == nil || doc == nil || ref.SourceID != coverage.sourceID {
+		return false
+	}
+	start, startOK := source.UnitIndex(doc, ref.StartUnitID)
+	end, endOK := source.UnitIndex(doc, ref.EndUnitID)
+	if !startOK || !endOK || start > end {
+		return false
+	}
+	for position := start; position <= end; position++ {
+		if _, found := coverage.unitIDs[doc.Units[position].ID]; !found {
+			return false
+		}
+	}
+	return true
+}
 func Spec() pipeline.ValidatorSpec {
 	return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
 }

internal/modules/dnd/validate/spells/source_refs/validator_test.go

@@ -48,6 +48,34 @@ func TestValidatorRejectsMissingSourceDocument(t *testing.T) {
 	}
 }
 
+func TestValidatorEnforcesCurrentChunkEvidenceDuringExtraction(t *testing.T) {
+	doc := validDocument()
+	req := requestWithValue(doc, source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2})
+	req.Stage = string(pipeline.StageExtract)
+	req.Chunk = &source.Chunk{SourceID: doc.ID, Units: append([]source.SourceUnit(nil), doc.Units...)}
+	result, err := New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("in-chunk extraction evidence = %#v, %v; want approval", result, err)
+	}
+
+	req.Chunk = &source.Chunk{SourceID: doc.ID, Units: []source.SourceUnit{{ID: 1}}}
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") {
+		t.Fatalf("off-chunk extraction evidence = %#v, %v; want rejection", result, err)
+	}
+
+	req.Chunk = nil
+	if _, err = New(Options{}).Validate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") {
+		t.Fatalf("missing extraction chunk error = %v", err)
+	}
+
+	req.Stage = string(pipeline.StageNormalize)
+	result, err = New(Options{}).Validate(context.Background(), req)
+	if err != nil || !result.Approved {
+		t.Fatalf("document-wide normalization evidence = %#v, %v; want approval", result, err)
+	}
+}
+
 func TestValidatorDefersMalformedShape(t *testing.T) {
 	value := dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "Cure Wounds"}}}
 	result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.SpellList]{Source: validDocument(), Value: value})

internal/modules/integration/dnd_combat_runner_test.go

@@ -51,8 +51,8 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
 	client := &fakeCombatLLMClient{responses: []string{
 		combatTestInvalidEnumResponse("unsupported"),
 		combatTestTurnResponse("mira thorn", "turn", 1),
-		combatTestTurnResponse("Mira Thorn", "reaction", 2),
-		combatTestTurnResponse("Hooded Guard", "turn", 3),
+		combatTestTurnResponse("Mira Thorn", "reaction", 3),
+		combatTestTurnResponse("Hooded Guard", "turn", 5),
 	}}
 	prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
 	if err != nil {