Expand module contracts for references and normalizer LLM access

docs/internal/llm.md

@@ -17,8 +17,8 @@ The request contains messages, optional model override, response schema name,
 and response schema JSON. The caller supplies a pointer target for decoded
 structured output.
 
-Extractors own prompts and schemas. Provider adapters should not contain
+Modules that call the LLM own their prompts and schemas. Provider adapters
-domain-specific prompt logic.
+should not contain domain-specific prompt logic.
 
 ## Production Client Construction
 

docs/internal/modules.md

@@ -20,20 +20,21 @@ A production module package should provide:
 Module specs should describe capabilities accurately. Resolution uses specs to
 reject incompatible pipelines before execution.
 
-Extractor modules that accept auxiliary reference material must declare slots
+Chunk, extract, and normalize modules that accept auxiliary reference material
-through both `ReferenceSlots()` and `ModuleSpec().ReferenceSlots`. The runtime
+must declare slots through both `ReferenceSlots()` and
-slot list and registry metadata should match so config validation can inspect
+`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
-slots without constructing extractor instances. A slot declaration names the
+should match so config validation can inspect slots without constructing module
-slot, whether it is required, accepted media types, whether multiple items are
+instances. A slot declaration names the slot, whether it is required, accepted
-allowed, and any byte limit. Empty `AcceptedMediaTypes` means any inferred
+media types, whether multiple items are allowed, and any byte limit. Empty
-media type is accepted, though the file must still be UTF-8 text. When a slot
+`AcceptedMediaTypes` means any inferred media type is accepted, though the file
-declares accepted media types, Notarius compares the canonical base media type
+must still be UTF-8 text. When a slot declares accepted media types, Notarius
-inferred from the file extension, case-insensitively and without parameters.
+compares the canonical base media type inferred from the file extension,
+case-insensitively and without parameters.
 
-Reference content is delivered only to the lane extractor through
+The current resolver materializes reference content only for lane extractors
-`contracts.ExtractionRequest.References`. It is not source evidence and must not
+through `contracts.ExtractionRequest.References`. It is not source evidence and
-be converted into `SourceRef` values. If a module prompt uses references, load
+must not be converted into `SourceRef` values. If a module prompt uses
-the prompt bundle with the same declared slots and render with
+references, load the prompt bundle with the same declared slots and render with
 `RenderUserSystemWithReferences`. Prompt templates may use the `reference`
 function for content and the `hasreference` function for conditional sections.
 Prompt metadata hashes remain based on template source, not rendered reference
@@ -44,6 +45,9 @@ when they need model-backed chunking. The pipeline runner validates generic
 chunk result invariants before extraction; module-owned policies may be stricter
 but must stay within the module package.
 
+Normalize modules receive the structured LLM client through
+`contracts.NormalizeRequest` when they need model-backed reconciliation.
+
 ## `seriatim` Input
 
 Package: `internal/modules/input/seriatim`

docs/internal/pipeline.md

@@ -74,9 +74,10 @@ Every production module registers a `ModuleSpec` with:
 - `Provides`: capabilities added after that module runs;
 - `Requires`: capabilities that must already be available.
 
-Extractor specs may also declare reference slots. Slot declarations are
+Chunk, extract, and normalize specs may also declare reference slots. Slot
-available from registry metadata without constructing extractor instances.
+declarations are available from registry metadata without constructing module
-Non-extractor module specs must not declare reference slots.
+instances. Input, merge, validate, and output specs must not declare reference
+slots.
 
 Capability checks prevent incompatible pipeline composition before a run starts.
 
@@ -114,10 +115,10 @@ The runner:
 ## Chunk Results
 
 Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
-with the validated source document, the structured LLM client, the configured
+with the validated source document, reference set, structured LLM client, the
-LLM profile, module options, and run metadata. Deterministic and LLM-backed
+configured LLM profile, module options, and run metadata. Deterministic and
-chunkers use the same contract; provider construction stays outside chunk
+LLM-backed chunkers use the same contract; provider construction stays outside
-modules.
+chunk modules.
 
 After `Chunk` returns, the runner appends chunker warnings before returning any
 chunker error. When chunking succeeds, the runner validates generic chunk

internal/cli/run_test.go

@@ -1961,6 +1961,10 @@ func (fakeRunChunker) Key() string {
 	return "generic"
 }
 
+func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{
 		Chunks: []contracts.SourceChunk{
@@ -2024,6 +2028,10 @@ func (fakeRunNormalizer) Key() string {
 	return "noop"
 }
 
+func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
 }

internal/framework/contracts/composition_test.go

@@ -162,6 +162,10 @@ func (chunker compositionChunker) Key() string {
 	return "generic-chunker"
 }
 
+func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	if req.Source == nil {
 		return contracts.ChunkResult{}, errors.New("source document is required")
@@ -264,6 +268,10 @@ func (normalizer compositionNormalizer) Key() string {
 	return "generic-normalizer"
 }
 
+func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	return contracts.NormalizeResult{Candidates: req.Candidates}, nil
 }

internal/framework/contracts/contracts.go

@@ -58,6 +58,7 @@ type SourceChunk struct {
 
 type ChunkRequest struct {
 	Source     *source.SourceDocument `json:"-"`
+	References ReferenceSet           `json:"references,omitempty"`
 	LLMClient  StructuredLLMClient    `json:"-"`
 	LLMProfile string                 `json:"llm_profile,omitempty"`
 	Options    map[string]any         `json:"options,omitempty"`
@@ -71,6 +72,7 @@ type ChunkResult struct {
 
 type Chunker interface {
 	Key() string
+	ReferenceSlots() []ReferenceSlot
 	Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
 }
 
@@ -165,6 +167,8 @@ type NormalizeRequest struct {
 	Source     *source.SourceDocument        `json:"-"`
 	LaneID     string                        `json:"lane_id"`
 	Candidates []artifacts.ArtifactCandidate `json:"candidates"`
+	References ReferenceSet                  `json:"references,omitempty"`
+	LLMClient  StructuredLLMClient           `json:"-"`
 	LLMProfile string                        `json:"llm_profile,omitempty"`
 	Options    map[string]any                `json:"options,omitempty"`
 	Metadata   map[string]any                `json:"metadata,omitempty"`
@@ -177,6 +181,7 @@ type NormalizeResult struct {
 
 type Normalizer interface {
 	Key() string
+	ReferenceSlots() []ReferenceSlot
 	Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
 }
 

internal/framework/contracts/contracts_test.go

@@ -378,6 +378,10 @@ func (chunker fakeChunker) Key() string {
 	return chunker.key
 }
 
+func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
+	return nil
+}
+
 func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
 	return ChunkResult{
 		Chunks: []SourceChunk{
@@ -400,6 +404,10 @@ func (chunker *recordingChunker) Key() string {
 	return chunker.key
 }
 
+func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
+	return nil
+}
+
 func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
 	chunker.request = req
 	return fakeChunker{key: chunker.key}.Chunk(ctx, req)
@@ -487,6 +495,10 @@ func (normalizer fakeNormalizer) Key() string {
 	return normalizer.key
 }
 
+func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
+	return nil
+}
+
 func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
 	return NormalizeResult{Candidates: req.Candidates}, nil
 }

internal/framework/pipeline/chunker_registry_test.go

@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
 	return chunker.key
 }
 
+func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{}, nil
 }
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
 	return normalizer.key
 }
 
+func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	return contracts.NormalizeResult{}, nil
 }

internal/framework/pipeline/module.go

@@ -87,7 +87,7 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
 	if spec.Stage != expectedStage {
 		return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
 	}
-	if spec.Stage != StageExtract && len(spec.ReferenceSlots) > 0 {
+	if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
 		return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
 	}
 	if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
@@ -96,6 +96,10 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
 	return nil
 }
 
+func referenceSlotStage(stage ModuleStage) bool {
+	return stage == StageChunk || stage == StageExtract || stage == StageNormalize
+}
+
 func sortedRegistryKeys[C any](constructors map[string]C) []string {
 	if len(constructors) == 0 {
 		return nil

internal/framework/pipeline/module_test.go

@@ -7,19 +7,117 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 )
 
-func TestValidateModuleSpecRejectsReferenceSlotsForNonExtractors(t *testing.T) {
+func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
-	err := validateModuleSpec("chunker", StageChunk, ModuleSpec{
+	tests := []struct {
-		Key:   "generic",
+		name  string
-		Stage: StageChunk,
+		kind  string
-		ReferenceSlots: []contracts.ReferenceSlot{
+		stage ModuleStage
-			{Name: "roster"},
+	}{
-		},
+		{name: "chunker", kind: "chunker", stage: StageChunk},
-	})
+		{name: "extractor", kind: "extractor", stage: StageExtract},
-
+		{name: "normalizer", kind: "normalizer", stage: StageNormalize},
-	if err == nil {
-		t.Fatal("validateModuleSpec() error = nil, want error")
 	}
-	if !strings.Contains(err.Error(), "reference slots") {
+
-		t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			spec := normalizeModuleSpec(ModuleSpec{
+				Key:   "module",
+				Stage: test.stage,
+				ReferenceSlots: []contracts.ReferenceSlot{
+					{Name: "roster", Description: "Character roster", MaxBytes: 1024},
+				},
+			})
+			err := validateModuleSpec(test.kind, test.stage, spec)
+			if err != nil {
+				t.Fatalf("validateModuleSpec() error = %v, want nil", err)
+			}
+		})
+	}
+}
+
+func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
+	tests := []struct {
+		name  string
+		kind  string
+		stage ModuleStage
+	}{
+		{name: "input", kind: "input adapter", stage: StageInput},
+		{name: "merge", kind: "merger", stage: StageMerge},
+		{name: "validate", kind: "validator", stage: StageValidate},
+		{name: "output", kind: "output encoder", stage: StageOutput},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			spec := normalizeModuleSpec(ModuleSpec{
+				Key:   "module",
+				Stage: test.stage,
+				ReferenceSlots: []contracts.ReferenceSlot{
+					{Name: "roster"},
+				},
+			})
+			err := validateModuleSpec(test.kind, test.stage, spec)
+			if err == nil {
+				t.Fatal("validateModuleSpec() error = nil, want error")
+			}
+			if !strings.Contains(err.Error(), "reference slots") {
+				t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
+			}
+		})
+	}
+}
+
+func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
+	invalidSlots := []struct {
+		name  string
+		slots []contracts.ReferenceSlot
+		want  string
+	}{
+		{
+			name:  "empty name",
+			slots: []contracts.ReferenceSlot{{Name: " "}},
+			want:  "name",
+		},
+		{
+			name: "duplicate name after trim",
+			slots: []contracts.ReferenceSlot{
+				{Name: "roster"},
+				{Name: " roster "},
+			},
+			want: "duplicated",
+		},
+		{
+			name:  "negative max bytes",
+			slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
+			want:  "max_bytes",
+		},
+	}
+	eligibleStages := []struct {
+		name  string
+		kind  string
+		stage ModuleStage
+	}{
+		{name: "chunk", kind: "chunker", stage: StageChunk},
+		{name: "extract", kind: "extractor", stage: StageExtract},
+		{name: "normalize", kind: "normalizer", stage: StageNormalize},
+	}
+
+	for _, stage := range eligibleStages {
+		for _, invalid := range invalidSlots {
+			t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
+				spec := normalizeModuleSpec(ModuleSpec{
+					Key:            "module",
+					Stage:          stage.stage,
+					ReferenceSlots: invalid.slots,
+				})
+				err := validateModuleSpec(stage.kind, stage.stage, spec)
+				if err == nil {
+					t.Fatal("validateModuleSpec() error = nil, want error")
+				}
+				if !strings.Contains(err.Error(), invalid.want) {
+					t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
+				}
+			})
+		}
 	}
 }

internal/framework/pipeline/registry_integration_test.go

@@ -118,6 +118,10 @@ func (chunker integrationChunker) Key() string {
 	return "chunk"
 }
 
+func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{
 		Chunks: []contracts.SourceChunk{
@@ -186,6 +190,10 @@ func (normalizer integrationNormalizer) Key() string {
 	return "normalize"
 }
 
+func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	return contracts.NormalizeResult{Candidates: req.Candidates}, nil
 }

internal/framework/pipeline/runner.go

@@ -224,6 +224,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
 		Source:     doc,
 		LaneID:     lane.ID,
 		Candidates: mergeResult.Candidates,
+		LLMClient:  input.LLMClient,
 		LLMProfile: lane.Normalize.LLMProfile,
 		Options:    cloneOptions(lane.Normalize.Options),
 		Metadata:   input.Metadata,

internal/framework/pipeline/runner_test.go

@@ -464,6 +464,10 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
 	if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
 		t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
 	}
+	normalizer := modules.normalizers["normalize"]
+	if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
+		t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
+	}
 	if extractor.seenMetadata[0]["request"] != "test" {
 		t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
 	}
@@ -1314,6 +1318,10 @@ func (chunker *runnerChunker) Key() string {
 	return chunker.key
 }
 
+func (chunker *runnerChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	chunker.requests = append(chunker.requests, req)
 	return contracts.ChunkResult{
@@ -1421,6 +1429,10 @@ func (normalizer *runnerNormalizer) Key() string {
 	return normalizer.key
 }
 
+func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	normalizer.requests = append(normalizer.requests, req)
 	candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)

internal/framework/pipeline/walking_skeleton_test.go

@@ -216,6 +216,10 @@ func (chunker walkingSkeletonChunker) Key() string {
 	return "fake/chunk"
 }
 
+func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	if len(req.Source.Units) < 3 {
 		return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
@@ -336,6 +340,10 @@ func (normalizer walkingSkeletonNormalizer) Key() string {
 	return DefaultNormalizeModule
 }
 
+func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	return contracts.NormalizeResult{Candidates: req.Candidates}, nil
 }

internal/modules/chunk/dnd/scenes/chunker.go

@@ -34,6 +34,10 @@ func (c *Chunker) Key() string {
 	return Key
 }
 
+func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (c *Chunker) ManifestMetadata() map[string]any {
 	promptMetadata := scenesPromptBundle.Metadata()
 	metadata := map[string]any{

internal/modules/chunk/dnd/scenes/chunker_test.go

@@ -57,6 +57,9 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
 	if built.Key() != Key {
 		t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
 	}
+	if slots := built.ReferenceSlots(); len(slots) != 0 {
+		t.Fatalf("ReferenceSlots() = %#v, want none", slots)
+	}
 }
 
 func TestRegisterNilRegistryReturnsError(t *testing.T) {

internal/modules/chunk/generic/chunker.go

@@ -31,6 +31,10 @@ func (c *Chunker) Key() string {
 	return Key
 }
 
+func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	if c == nil {
 		return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")

internal/modules/chunk/generic/chunker_test.go

@@ -41,6 +41,9 @@ func TestModuleSpecAndRegister(t *testing.T) {
 	if chunker.Key() != Key {
 		t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
 	}
+	if slots := chunker.ReferenceSlots(); len(slots) != 0 {
+		t.Fatalf("ReferenceSlots() = %#v, want none", slots)
+	}
 }
 
 func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {

internal/modules/extract/dnd/spells/config_test.go

@@ -236,6 +236,10 @@ func (dndSpellsChunker) Key() string {
 	return "fake/chunk"
 }
 
+func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{
 		Chunks: []contracts.SourceChunk{

internal/modules/input/seriatim/config_test.go

@@ -209,6 +209,8 @@ type fakeChunker struct{}
 
 func (fakeChunker) Key() string { return "fake/chunk" }
 
+func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
+
 func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{}, nil
 }

internal/modules/input/seriatim/runner_test.go

@@ -157,6 +157,10 @@ func (runnerSeriatimChunker) Key() string {
 	return "fake/chunk"
 }
 
+func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
 	return contracts.ChunkResult{
 		Chunks: []contracts.SourceChunk{

internal/modules/normalize/noop/normalizer.go

@@ -25,6 +25,10 @@ func (n *Normalizer) Key() string {
 	return Key
 }
 
+func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
 	if n == nil {
 		return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")

internal/modules/normalize/noop/normalizer_test.go

@@ -34,6 +34,13 @@ func TestModuleSpecAndRegister(t *testing.T) {
 	if !reflect.DeepEqual(spec, want) {
 		t.Fatalf("registered spec = %#v, want %#v", spec, want)
 	}
+	normalizer, err := registry.Build(Key)
+	if err != nil {
+		t.Fatalf("Build(%q) error = %v, want nil", Key, err)
+	}
+	if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
+		t.Fatalf("ReferenceSlots() = %#v, want none", slots)
+	}
 }
 
 func TestNormalizePassesThroughOrderAndValues(t *testing.T) {