Add reference contracts to extractor metadata

docs/internal/pipeline.md

@@ -44,6 +44,10 @@ Every production module registers a `ModuleSpec` with:
 - `Provides`: capabilities added after that module runs;
 - `Requires`: capabilities that must already be available.
 
+Extractor specs may also declare reference slots. Slot declarations are
+available from registry metadata without constructing extractor instances.
+Non-extractor module specs must not declare reference slots.
+
 Capability checks prevent incompatible pipeline composition before a run starts.
 
 ## Runner Input And Output

internal/framework/contracts/composition_test.go

@@ -203,6 +203,10 @@ func (extractor compositionExtractor) SchemaVersion() string {
 	return "v1"
 }
 
+func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (extractor compositionExtractor) Validators() []contracts.Validator {
 	return []contracts.Validator{compositionValidator{}}
 }

internal/framework/contracts/contracts.go

@@ -74,10 +74,49 @@ type Chunker interface {
 	Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
 }
 
+const (
+	ReferenceBindingSourceConfig = "config"
+	ReferenceBindingSourceCLI    = "cli"
+)
+
+type ReferenceSlot struct {
+	Name               string   `json:"name"`
+	Description        string   `json:"description,omitempty"`
+	Required           bool     `json:"required,omitempty"`
+	AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
+	Multiple           bool     `json:"multiple,omitempty"`
+	MaxBytes           int64    `json:"max_bytes,omitempty"`
+}
+
+type ReferenceOrigin struct {
+	Type string `json:"type"`
+	URI  string `json:"uri,omitempty"`
+}
+
+type ReferenceItem struct {
+	SlotName      string          `json:"slot_name"`
+	MediaType     string          `json:"media_type,omitempty"`
+	Content       []byte          `json:"-"`
+	Digest        string          `json:"digest,omitempty"`
+	Origin        ReferenceOrigin `json:"origin"`
+	SizeBytes     int64           `json:"size_bytes,omitempty"`
+	BindingSource string          `json:"binding_source,omitempty"`
+}
+
+type ResolvedReferenceSlot struct {
+	Slot  ReferenceSlot   `json:"slot"`
+	Items []ReferenceItem `json:"items,omitempty"`
+}
+
+type ReferenceSet struct {
+	Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
+}
+
 type ExtractionRequest struct {
 	Source         *source.SourceDocument `json:"-"`
 	Chunk          *SourceChunk           `json:"chunk,omitempty"`
 	AmbientContext map[string]any         `json:"ambient_context,omitempty"`
+	References     ReferenceSet           `json:"references,omitempty"`
 	LLMClient      StructuredLLMClient    `json:"-"`
 	LLMProfile     string                 `json:"llm_profile,omitempty"`
 	Options        map[string]any         `json:"options,omitempty"`
@@ -93,6 +132,7 @@ type Extractor interface {
 	Key() string
 	ArtifactType() string
 	SchemaVersion() string
+	ReferenceSlots() []ReferenceSlot
 	Validators() []Validator
 	Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
 }

internal/framework/contracts/contracts_test.go

@@ -186,6 +186,71 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
 	}
 }
 
+func TestReferenceSetDataTypes(t *testing.T) {
+	references := ReferenceSet{
+		Slots: map[string]ResolvedReferenceSlot{
+			"roster": {
+				Slot: ReferenceSlot{
+					Name:               "roster",
+					Description:        "Known characters",
+					Required:           true,
+					AcceptedMediaTypes: []string{"text/plain"},
+					Multiple:           true,
+					MaxBytes:           4096,
+				},
+				Items: []ReferenceItem{
+					{
+						SlotName:  "roster",
+						MediaType: "text/plain",
+						Content:   []byte("Aria\nBryn\n"),
+						Digest:    "sha256:reference",
+						Origin: ReferenceOrigin{
+							Type: "file",
+							URI:  "file:///tmp/roster.txt",
+						},
+						SizeBytes:     10,
+						BindingSource: ReferenceBindingSourceConfig,
+					},
+				},
+			},
+		},
+	}
+
+	item := references.Slots["roster"].Items[0]
+	if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
+		t.Fatalf("reference item = %#v, want constructed item fields", item)
+	}
+	if item.BindingSource != ReferenceBindingSourceConfig {
+		t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
+	}
+}
+
+func TestReferenceItemJSONOmitsContent(t *testing.T) {
+	item := ReferenceItem{
+		SlotName:  "roster",
+		MediaType: "text/plain",
+		Content:   []byte("reference content"),
+		Digest:    "sha256:reference",
+		Origin:    ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
+	}
+
+	encoded, err := json.Marshal(item)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v, want nil", err)
+	}
+
+	var got map[string]any
+	if err := json.Unmarshal(encoded, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v, want nil", err)
+	}
+	if _, ok := got["content"]; ok {
+		t.Fatalf("encoded reference item leaked content: %s", encoded)
+	}
+	if _, ok := got["Content"]; ok {
+		t.Fatalf("encoded reference item leaked Content: %s", encoded)
+	}
+}
+
 func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
 	candidate := artifacts.ArtifactCandidate{
 		Index:         0,
@@ -359,6 +424,10 @@ func (extractor fakeExtractor) SchemaVersion() string {
 	return extractor.schemaVersion
 }
 
+func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
+	return nil
+}
+
 func (extractor fakeExtractor) Validators() []Validator {
 	return extractor.validators
 }

internal/framework/pipeline/default_modules_test.go

@@ -117,6 +117,8 @@ func (defaultExtractor) ArtifactType() string { return "record" }
 
 func (defaultExtractor) SchemaVersion() string { return "v1" }
 
+func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
+
 func (defaultExtractor) Validators() []contracts.Validator { return nil }
 
 func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

internal/framework/pipeline/extractor_registry_test.go

@@ -49,6 +49,20 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
 		Stage:    StageExtract,
 		Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
 		Requires: []string{" source-document ", "source-document", ""},
+		ReferenceSlots: []contracts.ReferenceSlot{
+			{
+				Name:               " glossary ",
+				Description:        " Supporting terms ",
+				AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
+				MaxBytes:           1024,
+			},
+			{
+				Name:        " roster ",
+				Description: " Characters ",
+				Required:    true,
+				Multiple:    true,
+			},
+		},
 	}
 
 	if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
@@ -64,12 +78,28 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
 		Stage:    StageExtract,
 		Provides: []string{"generic-artifact", "source-citations"},
 		Requires: []string{"source-document"},
+		ReferenceSlots: []contracts.ReferenceSlot{
+			{
+				Name:               "glossary",
+				Description:        "Supporting terms",
+				AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
+				MaxBytes:           1024,
+			},
+			{
+				Name:        "roster",
+				Description: "Characters",
+				Required:    true,
+				Multiple:    true,
+			},
+		},
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Fatalf("Spec() = %#v, want %#v", got, want)
 	}
 
 	got.Provides[0] = "changed"
+	got.ReferenceSlots[0].Name = "changed"
+	got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
 	again, ok := registry.Spec("generic-extractor")
 	if !ok {
 		t.Fatal("Spec() after caller mutation ok = false, want true")
@@ -109,6 +139,50 @@ func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
 	}
 }
 
+func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
+	tests := []struct {
+		name  string
+		slots []contracts.ReferenceSlot
+		want  string
+	}{
+		{
+			name:  "empty name",
+			slots: []contracts.ReferenceSlot{{Name: " "}},
+			want:  "name",
+		},
+		{
+			name: "duplicate name after trim",
+			slots: []contracts.ReferenceSlot{
+				{Name: "roster"},
+				{Name: " roster "},
+			},
+			want: "duplicated",
+		},
+		{
+			name:  "negative max bytes",
+			slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
+			want:  "max_bytes",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			registry := NewExtractorRegistry()
+			err := registry.RegisterWithSpec(ModuleSpec{
+				Key:            "generic-extractor",
+				Stage:          StageExtract,
+				ReferenceSlots: test.slots,
+			}, fakeExtractorConstructor("generic-extractor"))
+			if err == nil {
+				t.Fatal("RegisterWithSpec() error = nil, want error")
+			}
+			if !strings.Contains(err.Error(), test.want) {
+				t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
+			}
+		})
+	}
+}
+
 func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
 	registry := NewExtractorRegistry()
 
@@ -301,6 +375,10 @@ func (extractor registryFakeExtractor) SchemaVersion() string {
 	return "v1"
 }
 
+func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (extractor registryFakeExtractor) Validators() []contracts.Validator {
 	return nil
 }

internal/framework/pipeline/module.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 )
 
 type ModuleStage string
@@ -19,10 +21,11 @@ const (
 )
 
 type ModuleSpec struct {
-	Key      string
+	Key            string
-	Stage    ModuleStage
+	Stage          ModuleStage
-	Provides []string
+	Provides       []string
-	Requires []string
+	Requires       []string
+	ReferenceSlots []contracts.ReferenceSlot
 }
 
 func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
@@ -34,10 +37,11 @@ func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
 
 func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
 	return ModuleSpec{
-		Key:      strings.TrimSpace(spec.Key),
+		Key:            strings.TrimSpace(spec.Key),
-		Stage:    spec.Stage,
+		Stage:          spec.Stage,
-		Provides: normalizeCapabilities(spec.Provides),
+		Provides:       normalizeCapabilities(spec.Provides),
-		Requires: normalizeCapabilities(spec.Requires),
+		Requires:       normalizeCapabilities(spec.Requires),
+		ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
 	}
 }
 
@@ -68,10 +72,11 @@ func normalizeCapabilities(values []string) []string {
 
 func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
 	return ModuleSpec{
-		Key:      spec.Key,
+		Key:            spec.Key,
-		Stage:    spec.Stage,
+		Stage:          spec.Stage,
-		Provides: append([]string(nil), spec.Provides...),
+		Provides:       append([]string(nil), spec.Provides...),
-		Requires: append([]string(nil), spec.Requires...),
+		Requires:       append([]string(nil), spec.Requires...),
+		ReferenceSlots: cloneReferenceSlots(spec.ReferenceSlots),
 	}
 }
 
@@ -82,6 +87,12 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
 	if spec.Stage != expectedStage {
 		return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
 	}
+	if spec.Stage != StageExtract && len(spec.ReferenceSlots) > 0 {
+		return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
+	}
+	if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
+		return fmt.Errorf("%s %q reference slots: %w", kind, spec.Key, err)
+	}
 	return nil
 }
 
@@ -97,3 +108,74 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
 	sort.Strings(keys)
 	return keys
 }
+
+func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
+	if len(slots) == 0 {
+		return nil
+	}
+
+	normalized := make([]contracts.ReferenceSlot, 0, len(slots))
+	for _, slot := range slots {
+		slot.Name = strings.TrimSpace(slot.Name)
+		slot.Description = strings.TrimSpace(slot.Description)
+		slot.AcceptedMediaTypes = normalizeStringSet(slot.AcceptedMediaTypes)
+		normalized = append(normalized, slot)
+	}
+	sort.SliceStable(normalized, func(i, j int) bool {
+		return normalized[i].Name < normalized[j].Name
+	})
+	return normalized
+}
+
+func normalizeStringSet(values []string) []string {
+	if len(values) == 0 {
+		return nil
+	}
+	seen := make(map[string]struct{}, len(values))
+	for _, value := range values {
+		normalized := strings.TrimSpace(value)
+		if normalized == "" {
+			continue
+		}
+		seen[normalized] = struct{}{}
+	}
+	if len(seen) == 0 {
+		return nil
+	}
+
+	out := make([]string, 0, len(seen))
+	for value := range seen {
+		out = append(out, value)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func validateReferenceSlots(slots []contracts.ReferenceSlot) error {
+	seen := make(map[string]struct{}, len(slots))
+	for i, slot := range slots {
+		if slot.Name == "" {
+			return fmt.Errorf("slot[%d].name must not be empty", i)
+		}
+		if _, ok := seen[slot.Name]; ok {
+			return fmt.Errorf("slot name %q is duplicated", slot.Name)
+		}
+		seen[slot.Name] = struct{}{}
+		if slot.MaxBytes < 0 {
+			return fmt.Errorf("slot %q max_bytes must not be negative", slot.Name)
+		}
+	}
+	return nil
+}
+
+func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
+	if len(slots) == 0 {
+		return nil
+	}
+	out := make([]contracts.ReferenceSlot, 0, len(slots))
+	for _, slot := range slots {
+		slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
+		out = append(out, slot)
+	}
+	return out
+}

internal/framework/pipeline/module_test.go

@@ -0,0 +1,25 @@
+package pipeline
+
+import (
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+)
+
+func TestValidateModuleSpecRejectsReferenceSlotsForNonExtractors(t *testing.T) {
+	err := validateModuleSpec("chunker", StageChunk, ModuleSpec{
+		Key:   "generic",
+		Stage: StageChunk,
+		ReferenceSlots: []contracts.ReferenceSlot{
+			{Name: "roster"},
+		},
+	})
+
+	if err == nil {
+		t.Fatal("validateModuleSpec() error = nil, want error")
+	}
+	if !strings.Contains(err.Error(), "reference slots") {
+		t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
+	}
+}

internal/framework/pipeline/registry_integration_test.go

@@ -149,6 +149,10 @@ func (extractor integrationExtractor) SchemaVersion() string {
 	return "v1"
 }
 
+func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (extractor integrationExtractor) Validators() []contracts.Validator {
 	return extractor.validators
 }

internal/framework/pipeline/runner_test.go

@@ -1269,6 +1269,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
 	return extractor.schemaVersion
 }
 
+func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
 	return extractor.manifestMetadata
 }

internal/framework/pipeline/walking_skeleton_test.go

@@ -252,6 +252,10 @@ func (extractor walkingSkeletonExtractor) SchemaVersion() string {
 	return "v1"
 }
 
+func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
 	return nil
 }

internal/modules/extract/dnd/spells/extractor.go

@@ -45,6 +45,10 @@ func (e *Extractor) SchemaVersion() string {
 	return SchemaVersion
 }
 
+func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (e *Extractor) ManifestMetadata() map[string]any {
 	promptMetadata := spellsPromptBundle.Metadata()
 	metadata := map[string]any{

internal/modules/extract/dnd/spells/registry_test.go

@@ -82,6 +82,15 @@ func TestRegisterStoresModuleSpec(t *testing.T) {
 	}
 }
 
+func TestRuntimeReferenceSlotsMatchModuleSpec(t *testing.T) {
+	extractor := New()
+	spec := ModuleSpec()
+
+	if !reflect.DeepEqual(extractor.ReferenceSlots(), spec.ReferenceSlots) {
+		t.Fatalf("ReferenceSlots() = %#v, want spec slots %#v", extractor.ReferenceSlots(), spec.ReferenceSlots)
+	}
+}
+
 func TestRegisterNilRegistryReturnsError(t *testing.T) {
 	err := Register(nil)
 	if err == nil {

internal/modules/input/seriatim/config_test.go

@@ -221,6 +221,8 @@ func (fakeExtractor) ArtifactType() string { return "fake" }
 
 func (fakeExtractor) SchemaVersion() string { return "v1" }
 
+func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
+
 func (fakeExtractor) Validators() []contracts.Validator { return nil }
 
 func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

internal/modules/input/seriatim/runner_test.go

@@ -186,6 +186,10 @@ func (e *runnerSeriatimExtractor) SchemaVersion() string {
 	return "v1"
 }
 
+func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
+	return nil
+}
+
 func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
 	return nil
 }