Record top-level module metadata in run manifests

docs/integrations/json-output.md

@@ -58,6 +58,18 @@ approved.
   "pipeline_digest": "sha256:...",
   "input_module": "seriatim",
   "chunker": "generic",
+  "module_metadata": {
+    "chunker": {
+      "prompt_id": "dnd.scenes",
+      "prompt_version": "v1",
+      "prompt_sha256": "sha256:...",
+      "response_schema_key": "dnd_scenes",
+      "response_schema_id": "schema-dnd-scenes",
+      "response_schema_name": "dnd_scenes",
+      "response_schema_version": "v1",
+      "response_schema_sha256": "sha256:..."
+    }
+  },
   "source_digests": ["sha256:..."],
   "extractors": ["dnd/spells"],
   "merger": "appendorder",
@@ -89,6 +101,10 @@ Fields with empty values may be omitted by JSON encoding.
 `validation_status` is `approved` when no candidates were rejected and
 `rejected` when one or more candidates were rejected.
 
+Top-level `module_metadata` is reserved for singleton pipeline modules
+(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
+`artifact_lanes[].metadata`.
+
 ## Artifact Files
 
 Each artifact file has this shape:

docs/internal/modules.md

@@ -87,8 +87,9 @@ unit IDs, and unit count. Boundary caveats become warnings with reason code
 `scene_boundary_caveat`.
 
 Malformed model output fails explicitly rather than falling back to another
-chunker. The chunker exposes prompt and response-schema provenance through its
+chunker. The chunker exposes prompt and response-schema provenance through
-metadata provider without raw prompts, raw schemas, source text, or secrets.
+top-level `module_metadata.chunker` without raw prompts, raw schemas, source
+text, or secrets.
 
 ## `dnd/spells` Extractor
 
@@ -114,7 +115,8 @@ Artifact type and schema version:
 - schema version: `v1`
 
 The extractor adds prompt and response-schema provenance to lane manifest
-metadata. Durable artifact payload details belong in the
+metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
+details belong in the
 [D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
 
 ## D&D Spell Validators

docs/internal/pipeline.md

@@ -145,9 +145,15 @@ On successful execution, the manifest validation status is:
 
 ## Manifest Population
 
-The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
+The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
-lanes, LLM profile metadata, source digest, validation status, and timing.
+module metadata, artifact lanes, LLM profile metadata, source digest,
+validation status, and timing.
 
-Modules can add non-secret manifest metadata by implementing
+Singleton pipeline modules may add non-secret metadata by implementing
-`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
+`contracts.ManifestMetadataProvider`. The runner records that metadata under
+`module_metadata` with stable keys for `input`, `chunker`, and `output`.
+
+Lane-owned modules may add non-secret metadata through
+`artifact_lanes[].metadata`. The runner records extractor, merger, and
+normalizer metadata there. The D&D spell extractor uses lane metadata for
 prompt and response-schema provenance.

docs/operations.md

@@ -34,8 +34,8 @@ The `json` output module writes these files:
 
 - `index.json`: file index with paths to the manifest, artifact files,
   rejected artifacts, and warnings.
-- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
+- `manifest.json`: run manifest with resolved pipeline provenance, top-level
-  validation status, and timing.
+  module metadata, module keys, validation status, and timing.
 - `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
   type. For the current D&D spell extractor, this includes
   `artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
@@ -63,7 +63,7 @@ Implemented diagnostics artifacts:
 - `effective-config.json`: resolved config with API keys redacted.
 - `resolved-pipeline.json`: resolved module bindings and pipeline digest.
 - `run-manifest.json`: the same run manifest written to durable output when it
-  is available.
+  is available, including top-level module metadata when present.
 - `warnings.json`: warning list.
 - `run-report.json`: counts, status, output path, diagnostics path, and run ID.
 - `error.log`: failure message, written after diagnostics directory creation

internal/cli/run_test.go

@@ -1094,6 +1094,37 @@ func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
 	if manifest.Chunker != scenes.Key {
 		t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
 	}
+	chunkerMetadata := manifest.ModuleMetadata["chunker"]
+	if chunkerMetadata == nil {
+		t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"])
+	}
+	wantMetadataKeys := []string{
+		"prompt_id",
+		"prompt_version",
+		"prompt_sha256",
+		"response_schema_key",
+		"response_schema_id",
+		"response_schema_name",
+		"response_schema_version",
+		"response_schema_sha256",
+	}
+	if len(chunkerMetadata) != len(wantMetadataKeys) {
+		t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys))
+	}
+	for _, key := range wantMetadataKeys {
+		value, ok := chunkerMetadata[key]
+		if !ok {
+			t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata)
+		}
+		if _, ok := value.(string); !ok {
+			t.Fatalf("chunker metadata[%q] = %#v, want string", key, value)
+		}
+	}
+	for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} {
+		if _, ok := chunkerMetadata[forbidden]; ok {
+			t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata)
+		}
+	}
 	for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
 		if strings.Contains(string(manifestBytes), forbidden) {
 			t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)

internal/core/artifacts/artifacts.go

@@ -49,22 +49,23 @@ type LLMProfileManifest struct {
 }
 
 type RunManifest struct {
-	RunID            string                 `json:"run_id,omitempty"`
+	RunID            string                    `json:"run_id,omitempty"`
-	PipelineID       string                 `json:"pipeline_id,omitempty"`
+	PipelineID       string                    `json:"pipeline_id,omitempty"`
-	PipelineDigest   string                 `json:"pipeline_digest,omitempty"`
+	PipelineDigest   string                    `json:"pipeline_digest,omitempty"`
-	InputModule      string                 `json:"input_module,omitempty"`
+	InputModule      string                    `json:"input_module,omitempty"`
-	Chunker          string                 `json:"chunker,omitempty"`
+	Chunker          string                    `json:"chunker,omitempty"`
-	SourceDigests    []string               `json:"source_digests,omitempty"`
+	SourceDigests    []string                  `json:"source_digests,omitempty"`
-	Extractors       []string               `json:"extractors,omitempty"`
+	Extractors       []string                  `json:"extractors,omitempty"`
-	Merger           string                 `json:"merger,omitempty"`
+	Merger           string                    `json:"merger,omitempty"`
-	Normalizer       string                 `json:"normalizer,omitempty"`
+	Normalizer       string                    `json:"normalizer,omitempty"`
-	OutputEncoder    string                 `json:"output_encoder,omitempty"`
+	OutputEncoder    string                    `json:"output_encoder,omitempty"`
-	ArtifactLanes    []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
+	ModuleMetadata   map[string]map[string]any `json:"module_metadata,omitempty"`
-	LLMProfiles      []LLMProfileManifest   `json:"llm_profiles,omitempty"`
+	ArtifactLanes    []ArtifactLaneManifest    `json:"artifact_lanes,omitempty"`
-	SchemaVersion    string                 `json:"schema_version,omitempty"`
+	LLMProfiles      []LLMProfileManifest      `json:"llm_profiles,omitempty"`
-	ValidationStatus string                 `json:"validation_status,omitempty"`
+	SchemaVersion    string                    `json:"schema_version,omitempty"`
-	StartedAt        *time.Time             `json:"started_at,omitempty"`
+	ValidationStatus string                    `json:"validation_status,omitempty"`
-	CompletedAt      *time.Time             `json:"completed_at,omitempty"`
+	StartedAt        *time.Time                `json:"started_at,omitempty"`
+	CompletedAt      *time.Time                `json:"completed_at,omitempty"`
 }
 
 func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {

internal/core/artifacts/artifacts_test.go

@@ -183,6 +183,42 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
 	assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
 }
 
+func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
+	manifest := RunManifest{
+		ModuleMetadata: map[string]map[string]any{
+			"chunker": {
+				"prompt_id":            "dnd.scenes",
+				"prompt_version":       "v1",
+				"prompt_sha256":        "sha256:abc123",
+				"response_schema_key":  "dnd_scenes",
+				"response_schema_name": "dnd_scenes",
+			},
+		},
+	}
+
+	gotJSON, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	var got map[string]any
+	if err := json.Unmarshal(gotJSON, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	moduleMetadata, ok := got["module_metadata"].(map[string]any)
+	if !ok {
+		t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
+	}
+	assertHasKeys(t, moduleMetadata, "chunker")
+
+	chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
+	if !ok {
+		t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
+	}
+	assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
+}
+
 func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
 	t.Helper()
 

internal/framework/pipeline/runner.go

@@ -69,6 +69,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
 	if err != nil {
 		return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
 	}
+	attachModuleManifestMetadata(&output, "input", adapter)
 	doc, err := adapter.Parse(ctx, contracts.ParseRequest{
 		SourceID:   input.SourceID,
 		Path:       input.Path,
@@ -89,6 +90,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
 	if err != nil {
 		return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
 	}
+	attachModuleManifestMetadata(&output, "chunker", chunker)
 	chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
 		Source:     doc,
 		LLMClient:  input.LLMClient,
@@ -125,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
 	if err != nil {
 		return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
 	}
+	attachModuleManifestMetadata(&output, "output", encoder)
 	encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
 		Manifest:   output.Manifest,
 		Approved:   output.Approved,
@@ -386,14 +389,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
 
 		metadata := make(map[string]any)
 		for _, module := range modules {
-			provider, ok := module.(contracts.ManifestMetadataProvider)
+			moduleMetadata, ok := moduleManifestMetadata(module)
 			if !ok {
 				continue
 			}
-			moduleMetadata := cloneMetadata(provider.ManifestMetadata())
-			if len(moduleMetadata) == 0 {
-				continue
-			}
 			key := manifestMetadataKey(module)
 			if key == "" {
 				continue
@@ -407,6 +406,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
 	}
 }
 
+func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
+	if output == nil {
+		return
+	}
+	moduleMetadata, ok := moduleManifestMetadata(module)
+	if !ok {
+		return
+	}
+	if output.Manifest.ModuleMetadata == nil {
+		output.Manifest.ModuleMetadata = make(map[string]map[string]any)
+	}
+	output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
+}
+
 func manifestMetadataKey(module any) string {
 	switch module.(type) {
 	case contracts.Extractor:
@@ -420,6 +433,19 @@ func manifestMetadataKey(module any) string {
 	}
 }
 
+func moduleManifestMetadata(module any) (map[string]any, bool) {
+	provider, ok := module.(contracts.ManifestMetadataProvider)
+	if !ok {
+		return nil, false
+	}
+
+	moduleMetadata := cloneMetadata(provider.ManifestMetadata())
+	if len(moduleMetadata) == 0 {
+		return nil, false
+	}
+	return moduleMetadata, true
+}
+
 func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
 	out := make([]contracts.OutputFile, 0, len(result.Files))
 	for _, file := range result.Files {

internal/framework/pipeline/runner_test.go

@@ -459,6 +459,47 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
 	}
 }
 
+func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
+	modules := defaultRunnerModules()
+	modules.input.manifestMetadata = map[string]any{
+		"input_profile": "input-metadata",
+	}
+	modules.chunker.manifestMetadata = map[string]any{
+		"prompt_id":            "dnd.scenes",
+		"prompt_version":       "v1",
+		"prompt_sha256":        "sha256:chunker-prompt",
+		"response_schema_key":  "dnd_scenes",
+		"response_schema_id":   "schema-dnd-scenes",
+		"response_schema_name": "dnd_scenes",
+	}
+	modules.output.manifestMetadata = map[string]any{
+		"output_profile": "output-metadata",
+	}
+
+	output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
+	if err != nil {
+		t.Fatalf("Run() error = %v, want nil", err)
+	}
+
+	if output.Manifest.ModuleMetadata == nil {
+		t.Fatal("ModuleMetadata = nil, want module metadata map")
+	}
+	if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
+		t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
+	}
+	if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
+		t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
+	}
+	if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
+		t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
+	}
+
+	modules.chunker.manifestMetadata["prompt_id"] = "changed"
+	if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
+		t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
+	}
+}
+
 func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
 	modules := defaultRunnerModules()
 
@@ -863,6 +904,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
 	if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
 		t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
 	}
+	if output.Manifest.ModuleMetadata != nil {
+		if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
+			t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
+		}
+	}
 }
 
 func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
@@ -1051,10 +1097,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
 }
 
 type runnerInputAdapter struct {
-	key      string
+	key              string
-	doc      *source.SourceDocument
+	doc              *source.SourceDocument
-	err      error
+	err              error
-	requests []contracts.ParseRequest
+	manifestMetadata map[string]any
+	requests         []contracts.ParseRequest
 }
 
 func (adapter *runnerInputAdapter) Key() string {
@@ -1066,12 +1113,17 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
 	return adapter.doc, adapter.err
 }
 
+func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
+	return adapter.manifestMetadata
+}
+
 type runnerChunker struct {
-	key      string
+	key              string
-	chunks   []contracts.SourceChunk
+	chunks           []contracts.SourceChunk
-	warnings []contracts.Warning
+	warnings         []contracts.Warning
-	err      error
+	err              error
-	requests []contracts.ChunkRequest
+	manifestMetadata map[string]any
+	requests         []contracts.ChunkRequest
 }
 
 func (chunker *runnerChunker) Key() string {
@@ -1086,6 +1138,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
 	}, chunker.err
 }
 
+func (chunker *runnerChunker) ManifestMetadata() map[string]any {
+	return chunker.manifestMetadata
+}
+
 type runnerExtractor struct {
 	key              string
 	artifactType     string
@@ -1226,11 +1282,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
 }
 
 type runnerOutputEncoder struct {
-	key      string
+	key              string
-	files    []contracts.OutputFile
+	files            []contracts.OutputFile
-	warnings []contracts.Warning
+	warnings         []contracts.Warning
-	err      error
+	err              error
-	requests []contracts.OutputRequest
+	manifestMetadata map[string]any
+	requests         []contracts.OutputRequest
 }
 
 func (encoder *runnerOutputEncoder) Key() string {
@@ -1245,6 +1302,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
 	}, encoder.err
 }
 
+func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
+	return encoder.manifestMetadata
+}
+
 type fakeLLMClient struct{}
 
 func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {