diff --git a/docs/integrations/json-output.md b/docs/integrations/json-output.md index 10f8ff5..12f8305 100644 --- a/docs/integrations/json-output.md +++ b/docs/integrations/json-output.md @@ -58,6 +58,18 @@ approved. "pipeline_digest": "sha256:...", "input_module": "seriatim", "chunker": "generic", + "module_metadata": { + "chunker": { + "prompt_id": "dnd.scenes", + "prompt_version": "v1", + "prompt_sha256": "sha256:...", + "response_schema_key": "dnd_scenes", + "response_schema_id": "schema-dnd-scenes", + "response_schema_name": "dnd_scenes", + "response_schema_version": "v1", + "response_schema_sha256": "sha256:..." + } + }, "source_digests": ["sha256:..."], "extractors": ["dnd/spells"], "merger": "appendorder", @@ -89,6 +101,10 @@ Fields with empty values may be omitted by JSON encoding. `validation_status` is `approved` when no candidates were rejected and `rejected` when one or more candidates were rejected. +Top-level `module_metadata` is reserved for singleton pipeline modules +(`input`, `chunker`, and `output`). Lane-owned module metadata remains under +`artifact_lanes[].metadata`. + ## Artifact Files Each artifact file has this shape: diff --git a/docs/internal/modules.md b/docs/internal/modules.md index c0692c6..b9d959d 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -87,8 +87,9 @@ unit IDs, and unit count. Boundary caveats become warnings with reason code `scene_boundary_caveat`. Malformed model output fails explicitly rather than falling back to another -chunker. The chunker exposes prompt and response-schema provenance through its -metadata provider without raw prompts, raw schemas, source text, or secrets. +chunker. The chunker exposes prompt and response-schema provenance through +top-level `module_metadata.chunker` without raw prompts, raw schemas, source +text, or secrets. ## `dnd/spells` Extractor @@ -114,7 +115,8 @@ Artifact type and schema version: - schema version: `v1` The extractor adds prompt and response-schema provenance to lane manifest -metadata. Durable artifact payload details belong in the +metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload +details belong in the [D&D spell artifact contract](../integrations/dnd-spell-artifacts.md). ## D&D Spell Validators diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index f624e10..655d6f5 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -145,9 +145,15 @@ On successful execution, the manifest validation status is: ## Manifest Population -The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact -lanes, LLM profile metadata, source digest, validation status, and timing. +The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level +module metadata, artifact lanes, LLM profile metadata, source digest, +validation status, and timing. -Modules can add non-secret manifest metadata by implementing -`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for +Singleton pipeline modules may add non-secret metadata by implementing +`contracts.ManifestMetadataProvider`. The runner records that metadata under +`module_metadata` with stable keys for `input`, `chunker`, and `output`. + +Lane-owned modules may add non-secret metadata through +`artifact_lanes[].metadata`. The runner records extractor, merger, and +normalizer metadata there. The D&D spell extractor uses lane metadata for prompt and response-schema provenance. diff --git a/docs/operations.md b/docs/operations.md index 130c3bb..689004c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -34,8 +34,8 @@ The `json` output module writes these files: - `index.json`: file index with paths to the manifest, artifact files, rejected artifacts, and warnings. -- `manifest.json`: run manifest with resolved pipeline provenance, module keys, - validation status, and timing. +- `manifest.json`: run manifest with resolved pipeline provenance, top-level + module metadata, module keys, validation status, and timing. - `artifacts/.json`: approved artifacts grouped by artifact type. For the current D&D spell extractor, this includes `artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved. @@ -63,7 +63,7 @@ Implemented diagnostics artifacts: - `effective-config.json`: resolved config with API keys redacted. - `resolved-pipeline.json`: resolved module bindings and pipeline digest. - `run-manifest.json`: the same run manifest written to durable output when it - is available. + is available, including top-level module metadata when present. - `warnings.json`: warning list. - `run-report.json`: counts, status, output path, diagnostics path, and run ID. - `error.log`: failure message, written after diagnostics directory creation diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 99dbc5b..a59b032 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1094,6 +1094,37 @@ func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) { if manifest.Chunker != scenes.Key { t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key) } + chunkerMetadata := manifest.ModuleMetadata["chunker"] + if chunkerMetadata == nil { + t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"]) + } + wantMetadataKeys := []string{ + "prompt_id", + "prompt_version", + "prompt_sha256", + "response_schema_key", + "response_schema_id", + "response_schema_name", + "response_schema_version", + "response_schema_sha256", + } + if len(chunkerMetadata) != len(wantMetadataKeys) { + t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys)) + } + for _, key := range wantMetadataKeys { + value, ok := chunkerMetadata[key] + if !ok { + t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata) + } + if _, ok := value.(string); !ok { + t.Fatalf("chunker metadata[%q] = %#v, want string", key, value) + } + } + for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} { + if _, ok := chunkerMetadata[forbidden]; ok { + t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata) + } + } for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} { if strings.Contains(string(manifestBytes), forbidden) { t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes) diff --git a/internal/core/artifacts/artifacts.go b/internal/core/artifacts/artifacts.go index 8e962eb..6724776 100644 --- a/internal/core/artifacts/artifacts.go +++ b/internal/core/artifacts/artifacts.go @@ -49,22 +49,23 @@ type LLMProfileManifest struct { } type RunManifest struct { - RunID string `json:"run_id,omitempty"` - PipelineID string `json:"pipeline_id,omitempty"` - PipelineDigest string `json:"pipeline_digest,omitempty"` - InputModule string `json:"input_module,omitempty"` - Chunker string `json:"chunker,omitempty"` - SourceDigests []string `json:"source_digests,omitempty"` - Extractors []string `json:"extractors,omitempty"` - Merger string `json:"merger,omitempty"` - Normalizer string `json:"normalizer,omitempty"` - OutputEncoder string `json:"output_encoder,omitempty"` - ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` - LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` - SchemaVersion string `json:"schema_version,omitempty"` - ValidationStatus string `json:"validation_status,omitempty"` - StartedAt *time.Time `json:"started_at,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` + RunID string `json:"run_id,omitempty"` + PipelineID string `json:"pipeline_id,omitempty"` + PipelineDigest string `json:"pipeline_digest,omitempty"` + InputModule string `json:"input_module,omitempty"` + Chunker string `json:"chunker,omitempty"` + SourceDigests []string `json:"source_digests,omitempty"` + Extractors []string `json:"extractors,omitempty"` + Merger string `json:"merger,omitempty"` + Normalizer string `json:"normalizer,omitempty"` + OutputEncoder string `json:"output_encoder,omitempty"` + ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"` + ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` + LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` + SchemaVersion string `json:"schema_version,omitempty"` + ValidationStatus string `json:"validation_status,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` } func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact { diff --git a/internal/core/artifacts/artifacts_test.go b/internal/core/artifacts/artifacts_test.go index a0be960..a8825ea 100644 --- a/internal/core/artifacts/artifacts_test.go +++ b/internal/core/artifacts/artifacts_test.go @@ -183,6 +183,42 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata") } +func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) { + manifest := RunManifest{ + ModuleMetadata: map[string]map[string]any{ + "chunker": { + "prompt_id": "dnd.scenes", + "prompt_version": "v1", + "prompt_sha256": "sha256:abc123", + "response_schema_key": "dnd_scenes", + "response_schema_name": "dnd_scenes", + }, + }, + } + + gotJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + + var got map[string]any + if err := json.Unmarshal(gotJSON, &got); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + + moduleMetadata, ok := got["module_metadata"].(map[string]any) + if !ok { + t.Fatalf("module_metadata = %#v, want object", got["module_metadata"]) + } + assertHasKeys(t, moduleMetadata, "chunker") + + chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any) + if !ok { + t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"]) + } + assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name") +} + func assertHasKeys(t *testing.T, values map[string]any, keys ...string) { t.Helper() diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 45379b6..c6b9706 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -69,6 +69,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { if err != nil { return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err) } + attachModuleManifestMetadata(&output, "input", adapter) doc, err := adapter.Parse(ctx, contracts.ParseRequest{ SourceID: input.SourceID, Path: input.Path, @@ -89,6 +90,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { if err != nil { return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err) } + attachModuleManifestMetadata(&output, "chunker", chunker) chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{ Source: doc, LLMClient: input.LLMClient, @@ -125,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { if err != nil { return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err) } + attachModuleManifestMetadata(&output, "output", encoder) encoded, err := encoder.Encode(ctx, contracts.OutputRequest{ Manifest: output.Manifest, Approved: output.Approved, @@ -386,14 +389,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) { metadata := make(map[string]any) for _, module := range modules { - provider, ok := module.(contracts.ManifestMetadataProvider) + moduleMetadata, ok := moduleManifestMetadata(module) if !ok { continue } - moduleMetadata := cloneMetadata(provider.ManifestMetadata()) - if len(moduleMetadata) == 0 { - continue - } key := manifestMetadataKey(module) if key == "" { continue @@ -407,6 +406,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) { } } +func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) { + if output == nil { + return + } + moduleMetadata, ok := moduleManifestMetadata(module) + if !ok { + return + } + if output.Manifest.ModuleMetadata == nil { + output.Manifest.ModuleMetadata = make(map[string]map[string]any) + } + output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata +} + func manifestMetadataKey(module any) string { switch module.(type) { case contracts.Extractor: @@ -420,6 +433,19 @@ func manifestMetadataKey(module any) string { } } +func moduleManifestMetadata(module any) (map[string]any, bool) { + provider, ok := module.(contracts.ManifestMetadataProvider) + if !ok { + return nil, false + } + + moduleMetadata := cloneMetadata(provider.ManifestMetadata()) + if len(moduleMetadata) == 0 { + return nil, false + } + return moduleMetadata, true +} + func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) { out := make([]contracts.OutputFile, 0, len(result.Files)) for _, file := range result.Files { diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index edaa7d5..2fc346b 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -459,6 +459,47 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) { } } +func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) { + modules := defaultRunnerModules() + modules.input.manifestMetadata = map[string]any{ + "input_profile": "input-metadata", + } + modules.chunker.manifestMetadata = map[string]any{ + "prompt_id": "dnd.scenes", + "prompt_version": "v1", + "prompt_sha256": "sha256:chunker-prompt", + "response_schema_key": "dnd_scenes", + "response_schema_id": "schema-dnd-scenes", + "response_schema_name": "dnd_scenes", + } + modules.output.manifestMetadata = map[string]any{ + "output_profile": "output-metadata", + } + + output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + if output.Manifest.ModuleMetadata == nil { + t.Fatal("ModuleMetadata = nil, want module metadata map") + } + if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) { + t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata) + } + if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) { + t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata) + } + if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) { + t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata) + } + + modules.chunker.manifestMetadata["prompt_id"] = "changed" + if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" { + t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"]) + } +} + func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) { modules := defaultRunnerModules() @@ -863,6 +904,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) { if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" { t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata) } + if output.Manifest.ModuleMetadata != nil { + if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok { + t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata) + } + } } func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { @@ -1051,10 +1097,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries { } type runnerInputAdapter struct { - key string - doc *source.SourceDocument - err error - requests []contracts.ParseRequest + key string + doc *source.SourceDocument + err error + manifestMetadata map[string]any + requests []contracts.ParseRequest } func (adapter *runnerInputAdapter) Key() string { @@ -1066,12 +1113,17 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars return adapter.doc, adapter.err } +func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any { + return adapter.manifestMetadata +} + type runnerChunker struct { - key string - chunks []contracts.SourceChunk - warnings []contracts.Warning - err error - requests []contracts.ChunkRequest + key string + chunks []contracts.SourceChunk + warnings []contracts.Warning + err error + manifestMetadata map[string]any + requests []contracts.ChunkRequest } func (chunker *runnerChunker) Key() string { @@ -1086,6 +1138,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ }, chunker.err } +func (chunker *runnerChunker) ManifestMetadata() map[string]any { + return chunker.manifestMetadata +} + type runnerExtractor struct { key string artifactType string @@ -1226,11 +1282,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va } type runnerOutputEncoder struct { - key string - files []contracts.OutputFile - warnings []contracts.Warning - err error - requests []contracts.OutputRequest + key string + files []contracts.OutputFile + warnings []contracts.Warning + err error + manifestMetadata map[string]any + requests []contracts.OutputRequest } func (encoder *runnerOutputEncoder) Key() string { @@ -1245,6 +1302,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou }, encoder.err } +func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any { + return encoder.manifestMetadata +} + type fakeLLMClient struct{} func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {