eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record top-level module metadata in run manifests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 13:23:56 +00:00
parent e19cc02c4d
commit e700df82d8
9 changed files with 224 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/cli/run_test.go
View File

@@ -1094,6 +1094,37 @@ func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
if manifest.Chunker != scenes.Key {
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
}
chunkerMetadata := manifest.ModuleMetadata["chunker"]
if chunkerMetadata == nil {
t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"])
}
wantMetadataKeys := []string{
"prompt_id",
"prompt_version",
"prompt_sha256",
"response_schema_key",
"response_schema_id",
"response_schema_name",
"response_schema_version",
"response_schema_sha256",
}
if len(chunkerMetadata) != len(wantMetadataKeys) {
t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys))
}
for _, key := range wantMetadataKeys {
value, ok := chunkerMetadata[key]
if !ok {
t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata)
}
if _, ok := value.(string); !ok {
t.Fatalf("chunker metadata[%q] = %#v, want string", key, value)
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} {
if _, ok := chunkerMetadata[forbidden]; ok {
t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata)
}
}
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
if strings.Contains(string(manifestBytes), forbidden) {
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)

33
internal/core/artifacts/artifacts.go
View File

@@ -49,22 +49,23 @@ type LLMProfileManifest struct {
}
type RunManifest struct {
RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
InputModule string `json:"input_module,omitempty"`
Chunker string `json:"chunker,omitempty"`
SourceDigests []string `json:"source_digests,omitempty"`
Extractors []string `json:"extractors,omitempty"`
Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
InputModule string `json:"input_module,omitempty"`
Chunker string `json:"chunker,omitempty"`
SourceDigests []string `json:"source_digests,omitempty"`
Extractors []string `json:"extractors,omitempty"`
Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
}
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {

36
internal/core/artifacts/artifacts_test.go
View File

@@ -183,6 +183,42 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
}
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
manifest := RunManifest{
ModuleMetadata: map[string]map[string]any{
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:abc123",
"response_schema_key": "dnd_scenes",
"response_schema_name": "dnd_scenes",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
moduleMetadata, ok := got["module_metadata"].(map[string]any)
if !ok {
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
}
assertHasKeys(t, moduleMetadata, "chunker")
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
if !ok {
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
}
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
}
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
t.Helper()

36
internal/framework/pipeline/runner.go
View File

@@ -69,6 +69,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
}
attachModuleManifestMetadata(&output, "input", adapter)
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
@@ -89,6 +90,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
attachModuleManifestMetadata(&output, "chunker", chunker)
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
LLMClient: input.LLMClient,
@@ -125,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
}
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
Approved: output.Approved,
@@ -386,14 +389,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
metadata := make(map[string]any)
for _, module := range modules {
provider, ok := module.(contracts.ManifestMetadataProvider)
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
continue
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
continue
}
key := manifestMetadataKey(module)
if key == "" {
continue
@@ -407,6 +406,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
}
}
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil {
return
}
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
return
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
}
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.Extractor:
@@ -420,6 +433,19 @@ func manifestMetadataKey(module any) string {
}
}
func moduleManifestMetadata(module any) (map[string]any, bool) {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
return nil, false
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
return nil, false
}
return moduleMetadata, true
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
out := make([]contracts.OutputFile, 0, len(result.Files))
for _, file := range result.Files {

89
internal/framework/pipeline/runner_test.go
View File

@@ -459,6 +459,47 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
}
}
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
modules := defaultRunnerModules()
modules.input.manifestMetadata = map[string]any{
"input_profile": "input-metadata",
}
modules.chunker.manifestMetadata = map[string]any{
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:chunker-prompt",
"response_schema_key": "dnd_scenes",
"response_schema_id": "schema-dnd-scenes",
"response_schema_name": "dnd_scenes",
}
modules.output.manifestMetadata = map[string]any{
"output_profile": "output-metadata",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ModuleMetadata == nil {
t.Fatal("ModuleMetadata = nil, want module metadata map")
}
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
}
modules.chunker.manifestMetadata["prompt_id"] = "changed"
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
@@ -863,6 +904,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
}
if output.Manifest.ModuleMetadata != nil {
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
}
}
}
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
@@ -1051,10 +1097,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
}
type runnerInputAdapter struct {
key string
doc *source.SourceDocument
err error
requests []contracts.ParseRequest
key string
doc *source.SourceDocument
err error
manifestMetadata map[string]any
requests []contracts.ParseRequest
}
func (adapter *runnerInputAdapter) Key() string {
@@ -1066,12 +1113,17 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
return adapter.doc, adapter.err
}
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
return adapter.manifestMetadata
}
type runnerChunker struct {
key string
chunks []contracts.SourceChunk
warnings []contracts.Warning
err error
requests []contracts.ChunkRequest
key string
chunks []contracts.SourceChunk
warnings []contracts.Warning
err error
manifestMetadata map[string]any
requests []contracts.ChunkRequest
}
func (chunker *runnerChunker) Key() string {
@@ -1086,6 +1138,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
}, chunker.err
}
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
return chunker.manifestMetadata
}
type runnerExtractor struct {
key string
artifactType string
@@ -1226,11 +1282,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
}
type runnerOutputEncoder struct {
key string
files []contracts.OutputFile
warnings []contracts.Warning
err error
requests []contracts.OutputRequest
key string
files []contracts.OutputFile
warnings []contracts.Warning
err error
manifestMetadata map[string]any
requests []contracts.OutputRequest
}
func (encoder *runnerOutputEncoder) Key() string {
@@ -1245,6 +1302,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
}, encoder.err
}
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
return encoder.manifestMetadata
}
type fakeLLMClient struct{}
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {