From bc4203a264e8881fce2c0a7f663b6b28c41f5d31 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 4 Jul 2026 00:45:21 +0000 Subject: [PATCH] Add run manifest and logical output file contracts --- internal/core/artifacts/artifacts.go | 18 +- internal/core/artifacts/artifacts_test.go | 23 ++- .../framework/contracts/composition_test.go | 18 +- internal/framework/contracts/contracts.go | 15 +- .../framework/contracts/contracts_test.go | 50 ++++- internal/framework/pipeline/runner.go | 177 ++++++++++++++++-- internal/framework/pipeline/runner_test.go | 157 ++++++++++++++-- internal/framework/prompt/registry.go | 8 + .../modules/extract/dnd/spells/extractor.go | 17 ++ .../extract/dnd/spells/extractor_test.go | 24 +++ 10 files changed, 455 insertions(+), 52 deletions(-) diff --git a/internal/core/artifacts/artifacts.go b/internal/core/artifacts/artifacts.go index 7f69d2e..8e962eb 100644 --- a/internal/core/artifacts/artifacts.go +++ b/internal/core/artifacts/artifacts.go @@ -34,11 +34,18 @@ type RejectedArtifact struct { } type ArtifactLaneManifest struct { - ID string `json:"id"` - Extractor string `json:"extractor"` - Merger string `json:"merger"` - Normalizer string `json:"normalizer"` - Validators []string `json:"validators,omitempty"` + ID string `json:"id"` + Extractor string `json:"extractor"` + Merger string `json:"merger"` + Normalizer string `json:"normalizer"` + Validators []string `json:"validators,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type LLMProfileManifest struct { + ID string `json:"id"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` } type RunManifest struct { @@ -53,6 +60,7 @@ type RunManifest struct { Normalizer string `json:"normalizer,omitempty"` OutputEncoder string `json:"output_encoder,omitempty"` ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` + LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` SchemaVersion string `json:"schema_version,omitempty"` ValidationStatus string `json:"validation_status,omitempty"` StartedAt *time.Time `json:"started_at,omitempty"` diff --git a/internal/core/artifacts/artifacts_test.go b/internal/core/artifacts/artifacts_test.go index 981f36d..a0be960 100644 --- a/internal/core/artifacts/artifacts_test.go +++ b/internal/core/artifacts/artifacts_test.go @@ -127,6 +127,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { manifest := RunManifest{ PipelineID: "pipeline-1", PipelineDigest: "sha256:abc123", + LLMProfiles: []LLMProfileManifest{ + {ID: "default", Provider: "openai-compatible", Model: "model-a"}, + }, ArtifactLanes: []ArtifactLaneManifest{ { ID: "events", @@ -134,6 +137,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { Merger: "appendorder", Normalizer: "noop", Validators: []string{"grounded"}, + Metadata: map[string]any{ + "extractor": map[string]any{"prompt_id": "test.prompt"}, + }, }, }, } @@ -148,7 +154,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { t.Fatalf("json.Unmarshal() error = %v", err) } - assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes") + assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles") + + profiles, ok := got["llm_profiles"].([]any) + if !ok { + t.Fatalf("llm_profiles = %#v, want array", got["llm_profiles"]) + } + if len(profiles) != 1 { + t.Fatalf("len(llm_profiles) = %d, want 1", len(profiles)) + } + profile, ok := profiles[0].(map[string]any) + if !ok { + t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0]) + } + assertHasKeys(t, profile, "id", "provider", "model") lanes, ok := got["artifact_lanes"].([]any) if !ok { @@ -161,7 +180,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { if !ok { t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0]) } - assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators") + assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata") } func assertHasKeys(t *testing.T, values map[string]any, keys ...string) { diff --git a/internal/framework/contracts/composition_test.go b/internal/framework/contracts/composition_test.go index 85a5e15..fb59166 100644 --- a/internal/framework/contracts/composition_test.go +++ b/internal/framework/contracts/composition_test.go @@ -124,10 +124,13 @@ func TestContractsComposeAcrossPackages(t *testing.T) { if err != nil { t.Fatalf("Encode() error = %v, want nil", err) } - if output.ContentType != "application/json" { - t.Fatalf("ContentType = %q, want application/json", output.ContentType) + if len(output.Files) != 1 { + t.Fatalf("len(Files) = %d, want 1", len(output.Files)) } - if len(output.Bytes) == 0 { + if output.Files[0].ContentType != "application/json" { + t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType) + } + if len(output.Files[0].Bytes) == 0 { t.Fatal("len(Bytes) = 0, want encoded bytes") } } @@ -293,7 +296,12 @@ func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contract } return contracts.OutputResult{ - Bytes: encoded, - ContentType: "application/json", + Files: []contracts.OutputFile{ + { + Name: "artifacts/generic.json", + ContentType: "application/json", + Bytes: encoded, + }, + }, }, nil } diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index c2c405e..6ab8d6d 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -182,8 +182,17 @@ type OutputRequest struct { Metadata map[string]any `json:"metadata,omitempty"` } +type OutputFile struct { + Name string `json:"name"` + ContentType string `json:"content_type,omitempty"` + Bytes []byte `json:"-"` +} + type OutputResult struct { - Bytes []byte `json:"-"` + Files []OutputFile `json:"files,omitempty"` + // Bytes is the legacy single-output payload. New encoders should return Files. + Bytes []byte `json:"-"` + // ContentType is the legacy single-output content type. New encoders should return Files. ContentType string `json:"content_type,omitempty"` Warnings []Warning `json:"warnings,omitempty"` } @@ -192,3 +201,7 @@ type OutputEncoder interface { Key() string Encode(ctx context.Context, req OutputRequest) (OutputResult, error) } + +type ManifestMetadataProvider interface { + ManifestMetadata() map[string]any +} diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index 8c59dcc..867b3bd 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -230,11 +230,44 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) { if encoder.Key() != "generic-output" { t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key()) } - if encoded.ContentType != "application/json" { - t.Fatalf("ContentType = %q, want application/json", encoded.ContentType) + if len(encoded.Files) != 1 { + t.Fatalf("len(Files) = %d, want 1", len(encoded.Files)) } - if string(encoded.Bytes) != `{"run_id":"run-1","approved_count":1}` { - t.Fatalf("Bytes = %s, want encoded output", encoded.Bytes) + if encoded.Files[0].ContentType != "application/json" { + t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType) + } + if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` { + t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes) + } +} + +func TestOutputFileJSONShapeOmitsBytes(t *testing.T) { + file := OutputFile{ + Name: "artifacts/events.json", + ContentType: "application/json", + Bytes: []byte(`{"ignored":true}`), + } + + encoded, err := json.Marshal(file) + if err != nil { + t.Fatalf("json.Marshal() error = %v, want nil", err) + } + + var got map[string]any + if err := json.Unmarshal(encoded, &got); err != nil { + t.Fatalf("json.Unmarshal() error = %v, want nil", err) + } + if got["name"] != "artifacts/events.json" { + t.Fatalf("name = %#v, want logical file name", got["name"]) + } + if got["content_type"] != "application/json" { + t.Fatalf("content_type = %#v, want application/json", got["content_type"]) + } + if _, ok := got["Bytes"]; ok { + t.Fatalf("encoded output file leaked Bytes: %s", encoded) + } + if _, ok := got["bytes"]; ok { + t.Fatalf("encoded output file leaked bytes: %s", encoded) } } @@ -397,7 +430,12 @@ func (encoder fakeOutputEncoder) Key() string { func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest) (OutputResult, error) { return OutputResult{ - Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`), - ContentType: "application/json", + Files: []OutputFile{ + { + Name: "artifacts/generic.json", + ContentType: "application/json", + Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`), + }, + }, }, nil } diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index d030e8f..db2d7de 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -3,6 +3,9 @@ package pipeline import ( "context" "fmt" + "path" + "strings" + "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/source" @@ -29,21 +32,27 @@ func New(registries Registries) *Runner { } type RunInput struct { - Pipeline ResolvedPipeline - SourceID string - Path string - RawInput []byte - LLMClient contracts.StructuredLLMClient - Metadata map[string]any + Pipeline ResolvedPipeline + SourceID string + Path string + RawInput []byte + LLMClient contracts.StructuredLLMClient + RunID string + StartedAt time.Time + LLMProfiles []artifacts.LLMProfileManifest + Metadata map[string]any } type RunOutput struct { - Manifest artifacts.RunManifest `json:"manifest"` - Approved []artifacts.Artifact `json:"approved,omitempty"` - Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"` - Warnings []contracts.Warning `json:"warnings,omitempty"` - EncodedOutput []byte `json:"-"` - ContentType string `json:"content_type,omitempty"` + Manifest artifacts.RunManifest `json:"manifest"` + Approved []artifacts.Artifact `json:"approved,omitempty"` + Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"` + Warnings []contracts.Warning `json:"warnings,omitempty"` + OutputFiles []contracts.OutputFile `json:"-"` + // EncodedOutput is the legacy single-output payload. New callers should use OutputFiles. + EncodedOutput []byte `json:"-"` + // ContentType is the legacy single-output content type. New callers should use OutputFiles. + ContentType string `json:"content_type,omitempty"` } func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { @@ -58,7 +67,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { return output, err } - output.Manifest = manifestFromPipeline(input.Pipeline) + output.Manifest = manifestFromPipeline(input) adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module) if err != nil { @@ -110,6 +119,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } else { output.Manifest.ValidationStatus = "approved" } + output.Manifest.CompletedAt = timePtr(time.Now().UTC()) encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module) if err != nil { @@ -128,8 +138,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { if err != nil { return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err) } - output.EncodedOutput = encoded.Bytes - output.ContentType = encoded.ContentType + files, err := outputFilesFromResult(encoded) + if err != nil { + return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err) + } + output.OutputFiles = files + if len(encoded.Files) == 0 { + output.EncodedOutput = append([]byte(nil), encoded.Bytes...) + output.ContentType = encoded.ContentType + } return output, nil } @@ -147,6 +164,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source if err != nil { return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err) } + setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer) var validators []validatorExecution if len(lane.Validators) > 0 { @@ -315,7 +333,17 @@ func validateRunInput(input RunInput) error { return nil } -func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest { +func manifestFromPipeline(input RunInput) artifacts.RunManifest { + startedAt := input.StartedAt + if startedAt.IsZero() { + startedAt = time.Now().UTC() + } + runID := strings.TrimSpace(input.RunID) + if runID == "" { + runID = fmt.Sprintf("run-%d", startedAt.UnixNano()) + } + + pipeline := input.Pipeline manifest := artifacts.RunManifest{ PipelineID: pipeline.ID, PipelineDigest: pipeline.Digest, @@ -323,6 +351,9 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest { Chunker: pipeline.Chunk.Module, OutputEncoder: pipeline.Output.Module, ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)), + RunID: runID, + StartedAt: timePtr(startedAt), + LLMProfiles: cloneLLMProfiles(input.LLMProfiles), } for _, lane := range pipeline.ArtifactLanes { @@ -343,10 +374,124 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest { func failOutput(output RunOutput) RunOutput { if output.Manifest.PipelineID != "" { output.Manifest.ValidationStatus = "failed" + output.Manifest.CompletedAt = timePtr(time.Now().UTC()) } return output } +func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) { + if output == nil { + return + } + for i := range output.Manifest.ArtifactLanes { + if output.Manifest.ArtifactLanes[i].ID != laneID { + continue + } + + metadata := make(map[string]any) + for _, module := range modules { + provider, ok := module.(contracts.ManifestMetadataProvider) + if !ok { + continue + } + moduleMetadata := cloneMetadata(provider.ManifestMetadata()) + if len(moduleMetadata) == 0 { + continue + } + key := manifestMetadataKey(module) + if key == "" { + continue + } + metadata[key] = moduleMetadata + } + if len(metadata) > 0 { + output.Manifest.ArtifactLanes[i].Metadata = metadata + } + return + } +} + +func manifestMetadataKey(module any) string { + switch module.(type) { + case contracts.Extractor: + return "extractor" + case contracts.Merger: + return "merger" + case contracts.Normalizer: + return "normalizer" + default: + return "" + } +} + +func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) { + files := result.Files + if len(files) == 0 && len(result.Bytes) > 0 { + files = []contracts.OutputFile{ + { + Name: "output", + ContentType: result.ContentType, + Bytes: result.Bytes, + }, + } + } + + out := make([]contracts.OutputFile, 0, len(files)) + for _, file := range files { + if err := validateOutputFileName(file.Name); err != nil { + return nil, err + } + out = append(out, contracts.OutputFile{ + Name: file.Name, + ContentType: file.ContentType, + Bytes: append([]byte(nil), file.Bytes...), + }) + } + return out, nil +} + +func validateOutputFileName(name string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("output file name must not be empty") + } + if strings.Contains(name, "\\") { + return fmt.Errorf("output file name %q must use slash-separated relative paths", name) + } + if path.IsAbs(name) { + return fmt.Errorf("output file name %q must be relative", name) + } + if strings.Contains(name, "..") { + return fmt.Errorf("output file name %q must not contain ..", name) + } + cleaned := path.Clean(name) + if cleaned == "." || cleaned != name { + return fmt.Errorf("output file name %q must be clean", name) + } + return nil +} + +func cloneMetadata(metadata map[string]any) map[string]any { + if len(metadata) == 0 { + return nil + } + out := make(map[string]any, len(metadata)) + for key, value := range metadata { + out[key] = value + } + return out +} + +func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest { + if len(profiles) == 0 { + return nil + } + return append([]artifacts.LLMProfileManifest(nil), profiles...) +} + +func timePtr(t time.Time) *time.Time { + return &t +} + func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool { for _, lane := range pipeline.ArtifactLanes { if len(lane.Validators) > 0 { diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 50961c1..9f056b2 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" "testing" + "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/source" @@ -32,6 +33,7 @@ func TestNewAndDataTypes(t *testing.T) { Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}}, Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}}, Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}}, + OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}}, EncodedOutput: []byte(`{}`), ContentType: "application/json", } @@ -39,7 +41,7 @@ func TestNewAndDataTypes(t *testing.T) { if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" { t.Fatalf("RunInput = %#v, want constructed fields", input) } - if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 { + if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 { t.Fatalf("RunOutput = %#v, want constructed fields", output) } } @@ -604,11 +606,18 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) { t.Fatalf("Run() error = %v, want nil", err) } - if output.ContentType != "application/json" { - t.Fatalf("ContentType = %q, want application/json", output.ContentType) + if len(output.OutputFiles) != 1 { + t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } - if string(output.EncodedOutput) != `{"encoded":true}` { - t.Fatalf("EncodedOutput = %s, want encoded payload", output.EncodedOutput) + file := output.OutputFiles[0] + if file.Name != "artifacts/generic.json" { + t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name) + } + if file.ContentType != "application/json" { + t.Fatalf("ContentType = %q, want application/json", file.ContentType) + } + if string(file.Bytes) != `{"encoded":true}` { + t.Fatalf("OutputFiles[0].Bytes = %s, want encoded payload", file.Bytes) } if len(modules.output.requests) != 1 { t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests)) @@ -622,6 +631,35 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) { } } +func TestRunRejectsUnsafeOutputFileNames(t *testing.T) { + tests := []struct { + name string + fileName string + }{ + {name: "empty", fileName: ""}, + {name: "absolute", fileName: "/tmp/output.json"}, + {name: "parent", fileName: "artifacts/../manifest.json"}, + {name: "backslash", fileName: `artifacts\manifest.json`}, + {name: "unclean", fileName: "artifacts//manifest.json"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + modules := defaultRunnerModules() + modules.output.files = []contracts.OutputFile{ + {Name: test.fileName, ContentType: "application/json", Bytes: []byte(`{}`)}, + } + + output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) + + assertRunError(t, err, "output file name") + if output.Manifest.ValidationStatus != "failed" { + t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus) + } + }) + } +} + func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) { modules := defaultRunnerModules() modules.output.err = errors.New("encode failed") @@ -632,6 +670,9 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) { if output.Manifest.ValidationStatus != "failed" { t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus) } + if output.Manifest.CompletedAt == nil { + t.Fatal("CompletedAt = nil, want failed run completion timestamp") + } if len(output.Approved) != 2 { t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved)) } @@ -668,6 +709,76 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { } } +func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) { + startedAt := time.Now().Add(-time.Minute).UTC() + profiles := []artifacts.LLMProfileManifest{ + {ID: "default", Provider: "openai-compatible", Model: "model-a"}, + } + + output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{ + Pipeline: resolvedPipeline(), + RunID: "run-test", + StartedAt: startedAt, + LLMProfiles: profiles, + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + manifest := output.Manifest + if manifest.RunID != "run-test" { + t.Fatalf("RunID = %q, want run-test", manifest.RunID) + } + if manifest.StartedAt == nil || !manifest.StartedAt.Equal(startedAt) { + t.Fatalf("StartedAt = %v, want %s", manifest.StartedAt, startedAt) + } + if manifest.CompletedAt == nil || manifest.CompletedAt.Before(startedAt) { + t.Fatalf("CompletedAt = %v, want timestamp after start", manifest.CompletedAt) + } + if !reflect.DeepEqual(manifest.LLMProfiles, profiles) { + t.Fatalf("LLMProfiles = %#v, want %#v", manifest.LLMProfiles, profiles) + } +} + +func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) { + output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + if !strings.HasPrefix(output.Manifest.RunID, "run-") { + t.Fatalf("RunID = %q, want generated run ID", output.Manifest.RunID) + } + if output.Manifest.StartedAt == nil { + t.Fatal("StartedAt = nil, want generated timestamp") + } + if output.Manifest.CompletedAt == nil { + t.Fatal("CompletedAt = nil, want generated timestamp") + } +} + +func TestRunManifestIncludesExtractorMetadata(t *testing.T) { + modules := defaultRunnerModules() + modules.extractors["extract-alpha"].manifestMetadata = map[string]any{ + "prompt_id": "test.prompt", + "response_schema_name": "test_schema", + } + + output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + lane := output.Manifest.ArtifactLanes[0] + extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) + if !ok { + t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata) + } + if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" { + t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata) + } +} + func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { modules := defaultRunnerModules() modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")} @@ -783,7 +894,12 @@ func defaultRunnerModules() *runnerModules { "configured": {name: "configured", decisions: approveAll}, "second-validator": {name: "second-validator", decisions: approveAll}, }, - output: &runnerOutputEncoder{key: "output", bytes: []byte(`{"encoded":true}`), contentType: "application/json"}, + output: &runnerOutputEncoder{ + key: "output", + files: []contracts.OutputFile{ + {Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)}, + }, + }, } } @@ -885,17 +1001,18 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ } type runnerExtractor struct { - key string - artifactType string - schemaVersion string - candidates []artifacts.ArtifactCandidate - validators []contracts.Validator - warnings []contracts.Warning - err error - requests []contracts.ExtractionRequest - seenChunkIDs []string - seenLLMClients []contracts.StructuredLLMClient - seenMetadata []map[string]any + key string + artifactType string + schemaVersion string + manifestMetadata map[string]any + candidates []artifacts.ArtifactCandidate + validators []contracts.Validator + warnings []contracts.Warning + err error + requests []contracts.ExtractionRequest + seenChunkIDs []string + seenLLMClients []contracts.StructuredLLMClient + seenMetadata []map[string]any } func (extractor *runnerExtractor) Key() string { @@ -910,6 +1027,10 @@ func (extractor *runnerExtractor) SchemaVersion() string { return extractor.schemaVersion } +func (extractor *runnerExtractor) ManifestMetadata() map[string]any { + return extractor.manifestMetadata +} + func (extractor *runnerExtractor) Validators() []contracts.Validator { return extractor.validators } @@ -1020,6 +1141,7 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va type runnerOutputEncoder struct { key string + files []contracts.OutputFile bytes []byte contentType string warnings []contracts.Warning @@ -1034,6 +1156,7 @@ func (encoder *runnerOutputEncoder) Key() string { func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { encoder.requests = append(encoder.requests, req) return contracts.OutputResult{ + Files: encoder.files, Bytes: encoder.bytes, ContentType: encoder.contentType, Warnings: encoder.warnings, diff --git a/internal/framework/prompt/registry.go b/internal/framework/prompt/registry.go index 530a6a4..4644ab4 100644 --- a/internal/framework/prompt/registry.go +++ b/internal/framework/prompt/registry.go @@ -57,6 +57,14 @@ type Bundle struct { metadata Metadata } +// Metadata returns metadata for the compiled prompt bundle. +func (b *Bundle) Metadata() Metadata { + if b == nil { + return Metadata{} + } + return b.metadata +} + var promptRegistry map[string]*Bundle var sharedHardening string diff --git a/internal/modules/extract/dnd/spells/extractor.go b/internal/modules/extract/dnd/spells/extractor.go index 6e4c3d3..62f6624 100644 --- a/internal/modules/extract/dnd/spells/extractor.go +++ b/internal/modules/extract/dnd/spells/extractor.go @@ -45,6 +45,23 @@ func (e *Extractor) SchemaVersion() string { return SchemaVersion } +func (e *Extractor) ManifestMetadata() map[string]any { + promptMetadata := spellsPromptBundle.Metadata() + metadata := map[string]any{ + "prompt_id": PromptID, + "prompt_version": promptMetadata.PromptVersion, + "prompt_sha256": promptMetadata.SHA256, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + } + if schema, err := loadResponseSchema(); err == nil { + metadata["response_schema_version"] = schema.Version + metadata["response_schema_sha256"] = schema.SHA256 + } + return metadata +} + func (e *Extractor) Validators() []contracts.Validator { return []contracts.Validator{ ShapeValidator{}, diff --git a/internal/modules/extract/dnd/spells/extractor_test.go b/internal/modules/extract/dnd/spells/extractor_test.go index 74ec9a1..fcf1e9a 100644 --- a/internal/modules/extract/dnd/spells/extractor_test.go +++ b/internal/modules/extract/dnd/spells/extractor_test.go @@ -92,6 +92,30 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) { } } +func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) { + metadata := New().ManifestMetadata() + + tests := map[string]string{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + } + for key, want := range tests { + if metadata[key] != want { + t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) + } + } + for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { + value, ok := metadata[key].(string) + if !ok || !strings.HasPrefix(value, "sha256:") { + t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key]) + } + } +} + func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) { client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}