Add run manifest and logical output file contracts

This commit is contained in:
2026-07-04 00:45:21 +00:00
parent ac14667797
commit bc4203a264
10 changed files with 455 additions and 52 deletions

View File

@@ -34,11 +34,18 @@ type RejectedArtifact struct {
} }
type ArtifactLaneManifest struct { type ArtifactLaneManifest struct {
ID string `json:"id"` ID string `json:"id"`
Extractor string `json:"extractor"` Extractor string `json:"extractor"`
Merger string `json:"merger"` Merger string `json:"merger"`
Normalizer string `json:"normalizer"` Normalizer string `json:"normalizer"`
Validators []string `json:"validators,omitempty"` Validators []string `json:"validators,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type LLMProfileManifest struct {
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
} }
type RunManifest struct { type RunManifest struct {
@@ -53,6 +60,7 @@ type RunManifest struct {
Normalizer string `json:"normalizer,omitempty"` Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"` OutputEncoder string `json:"output_encoder,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"` SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"` ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"` StartedAt *time.Time `json:"started_at,omitempty"`

View File

@@ -127,6 +127,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
manifest := RunManifest{ manifest := RunManifest{
PipelineID: "pipeline-1", PipelineID: "pipeline-1",
PipelineDigest: "sha256:abc123", PipelineDigest: "sha256:abc123",
LLMProfiles: []LLMProfileManifest{
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
},
ArtifactLanes: []ArtifactLaneManifest{ ArtifactLanes: []ArtifactLaneManifest{
{ {
ID: "events", ID: "events",
@@ -134,6 +137,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
Merger: "appendorder", Merger: "appendorder",
Normalizer: "noop", Normalizer: "noop",
Validators: []string{"grounded"}, Validators: []string{"grounded"},
Metadata: map[string]any{
"extractor": map[string]any{"prompt_id": "test.prompt"},
},
}, },
}, },
} }
@@ -148,7 +154,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err) t.Fatalf("json.Unmarshal() error = %v", err)
} }
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes") assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
profiles, ok := got["llm_profiles"].([]any)
if !ok {
t.Fatalf("llm_profiles = %#v, want array", got["llm_profiles"])
}
if len(profiles) != 1 {
t.Fatalf("len(llm_profiles) = %d, want 1", len(profiles))
}
profile, ok := profiles[0].(map[string]any)
if !ok {
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
}
assertHasKeys(t, profile, "id", "provider", "model")
lanes, ok := got["artifact_lanes"].([]any) lanes, ok := got["artifact_lanes"].([]any)
if !ok { if !ok {
@@ -161,7 +180,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok { if !ok {
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0]) t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
} }
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators") assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
} }
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) { func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {

View File

@@ -124,10 +124,13 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Encode() error = %v, want nil", err) t.Fatalf("Encode() error = %v, want nil", err)
} }
if output.ContentType != "application/json" { if len(output.Files) != 1 {
t.Fatalf("ContentType = %q, want application/json", output.ContentType) t.Fatalf("len(Files) = %d, want 1", len(output.Files))
} }
if len(output.Bytes) == 0 { if output.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType)
}
if len(output.Files[0].Bytes) == 0 {
t.Fatal("len(Bytes) = 0, want encoded bytes") t.Fatal("len(Bytes) = 0, want encoded bytes")
} }
} }
@@ -293,7 +296,12 @@ func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contract
} }
return contracts.OutputResult{ return contracts.OutputResult{
Bytes: encoded, Files: []contracts.OutputFile{
ContentType: "application/json", {
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: encoded,
},
},
}, nil }, nil
} }

View File

@@ -182,8 +182,17 @@ type OutputRequest struct {
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
type OutputFile struct {
Name string `json:"name"`
ContentType string `json:"content_type,omitempty"`
Bytes []byte `json:"-"`
}
type OutputResult struct { type OutputResult struct {
Bytes []byte `json:"-"` Files []OutputFile `json:"files,omitempty"`
// Bytes is the legacy single-output payload. New encoders should return Files.
Bytes []byte `json:"-"`
// ContentType is the legacy single-output content type. New encoders should return Files.
ContentType string `json:"content_type,omitempty"` ContentType string `json:"content_type,omitempty"`
Warnings []Warning `json:"warnings,omitempty"` Warnings []Warning `json:"warnings,omitempty"`
} }
@@ -192,3 +201,7 @@ type OutputEncoder interface {
Key() string Key() string
Encode(ctx context.Context, req OutputRequest) (OutputResult, error) Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
} }
type ManifestMetadataProvider interface {
ManifestMetadata() map[string]any
}

View File

@@ -230,11 +230,44 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if encoder.Key() != "generic-output" { if encoder.Key() != "generic-output" {
t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key()) t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key())
} }
if encoded.ContentType != "application/json" { if len(encoded.Files) != 1 {
t.Fatalf("ContentType = %q, want application/json", encoded.ContentType) t.Fatalf("len(Files) = %d, want 1", len(encoded.Files))
} }
if string(encoded.Bytes) != `{"run_id":"run-1","approved_count":1}` { if encoded.Files[0].ContentType != "application/json" {
t.Fatalf("Bytes = %s, want encoded output", encoded.Bytes) t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
}
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
}
}
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
file := OutputFile{
Name: "artifacts/events.json",
ContentType: "application/json",
Bytes: []byte(`{"ignored":true}`),
}
encoded, err := json.Marshal(file)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if got["name"] != "artifacts/events.json" {
t.Fatalf("name = %#v, want logical file name", got["name"])
}
if got["content_type"] != "application/json" {
t.Fatalf("content_type = %#v, want application/json", got["content_type"])
}
if _, ok := got["Bytes"]; ok {
t.Fatalf("encoded output file leaked Bytes: %s", encoded)
}
if _, ok := got["bytes"]; ok {
t.Fatalf("encoded output file leaked bytes: %s", encoded)
} }
} }
@@ -397,7 +430,12 @@ func (encoder fakeOutputEncoder) Key() string {
func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest) (OutputResult, error) { func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest) (OutputResult, error) {
return OutputResult{ return OutputResult{
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`), Files: []OutputFile{
ContentType: "application/json", {
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
},
},
}, nil }, nil
} }

View File

@@ -3,6 +3,9 @@ package pipeline
import ( import (
"context" "context"
"fmt" "fmt"
"path"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -29,21 +32,27 @@ func New(registries Registries) *Runner {
} }
type RunInput struct { type RunInput struct {
Pipeline ResolvedPipeline Pipeline ResolvedPipeline
SourceID string SourceID string
Path string Path string
RawInput []byte RawInput []byte
LLMClient contracts.StructuredLLMClient LLMClient contracts.StructuredLLMClient
Metadata map[string]any RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
} }
type RunOutput struct { type RunOutput struct {
Manifest artifacts.RunManifest `json:"manifest"` Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"` Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"` Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"` Warnings []contracts.Warning `json:"warnings,omitempty"`
EncodedOutput []byte `json:"-"` OutputFiles []contracts.OutputFile `json:"-"`
ContentType string `json:"content_type,omitempty"` // EncodedOutput is the legacy single-output payload. New callers should use OutputFiles.
EncodedOutput []byte `json:"-"`
// ContentType is the legacy single-output content type. New callers should use OutputFiles.
ContentType string `json:"content_type,omitempty"`
} }
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
@@ -58,7 +67,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return output, err return output, err
} }
output.Manifest = manifestFromPipeline(input.Pipeline) output.Manifest = manifestFromPipeline(input)
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module) adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil { if err != nil {
@@ -110,6 +119,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
} else { } else {
output.Manifest.ValidationStatus = "approved" output.Manifest.ValidationStatus = "approved"
} }
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module) encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module)
if err != nil { if err != nil {
@@ -128,8 +138,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err) return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err)
} }
output.EncodedOutput = encoded.Bytes files, err := outputFilesFromResult(encoded)
output.ContentType = encoded.ContentType if err != nil {
return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
}
output.OutputFiles = files
if len(encoded.Files) == 0 {
output.EncodedOutput = append([]byte(nil), encoded.Bytes...)
output.ContentType = encoded.ContentType
}
return output, nil return output, nil
} }
@@ -147,6 +164,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
if err != nil { if err != nil {
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err) return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
} }
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
var validators []validatorExecution var validators []validatorExecution
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {
@@ -315,7 +333,17 @@ func validateRunInput(input RunInput) error {
return nil return nil
} }
func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest { func manifestFromPipeline(input RunInput) artifacts.RunManifest {
startedAt := input.StartedAt
if startedAt.IsZero() {
startedAt = time.Now().UTC()
}
runID := strings.TrimSpace(input.RunID)
if runID == "" {
runID = fmt.Sprintf("run-%d", startedAt.UnixNano())
}
pipeline := input.Pipeline
manifest := artifacts.RunManifest{ manifest := artifacts.RunManifest{
PipelineID: pipeline.ID, PipelineID: pipeline.ID,
PipelineDigest: pipeline.Digest, PipelineDigest: pipeline.Digest,
@@ -323,6 +351,9 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest {
Chunker: pipeline.Chunk.Module, Chunker: pipeline.Chunk.Module,
OutputEncoder: pipeline.Output.Module, OutputEncoder: pipeline.Output.Module,
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)), ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID,
StartedAt: timePtr(startedAt),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
} }
for _, lane := range pipeline.ArtifactLanes { for _, lane := range pipeline.ArtifactLanes {
@@ -343,10 +374,124 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest {
func failOutput(output RunOutput) RunOutput { func failOutput(output RunOutput) RunOutput {
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
output.Manifest.ValidationStatus = "failed" output.Manifest.ValidationStatus = "failed"
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
} }
return output return output
} }
func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
if output == nil {
return
}
for i := range output.Manifest.ArtifactLanes {
if output.Manifest.ArtifactLanes[i].ID != laneID {
continue
}
metadata := make(map[string]any)
for _, module := range modules {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
continue
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
continue
}
key := manifestMetadataKey(module)
if key == "" {
continue
}
metadata[key] = moduleMetadata
}
if len(metadata) > 0 {
output.Manifest.ArtifactLanes[i].Metadata = metadata
}
return
}
}
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.Extractor:
return "extractor"
case contracts.Merger:
return "merger"
case contracts.Normalizer:
return "normalizer"
default:
return ""
}
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
files := result.Files
if len(files) == 0 && len(result.Bytes) > 0 {
files = []contracts.OutputFile{
{
Name: "output",
ContentType: result.ContentType,
Bytes: result.Bytes,
},
}
}
out := make([]contracts.OutputFile, 0, len(files))
for _, file := range files {
if err := validateOutputFileName(file.Name); err != nil {
return nil, err
}
out = append(out, contracts.OutputFile{
Name: file.Name,
ContentType: file.ContentType,
Bytes: append([]byte(nil), file.Bytes...),
})
}
return out, nil
}
func validateOutputFileName(name string) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("output file name must not be empty")
}
if strings.Contains(name, "\\") {
return fmt.Errorf("output file name %q must use slash-separated relative paths", name)
}
if path.IsAbs(name) {
return fmt.Errorf("output file name %q must be relative", name)
}
if strings.Contains(name, "..") {
return fmt.Errorf("output file name %q must not contain ..", name)
}
cleaned := path.Clean(name)
if cleaned == "." || cleaned != name {
return fmt.Errorf("output file name %q must be clean", name)
}
return nil
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
if len(profiles) == 0 {
return nil
}
return append([]artifacts.LLMProfileManifest(nil), profiles...)
}
func timePtr(t time.Time) *time.Time {
return &t
}
func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool { func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool {
for _, lane := range pipeline.ArtifactLanes { for _, lane := range pipeline.ArtifactLanes {
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {

View File

@@ -6,6 +6,7 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -32,6 +33,7 @@ func TestNewAndDataTypes(t *testing.T) {
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}}, Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}}, Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}}, Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
EncodedOutput: []byte(`{}`), EncodedOutput: []byte(`{}`),
ContentType: "application/json", ContentType: "application/json",
} }
@@ -39,7 +41,7 @@ func TestNewAndDataTypes(t *testing.T) {
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" { if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
t.Fatalf("RunInput = %#v, want constructed fields", input) t.Fatalf("RunInput = %#v, want constructed fields", input)
} }
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 { if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
t.Fatalf("RunOutput = %#v, want constructed fields", output) t.Fatalf("RunOutput = %#v, want constructed fields", output)
} }
} }
@@ -604,11 +606,18 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err) t.Fatalf("Run() error = %v, want nil", err)
} }
if output.ContentType != "application/json" { if len(output.OutputFiles) != 1 {
t.Fatalf("ContentType = %q, want application/json", output.ContentType) t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
} }
if string(output.EncodedOutput) != `{"encoded":true}` { file := output.OutputFiles[0]
t.Fatalf("EncodedOutput = %s, want encoded payload", output.EncodedOutput) if file.Name != "artifacts/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name)
}
if file.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", file.ContentType)
}
if string(file.Bytes) != `{"encoded":true}` {
t.Fatalf("OutputFiles[0].Bytes = %s, want encoded payload", file.Bytes)
} }
if len(modules.output.requests) != 1 { if len(modules.output.requests) != 1 {
t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests)) t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests))
@@ -622,6 +631,35 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
} }
} }
func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
tests := []struct {
name string
fileName string
}{
{name: "empty", fileName: ""},
{name: "absolute", fileName: "/tmp/output.json"},
{name: "parent", fileName: "artifacts/../manifest.json"},
{name: "backslash", fileName: `artifacts\manifest.json`},
{name: "unclean", fileName: "artifacts//manifest.json"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.output.files = []contracts.OutputFile{
{Name: test.fileName, ContentType: "application/json", Bytes: []byte(`{}`)},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, "output file name")
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
})
}
}
func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) { func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
modules.output.err = errors.New("encode failed") modules.output.err = errors.New("encode failed")
@@ -632,6 +670,9 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
if output.Manifest.ValidationStatus != "failed" { if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus) t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
} }
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want failed run completion timestamp")
}
if len(output.Approved) != 2 { if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved)) t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
} }
@@ -668,6 +709,76 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
} }
} }
func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
startedAt := time.Now().Add(-time.Minute).UTC()
profiles := []artifacts.LLMProfileManifest{
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
RunID: "run-test",
StartedAt: startedAt,
LLMProfiles: profiles,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
manifest := output.Manifest
if manifest.RunID != "run-test" {
t.Fatalf("RunID = %q, want run-test", manifest.RunID)
}
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(startedAt) {
t.Fatalf("StartedAt = %v, want %s", manifest.StartedAt, startedAt)
}
if manifest.CompletedAt == nil || manifest.CompletedAt.Before(startedAt) {
t.Fatalf("CompletedAt = %v, want timestamp after start", manifest.CompletedAt)
}
if !reflect.DeepEqual(manifest.LLMProfiles, profiles) {
t.Fatalf("LLMProfiles = %#v, want %#v", manifest.LLMProfiles, profiles)
}
}
func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if !strings.HasPrefix(output.Manifest.RunID, "run-") {
t.Fatalf("RunID = %q, want generated run ID", output.Manifest.RunID)
}
if output.Manifest.StartedAt == nil {
t.Fatal("StartedAt = nil, want generated timestamp")
}
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want generated timestamp")
}
}
func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].manifestMetadata = map[string]any{
"prompt_id": "test.prompt",
"response_schema_name": "test_schema",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
lane := output.Manifest.ArtifactLanes[0]
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
if !ok {
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
}
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
}
}
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")} modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
@@ -783,7 +894,12 @@ func defaultRunnerModules() *runnerModules {
"configured": {name: "configured", decisions: approveAll}, "configured": {name: "configured", decisions: approveAll},
"second-validator": {name: "second-validator", decisions: approveAll}, "second-validator": {name: "second-validator", decisions: approveAll},
}, },
output: &runnerOutputEncoder{key: "output", bytes: []byte(`{"encoded":true}`), contentType: "application/json"}, output: &runnerOutputEncoder{
key: "output",
files: []contracts.OutputFile{
{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
},
} }
} }
@@ -885,17 +1001,18 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
} }
type runnerExtractor struct { type runnerExtractor struct {
key string key string
artifactType string artifactType string
schemaVersion string schemaVersion string
candidates []artifacts.ArtifactCandidate manifestMetadata map[string]any
validators []contracts.Validator candidates []artifacts.ArtifactCandidate
warnings []contracts.Warning validators []contracts.Validator
err error warnings []contracts.Warning
requests []contracts.ExtractionRequest err error
seenChunkIDs []string requests []contracts.ExtractionRequest
seenLLMClients []contracts.StructuredLLMClient seenChunkIDs []string
seenMetadata []map[string]any seenLLMClients []contracts.StructuredLLMClient
seenMetadata []map[string]any
} }
func (extractor *runnerExtractor) Key() string { func (extractor *runnerExtractor) Key() string {
@@ -910,6 +1027,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion return extractor.schemaVersion
} }
func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata
}
func (extractor *runnerExtractor) Validators() []contracts.Validator { func (extractor *runnerExtractor) Validators() []contracts.Validator {
return extractor.validators return extractor.validators
} }
@@ -1020,6 +1141,7 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
type runnerOutputEncoder struct { type runnerOutputEncoder struct {
key string key string
files []contracts.OutputFile
bytes []byte bytes []byte
contentType string contentType string
warnings []contracts.Warning warnings []contracts.Warning
@@ -1034,6 +1156,7 @@ func (encoder *runnerOutputEncoder) Key() string {
func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
encoder.requests = append(encoder.requests, req) encoder.requests = append(encoder.requests, req)
return contracts.OutputResult{ return contracts.OutputResult{
Files: encoder.files,
Bytes: encoder.bytes, Bytes: encoder.bytes,
ContentType: encoder.contentType, ContentType: encoder.contentType,
Warnings: encoder.warnings, Warnings: encoder.warnings,

View File

@@ -57,6 +57,14 @@ type Bundle struct {
metadata Metadata metadata Metadata
} }
// Metadata returns metadata for the compiled prompt bundle.
func (b *Bundle) Metadata() Metadata {
if b == nil {
return Metadata{}
}
return b.metadata
}
var promptRegistry map[string]*Bundle var promptRegistry map[string]*Bundle
var sharedHardening string var sharedHardening string

View File

@@ -45,6 +45,23 @@ func (e *Extractor) SchemaVersion() string {
return SchemaVersion return SchemaVersion
} }
func (e *Extractor) ManifestMetadata() map[string]any {
promptMetadata := spellsPromptBundle.Metadata()
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_sha256": promptMetadata.SHA256,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
}
if schema, err := loadResponseSchema(); err == nil {
metadata["response_schema_version"] = schema.Version
metadata["response_schema_sha256"] = schema.SHA256
}
return metadata
}
func (e *Extractor) Validators() []contracts.Validator { func (e *Extractor) Validators() []contracts.Validator {
return []contracts.Validator{ return []contracts.Validator{
ShapeValidator{}, ShapeValidator{},

View File

@@ -92,6 +92,30 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
} }
} }
func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := New().ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) { func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}