eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add run manifest and logical output file contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-04 00:45:21 +00:00
parent ac14667797
commit bc4203a264
10 changed files with 455 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
internal/framework/pipeline/runner_test.go
View File

@@ -6,6 +6,7 @@ import (
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -32,6 +33,7 @@ func TestNewAndDataTypes(t *testing.T) {
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
EncodedOutput: []byte(`{}`),
ContentType: "application/json",
}
@@ -39,7 +41,7 @@ func TestNewAndDataTypes(t *testing.T) {
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
t.Fatalf("RunInput = %#v, want constructed fields", input)
}
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 {
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
t.Fatalf("RunOutput = %#v, want constructed fields", output)
}
}
@@ -604,11 +606,18 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.ContentType)
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if string(output.EncodedOutput) != `{"encoded":true}` {
t.Fatalf("EncodedOutput = %s, want encoded payload", output.EncodedOutput)
file := output.OutputFiles[0]
if file.Name != "artifacts/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name)
}
if file.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", file.ContentType)
}
if string(file.Bytes) != `{"encoded":true}` {
t.Fatalf("OutputFiles[0].Bytes = %s, want encoded payload", file.Bytes)
}
if len(modules.output.requests) != 1 {
t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests))
@@ -622,6 +631,35 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
}
}
func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
tests := []struct {
name string
fileName string
}{
{name: "empty", fileName: ""},
{name: "absolute", fileName: "/tmp/output.json"},
{name: "parent", fileName: "artifacts/../manifest.json"},
{name: "backslash", fileName: `artifacts\manifest.json`},
{name: "unclean", fileName: "artifacts//manifest.json"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.output.files = []contracts.OutputFile{
{Name: test.fileName, ContentType: "application/json", Bytes: []byte(`{}`)},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, "output file name")
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
})
}
}
func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
modules := defaultRunnerModules()
modules.output.err = errors.New("encode failed")
@@ -632,6 +670,9 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want failed run completion timestamp")
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
}
@@ -668,6 +709,76 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
}
}
func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
startedAt := time.Now().Add(-time.Minute).UTC()
profiles := []artifacts.LLMProfileManifest{
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
RunID: "run-test",
StartedAt: startedAt,
LLMProfiles: profiles,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
manifest := output.Manifest
if manifest.RunID != "run-test" {
t.Fatalf("RunID = %q, want run-test", manifest.RunID)
}
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(startedAt) {
t.Fatalf("StartedAt = %v, want %s", manifest.StartedAt, startedAt)
}
if manifest.CompletedAt == nil || manifest.CompletedAt.Before(startedAt) {
t.Fatalf("CompletedAt = %v, want timestamp after start", manifest.CompletedAt)
}
if !reflect.DeepEqual(manifest.LLMProfiles, profiles) {
t.Fatalf("LLMProfiles = %#v, want %#v", manifest.LLMProfiles, profiles)
}
}
func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if !strings.HasPrefix(output.Manifest.RunID, "run-") {
t.Fatalf("RunID = %q, want generated run ID", output.Manifest.RunID)
}
if output.Manifest.StartedAt == nil {
t.Fatal("StartedAt = nil, want generated timestamp")
}
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want generated timestamp")
}
}
func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].manifestMetadata = map[string]any{
"prompt_id": "test.prompt",
"response_schema_name": "test_schema",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
lane := output.Manifest.ArtifactLanes[0]
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
if !ok {
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
}
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
}
}
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
@@ -783,7 +894,12 @@ func defaultRunnerModules() *runnerModules {
"configured": {name: "configured", decisions: approveAll},
"second-validator": {name: "second-validator", decisions: approveAll},
},
output: &runnerOutputEncoder{key: "output", bytes: []byte(`{"encoded":true}`), contentType: "application/json"},
output: &runnerOutputEncoder{
key: "output",
files: []contracts.OutputFile{
{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
},
}
}
@@ -885,17 +1001,18 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
}
type runnerExtractor struct {
key string
artifactType string
schemaVersion string
candidates []artifacts.ArtifactCandidate
validators []contracts.Validator
warnings []contracts.Warning
err error
requests []contracts.ExtractionRequest
seenChunkIDs []string
seenLLMClients []contracts.StructuredLLMClient
seenMetadata []map[string]any
key string
artifactType string
schemaVersion string
manifestMetadata map[string]any
candidates []artifacts.ArtifactCandidate
validators []contracts.Validator
warnings []contracts.Warning
err error
requests []contracts.ExtractionRequest
seenChunkIDs []string
seenLLMClients []contracts.StructuredLLMClient
seenMetadata []map[string]any
}
func (extractor *runnerExtractor) Key() string {
@@ -910,6 +1027,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata
}
func (extractor *runnerExtractor) Validators() []contracts.Validator {
return extractor.validators
}
@@ -1020,6 +1141,7 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
type runnerOutputEncoder struct {
key string
files []contracts.OutputFile
bytes []byte
contentType string
warnings []contracts.Warning
@@ -1034,6 +1156,7 @@ func (encoder *runnerOutputEncoder) Key() string {
func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
encoder.requests = append(encoder.requests, req)
return contracts.OutputResult{
Files: encoder.files,
Bytes: encoder.bytes,
ContentType: encoder.contentType,
Warnings: encoder.warnings,