Add run manifest and logical output file contracts
This commit is contained in:
@@ -34,11 +34,18 @@ type RejectedArtifact struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ArtifactLaneManifest struct {
|
type ArtifactLaneManifest struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Extractor string `json:"extractor"`
|
Extractor string `json:"extractor"`
|
||||||
Merger string `json:"merger"`
|
Merger string `json:"merger"`
|
||||||
Normalizer string `json:"normalizer"`
|
Normalizer string `json:"normalizer"`
|
||||||
Validators []string `json:"validators,omitempty"`
|
Validators []string `json:"validators,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type LLMProfileManifest struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Provider string `json:"provider,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
@@ -53,6 +60,7 @@ type RunManifest struct {
|
|||||||
Normalizer string `json:"normalizer,omitempty"`
|
Normalizer string `json:"normalizer,omitempty"`
|
||||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||||
|
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||||
SchemaVersion string `json:"schema_version,omitempty"`
|
SchemaVersion string `json:"schema_version,omitempty"`
|
||||||
ValidationStatus string `json:"validation_status,omitempty"`
|
ValidationStatus string `json:"validation_status,omitempty"`
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
|
|||||||
@@ -127,6 +127,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
manifest := RunManifest{
|
manifest := RunManifest{
|
||||||
PipelineID: "pipeline-1",
|
PipelineID: "pipeline-1",
|
||||||
PipelineDigest: "sha256:abc123",
|
PipelineDigest: "sha256:abc123",
|
||||||
|
LLMProfiles: []LLMProfileManifest{
|
||||||
|
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
|
||||||
|
},
|
||||||
ArtifactLanes: []ArtifactLaneManifest{
|
ArtifactLanes: []ArtifactLaneManifest{
|
||||||
{
|
{
|
||||||
ID: "events",
|
ID: "events",
|
||||||
@@ -134,6 +137,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
Merger: "appendorder",
|
Merger: "appendorder",
|
||||||
Normalizer: "noop",
|
Normalizer: "noop",
|
||||||
Validators: []string{"grounded"},
|
Validators: []string{"grounded"},
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -148,7 +154,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes")
|
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
|
||||||
|
|
||||||
|
profiles, ok := got["llm_profiles"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("llm_profiles = %#v, want array", got["llm_profiles"])
|
||||||
|
}
|
||||||
|
if len(profiles) != 1 {
|
||||||
|
t.Fatalf("len(llm_profiles) = %d, want 1", len(profiles))
|
||||||
|
}
|
||||||
|
profile, ok := profiles[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, profile, "id", "provider", "model")
|
||||||
|
|
||||||
lanes, ok := got["artifact_lanes"].([]any)
|
lanes, ok := got["artifact_lanes"].([]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -161,7 +180,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
||||||
}
|
}
|
||||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators")
|
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
||||||
|
|||||||
@@ -124,10 +124,13 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Encode() error = %v, want nil", err)
|
t.Fatalf("Encode() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if output.ContentType != "application/json" {
|
if len(output.Files) != 1 {
|
||||||
t.Fatalf("ContentType = %q, want application/json", output.ContentType)
|
t.Fatalf("len(Files) = %d, want 1", len(output.Files))
|
||||||
}
|
}
|
||||||
if len(output.Bytes) == 0 {
|
if output.Files[0].ContentType != "application/json" {
|
||||||
|
t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType)
|
||||||
|
}
|
||||||
|
if len(output.Files[0].Bytes) == 0 {
|
||||||
t.Fatal("len(Bytes) = 0, want encoded bytes")
|
t.Fatal("len(Bytes) = 0, want encoded bytes")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -293,7 +296,12 @@ func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contract
|
|||||||
}
|
}
|
||||||
|
|
||||||
return contracts.OutputResult{
|
return contracts.OutputResult{
|
||||||
Bytes: encoded,
|
Files: []contracts.OutputFile{
|
||||||
ContentType: "application/json",
|
{
|
||||||
|
Name: "artifacts/generic.json",
|
||||||
|
ContentType: "application/json",
|
||||||
|
Bytes: encoded,
|
||||||
|
},
|
||||||
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -182,8 +182,17 @@ type OutputRequest struct {
|
|||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type OutputFile struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
ContentType string `json:"content_type,omitempty"`
|
||||||
|
Bytes []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
type OutputResult struct {
|
type OutputResult struct {
|
||||||
Bytes []byte `json:"-"`
|
Files []OutputFile `json:"files,omitempty"`
|
||||||
|
// Bytes is the legacy single-output payload. New encoders should return Files.
|
||||||
|
Bytes []byte `json:"-"`
|
||||||
|
// ContentType is the legacy single-output content type. New encoders should return Files.
|
||||||
ContentType string `json:"content_type,omitempty"`
|
ContentType string `json:"content_type,omitempty"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -192,3 +201,7 @@ type OutputEncoder interface {
|
|||||||
Key() string
|
Key() string
|
||||||
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
|
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ManifestMetadataProvider interface {
|
||||||
|
ManifestMetadata() map[string]any
|
||||||
|
}
|
||||||
|
|||||||
@@ -230,11 +230,44 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
|||||||
if encoder.Key() != "generic-output" {
|
if encoder.Key() != "generic-output" {
|
||||||
t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key())
|
t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key())
|
||||||
}
|
}
|
||||||
if encoded.ContentType != "application/json" {
|
if len(encoded.Files) != 1 {
|
||||||
t.Fatalf("ContentType = %q, want application/json", encoded.ContentType)
|
t.Fatalf("len(Files) = %d, want 1", len(encoded.Files))
|
||||||
}
|
}
|
||||||
if string(encoded.Bytes) != `{"run_id":"run-1","approved_count":1}` {
|
if encoded.Files[0].ContentType != "application/json" {
|
||||||
t.Fatalf("Bytes = %s, want encoded output", encoded.Bytes)
|
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
|
||||||
|
}
|
||||||
|
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
|
||||||
|
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
|
||||||
|
file := OutputFile{
|
||||||
|
Name: "artifacts/events.json",
|
||||||
|
ContentType: "application/json",
|
||||||
|
Bytes: []byte(`{"ignored":true}`),
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got["name"] != "artifacts/events.json" {
|
||||||
|
t.Fatalf("name = %#v, want logical file name", got["name"])
|
||||||
|
}
|
||||||
|
if got["content_type"] != "application/json" {
|
||||||
|
t.Fatalf("content_type = %#v, want application/json", got["content_type"])
|
||||||
|
}
|
||||||
|
if _, ok := got["Bytes"]; ok {
|
||||||
|
t.Fatalf("encoded output file leaked Bytes: %s", encoded)
|
||||||
|
}
|
||||||
|
if _, ok := got["bytes"]; ok {
|
||||||
|
t.Fatalf("encoded output file leaked bytes: %s", encoded)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -397,7 +430,12 @@ func (encoder fakeOutputEncoder) Key() string {
|
|||||||
|
|
||||||
func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest) (OutputResult, error) {
|
func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest) (OutputResult, error) {
|
||||||
return OutputResult{
|
return OutputResult{
|
||||||
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
|
Files: []OutputFile{
|
||||||
ContentType: "application/json",
|
{
|
||||||
|
Name: "artifacts/generic.json",
|
||||||
|
ContentType: "application/json",
|
||||||
|
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
|
||||||
|
},
|
||||||
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package pipeline
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
@@ -29,21 +32,27 @@ func New(registries Registries) *Runner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type RunInput struct {
|
type RunInput struct {
|
||||||
Pipeline ResolvedPipeline
|
Pipeline ResolvedPipeline
|
||||||
SourceID string
|
SourceID string
|
||||||
Path string
|
Path string
|
||||||
RawInput []byte
|
RawInput []byte
|
||||||
LLMClient contracts.StructuredLLMClient
|
LLMClient contracts.StructuredLLMClient
|
||||||
Metadata map[string]any
|
RunID string
|
||||||
|
StartedAt time.Time
|
||||||
|
LLMProfiles []artifacts.LLMProfileManifest
|
||||||
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type RunOutput struct {
|
type RunOutput struct {
|
||||||
Manifest artifacts.RunManifest `json:"manifest"`
|
Manifest artifacts.RunManifest `json:"manifest"`
|
||||||
Approved []artifacts.Artifact `json:"approved,omitempty"`
|
Approved []artifacts.Artifact `json:"approved,omitempty"`
|
||||||
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
|
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
|
||||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||||
EncodedOutput []byte `json:"-"`
|
OutputFiles []contracts.OutputFile `json:"-"`
|
||||||
ContentType string `json:"content_type,omitempty"`
|
// EncodedOutput is the legacy single-output payload. New callers should use OutputFiles.
|
||||||
|
EncodedOutput []byte `json:"-"`
|
||||||
|
// ContentType is the legacy single-output content type. New callers should use OutputFiles.
|
||||||
|
ContentType string `json:"content_type,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||||
@@ -58,7 +67,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
return output, err
|
return output, err
|
||||||
}
|
}
|
||||||
|
|
||||||
output.Manifest = manifestFromPipeline(input.Pipeline)
|
output.Manifest = manifestFromPipeline(input)
|
||||||
|
|
||||||
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
|
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -110,6 +119,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
} else {
|
} else {
|
||||||
output.Manifest.ValidationStatus = "approved"
|
output.Manifest.ValidationStatus = "approved"
|
||||||
}
|
}
|
||||||
|
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
|
||||||
|
|
||||||
encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module)
|
encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -128,8 +138,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err)
|
return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err)
|
||||||
}
|
}
|
||||||
output.EncodedOutput = encoded.Bytes
|
files, err := outputFilesFromResult(encoded)
|
||||||
output.ContentType = encoded.ContentType
|
if err != nil {
|
||||||
|
return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
|
||||||
|
}
|
||||||
|
output.OutputFiles = files
|
||||||
|
if len(encoded.Files) == 0 {
|
||||||
|
output.EncodedOutput = append([]byte(nil), encoded.Bytes...)
|
||||||
|
output.ContentType = encoded.ContentType
|
||||||
|
}
|
||||||
|
|
||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
@@ -147,6 +164,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
|
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
|
||||||
}
|
}
|
||||||
|
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
|
||||||
|
|
||||||
var validators []validatorExecution
|
var validators []validatorExecution
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
@@ -315,7 +333,17 @@ func validateRunInput(input RunInput) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest {
|
func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
||||||
|
startedAt := input.StartedAt
|
||||||
|
if startedAt.IsZero() {
|
||||||
|
startedAt = time.Now().UTC()
|
||||||
|
}
|
||||||
|
runID := strings.TrimSpace(input.RunID)
|
||||||
|
if runID == "" {
|
||||||
|
runID = fmt.Sprintf("run-%d", startedAt.UnixNano())
|
||||||
|
}
|
||||||
|
|
||||||
|
pipeline := input.Pipeline
|
||||||
manifest := artifacts.RunManifest{
|
manifest := artifacts.RunManifest{
|
||||||
PipelineID: pipeline.ID,
|
PipelineID: pipeline.ID,
|
||||||
PipelineDigest: pipeline.Digest,
|
PipelineDigest: pipeline.Digest,
|
||||||
@@ -323,6 +351,9 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest {
|
|||||||
Chunker: pipeline.Chunk.Module,
|
Chunker: pipeline.Chunk.Module,
|
||||||
OutputEncoder: pipeline.Output.Module,
|
OutputEncoder: pipeline.Output.Module,
|
||||||
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
||||||
|
RunID: runID,
|
||||||
|
StartedAt: timePtr(startedAt),
|
||||||
|
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, lane := range pipeline.ArtifactLanes {
|
for _, lane := range pipeline.ArtifactLanes {
|
||||||
@@ -343,10 +374,124 @@ func manifestFromPipeline(pipeline ResolvedPipeline) artifacts.RunManifest {
|
|||||||
func failOutput(output RunOutput) RunOutput {
|
func failOutput(output RunOutput) RunOutput {
|
||||||
if output.Manifest.PipelineID != "" {
|
if output.Manifest.PipelineID != "" {
|
||||||
output.Manifest.ValidationStatus = "failed"
|
output.Manifest.ValidationStatus = "failed"
|
||||||
|
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
|
||||||
}
|
}
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
||||||
|
if output == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := range output.Manifest.ArtifactLanes {
|
||||||
|
if output.Manifest.ArtifactLanes[i].ID != laneID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata := make(map[string]any)
|
||||||
|
for _, module := range modules {
|
||||||
|
provider, ok := module.(contracts.ManifestMetadataProvider)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
|
||||||
|
if len(moduleMetadata) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := manifestMetadataKey(module)
|
||||||
|
if key == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
metadata[key] = moduleMetadata
|
||||||
|
}
|
||||||
|
if len(metadata) > 0 {
|
||||||
|
output.Manifest.ArtifactLanes[i].Metadata = metadata
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestMetadataKey(module any) string {
|
||||||
|
switch module.(type) {
|
||||||
|
case contracts.Extractor:
|
||||||
|
return "extractor"
|
||||||
|
case contracts.Merger:
|
||||||
|
return "merger"
|
||||||
|
case contracts.Normalizer:
|
||||||
|
return "normalizer"
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
|
||||||
|
files := result.Files
|
||||||
|
if len(files) == 0 && len(result.Bytes) > 0 {
|
||||||
|
files = []contracts.OutputFile{
|
||||||
|
{
|
||||||
|
Name: "output",
|
||||||
|
ContentType: result.ContentType,
|
||||||
|
Bytes: result.Bytes,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]contracts.OutputFile, 0, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
if err := validateOutputFileName(file.Name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, contracts.OutputFile{
|
||||||
|
Name: file.Name,
|
||||||
|
ContentType: file.ContentType,
|
||||||
|
Bytes: append([]byte(nil), file.Bytes...),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOutputFileName(name string) error {
|
||||||
|
if strings.TrimSpace(name) == "" {
|
||||||
|
return fmt.Errorf("output file name must not be empty")
|
||||||
|
}
|
||||||
|
if strings.Contains(name, "\\") {
|
||||||
|
return fmt.Errorf("output file name %q must use slash-separated relative paths", name)
|
||||||
|
}
|
||||||
|
if path.IsAbs(name) {
|
||||||
|
return fmt.Errorf("output file name %q must be relative", name)
|
||||||
|
}
|
||||||
|
if strings.Contains(name, "..") {
|
||||||
|
return fmt.Errorf("output file name %q must not contain ..", name)
|
||||||
|
}
|
||||||
|
cleaned := path.Clean(name)
|
||||||
|
if cleaned == "." || cleaned != name {
|
||||||
|
return fmt.Errorf("output file name %q must be clean", name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||||
|
if len(metadata) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(metadata))
|
||||||
|
for key, value := range metadata {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
|
||||||
|
if len(profiles) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return append([]artifacts.LLMProfileManifest(nil), profiles...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func timePtr(t time.Time) *time.Time {
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool {
|
func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool {
|
||||||
for _, lane := range pipeline.ArtifactLanes {
|
for _, lane := range pipeline.ArtifactLanes {
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
@@ -32,6 +33,7 @@ func TestNewAndDataTypes(t *testing.T) {
|
|||||||
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
|
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
|
||||||
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
|
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
|
||||||
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
|
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
|
||||||
|
OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
|
||||||
EncodedOutput: []byte(`{}`),
|
EncodedOutput: []byte(`{}`),
|
||||||
ContentType: "application/json",
|
ContentType: "application/json",
|
||||||
}
|
}
|
||||||
@@ -39,7 +41,7 @@ func TestNewAndDataTypes(t *testing.T) {
|
|||||||
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
|
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
|
||||||
t.Fatalf("RunInput = %#v, want constructed fields", input)
|
t.Fatalf("RunInput = %#v, want constructed fields", input)
|
||||||
}
|
}
|
||||||
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 {
|
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
|
||||||
t.Fatalf("RunOutput = %#v, want constructed fields", output)
|
t.Fatalf("RunOutput = %#v, want constructed fields", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -604,11 +606,18 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v, want nil", err)
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if output.ContentType != "application/json" {
|
if len(output.OutputFiles) != 1 {
|
||||||
t.Fatalf("ContentType = %q, want application/json", output.ContentType)
|
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||||
}
|
}
|
||||||
if string(output.EncodedOutput) != `{"encoded":true}` {
|
file := output.OutputFiles[0]
|
||||||
t.Fatalf("EncodedOutput = %s, want encoded payload", output.EncodedOutput)
|
if file.Name != "artifacts/generic.json" {
|
||||||
|
t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name)
|
||||||
|
}
|
||||||
|
if file.ContentType != "application/json" {
|
||||||
|
t.Fatalf("ContentType = %q, want application/json", file.ContentType)
|
||||||
|
}
|
||||||
|
if string(file.Bytes) != `{"encoded":true}` {
|
||||||
|
t.Fatalf("OutputFiles[0].Bytes = %s, want encoded payload", file.Bytes)
|
||||||
}
|
}
|
||||||
if len(modules.output.requests) != 1 {
|
if len(modules.output.requests) != 1 {
|
||||||
t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests))
|
t.Fatalf("len(output requests) = %d, want 1", len(modules.output.requests))
|
||||||
@@ -622,6 +631,35 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fileName string
|
||||||
|
}{
|
||||||
|
{name: "empty", fileName: ""},
|
||||||
|
{name: "absolute", fileName: "/tmp/output.json"},
|
||||||
|
{name: "parent", fileName: "artifacts/../manifest.json"},
|
||||||
|
{name: "backslash", fileName: `artifacts\manifest.json`},
|
||||||
|
{name: "unclean", fileName: "artifacts//manifest.json"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.output.files = []contracts.OutputFile{
|
||||||
|
{Name: test.fileName, ContentType: "application/json", Bytes: []byte(`{}`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
|
||||||
|
assertRunError(t, err, "output file name")
|
||||||
|
if output.Manifest.ValidationStatus != "failed" {
|
||||||
|
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
|
func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
modules.output.err = errors.New("encode failed")
|
modules.output.err = errors.New("encode failed")
|
||||||
@@ -632,6 +670,9 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
|
|||||||
if output.Manifest.ValidationStatus != "failed" {
|
if output.Manifest.ValidationStatus != "failed" {
|
||||||
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
||||||
}
|
}
|
||||||
|
if output.Manifest.CompletedAt == nil {
|
||||||
|
t.Fatal("CompletedAt = nil, want failed run completion timestamp")
|
||||||
|
}
|
||||||
if len(output.Approved) != 2 {
|
if len(output.Approved) != 2 {
|
||||||
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
||||||
}
|
}
|
||||||
@@ -668,6 +709,76 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
|
||||||
|
startedAt := time.Now().Add(-time.Minute).UTC()
|
||||||
|
profiles := []artifacts.LLMProfileManifest{
|
||||||
|
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
|
||||||
|
Pipeline: resolvedPipeline(),
|
||||||
|
RunID: "run-test",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
LLMProfiles: profiles,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest := output.Manifest
|
||||||
|
if manifest.RunID != "run-test" {
|
||||||
|
t.Fatalf("RunID = %q, want run-test", manifest.RunID)
|
||||||
|
}
|
||||||
|
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(startedAt) {
|
||||||
|
t.Fatalf("StartedAt = %v, want %s", manifest.StartedAt, startedAt)
|
||||||
|
}
|
||||||
|
if manifest.CompletedAt == nil || manifest.CompletedAt.Before(startedAt) {
|
||||||
|
t.Fatalf("CompletedAt = %v, want timestamp after start", manifest.CompletedAt)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(manifest.LLMProfiles, profiles) {
|
||||||
|
t.Fatalf("LLMProfiles = %#v, want %#v", manifest.LLMProfiles, profiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) {
|
||||||
|
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(output.Manifest.RunID, "run-") {
|
||||||
|
t.Fatalf("RunID = %q, want generated run ID", output.Manifest.RunID)
|
||||||
|
}
|
||||||
|
if output.Manifest.StartedAt == nil {
|
||||||
|
t.Fatal("StartedAt = nil, want generated timestamp")
|
||||||
|
}
|
||||||
|
if output.Manifest.CompletedAt == nil {
|
||||||
|
t.Fatal("CompletedAt = nil, want generated timestamp")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.extractors["extract-alpha"].manifestMetadata = map[string]any{
|
||||||
|
"prompt_id": "test.prompt",
|
||||||
|
"response_schema_name": "test_schema",
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lane := output.Manifest.ArtifactLanes[0]
|
||||||
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||||
|
}
|
||||||
|
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
|
||||||
|
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
|
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
|
||||||
@@ -783,7 +894,12 @@ func defaultRunnerModules() *runnerModules {
|
|||||||
"configured": {name: "configured", decisions: approveAll},
|
"configured": {name: "configured", decisions: approveAll},
|
||||||
"second-validator": {name: "second-validator", decisions: approveAll},
|
"second-validator": {name: "second-validator", decisions: approveAll},
|
||||||
},
|
},
|
||||||
output: &runnerOutputEncoder{key: "output", bytes: []byte(`{"encoded":true}`), contentType: "application/json"},
|
output: &runnerOutputEncoder{
|
||||||
|
key: "output",
|
||||||
|
files: []contracts.OutputFile{
|
||||||
|
{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -885,17 +1001,18 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
|
|||||||
}
|
}
|
||||||
|
|
||||||
type runnerExtractor struct {
|
type runnerExtractor struct {
|
||||||
key string
|
key string
|
||||||
artifactType string
|
artifactType string
|
||||||
schemaVersion string
|
schemaVersion string
|
||||||
candidates []artifacts.ArtifactCandidate
|
manifestMetadata map[string]any
|
||||||
validators []contracts.Validator
|
candidates []artifacts.ArtifactCandidate
|
||||||
warnings []contracts.Warning
|
validators []contracts.Validator
|
||||||
err error
|
warnings []contracts.Warning
|
||||||
requests []contracts.ExtractionRequest
|
err error
|
||||||
seenChunkIDs []string
|
requests []contracts.ExtractionRequest
|
||||||
seenLLMClients []contracts.StructuredLLMClient
|
seenChunkIDs []string
|
||||||
seenMetadata []map[string]any
|
seenLLMClients []contracts.StructuredLLMClient
|
||||||
|
seenMetadata []map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor *runnerExtractor) Key() string {
|
func (extractor *runnerExtractor) Key() string {
|
||||||
@@ -910,6 +1027,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
|
|||||||
return extractor.schemaVersion
|
return extractor.schemaVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
|
||||||
|
return extractor.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
func (extractor *runnerExtractor) Validators() []contracts.Validator {
|
func (extractor *runnerExtractor) Validators() []contracts.Validator {
|
||||||
return extractor.validators
|
return extractor.validators
|
||||||
}
|
}
|
||||||
@@ -1020,6 +1141,7 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
|
|||||||
|
|
||||||
type runnerOutputEncoder struct {
|
type runnerOutputEncoder struct {
|
||||||
key string
|
key string
|
||||||
|
files []contracts.OutputFile
|
||||||
bytes []byte
|
bytes []byte
|
||||||
contentType string
|
contentType string
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
@@ -1034,6 +1156,7 @@ func (encoder *runnerOutputEncoder) Key() string {
|
|||||||
func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||||
encoder.requests = append(encoder.requests, req)
|
encoder.requests = append(encoder.requests, req)
|
||||||
return contracts.OutputResult{
|
return contracts.OutputResult{
|
||||||
|
Files: encoder.files,
|
||||||
Bytes: encoder.bytes,
|
Bytes: encoder.bytes,
|
||||||
ContentType: encoder.contentType,
|
ContentType: encoder.contentType,
|
||||||
Warnings: encoder.warnings,
|
Warnings: encoder.warnings,
|
||||||
|
|||||||
@@ -57,6 +57,14 @@ type Bundle struct {
|
|||||||
metadata Metadata
|
metadata Metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Metadata returns metadata for the compiled prompt bundle.
|
||||||
|
func (b *Bundle) Metadata() Metadata {
|
||||||
|
if b == nil {
|
||||||
|
return Metadata{}
|
||||||
|
}
|
||||||
|
return b.metadata
|
||||||
|
}
|
||||||
|
|
||||||
var promptRegistry map[string]*Bundle
|
var promptRegistry map[string]*Bundle
|
||||||
var sharedHardening string
|
var sharedHardening string
|
||||||
|
|
||||||
|
|||||||
@@ -45,6 +45,23 @@ func (e *Extractor) SchemaVersion() string {
|
|||||||
return SchemaVersion
|
return SchemaVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||||
|
promptMetadata := spellsPromptBundle.Metadata()
|
||||||
|
metadata := map[string]any{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": promptMetadata.PromptVersion,
|
||||||
|
"prompt_sha256": promptMetadata.SHA256,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
}
|
||||||
|
if schema, err := loadResponseSchema(); err == nil {
|
||||||
|
metadata["response_schema_version"] = schema.Version
|
||||||
|
metadata["response_schema_sha256"] = schema.SHA256
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
|
|
||||||
func (e *Extractor) Validators() []contracts.Validator {
|
func (e *Extractor) Validators() []contracts.Validator {
|
||||||
return []contracts.Validator{
|
return []contracts.Validator{
|
||||||
ShapeValidator{},
|
ShapeValidator{},
|
||||||
|
|||||||
@@ -92,6 +92,30 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
|
||||||
|
metadata := New().ManifestMetadata()
|
||||||
|
|
||||||
|
tests := map[string]string{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": SchemaVersion,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
"response_schema_version": SchemaVersion,
|
||||||
|
}
|
||||||
|
for key, want := range tests {
|
||||||
|
if metadata[key] != want {
|
||||||
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||||
|
value, ok := metadata[key].(string)
|
||||||
|
if !ok || !strings.HasPrefix(value, "sha256:") {
|
||||||
|
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
|
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
|
||||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user