eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raw module output contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 18:58:23 +00:00
parent 9e3f8809b3
commit c05ecb58d8
33 changed files with 1166 additions and 1780 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
internal/framework/contracts/composition_test.go
View File

@@ -27,7 +27,6 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
extractor := compositionExtractor{}
merger := compositionMerger{}
normalizer := compositionNormalizer{}
validator := compositionValidator{}
encoder := compositionOutputEncoder{}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
@@ -58,70 +57,37 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(extraction.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(extraction.Candidates))
}
candidate := extraction.Candidates[0]
for _, ref := range candidate.SourceRefs {
if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if extraction.Output.Payload.MediaType != "application/json" {
t.Fatalf("extract media type = %q, want application/json", extraction.Output.Payload.MediaType)
}
merge, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
ChunkArtifacts: []contracts.ChunkArtifacts{
{
Chunk: chunking.Chunks[0],
Candidates: extraction.Candidates,
},
},
Source: doc,
LaneID: "generic-lane",
ExtractOutputs: []contracts.ExtractOutput{extraction.Output},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(merge.Candidates) != 1 {
t.Fatalf("len(merge.Candidates) = %d, want 1", len(merge.Candidates))
if string(merge.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merge output = %s, want extract payload", merge.Output.Payload.Content)
}
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
Candidates: merge.Candidates,
Source: doc,
LaneID: "generic-lane",
MergeOutput: merge.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(normalize.Candidates) != 1 {
t.Fatalf("len(normalize.Candidates) = %d, want 1", len(normalize.Candidates))
}
validation, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: normalize.Candidates,
})
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if len(validation.Decisions) != 1 {
t.Fatalf("len(Decisions) = %d, want 1", len(validation.Decisions))
}
decision := validation.Decisions[0]
if !decision.Approved {
t.Fatal("Approved = false, want true")
}
if decision.CandidateIndex != candidate.Index {
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, candidate.Index)
if string(normalize.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalize output = %s, want merge payload", normalize.Output.Payload.Content)
}
output, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalize.Candidates[0]),
},
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.NormalizeOutput{normalize.Output},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -203,49 +169,24 @@ func (extractor compositionExtractor) Key() string {
return "generic-extractor"
}
func (extractor compositionExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor compositionExtractor) SchemaVersion() string {
return "v1"
}
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}}
}
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if req.Source == nil {
return contracts.ExtractionResult{}, errors.New("source document is required")
}
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
if req.AmbientContext["synopsis"] == "" {
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.Key(),
ArtifactType: extractor.ArtifactType(),
SchemaVersion: extractor.SchemaVersion(),
Payload: json.RawMessage(`{"value":"example"}`),
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
},
},
}, nil
@@ -258,12 +199,14 @@ func (merger compositionMerger) Key() string {
}
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return contracts.MergeResult{Candidates: candidates}, nil
output := req.ExtractOutputs[0]
return contracts.MergeResult{Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.Key(),
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneCompositionPayload(output.Payload),
}}, nil
}
type compositionNormalizer struct{}
@@ -277,7 +220,33 @@ func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSl
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.Key(),
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneCompositionPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneCompositionPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneCompositionMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneCompositionMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type compositionValidator struct{}
@@ -311,11 +280,11 @@ func (encoder compositionOutputEncoder) Key() string {
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
payload := struct {
RunID string `json:"run_id"`
ApprovedCount int `json:"approved_count"`
RunID string `json:"run_id"`
OutputCount int `json:"output_count"`
}{
RunID: req.Manifest.RunID,
ApprovedCount: len(req.Approved),
RunID: req.Manifest.RunID,
OutputCount: len(req.NormalizeOutputs),
}
encoded, err := json.Marshal(payload)
if err != nil {

104
internal/framework/contracts/contracts.go
View File

@@ -186,36 +186,59 @@ type ExtractionRequest struct {
}
type ExtractionResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
Output ExtractOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Extractor interface {
Key() string
ArtifactType() string
SchemaVersion() string
ReferenceSlots() []ReferenceSlot
Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
}
type ChunkArtifacts struct {
Chunk SourceChunk `json:"chunk"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
type RawPayload struct {
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
}
type ResponseSchema struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
}
type ExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type MergeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
ChunkArtifacts []ChunkArtifacts `json:"chunk_artifacts"`
ExtractOutputs []ExtractOutput `json:"extract_outputs"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type MergeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
Output MergeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type MergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type Merger interface {
@@ -224,21 +247,29 @@ type Merger interface {
}
type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
MergeOutput MergeOutput `json:"merge_output"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type NormalizeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
Output NormalizeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type NormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type Normalizer interface {
@@ -281,13 +312,13 @@ type Warning struct {
}
type OutputRequest struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []NormalizeOutput `json:"normalize_outputs,omitempty"`
Rejected []RejectedOutput `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type OutputFile struct {
@@ -306,6 +337,19 @@ type OutputEncoder interface {
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
}
type RejectedOutput struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ManifestMetadataProvider interface {
ManifestMetadata() map[string]any
}

203
internal/framework/contracts/contracts_test.go
View File

@@ -19,13 +19,9 @@ var _ Validator = fakeValidator{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
validator := fakeValidator{name: "generic-validator"}
func TestFakeExtractorReturnsRawOutput(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
validators: []Validator{validator},
key: "generic-extractor",
}
doc := &source.SourceDocument{
ID: "source-1",
@@ -45,37 +41,14 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
}
if extractor.ArtifactType() != "generic-artifact" {
t.Fatalf("ArtifactType() = %q, want generic-artifact", extractor.ArtifactType())
if result.Output.ExtractorKey != "" {
t.Fatalf("ExtractorKey = %q, want runner-owned empty value", result.Output.ExtractorKey)
}
if extractor.SchemaVersion() != "v1" {
t.Fatalf("SchemaVersion() = %q, want v1", extractor.SchemaVersion())
if result.Output.Schema.Version != "v1" {
t.Fatalf("Schema.Version = %q, want v1", result.Output.Schema.Version)
}
if len(extractor.Validators()) != 1 {
t.Fatalf("len(Validators()) = %d, want 1", len(extractor.Validators()))
}
if extractor.Validators()[0].Name() != "generic-validator" {
t.Fatalf("Validators()[0].Name() = %q, want generic-validator", extractor.Validators()[0].Name())
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
}
candidate := result.Candidates[0]
if candidate.Index != 0 {
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType != extractor.ArtifactType() {
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
}
if string(candidate.Payload) != `{"value":"example"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
if result.Output.Payload.MediaType != "application/json" || string(result.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("payload = %q %s, want JSON raw output", result.Output.Payload.MediaType, result.Output.Payload.Content)
}
}
@@ -146,11 +119,7 @@ func TestFakeChunkerReceivesLLMClient(t *testing.T) {
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
}
extractor := fakeExtractor{key: "generic-extractor"}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
@@ -180,20 +149,11 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
if result.Output.ChunkID != "" || result.Output.ChunkIndex != 0 {
t.Fatalf("chunk provenance = %q/%d, want runner-owned zero values", result.Output.ChunkID, result.Output.ChunkIndex)
}
candidate := result.Candidates[0]
if string(candidate.Payload) != `{"value":"chunked"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want chunked payload", candidate.Payload)
}
if len(candidate.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs))
}
ref := candidate.SourceRefs[0]
if ref.StartUnitID != 2 || ref.EndUnitID != 2 {
t.Fatalf("SourceRef = %+v, want unit 2 range", ref)
if string(result.Output.Payload.Content) != `{"value":"chunked"}` {
t.Fatalf("Payload.Content = %s, want chunked payload", result.Output.Payload.Content)
}
}
@@ -363,19 +323,17 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{
Index: 0,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"value":"example"}`),
}
chunk := SourceChunk{
ID: "source-1:chunk:0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "Source text."},
extractOutput := ExtractOutput{
LaneID: "generic-lane",
ExtractorKey: "generic-extractor",
SourceID: "source-1",
ChunkID: "source-1:chunk:0",
ChunkIndex: 0,
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
Metadata: map[string]any{"confidence": 0.75},
},
}
merger := fakeMerger{key: "generic-merger"}
@@ -383,13 +341,8 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), MergeRequest{
LaneID: "generic-artifact",
ChunkArtifacts: []ChunkArtifacts{
{
Chunk: chunk,
Candidates: []artifacts.ArtifactCandidate{candidate},
},
},
LaneID: "generic-lane",
ExtractOutputs: []ExtractOutput{extractOutput},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
@@ -397,13 +350,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
}
if len(merged.Candidates) != 1 {
t.Fatalf("len(merged.Candidates) = %d, want 1", len(merged.Candidates))
if string(merged.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merged content = %s, want raw extract content", merged.Output.Payload.Content)
}
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{
LaneID: "generic-artifact",
Candidates: merged.Candidates,
LaneID: "generic-lane",
MergeOutput: merged.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
@@ -411,15 +364,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
}
if len(normalized.Candidates) != 1 {
t.Fatalf("len(normalized.Candidates) = %d, want 1", len(normalized.Candidates))
if string(normalized.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalized content = %s, want raw merge content", normalized.Output.Payload.Content)
}
encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalized.Candidates[0]),
},
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []NormalizeOutput{normalized.Output},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -433,7 +384,7 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if encoded.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
}
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
}
}
@@ -529,57 +480,29 @@ func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (C
}
type fakeExtractor struct {
key string
artifactType string
schemaVersion string
validators []Validator
key string
}
func (extractor fakeExtractor) Key() string {
return extractor.key
}
func (extractor fakeExtractor) ArtifactType() string {
return extractor.artifactType
}
func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators
}
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
payload := json.RawMessage(`{"value":"example"}`)
if req.AmbientContext["mode"] == "chunked" {
payload = json.RawMessage(`{"value":"chunked"}`)
}
return ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.key,
ArtifactType: extractor.artifactType,
SchemaVersion: extractor.schemaVersion,
Payload: payload,
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
Output: ExtractOutput{
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: append([]byte(nil), payload...),
MediaType: "application/json",
},
},
}, nil
@@ -594,12 +517,14 @@ func (merger fakeMerger) Key() string {
}
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return MergeResult{Candidates: candidates}, nil
output := req.ExtractOutputs[0]
return MergeResult{Output: MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.key,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneTestRawPayload(output.Payload),
}}, nil
}
type fakeNormalizer struct {
@@ -615,7 +540,33 @@ func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil
return NormalizeResult{Output: NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneTestRawPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneTestRawPayload(payload RawPayload) RawPayload {
return RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneTestMetadata(payload.Metadata),
Warnings: append([]Warning(nil), payload.Warnings...),
}
}
func cloneTestMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type fakeValidator struct {
@@ -665,7 +616,7 @@ func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest)
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","output_count":1}`),
},
},
}, nil

6
internal/framework/pipeline/default_modules_test.go
View File

@@ -113,14 +113,8 @@ type defaultExtractor struct{}
func (defaultExtractor) Key() string { return "extract" }
func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

12
internal/framework/pipeline/extractor_registry_test.go
View File

@@ -367,22 +367,10 @@ func (extractor registryFakeExtractor) Key() string {
return extractor.key
}
func (extractor registryFakeExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1"
}
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil
}
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

114
internal/framework/pipeline/registry_integration_test.go
View File

@@ -5,11 +5,8 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestRunnerUsesRegistries(t *testing.T) {
@@ -33,11 +30,11 @@ func TestRunnerUsesRegistries(t *testing.T) {
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
}
if got := artifactKeys(output.Approved); !reflect.DeepEqual(got, []string{"extract-first"}) {
t.Fatalf("approved keys = %#v, want [extract-first]", got)
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
}
if got := rejectedKeys(output.Rejected); !reflect.DeepEqual(got, []string{"extract-second"}) {
t.Fatalf("rejected keys = %#v, want [extract-second]", got)
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
}
}
@@ -64,12 +61,8 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed, []contracts.Validator{
integrationValidator{name: "approve-first", approve: true},
})
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed, []contracts.Validator{
integrationValidator{name: "reject-second", approve: false},
})
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
if err := registries.Mergers.Register("merge", func() (contracts.Merger, error) {
*built = append(*built, "merge")
return integrationMerger{}, nil
@@ -91,12 +84,12 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
return registries
}
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string, validators []contracts.Validator) {
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
t.Helper()
if err := registry.Register(key, func() (contracts.Extractor, error) {
*built = append(*built, key)
return integrationExtractor{key: key, executed: executed, validators: validators}, nil
return integrationExtractor{key: key, executed: executed}, nil
}); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
@@ -140,36 +133,27 @@ func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.Chunk
}
type integrationExtractor struct {
key string
executed *[]string
validators []contracts.Validator
key string
executed *[]string
}
func (extractor integrationExtractor) Key() string {
return extractor.key
}
func (extractor integrationExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor integrationExtractor) SchemaVersion() string {
return "v1"
}
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Validators() []contracts.Validator {
return extractor.validators
}
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{Payload: []byte(`{"value":true}`)},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
}
@@ -183,11 +167,20 @@ func (merger integrationMerger) Key() string {
}
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
output := contracts.MergeOutput{
LaneID: req.LaneID,
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
return contracts.MergeResult{Candidates: candidates}, nil
if len(req.ExtractOutputs) > 0 {
output.SourceID = req.ExtractOutputs[0].SourceID
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
func (normalizer integrationNormalizer) Key() string {
@@ -199,7 +192,14 @@ func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSl
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type integrationOutput struct{}
@@ -216,30 +216,6 @@ func (output integrationOutput) Encode(ctx context.Context, req contracts.Output
}, nil
}
type integrationValidator struct {
name string
approve bool
}
func (validator integrationValidator) Name() string {
return validator.name
}
func (validator integrationValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
if validator.approve {
decisions = append(decisions, validate.Approved(candidate.Index))
} else {
decisions = append(decisions, validate.Rejected(candidate.Index, "invalid", "not accepted"))
}
}
return contracts.ValidationResult{
ValidatorName: validator.name,
Decisions: decisions,
}, nil
}
func integrationPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
@@ -276,18 +252,10 @@ func integrationSourceDocument() *source.SourceDocument {
}
}
func artifactKeys(approved []artifacts.Artifact) []string {
keys := make([]string, 0, len(approved))
for _, artifact := range approved {
keys = append(keys, artifact.ExtractorKey)
}
return keys
}
func rejectedKeys(rejected []artifacts.RejectedArtifact) []string {
keys := make([]string, 0, len(rejected))
for _, artifact := range rejected {
keys = append(keys, artifact.Candidate.ExtractorKey)
func normalizeOutputKeys(outputs []contracts.NormalizeOutput) []string {
keys := make([]string, 0, len(outputs))
for _, output := range outputs {
keys = append(keys, output.NormalizerKey)
}
return keys
}

251
internal/framework/pipeline/runner.go
View File

@@ -15,7 +15,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
type Registries struct {
@@ -51,11 +50,11 @@ type RunInput struct {
}
type RunOutput struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []contracts.NormalizeOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
}
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
@@ -126,9 +125,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
nextCandidateIndex := 0
for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
return failOutput(output), err
}
}
@@ -146,13 +144,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
Approved: output.Approved,
Rejected: output.Rejected,
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
Metadata: input.Metadata,
Manifest: output.Manifest,
NormalizeOutputs: cloneNormalizeOutputs(output.NormalizeOutputs),
Rejected: cloneRejectedOutputs(output.Rejected),
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, encoded.Warnings...)
if err != nil {
@@ -167,7 +165,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput, nextCandidateIndex *int) error {
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
@@ -182,19 +180,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
}
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
var validators []validatorExecution
if len(lane.Validators) > 0 {
validators, err = r.buildConfiguredValidators(lane)
if err != nil {
return err
}
} else {
for _, validator := range extractor.Validators() {
validators = append(validators, validatorExecution{validator: validator})
}
}
chunkArtifacts := make([]contracts.ChunkArtifacts, 0, len(chunks))
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
for index := range chunks {
chunk := chunks[index]
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
@@ -212,21 +198,20 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
if err != nil {
return fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
}
candidates, err := normalizeCandidates(extractor, result.Candidates, nextCandidateIndex)
if err != nil {
return err
}
chunkArtifacts = append(chunkArtifacts, contracts.ChunkArtifacts{
Chunk: chunk,
Candidates: candidates,
})
extractOutput := result.Output
extractOutput.LaneID = lane.ID
extractOutput.ExtractorKey = extractor.Key()
extractOutput.SourceID = doc.ID
extractOutput.ChunkID = chunk.ID
extractOutput.ChunkIndex = chunk.Index
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
extractOutputs = append(extractOutputs, cloneExtractOutput(extractOutput))
}
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: lane.ID,
ChunkArtifacts: chunkArtifacts,
ExtractOutputs: cloneExtractOutputs(extractOutputs),
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
@@ -239,7 +224,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: lane.ID,
Candidates: mergeResult.Candidates,
MergeOutput: cloneMergeOutput(mergeResult.Output),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
@@ -252,21 +237,12 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
if err != nil {
return fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
if err := validateCandidateEnvelope(extractor, normalizeResult.Candidates); err != nil {
return fmt.Errorf("validate normalized candidates for lane %q: %w", lane.ID, err)
}
approved, rejected, warnings, err := runValidators(ctx, extractor.Key(), validators, doc, normalizeResult.Candidates, input.Metadata)
output.Warnings = append(output.Warnings, warnings...)
output.Rejected = append(output.Rejected, rejected...)
if err != nil {
return err
}
for _, candidate := range approved {
output.Approved = append(output.Approved, artifacts.ArtifactFromCandidate(candidate))
}
normalizeOutput := normalizeResult.Output
normalizeOutput.LaneID = lane.ID
normalizeOutput.NormalizerKey = normalizer.Key()
normalizeOutput.SourceID = doc.ID
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
output.NormalizeOutputs = append(output.NormalizeOutputs, cloneNormalizeOutput(normalizeOutput))
return nil
}
@@ -628,6 +604,59 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
return append([]contracts.Warning(nil), warnings...)
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: cloneWarnings(payload.Warnings),
}
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneExtractOutput(output))
}
return out
}
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.NormalizeOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneNormalizeOutput(output))
}
return out
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
return append([]contracts.RejectedOutput(nil), rejected...)
}
func timePtr(t time.Time) *time.Time {
return &t
}
@@ -640,115 +669,3 @@ func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool {
}
return false
}
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate, nextIndex *int) ([]artifacts.ArtifactCandidate, error) {
normalized := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
candidate.Index = *nextIndex
*nextIndex = *nextIndex + 1
if candidate.ExtractorKey == "" {
candidate.ExtractorKey = extractor.Key()
} else if candidate.ExtractorKey != extractor.Key() {
return nil, fmt.Errorf("candidate extractor_key %q does not match extractor %q", candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType == "" {
candidate.ArtifactType = extractor.ArtifactType()
} else if candidate.ArtifactType != extractor.ArtifactType() {
return nil, fmt.Errorf("candidate artifact_type %q does not match extractor %q artifact type %q", candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
}
if candidate.SchemaVersion == "" {
candidate.SchemaVersion = extractor.SchemaVersion()
} else if candidate.SchemaVersion != extractor.SchemaVersion() {
return nil, fmt.Errorf("candidate schema_version %q does not match extractor %q schema version %q", candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
}
normalized = append(normalized, candidate)
}
return normalized, nil
}
func validateCandidateEnvelope(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate) error {
seen := make(map[int]struct{}, len(candidates))
for _, candidate := range candidates {
if _, ok := seen[candidate.Index]; ok {
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
}
seen[candidate.Index] = struct{}{}
if candidate.ExtractorKey == "" {
return fmt.Errorf("candidate index %d extractor_key must not be empty", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
return fmt.Errorf("candidate index %d extractor_key %q does not match extractor %q", candidate.Index, candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType == "" {
return fmt.Errorf("candidate index %d artifact_type must not be empty", candidate.Index)
}
if candidate.ArtifactType != extractor.ArtifactType() {
return fmt.Errorf("candidate index %d artifact_type %q does not match extractor %q artifact type %q", candidate.Index, candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
}
if candidate.SchemaVersion == "" {
return fmt.Errorf("candidate index %d schema_version must not be empty", candidate.Index)
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
return fmt.Errorf("candidate index %d schema_version %q does not match extractor %q schema version %q", candidate.Index, candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
}
}
return nil
}
func runValidators(ctx context.Context, extractorKey string, validators []validatorExecution, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
eligible := candidates
var rejected []artifacts.RejectedArtifact
var warnings []contracts.Warning
for validatorIndex, execution := range validators {
validator := execution.validator
if validator == nil {
return nil, rejected, warnings, fmt.Errorf("extractor %q validator[%d] must not be nil", extractorKey, validatorIndex)
}
result, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: eligible,
LLMProfile: execution.binding.LLMProfile,
Options: cloneOptions(execution.binding.Options),
Metadata: metadata,
})
warnings = append(warnings, result.Warnings...)
if err != nil {
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractorKey, validator.Name(), err)
}
if result.ValidatorName != validator.Name() {
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
}
if err := validate.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractorKey, validator.Name(), err)
}
decisions := make(map[int]contracts.ValidationDecision, len(result.Decisions))
for _, decision := range result.Decisions {
decisions[decision.CandidateIndex] = decision
}
nextEligible := make([]artifacts.ArtifactCandidate, 0, len(eligible))
for _, candidate := range eligible {
decision := decisions[candidate.Index]
if decision.Approved {
nextEligible = append(nextEligible, candidate)
continue
}
rejected = append(rejected, artifacts.RejectedArtifact{
Candidate: candidate,
ValidatorName: result.ValidatorName,
ReasonCode: decision.ReasonCode,
Message: decision.Message,
})
}
eligible = nextEligible
}
return eligible, rejected, warnings, nil
}

391
internal/framework/pipeline/runner_test.go
View File

@@ -11,7 +11,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestNewAndDataTypes(t *testing.T) {
@@ -29,17 +28,17 @@ func TestNewAndDataTypes(t *testing.T) {
Metadata: map[string]any{"request": "test"},
}
output := RunOutput{
Manifest: artifacts.RunManifest{PipelineID: "pipeline-1"},
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
Manifest: artifacts.RunManifest{PipelineID: "pipeline-1"},
NormalizeOutputs: []contracts.NormalizeOutput{{NormalizerKey: "normalize"}},
Rejected: []contracts.RejectedOutput{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "outputs/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
}
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
t.Fatalf("RunInput = %#v, want constructed fields", input)
}
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
if output.Manifest.PipelineID != "pipeline-1" || len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
t.Fatalf("RunOutput = %#v, want constructed fields", output)
}
}
@@ -361,8 +360,8 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want one lane output", len(output.NormalizeOutputs))
}
}
@@ -397,8 +396,8 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
extractor := modules.extractors["extract-alpha"]
@@ -504,8 +503,8 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want one lane output", len(output.NormalizeOutputs))
}
}
@@ -660,12 +659,6 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
if got := modules.normalizers["normalize"].requests[0].Options["normalize_option"]; got != "normalize-value" {
t.Fatalf("normalize Options = %#v, want normalize option", modules.normalizers["normalize"].requests[0].Options)
}
if got := modules.validators["configured"].requests[0].LLMProfile; got != "validator-profile" {
t.Fatalf("validator LLMProfile = %q, want validator-profile", got)
}
if got := modules.validators["configured"].requests[0].Options["validator_option"]; got != "validator-value" {
t.Fatalf("validator Options = %#v, want validator option", modules.validators["configured"].requests[0].Options)
}
if got := modules.output.requests[0].LLMProfile; got != "output-profile" {
t.Fatalf("output LLMProfile = %q, want output-profile", got)
}
@@ -801,7 +794,7 @@ func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
func TestRunPassesPerChunkRawOutputsToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -813,111 +806,31 @@ func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
if len(merger.requests) != 1 {
t.Fatalf("len(merge requests) = %d, want 1", len(merger.requests))
}
chunkArtifacts := merger.requests[0].ChunkArtifacts
if len(chunkArtifacts) != 2 {
t.Fatalf("len(ChunkArtifacts) = %d, want 2", len(chunkArtifacts))
extractOutputs := merger.requests[0].ExtractOutputs
if len(extractOutputs) != 2 {
t.Fatalf("len(ExtractOutputs) = %d, want 2", len(extractOutputs))
}
if chunkArtifacts[0].Chunk.ID != "chunk-0" || chunkArtifacts[1].Chunk.ID != "chunk-1" {
t.Fatalf("merge chunks = %#v, want chunk order", chunkArtifacts)
if extractOutputs[0].ChunkID != "chunk-0" || extractOutputs[1].ChunkID != "chunk-1" {
t.Fatalf("merge chunks = %#v, want chunk order", extractOutputs)
}
if got := candidateIndices(chunkArtifacts[0].Candidates); !reflect.DeepEqual(got, []int{0}) {
t.Fatalf("first chunk candidate indices = %#v, want [0]", got)
if extractOutputs[0].ChunkIndex != 0 || string(extractOutputs[0].Payload.Content) != `{"chunk":"chunk-0"}` {
t.Fatalf("first extract output = %#v, want first chunk payload", extractOutputs[0])
}
if got := candidateIndices(chunkArtifacts[1].Candidates); !reflect.DeepEqual(got, []int{1}) {
t.Fatalf("second chunk candidate indices = %#v, want [1]", got)
if extractOutputs[1].ChunkIndex != 1 || string(extractOutputs[1].Payload.Content) != `{"chunk":"chunk-1"}` {
t.Fatalf("second extract output = %#v, want second chunk payload", extractOutputs[1])
}
normalizer := modules.normalizers["normalize"]
if len(normalizer.requests) != 1 {
t.Fatalf("len(normalize requests) = %d, want 1", len(normalizer.requests))
}
if got := candidateIndices(normalizer.requests[0].Candidates); !reflect.DeepEqual(got, []int{0, 1}) {
t.Fatalf("normalize candidate indices = %#v, want merged candidates", got)
if string(normalizer.requests[0].MergeOutput.Payload.Content) != `{"merged":true}` {
t.Fatalf("normalize merge output = %#v, want merged raw output", normalizer.requests[0].MergeOutput)
}
}
func TestRunRejectsInvalidPostNormalizeCandidateEnvelope(t *testing.T) {
tests := []struct {
name string
candidates []artifacts.ArtifactCandidate
want string
}{
{
name: "duplicate index",
candidates: []artifacts.ArtifactCandidate{
runnerCandidate(0),
runnerCandidate(0),
},
want: "duplicated",
},
{
name: "missing extractor key",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ArtifactType: "artifact", SchemaVersion: "v1", Payload: []byte(`{"value":true}`)},
},
want: "extractor_key",
},
{
name: "mismatched schema version",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ExtractorKey: "extract-alpha", ArtifactType: "artifact", SchemaVersion: "other", Payload: []byte(`{"value":true}`)},
},
want: "schema_version",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.normalizers["normalize"].result = test.candidates
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want no approved artifacts", len(output.Approved))
}
})
}
}
func TestRunValidatorApprovalAndRejection(t *testing.T) {
func TestRunRecordsConfiguredValidatorsInManifest(t *testing.T) {
modules := defaultRunnerModules()
rejectFirst := &runnerValidator{
name: "default-validator",
decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
return []contracts.ValidationDecision{
validate.Rejected(candidates[0].Index, "invalid", "not accepted"),
validate.Approved(candidates[1].Index),
}
},
}
modules.extractors["extract-alpha"].validators = []contracts.Validator{rejectFirst}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 1 || len(output.Rejected) != 1 {
t.Fatalf("approved/rejected = %d/%d, want 1/1", len(output.Approved), len(output.Rejected))
}
if output.Rejected[0].ValidatorName != "default-validator" || output.Rejected[0].ReasonCode != "invalid" {
t.Fatalf("Rejected[0] = %#v, want rejection details", output.Rejected[0])
}
}
func TestRunUsesConfiguredValidatorsInLaneOrder(t *testing.T) {
modules := defaultRunnerModules()
var order []string
modules.validators["configured"] = &runnerValidator{name: "configured", decisions: approveAll, order: &order}
modules.validators["second-validator"] = &runnerValidator{name: "second-validator", decisions: approveAll, order: &order}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
@@ -926,93 +839,17 @@ func TestRunUsesConfiguredValidatorsInLaneOrder(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if !reflect.DeepEqual(order, []string{"configured", "second-validator"}) {
t.Fatalf("validator order = %#v, want configured order", order)
}
if got := output.Manifest.ArtifactLanes[0].Validators; !reflect.DeepEqual(got, []string{"configured", "second-validator"}) {
t.Fatalf("manifest validators = %#v, want configured validators", got)
}
}
func TestRunUsesDefaultValidatorsWhenLaneDoesNotConfigureValidators(t *testing.T) {
modules := defaultRunnerModules()
defaultValidator := &runnerValidator{name: "default-validator", decisions: approveAll}
configuredValidator := &runnerValidator{name: "configured", decisions: approveAll}
modules.extractors["extract-alpha"].validators = []contracts.Validator{defaultValidator}
modules.validators["configured"] = configuredValidator
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if defaultValidator.calls != 1 {
t.Fatalf("default validator calls = %d, want 1", defaultValidator.calls)
}
if configuredValidator.calls != 0 {
t.Fatalf("configured validator calls = %d, want 0", configuredValidator.calls)
}
}
func TestRunConfiguredValidatorsReplaceExtractorDefaults(t *testing.T) {
modules := defaultRunnerModules()
defaultValidator := &runnerValidator{name: "default-validator", decisions: approveAll}
configuredValidator := &runnerValidator{name: "configured", decisions: approveAll}
modules.extractors["extract-alpha"].validators = []contracts.Validator{defaultValidator}
modules.validators["configured"] = configuredValidator
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipelineWithValidators("configured")})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if defaultValidator.calls != 0 {
t.Fatalf("default validator calls = %d, want 0", defaultValidator.calls)
}
if configuredValidator.calls != 1 {
t.Fatalf("configured validator calls = %d, want 1", configuredValidator.calls)
}
}
func TestRunAssignsGlobalCandidateIndicesAcrossLanesAndChunks(t *testing.T) {
modules := defaultRunnerModules()
var seenIndices []int
recordIndices := func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
seenIndices = append(seenIndices, candidateIndices(candidates)...)
return approveAll(candidates)
}
modules.extractors["extract-alpha"].validators = []contracts.Validator{&runnerValidator{name: "alpha-validator", decisions: recordIndices}}
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", validators: []contracts.Validator{&runnerValidator{name: "beta-validator", decisions: recordIndices}}}
pipeline := resolvedPipeline()
pipeline.ArtifactLanes = append(pipeline.ArtifactLanes, ResolvedArtifactLane{
ID: "beta",
Extract: Binding("extract-beta"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
})
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if !reflect.DeepEqual(seenIndices, []int{0, 1, 2, 3}) {
t.Fatalf("seen indices = %#v, want global indices", seenIndices)
}
if len(output.Approved) != 4 {
t.Fatalf("len(Approved) = %d, want 4", len(output.Approved))
}
}
func TestRunCollectsStageWarnings(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.warnings = []contracts.Warning{{ReasonCode: "chunk-warning", Message: "chunk warning"}}
modules.extractors["extract-alpha"].warnings = []contracts.Warning{{ReasonCode: "extract-warning", Message: "extract warning"}}
modules.mergers["merge"].warnings = []contracts.Warning{{ReasonCode: "merge-warning", Message: "merge warning"}}
modules.normalizers["normalize"].warnings = []contracts.Warning{{ReasonCode: "normalize-warning", Message: "normalize warning"}}
modules.extractors["extract-alpha"].validators = []contracts.Validator{&runnerValidator{
name: "default-validator",
decisions: approveAll,
warnings: []contracts.Warning{{ReasonCode: "validator-warning", Message: "validator warning"}},
}}
modules.output.warnings = []contracts.Warning{{ReasonCode: "output-warning", Message: "output warning"}}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -1020,13 +857,13 @@ func TestRunCollectsStageWarnings(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
want := []string{"chunk-warning", "extract-warning", "extract-warning", "merge-warning", "normalize-warning", "validator-warning", "output-warning"}
want := []string{"chunk-warning", "extract-warning", "extract-warning", "merge-warning", "normalize-warning", "output-warning"}
if got := warningReasons(output.Warnings); !reflect.DeepEqual(got, want) {
t.Fatalf("warning reasons = %#v, want %#v", got, want)
}
}
func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
func TestRunOutputEncoderReceivesManifestAndRawOutputs(t *testing.T) {
modules := defaultRunnerModules()
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -1038,8 +875,8 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
file := output.OutputFiles[0]
if file.Name != "artifacts/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name)
if file.Name != "outputs/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want outputs/generic.json", file.Name)
}
if file.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", file.ContentType)
@@ -1054,8 +891,11 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
if req.Manifest.PipelineID != "pipeline-1" || req.Manifest.PipelineDigest != "sha256:pipeline" {
t.Fatalf("output manifest = %#v, want pipeline details", req.Manifest)
}
if len(req.Approved) != 2 {
t.Fatalf("len(output approved) = %d, want 2", len(req.Approved))
if len(req.NormalizeOutputs) != 1 {
t.Fatalf("len(output NormalizeOutputs) = %d, want 1", len(req.NormalizeOutputs))
}
if req.NormalizeOutputs[0].LaneID != "alpha" || req.NormalizeOutputs[0].NormalizerKey != "normalize" {
t.Fatalf("NormalizeOutputs[0] = %#v, want normalized alpha output", req.NormalizeOutputs[0])
}
}
@@ -1066,9 +906,9 @@ func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
}{
{name: "empty", fileName: ""},
{name: "absolute", fileName: "/tmp/output.json"},
{name: "parent", fileName: "artifacts/../manifest.json"},
{name: "backslash", fileName: `artifacts\manifest.json`},
{name: "unclean", fileName: "artifacts//manifest.json"},
{name: "parent", fileName: "outputs/../manifest.json"},
{name: "backslash", fileName: `outputs\manifest.json`},
{name: "unclean", fileName: "outputs//manifest.json"},
}
for _, test := range tests {
@@ -1101,8 +941,8 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want failed run completion timestamp")
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want partial normalized output", len(output.NormalizeOutputs))
}
}
@@ -1313,7 +1153,7 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", err: errors.New("extract failed")}
pipeline := resolvedPipeline()
pipeline.ArtifactLanes = append(pipeline.ArtifactLanes, ResolvedArtifactLane{
ID: "beta",
@@ -1328,46 +1168,8 @@ func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want first lane approved output", len(output.Approved))
}
}
func TestRunSurfacesValidatorErrors(t *testing.T) {
tests := []struct {
name string
validator *runnerValidator
want string
}{
{name: "name mismatch", validator: &runnerValidator{name: "default-validator", resultName: "other", decisions: approveAll}, want: "returned result"},
{name: "cardinality", validator: &runnerValidator{name: "default-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision { return nil }}, want: "0 decisions"},
{name: "error", validator: &runnerValidator{name: "default-validator", decisions: approveAll, err: errors.New("validator failed")}, want: "validator failed"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].validators = []contracts.Validator{test.validator}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
})
}
}
func TestRunRejectsNilDefaultValidator(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].validators = []contracts.Validator{nil}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, "validator[0] must not be nil")
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want first lane output", len(output.NormalizeOutputs))
}
}
@@ -1438,7 +1240,7 @@ func defaultRunnerModules() *runnerModules {
input: &runnerInputAdapter{key: "input", doc: validSourceDocument()},
chunker: &runnerChunker{key: "chunk", chunks: []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
extractors: map[string]*runnerExtractor{
"extract-alpha": {key: "extract-alpha", artifactType: "artifact", schemaVersion: "v1"},
"extract-alpha": {key: "extract-alpha"},
},
mergers: map[string]*runnerMerger{
"merge": {key: "merge"},
@@ -1447,13 +1249,13 @@ func defaultRunnerModules() *runnerModules {
"normalize": {key: "normalize"},
},
validators: map[string]*runnerValidator{
"configured": {name: "configured", decisions: approveAll},
"second-validator": {name: "second-validator", decisions: approveAll},
"configured": {name: "configured"},
"second-validator": {name: "second-validator"},
},
output: &runnerOutputEncoder{
key: "output",
files: []contracts.OutputFile{
{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
{Name: "outputs/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
},
}
@@ -1572,11 +1374,8 @@ func (chunker *runnerChunker) ManifestMetadata() map[string]any {
type runnerExtractor struct {
key string
artifactType string
schemaVersion string
manifestMetadata map[string]any
candidates []artifacts.ArtifactCandidate
validators []contracts.Validator
output *contracts.ExtractOutput
warnings []contracts.Warning
err error
requests []contracts.ExtractionRequest
@@ -1589,14 +1388,6 @@ func (extractor *runnerExtractor) Key() string {
return extractor.key
}
func (extractor *runnerExtractor) ArtifactType() string {
return extractor.artifactType
}
func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
@@ -1605,10 +1396,6 @@ func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata
}
func (extractor *runnerExtractor) Validators() []contracts.Validator {
return extractor.validators
}
func (extractor *runnerExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
extractor.requests = append(extractor.requests, req)
if req.Chunk != nil {
@@ -1617,19 +1404,28 @@ func (extractor *runnerExtractor) Extract(ctx context.Context, req contracts.Ext
extractor.seenLLMClients = append(extractor.seenLLMClients, req.LLMClient)
extractor.seenMetadata = append(extractor.seenMetadata, req.Metadata)
candidates := append([]artifacts.ArtifactCandidate(nil), extractor.candidates...)
if len(candidates) == 0 {
candidates = []artifacts.ArtifactCandidate{{Payload: []byte(`{"value":true}`)}}
output := contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "runner.raw", Name: "runner_raw", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
}
if req.Chunk != nil {
output.Payload.Content = []byte(`{"chunk":"` + req.Chunk.ID + `"}`)
}
if extractor.output != nil {
output = *extractor.output
}
return contracts.ExtractionResult{
Candidates: candidates,
Warnings: extractor.warnings,
Output: output,
Warnings: extractor.warnings,
}, extractor.err
}
type runnerMerger struct {
key string
result []artifacts.ArtifactCandidate
result *contracts.MergeOutput
warnings []contracts.Warning
err error
requests []contracts.MergeRequest
@@ -1641,21 +1437,27 @@ func (merger *runnerMerger) Key() string {
func (merger *runnerMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
merger.requests = append(merger.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), merger.result...)
if candidates == nil {
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
output := contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "runner.raw", Name: "runner_raw", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
if merger.result != nil {
output = *merger.result
}
return contracts.MergeResult{
Candidates: candidates,
Warnings: merger.warnings,
Output: output,
Warnings: merger.warnings,
}, merger.err
}
type runnerNormalizer struct {
key string
result []artifacts.ArtifactCandidate
result *contracts.NormalizeOutput
warnings []contracts.Warning
err error
requests []contracts.NormalizeRequest
@@ -1671,13 +1473,18 @@ func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
normalizer.requests = append(normalizer.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
if candidates == nil {
candidates = append(candidates, req.Candidates...)
output := contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
}
if normalizer.result != nil {
output = *normalizer.result
}
return contracts.NormalizeResult{
Candidates: candidates,
Warnings: normalizer.warnings,
Output: output,
Warnings: normalizer.warnings,
}, normalizer.err
}
@@ -1757,14 +1564,6 @@ func (client manifestReportingLLMClient) LLMProfileManifests() []artifacts.LLMPr
return append([]artifacts.LLMProfileManifest(nil), client.profiles...)
}
func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
for _, candidate := range candidates {
decisions = append(decisions, validate.Approved(candidate.Index))
}
return decisions
}
func validSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
@@ -1866,24 +1665,6 @@ func warningReasons(warnings []contracts.Warning) []string {
return reasons
}
func candidateIndices(candidates []artifacts.ArtifactCandidate) []int {
indices := make([]int, 0, len(candidates))
for _, candidate := range candidates {
indices = append(indices, candidate.Index)
}
return indices
}
func runnerCandidate(index int) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "extract-alpha",
ArtifactType: "artifact",
SchemaVersion: "v1",
Payload: []byte(`{"value":true}`),
}
}
func assertRunError(t *testing.T, err error, want string) {
t.Helper()

55
internal/framework/pipeline/testdata/walking_skeleton_output.json vendored
View File

@@ -12,40 +12,31 @@
}
]
},
"approved": [
"normalize_outputs": [
{
"extractor_key": "fake/extract",
"artifact_type": "fake_event",
"schema_version": "v1",
"payload": {
"chunk_id": "fixture-source:chunk:0",
"llm_call": 1,
"text": "First event. Second event."
"lane_id": "events",
"normalizer_key": "noop",
"source_id": "fixture-source",
"schema": {
"id": "fake_event",
"name": "fake_event",
"version": "v1"
},
"source_refs": [
{
"source_id": "fixture-source",
"start_unit_id": 1,
"end_unit_id": 2
}
]
},
{
"extractor_key": "fake/extract",
"artifact_type": "fake_event",
"schema_version": "v1",
"payload": {
"chunk_id": "fixture-source:chunk:1",
"llm_call": 2,
"text": "Third event."
},
"source_refs": [
{
"source_id": "fixture-source",
"start_unit_id": 3,
"end_unit_id": 3
}
]
"media_type": "application/json",
"content": {
"outputs": [
{
"chunk_id": "fixture-source:chunk:0",
"llm_call": 1,
"text": "First event. Second event."
},
{
"chunk_id": "fixture-source:chunk:1",
"llm_call": 2,
"text": "Third event."
}
]
}
}
]
}

83
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -256,22 +256,10 @@ func (extractor walkingSkeletonExtractor) Key() string {
return "fake/extract"
}
func (extractor walkingSkeletonExtractor) ArtifactType() string {
return "fake_event"
}
func (extractor walkingSkeletonExtractor) SchemaVersion() string {
return "v1"
}
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
return nil
}
func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
var response struct {
Call int `json:"call"`
@@ -294,16 +282,11 @@ func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contr
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Payload: payload,
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
@@ -336,11 +319,25 @@ func (merger walkingSkeletonMerger) Key() string {
}
func (merger walkingSkeletonMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
outputs := make([]json.RawMessage, 0, len(req.ExtractOutputs))
for _, output := range req.ExtractOutputs {
outputs = append(outputs, json.RawMessage(output.Payload.Content))
}
return contracts.MergeResult{Candidates: candidates}, nil
content, err := json.Marshal(map[string]any{"outputs": outputs})
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonNormalizer struct{}
@@ -364,7 +361,14 @@ func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req c
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type walkingSkeletonOutput struct{}
@@ -374,9 +378,28 @@ func (output walkingSkeletonOutput) Key() string {
}
func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
type rawOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id"`
Schema contracts.ResponseSchema `json:"schema"`
MediaType string `json:"media_type"`
Content json.RawMessage `json:"content"`
}
rawOutputs := make([]rawOutput, 0, len(req.NormalizeOutputs))
for _, output := range req.NormalizeOutputs {
rawOutputs = append(rawOutputs, rawOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: output.Schema,
MediaType: output.Payload.MediaType,
Content: json.RawMessage(output.Payload.Content),
})
}
encoded, err := json.Marshal(struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []rawOutput `json:"normalize_outputs"`
}{
Manifest: artifacts.RunManifest{
PipelineID: req.Manifest.PipelineID,
@@ -384,7 +407,7 @@ func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.Ou
ArtifactLanes: req.Manifest.ArtifactLanes,
ValidationStatus: req.Manifest.ValidationStatus,
},
Approved: req.Approved,
NormalizeOutputs: rawOutputs,
})
if err != nil {
return contracts.OutputResult{}, err