Add framework composition runner
This commit is contained in:
141
internal/framework/runner/registry_integration_test.go
Normal file
141
internal/framework/runner/registry_integration_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/extractorregistry"
|
||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
)
|
||||
|
||||
func TestRunnerUsesExtractorRegistry(t *testing.T) {
|
||||
var builtKeys []string
|
||||
var executedKeys []string
|
||||
registry := extractorregistry.New()
|
||||
|
||||
registerIntegrationExtractor(t, registry, "second", &builtKeys, &executedKeys, []contracts.Validator{
|
||||
integrationValidator{name: "reject-second", approve: false},
|
||||
})
|
||||
registerIntegrationExtractor(t, registry, "first", &builtKeys, &executedKeys, []contracts.Validator{
|
||||
integrationValidator{name: "approve-first", approve: true},
|
||||
})
|
||||
|
||||
output, err := New(registry).Run(context.Background(), RunInput{
|
||||
Source: integrationSourceDocument(),
|
||||
ExtractorKeys: []string{"second", "first"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(builtKeys, []string{"second", "first"}) {
|
||||
t.Fatalf("built keys = %#v, want configured order", builtKeys)
|
||||
}
|
||||
if !reflect.DeepEqual(executedKeys, []string{"second", "first"}) {
|
||||
t.Fatalf("executed keys = %#v, want configured order", executedKeys)
|
||||
}
|
||||
if got := artifactKeys(output.Approved); !reflect.DeepEqual(got, []string{"first"}) {
|
||||
t.Fatalf("approved keys = %#v, want [first]", got)
|
||||
}
|
||||
if got := rejectedKeys(output.Rejected); !reflect.DeepEqual(got, []string{"second"}) {
|
||||
t.Fatalf("rejected keys = %#v, want [second]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func registerIntegrationExtractor(t *testing.T, registry *extractorregistry.Registry, key string, builtKeys *[]string, executedKeys *[]string, validators []contracts.Validator) {
|
||||
t.Helper()
|
||||
|
||||
if err := registry.Register(key, func() (contracts.Extractor, error) {
|
||||
*builtKeys = append(*builtKeys, key)
|
||||
return integrationExtractor{key: key, executedKeys: executedKeys, validators: validators}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
type integrationExtractor struct {
|
||||
key string
|
||||
executedKeys *[]string
|
||||
validators []contracts.Validator
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) ArtifactType() string {
|
||||
return "generic-artifact"
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) SchemaVersion() string {
|
||||
return "v1"
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Validators() []contracts.Validator {
|
||||
return extractor.validators
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
*extractor.executedKeys = append(*extractor.executedKeys, extractor.key)
|
||||
return contracts.ExtractionResult{
|
||||
Candidates: []artifacts.Candidate{
|
||||
{Payload: []byte(`{"value":true}`)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationValidator struct {
|
||||
name string
|
||||
approve bool
|
||||
}
|
||||
|
||||
func (validator integrationValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator integrationValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
||||
for _, candidate := range req.Candidates {
|
||||
if validator.approve {
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
} else {
|
||||
decisions = append(decisions, validationhelpers.Rejected(candidate.Index, "invalid", "not accepted"))
|
||||
}
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
ValidatorName: validator.name,
|
||||
Decisions: decisions,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func integrationSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func artifactKeys(approved []artifacts.Artifact) []string {
|
||||
keys := make([]string, 0, len(approved))
|
||||
for _, artifact := range approved {
|
||||
keys = append(keys, artifact.ExtractorKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func rejectedKeys(rejected []artifacts.RejectedArtifact) []string {
|
||||
keys := make([]string, 0, len(rejected))
|
||||
for _, artifact := range rejected {
|
||||
keys = append(keys, artifact.Candidate.ExtractorKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
164
internal/framework/runner/runner.go
Normal file
164
internal/framework/runner/runner.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
)
|
||||
|
||||
type ExtractorFactory interface {
|
||||
Build(key string) (contracts.Extractor, error)
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
extractors ExtractorFactory
|
||||
}
|
||||
|
||||
func New(extractors ExtractorFactory) *Runner {
|
||||
return &Runner{extractors: extractors}
|
||||
}
|
||||
|
||||
type RunInput struct {
|
||||
Source *source.SourceDocument
|
||||
ExtractorKeys []string
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type RunOutput struct {
|
||||
Approved []artifacts.Artifact `json:"approved,omitempty"`
|
||||
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
var output RunOutput
|
||||
if r == nil {
|
||||
return output, fmt.Errorf("runner must not be nil")
|
||||
}
|
||||
if r.extractors == nil {
|
||||
return output, fmt.Errorf("runner extractor factory must not be nil")
|
||||
}
|
||||
if err := source.ValidateDocument(input.Source); err != nil {
|
||||
return output, fmt.Errorf("validate source document: %w", err)
|
||||
}
|
||||
if len(input.ExtractorKeys) == 0 {
|
||||
return output, fmt.Errorf("extractor keys must not be empty")
|
||||
}
|
||||
|
||||
nextCandidateIndex := 0
|
||||
for _, extractorKey := range input.ExtractorKeys {
|
||||
extractor, err := r.extractors.Build(extractorKey)
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("build extractor %q: %w", extractorKey, err)
|
||||
}
|
||||
|
||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||
Source: input.Source,
|
||||
LLMClient: input.LLMClient,
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
output.Warnings = append(output.Warnings, result.Warnings...)
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("extract with extractor %q: %w", extractor.Key(), err)
|
||||
}
|
||||
|
||||
candidates, err := normalizeCandidates(extractor, result.Candidates, &nextCandidateIndex)
|
||||
if err != nil {
|
||||
return output, err
|
||||
}
|
||||
|
||||
approved, rejected, warnings, err := runValidators(ctx, extractor, input.Source, candidates, input.Metadata)
|
||||
output.Warnings = append(output.Warnings, warnings...)
|
||||
output.Rejected = append(output.Rejected, rejected...)
|
||||
if err != nil {
|
||||
return output, err
|
||||
}
|
||||
|
||||
for _, candidate := range approved {
|
||||
output.Approved = append(output.Approved, artifacts.ArtifactFromCandidate(candidate))
|
||||
}
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.Candidate, nextIndex *int) ([]artifacts.Candidate, error) {
|
||||
normalized := make([]artifacts.Candidate, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
candidate.Index = *nextIndex
|
||||
*nextIndex = *nextIndex + 1
|
||||
|
||||
if candidate.ExtractorKey == "" {
|
||||
candidate.ExtractorKey = extractor.Key()
|
||||
} else if candidate.ExtractorKey != extractor.Key() {
|
||||
return nil, fmt.Errorf("candidate extractor_key %q does not match extractor %q", candidate.ExtractorKey, extractor.Key())
|
||||
}
|
||||
|
||||
if candidate.ArtifactType == "" {
|
||||
candidate.ArtifactType = extractor.ArtifactType()
|
||||
} else if candidate.ArtifactType != extractor.ArtifactType() {
|
||||
return nil, fmt.Errorf("candidate artifact_type %q does not match extractor %q artifact type %q", candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
|
||||
}
|
||||
|
||||
if candidate.SchemaVersion == "" {
|
||||
candidate.SchemaVersion = extractor.SchemaVersion()
|
||||
} else if candidate.SchemaVersion != extractor.SchemaVersion() {
|
||||
return nil, fmt.Errorf("candidate schema_version %q does not match extractor %q schema version %q", candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
|
||||
}
|
||||
|
||||
normalized = append(normalized, candidate)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func runValidators(ctx context.Context, extractor contracts.Extractor, doc *source.SourceDocument, candidates []artifacts.Candidate, metadata map[string]any) ([]artifacts.Candidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
|
||||
eligible := candidates
|
||||
var rejected []artifacts.RejectedArtifact
|
||||
var warnings []contracts.Warning
|
||||
|
||||
for _, validator := range extractor.Validators() {
|
||||
result, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
Source: doc,
|
||||
Candidates: eligible,
|
||||
Metadata: metadata,
|
||||
})
|
||||
warnings = append(warnings, result.Warnings...)
|
||||
if err != nil {
|
||||
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractor.Key(), validator.Name(), err)
|
||||
}
|
||||
if result.ValidatorName != validator.Name() {
|
||||
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
|
||||
}
|
||||
if err := validators.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
|
||||
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractor.Key(), validator.Name(), err)
|
||||
}
|
||||
|
||||
decisions := make(map[int]contracts.ValidationDecision, len(result.Decisions))
|
||||
for _, decision := range result.Decisions {
|
||||
decisions[decision.CandidateIndex] = decision
|
||||
}
|
||||
|
||||
nextEligible := make([]artifacts.Candidate, 0, len(eligible))
|
||||
for _, candidate := range eligible {
|
||||
decision := decisions[candidate.Index]
|
||||
if decision.Approved {
|
||||
nextEligible = append(nextEligible, candidate)
|
||||
continue
|
||||
}
|
||||
rejected = append(rejected, artifacts.RejectedArtifact{
|
||||
Candidate: candidate,
|
||||
ValidatorName: result.ValidatorName,
|
||||
ReasonCode: decision.ReasonCode,
|
||||
Message: decision.Message,
|
||||
})
|
||||
}
|
||||
eligible = nextEligible
|
||||
}
|
||||
|
||||
return eligible, rejected, warnings, nil
|
||||
}
|
||||
444
internal/framework/runner/runner_test.go
Normal file
444
internal/framework/runner/runner_test.go
Normal file
@@ -0,0 +1,444 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
||||
)
|
||||
|
||||
func TestNewAndDataTypes(t *testing.T) {
|
||||
r := New(fakeFactory{})
|
||||
if r == nil {
|
||||
t.Fatal("New() = nil, want runner")
|
||||
}
|
||||
|
||||
input := RunInput{
|
||||
Source: validSourceDocument(),
|
||||
ExtractorKeys: []string{"generic-extractor"},
|
||||
Metadata: map[string]any{"request": "test"},
|
||||
}
|
||||
output := RunOutput{
|
||||
Approved: []artifacts.Artifact{{ExtractorKey: "generic-extractor"}},
|
||||
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "generic-validator"}},
|
||||
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
|
||||
}
|
||||
|
||||
if input.Source.ID != "source-1" {
|
||||
t.Fatalf("RunInput.Source.ID = %q, want source-1", input.Source.ID)
|
||||
}
|
||||
if len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 {
|
||||
t.Fatalf("RunOutput = %#v, want constructed fields", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsInvalidSetup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
run func() (RunOutput, error)
|
||||
error string
|
||||
}{
|
||||
{
|
||||
name: "nil runner",
|
||||
run: func() (RunOutput, error) { return (*Runner)(nil).Run(context.Background(), RunInput{}) },
|
||||
error: "runner must not be nil",
|
||||
},
|
||||
{
|
||||
name: "nil factory",
|
||||
run: func() (RunOutput, error) { return New(nil).Run(context.Background(), RunInput{}) },
|
||||
error: "factory",
|
||||
},
|
||||
{
|
||||
name: "invalid source",
|
||||
run: func() (RunOutput, error) {
|
||||
return New(fakeFactory{}).Run(context.Background(), RunInput{Source: &source.SourceDocument{}, ExtractorKeys: []string{"generic-extractor"}})
|
||||
},
|
||||
error: "validate source document",
|
||||
},
|
||||
{
|
||||
name: "empty extractors",
|
||||
run: func() (RunOutput, error) {
|
||||
return New(fakeFactory{}).Run(context.Background(), RunInput{Source: validSourceDocument()})
|
||||
},
|
||||
error: "extractor keys must not be empty",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := tt.run()
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.error) {
|
||||
t.Fatalf("Run() error = %q, want substring %q", err.Error(), tt.error)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
||||
var order []string
|
||||
var seenIndices []int
|
||||
recordIndices := func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
seenIndices = append(seenIndices, candidate.Index)
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
}
|
||||
return decisions
|
||||
}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"second": fakeExtractor{key: "second", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{fakeValidator{name: "recorder-second", decisions: recordIndices}}, order: &order},
|
||||
"first": fakeExtractor{key: "first", artifactType: "artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{fakeValidator{name: "recorder-first", decisions: recordIndices}}, order: &order},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{
|
||||
Source: validSourceDocument(),
|
||||
ExtractorKeys: []string{"second", "first"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(order, []string{"second", "first"}) {
|
||||
t.Fatalf("order = %#v, want configured order", order)
|
||||
}
|
||||
if !reflect.DeepEqual(seenIndices, []int{0, 1, 2}) {
|
||||
t.Fatalf("seen indices = %#v, want [0 1 2]", seenIndices)
|
||||
}
|
||||
if len(output.Approved) != 3 {
|
||||
t.Fatalf("len(Approved) = %d, want 3", len(output.Approved))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{{Payload: []byte(`{"value":true}`)}}},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
artifact := output.Approved[0]
|
||||
if artifact.ExtractorKey != "generic-extractor" || artifact.ArtifactType != "generic-artifact" || artifact.SchemaVersion != "v1" {
|
||||
t.Fatalf("approved artifact metadata = %#v, want extractor metadata", artifact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
candidate artifacts.Candidate
|
||||
error string
|
||||
}{
|
||||
{name: "extractor key", candidate: artifacts.Candidate{ExtractorKey: "other"}, error: "extractor_key"},
|
||||
{name: "artifact type", candidate: artifacts.Candidate{ArtifactType: "other"}, error: "artifact_type"},
|
||||
{name: "schema version", candidate: artifacts.Candidate{SchemaVersion: "other"}, error: "schema_version"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{tt.candidate}},
|
||||
}}
|
||||
|
||||
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.error) {
|
||||
t.Fatalf("Run() error = %q, want substring %q", err.Error(), tt.error)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunApprovesCandidatesWithoutValidators(t *testing.T) {
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.Approved) != 2 {
|
||||
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
|
||||
}
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||
}}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{validator}},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.Approved) != 1 {
|
||||
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
||||
var laterSeen int
|
||||
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
return []contracts.ValidationDecision{
|
||||
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
||||
validationhelpers.Approved(candidates[1].Index),
|
||||
}
|
||||
}}
|
||||
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
laterSeen = len(candidates)
|
||||
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
||||
}}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{rejectFirst, approveRemaining}},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if laterSeen != 1 {
|
||||
t.Fatalf("later validator saw %d candidates, want 1", laterSeen)
|
||||
}
|
||||
if len(output.Rejected) != 1 {
|
||||
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
|
||||
}
|
||||
if output.Rejected[0].ValidatorName != "reject-first" || output.Rejected[0].ReasonCode != "invalid" {
|
||||
t.Fatalf("Rejected[0] = %#v, want rejection details", output.Rejected[0])
|
||||
}
|
||||
if len(output.Approved) != 1 {
|
||||
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSurfacesValidatorNameMismatch(t *testing.T) {
|
||||
validator := fakeValidator{name: "generic-validator", resultName: "other-validator", decisions: approveAll}
|
||||
factory := factoryWithValidator(validator)
|
||||
|
||||
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
|
||||
assertRunError(t, err, "returned result")
|
||||
}
|
||||
|
||||
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
||||
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
return nil
|
||||
}}
|
||||
factory := factoryWithValidator(validator)
|
||||
|
||||
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
|
||||
assertRunError(t, err, "0 decisions for 1 candidates")
|
||||
}
|
||||
|
||||
func TestRunCollectsExtractorAndValidatorWarnings(t *testing.T) {
|
||||
validator := fakeValidator{
|
||||
name: "generic-validator",
|
||||
decisions: approveAll,
|
||||
warnings: []contracts.Warning{{ReasonCode: "validator-warning", Message: "validator warning"}},
|
||||
}
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{
|
||||
key: "generic-extractor",
|
||||
artifactType: "generic-artifact",
|
||||
schemaVersion: "v1",
|
||||
candidateCount: 1,
|
||||
validators: []contracts.Validator{validator},
|
||||
warnings: []contracts.Warning{{ReasonCode: "extractor-warning", Message: "extractor warning"}},
|
||||
},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if got := warningReasons(output.Warnings); !reflect.DeepEqual(got, []string{"extractor-warning", "validator-warning"}) {
|
||||
t.Fatalf("warning reasons = %#v, want extractor and validator warnings", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReturnsPartialOutputWhenLaterExtractorFails(t *testing.T) {
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"ok": fakeExtractor{key: "ok", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1},
|
||||
"fail": fakeExtractor{key: "fail", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"ok", "fail"}})
|
||||
|
||||
assertRunError(t, err, "extract with extractor")
|
||||
if len(output.Approved) != 1 {
|
||||
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReturnsPartialOutputWhenLaterValidatorFails(t *testing.T) {
|
||||
validatorErr := errors.New("validator failed")
|
||||
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"ok": fakeExtractor{key: "ok", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1},
|
||||
"fail": fakeExtractor{key: "fail", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{fakeValidator{name: "failing-validator", err: validatorErr}}},
|
||||
}}
|
||||
|
||||
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"ok", "fail"}})
|
||||
|
||||
assertRunError(t, err, "failing-validator")
|
||||
if len(output.Approved) != 1 {
|
||||
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
||||
}
|
||||
}
|
||||
|
||||
type fakeFactory struct {
|
||||
extractors map[string]contracts.Extractor
|
||||
err error
|
||||
}
|
||||
|
||||
func (factory fakeFactory) Build(key string) (contracts.Extractor, error) {
|
||||
if factory.err != nil {
|
||||
return nil, factory.err
|
||||
}
|
||||
extractor, ok := factory.extractors[key]
|
||||
if !ok {
|
||||
return nil, errors.New("missing extractor")
|
||||
}
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
type fakeExtractor struct {
|
||||
key string
|
||||
artifactType string
|
||||
schemaVersion string
|
||||
candidateCount int
|
||||
candidates []artifacts.Candidate
|
||||
validators []contracts.Validator
|
||||
warnings []contracts.Warning
|
||||
err error
|
||||
order *[]string
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) ArtifactType() string {
|
||||
return extractor.artifactType
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) SchemaVersion() string {
|
||||
return extractor.schemaVersion
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Validators() []contracts.Validator {
|
||||
return extractor.validators
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
if extractor.order != nil {
|
||||
*extractor.order = append(*extractor.order, extractor.key)
|
||||
}
|
||||
candidates := append([]artifacts.Candidate(nil), extractor.candidates...)
|
||||
for len(candidates) < extractor.candidateCount {
|
||||
candidates = append(candidates, artifacts.Candidate{Payload: []byte(`{"value":true}`)})
|
||||
}
|
||||
return contracts.ExtractionResult{
|
||||
Candidates: candidates,
|
||||
Warnings: extractor.warnings,
|
||||
}, extractor.err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
name string
|
||||
resultName string
|
||||
decisions func([]artifacts.Candidate) []contracts.ValidationDecision
|
||||
warnings []contracts.Warning
|
||||
err error
|
||||
}
|
||||
|
||||
func (validator fakeValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
resultName := validator.resultName
|
||||
if resultName == "" {
|
||||
resultName = validator.name
|
||||
}
|
||||
var decisions []contracts.ValidationDecision
|
||||
if validator.decisions != nil {
|
||||
decisions = validator.decisions(req.Candidates)
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
ValidatorName: resultName,
|
||||
Decisions: decisions,
|
||||
Warnings: validator.warnings,
|
||||
}, validator.err
|
||||
}
|
||||
|
||||
func factoryWithValidator(validator contracts.Validator) fakeFactory {
|
||||
return fakeFactory{extractors: map[string]contracts.Extractor{
|
||||
"generic-extractor": fakeExtractor{
|
||||
key: "generic-extractor",
|
||||
artifactType: "generic-artifact",
|
||||
schemaVersion: "v1",
|
||||
candidateCount: 1,
|
||||
validators: []contracts.Validator{validator},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func approveAll(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
||||
}
|
||||
return decisions
|
||||
}
|
||||
|
||||
func validSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func warningReasons(warnings []contracts.Warning) []string {
|
||||
reasons := make([]string, 0, len(warnings))
|
||||
for _, warning := range warnings {
|
||||
reasons = append(reasons, warning.ReasonCode)
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
func assertRunError(t *testing.T, err error, want string) {
|
||||
t.Helper()
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Run() error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user