445 lines
16 KiB
Go
445 lines
16 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
validationhelpers "gitea.maximumdirect.net/eric/notarius/internal/framework/validators"
|
|
)
|
|
|
|
func TestNewAndDataTypes(t *testing.T) {
|
|
r := New(fakeFactory{})
|
|
if r == nil {
|
|
t.Fatal("New() = nil, want runner")
|
|
}
|
|
|
|
input := RunInput{
|
|
Source: validSourceDocument(),
|
|
ExtractorKeys: []string{"generic-extractor"},
|
|
Metadata: map[string]any{"request": "test"},
|
|
}
|
|
output := RunOutput{
|
|
Approved: []artifacts.Artifact{{ExtractorKey: "generic-extractor"}},
|
|
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "generic-validator"}},
|
|
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
|
|
}
|
|
|
|
if input.Source.ID != "source-1" {
|
|
t.Fatalf("RunInput.Source.ID = %q, want source-1", input.Source.ID)
|
|
}
|
|
if len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 {
|
|
t.Fatalf("RunOutput = %#v, want constructed fields", output)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsInvalidSetup(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
run func() (RunOutput, error)
|
|
error string
|
|
}{
|
|
{
|
|
name: "nil runner",
|
|
run: func() (RunOutput, error) { return (*Runner)(nil).Run(context.Background(), RunInput{}) },
|
|
error: "runner must not be nil",
|
|
},
|
|
{
|
|
name: "nil factory",
|
|
run: func() (RunOutput, error) { return New(nil).Run(context.Background(), RunInput{}) },
|
|
error: "factory",
|
|
},
|
|
{
|
|
name: "invalid source",
|
|
run: func() (RunOutput, error) {
|
|
return New(fakeFactory{}).Run(context.Background(), RunInput{Source: &source.SourceDocument{}, ExtractorKeys: []string{"generic-extractor"}})
|
|
},
|
|
error: "validate source document",
|
|
},
|
|
{
|
|
name: "empty extractors",
|
|
run: func() (RunOutput, error) {
|
|
return New(fakeFactory{}).Run(context.Background(), RunInput{Source: validSourceDocument()})
|
|
},
|
|
error: "extractor keys must not be empty",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
_, err := tt.run()
|
|
if err == nil {
|
|
t.Fatal("Run() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), tt.error) {
|
|
t.Fatalf("Run() error = %q, want substring %q", err.Error(), tt.error)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunUsesConfiguredExtractorOrderAndAssignsGlobalIndices(t *testing.T) {
|
|
var order []string
|
|
var seenIndices []int
|
|
recordIndices := func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
|
for _, candidate := range candidates {
|
|
seenIndices = append(seenIndices, candidate.Index)
|
|
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
|
}
|
|
return decisions
|
|
}
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"second": fakeExtractor{key: "second", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{fakeValidator{name: "recorder-second", decisions: recordIndices}}, order: &order},
|
|
"first": fakeExtractor{key: "first", artifactType: "artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{fakeValidator{name: "recorder-first", decisions: recordIndices}}, order: &order},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{
|
|
Source: validSourceDocument(),
|
|
ExtractorKeys: []string{"second", "first"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(order, []string{"second", "first"}) {
|
|
t.Fatalf("order = %#v, want configured order", order)
|
|
}
|
|
if !reflect.DeepEqual(seenIndices, []int{0, 1, 2}) {
|
|
t.Fatalf("seen indices = %#v, want [0 1 2]", seenIndices)
|
|
}
|
|
if len(output.Approved) != 3 {
|
|
t.Fatalf("len(Approved) = %d, want 3", len(output.Approved))
|
|
}
|
|
}
|
|
|
|
func TestRunFillsEmptyCandidateExtractorMetadata(t *testing.T) {
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{{Payload: []byte(`{"value":true}`)}}},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
|
|
artifact := output.Approved[0]
|
|
if artifact.ExtractorKey != "generic-extractor" || artifact.ArtifactType != "generic-artifact" || artifact.SchemaVersion != "v1" {
|
|
t.Fatalf("approved artifact metadata = %#v, want extractor metadata", artifact)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsCandidateMetadataMismatches(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
candidate artifacts.Candidate
|
|
error string
|
|
}{
|
|
{name: "extractor key", candidate: artifacts.Candidate{ExtractorKey: "other"}, error: "extractor_key"},
|
|
{name: "artifact type", candidate: artifacts.Candidate{ArtifactType: "other"}, error: "artifact_type"},
|
|
{name: "schema version", candidate: artifacts.Candidate{SchemaVersion: "other"}, error: "schema_version"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidates: []artifacts.Candidate{tt.candidate}},
|
|
}}
|
|
|
|
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
|
|
if err == nil {
|
|
t.Fatal("Run() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), tt.error) {
|
|
t.Fatalf("Run() error = %q, want substring %q", err.Error(), tt.error)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunApprovesCandidatesWithoutValidators(t *testing.T) {
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if len(output.Approved) != 2 {
|
|
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
|
|
}
|
|
if len(output.Rejected) != 0 {
|
|
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
|
|
}
|
|
}
|
|
|
|
func TestRunValidatorApprovalProducesApprovedArtifacts(t *testing.T) {
|
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
|
}}
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{validator}},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if len(output.Approved) != 1 {
|
|
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
|
}
|
|
}
|
|
|
|
func TestRunValidatorRejectionRemovesCandidateFromLaterValidators(t *testing.T) {
|
|
var laterSeen int
|
|
rejectFirst := fakeValidator{name: "reject-first", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
return []contracts.ValidationDecision{
|
|
validationhelpers.Rejected(candidates[0].Index, "invalid", "not accepted"),
|
|
validationhelpers.Approved(candidates[1].Index),
|
|
}
|
|
}}
|
|
approveRemaining := fakeValidator{name: "approve-remaining", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
laterSeen = len(candidates)
|
|
return []contracts.ValidationDecision{validationhelpers.Approved(candidates[0].Index)}
|
|
}}
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{key: "generic-extractor", artifactType: "generic-artifact", schemaVersion: "v1", candidateCount: 2, validators: []contracts.Validator{rejectFirst, approveRemaining}},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if laterSeen != 1 {
|
|
t.Fatalf("later validator saw %d candidates, want 1", laterSeen)
|
|
}
|
|
if len(output.Rejected) != 1 {
|
|
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
|
|
}
|
|
if output.Rejected[0].ValidatorName != "reject-first" || output.Rejected[0].ReasonCode != "invalid" {
|
|
t.Fatalf("Rejected[0] = %#v, want rejection details", output.Rejected[0])
|
|
}
|
|
if len(output.Approved) != 1 {
|
|
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
|
}
|
|
}
|
|
|
|
func TestRunSurfacesValidatorNameMismatch(t *testing.T) {
|
|
validator := fakeValidator{name: "generic-validator", resultName: "other-validator", decisions: approveAll}
|
|
factory := factoryWithValidator(validator)
|
|
|
|
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
|
|
assertRunError(t, err, "returned result")
|
|
}
|
|
|
|
func TestRunSurfacesValidatorCardinalityError(t *testing.T) {
|
|
validator := fakeValidator{name: "generic-validator", decisions: func(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
return nil
|
|
}}
|
|
factory := factoryWithValidator(validator)
|
|
|
|
_, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
|
|
assertRunError(t, err, "0 decisions for 1 candidates")
|
|
}
|
|
|
|
func TestRunCollectsExtractorAndValidatorWarnings(t *testing.T) {
|
|
validator := fakeValidator{
|
|
name: "generic-validator",
|
|
decisions: approveAll,
|
|
warnings: []contracts.Warning{{ReasonCode: "validator-warning", Message: "validator warning"}},
|
|
}
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{
|
|
key: "generic-extractor",
|
|
artifactType: "generic-artifact",
|
|
schemaVersion: "v1",
|
|
candidateCount: 1,
|
|
validators: []contracts.Validator{validator},
|
|
warnings: []contracts.Warning{{ReasonCode: "extractor-warning", Message: "extractor warning"}},
|
|
},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"generic-extractor"}})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if got := warningReasons(output.Warnings); !reflect.DeepEqual(got, []string{"extractor-warning", "validator-warning"}) {
|
|
t.Fatalf("warning reasons = %#v, want extractor and validator warnings", got)
|
|
}
|
|
}
|
|
|
|
func TestRunReturnsPartialOutputWhenLaterExtractorFails(t *testing.T) {
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"ok": fakeExtractor{key: "ok", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1},
|
|
"fail": fakeExtractor{key: "fail", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"ok", "fail"}})
|
|
|
|
assertRunError(t, err, "extract with extractor")
|
|
if len(output.Approved) != 1 {
|
|
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
|
}
|
|
}
|
|
|
|
func TestRunReturnsPartialOutputWhenLaterValidatorFails(t *testing.T) {
|
|
validatorErr := errors.New("validator failed")
|
|
factory := fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"ok": fakeExtractor{key: "ok", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1},
|
|
"fail": fakeExtractor{key: "fail", artifactType: "artifact", schemaVersion: "v1", candidateCount: 1, validators: []contracts.Validator{fakeValidator{name: "failing-validator", err: validatorErr}}},
|
|
}}
|
|
|
|
output, err := New(factory).Run(context.Background(), RunInput{Source: validSourceDocument(), ExtractorKeys: []string{"ok", "fail"}})
|
|
|
|
assertRunError(t, err, "failing-validator")
|
|
if len(output.Approved) != 1 {
|
|
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
|
|
}
|
|
}
|
|
|
|
type fakeFactory struct {
|
|
extractors map[string]contracts.Extractor
|
|
err error
|
|
}
|
|
|
|
func (factory fakeFactory) Build(key string) (contracts.Extractor, error) {
|
|
if factory.err != nil {
|
|
return nil, factory.err
|
|
}
|
|
extractor, ok := factory.extractors[key]
|
|
if !ok {
|
|
return nil, errors.New("missing extractor")
|
|
}
|
|
return extractor, nil
|
|
}
|
|
|
|
type fakeExtractor struct {
|
|
key string
|
|
artifactType string
|
|
schemaVersion string
|
|
candidateCount int
|
|
candidates []artifacts.Candidate
|
|
validators []contracts.Validator
|
|
warnings []contracts.Warning
|
|
err error
|
|
order *[]string
|
|
}
|
|
|
|
func (extractor fakeExtractor) Key() string {
|
|
return extractor.key
|
|
}
|
|
|
|
func (extractor fakeExtractor) ArtifactType() string {
|
|
return extractor.artifactType
|
|
}
|
|
|
|
func (extractor fakeExtractor) SchemaVersion() string {
|
|
return extractor.schemaVersion
|
|
}
|
|
|
|
func (extractor fakeExtractor) Validators() []contracts.Validator {
|
|
return extractor.validators
|
|
}
|
|
|
|
func (extractor fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
|
if extractor.order != nil {
|
|
*extractor.order = append(*extractor.order, extractor.key)
|
|
}
|
|
candidates := append([]artifacts.Candidate(nil), extractor.candidates...)
|
|
for len(candidates) < extractor.candidateCount {
|
|
candidates = append(candidates, artifacts.Candidate{Payload: []byte(`{"value":true}`)})
|
|
}
|
|
return contracts.ExtractionResult{
|
|
Candidates: candidates,
|
|
Warnings: extractor.warnings,
|
|
}, extractor.err
|
|
}
|
|
|
|
type fakeValidator struct {
|
|
name string
|
|
resultName string
|
|
decisions func([]artifacts.Candidate) []contracts.ValidationDecision
|
|
warnings []contracts.Warning
|
|
err error
|
|
}
|
|
|
|
func (validator fakeValidator) Name() string {
|
|
return validator.name
|
|
}
|
|
|
|
func (validator fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
|
resultName := validator.resultName
|
|
if resultName == "" {
|
|
resultName = validator.name
|
|
}
|
|
var decisions []contracts.ValidationDecision
|
|
if validator.decisions != nil {
|
|
decisions = validator.decisions(req.Candidates)
|
|
}
|
|
return contracts.ValidationResult{
|
|
ValidatorName: resultName,
|
|
Decisions: decisions,
|
|
Warnings: validator.warnings,
|
|
}, validator.err
|
|
}
|
|
|
|
func factoryWithValidator(validator contracts.Validator) fakeFactory {
|
|
return fakeFactory{extractors: map[string]contracts.Extractor{
|
|
"generic-extractor": fakeExtractor{
|
|
key: "generic-extractor",
|
|
artifactType: "generic-artifact",
|
|
schemaVersion: "v1",
|
|
candidateCount: 1,
|
|
validators: []contracts.Validator{validator},
|
|
},
|
|
}}
|
|
}
|
|
|
|
func approveAll(candidates []artifacts.Candidate) []contracts.ValidationDecision {
|
|
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
|
|
for _, candidate := range candidates {
|
|
decisions = append(decisions, validationhelpers.Approved(candidate.Index))
|
|
}
|
|
return decisions
|
|
}
|
|
|
|
func validSourceDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{
|
|
ID: "source-1",
|
|
Kind: "document",
|
|
Format: "text/plain",
|
|
Digest: "sha256:abc123",
|
|
Units: []source.SourceUnit{
|
|
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
|
},
|
|
}
|
|
}
|
|
|
|
func warningReasons(warnings []contracts.Warning) []string {
|
|
reasons := make([]string, 0, len(warnings))
|
|
for _, warning := range warnings {
|
|
reasons = append(reasons, warning.ReasonCode)
|
|
}
|
|
return reasons
|
|
}
|
|
|
|
func assertRunError(t *testing.T, err error, want string) {
|
|
t.Helper()
|
|
|
|
if err == nil {
|
|
t.Fatal("Run() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("Run() error = %q, want substring %q", err.Error(), want)
|
|
}
|
|
}
|