eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add type-safe artifact lane resolution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:57:31 +00:00
parent fc1b57bde2
commit 1c84d19e5f
51 changed files with 1402 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/framework/contracts/composition_test.go
View File

@@ -13,10 +13,10 @@ import (
var _ contracts.InputAdapter = compositionAdapter{}
var _ contracts.Chunker = compositionChunker{}
var _ contracts.Extractor = compositionExtractor{}
var _ contracts.Merger = compositionMerger{}
var _ contracts.Normalizer = compositionNormalizer{}
var _ contracts.Validator = compositionValidator{}
var _ contracts.LegacyRawExtractor = compositionExtractor{}
var _ contracts.LegacyRawMerger = compositionMerger{}
var _ contracts.LegacyRawNormalizer = compositionNormalizer{}
var _ contracts.LegacyRawValidator = compositionValidator{}
var _ contracts.StructuredLLMClient = compositionLLMClient{}
var _ contracts.OutputEncoder = compositionOutputEncoder{}

8
internal/framework/contracts/contracts.go
View File

@@ -228,7 +228,7 @@ type ExtractionResult struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type Extractor interface {
type LegacyRawExtractor interface {
Key() string
ReferenceSlots() []ReferenceSlot
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
@@ -279,7 +279,7 @@ type ValidationResult struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type Validator interface {
type LegacyRawValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
@@ -328,7 +328,7 @@ type MergeOutput struct {
Payload RawPayload `json:"payload"`
}
type Merger interface {
type LegacyRawMerger interface {
Key() string
Merge(ctx context.Context, req MergeRequest) (MergeResult, error)
}
@@ -359,7 +359,7 @@ type NormalizeOutput struct {
Payload RawPayload `json:"payload"`
}
type Normalizer interface {
type LegacyRawNormalizer interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)

8
internal/framework/contracts/contracts_test.go
View File

@@ -12,10 +12,10 @@ import (
var _ InputAdapter = fakeAdapter{}
var _ Chunker = fakeChunker{}
var _ Extractor = fakeExtractor{}
var _ Merger = fakeMerger{}
var _ Normalizer = fakeNormalizer{}
var _ Validator = fakeValidator{}
var _ LegacyRawExtractor = fakeExtractor{}
var _ LegacyRawMerger = fakeMerger{}
var _ LegacyRawNormalizer = fakeNormalizer{}
var _ LegacyRawValidator = fakeValidator{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}

164
internal/framework/contracts/typed_pipeline.go Normal file
View File

@@ -0,0 +1,164 @@
package contracts
import (
"context"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
// ExtractArtifact carries a typed per-chunk value with framework provenance.
type ExtractArtifact[T any] struct {
LaneID string
ExtractorKey string
SourceID string
ChunkID string
ChunkIndex int
ChunkRef source.SourceRef
Value T
}
// MergeArtifact carries a typed merged value with framework provenance.
type MergeArtifact[T any] struct {
LaneID string
MergerKey string
SourceID string
Value T
}
// NormalizeArtifact carries a typed normalized value with framework provenance.
type NormalizeArtifact[T any] struct {
LaneID string
NormalizerKey string
SourceID string
Value T
}
type TypedExtractionRequest struct {
Source *source.SourceDocument
Chunk *source.Chunk
AmbientContext map[string]any
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedExtractionResult[T any] struct {
Value T
Warnings []Warning
}
type Extractor[T any] interface {
Key() string
ReferenceSlots() []ReferenceSlot
Extract(context.Context, TypedExtractionRequest) (TypedExtractionResult[T], error)
}
type TypedMergeRequest[T any] struct {
Source *source.SourceDocument
LaneID string
ExtractOutputs []ExtractArtifact[T]
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedMergeResult[T any] struct {
Value T
Warnings []Warning
}
type Merger[T any] interface {
Key() string
Merge(context.Context, TypedMergeRequest[T]) (TypedMergeResult[T], error)
}
type TypedNormalizeRequest[T any] struct {
Source *source.SourceDocument
LaneID string
MergeOutput MergeArtifact[T]
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedNormalizeResult[T any] struct {
Value T
Warnings []Warning
}
type Normalizer[T any] interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(context.Context, TypedNormalizeRequest[T]) (TypedNormalizeResult[T], error)
}
type TypedValidationRequest[T any] struct {
Stage string
LaneID string
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunk *source.Chunk
Chunks []source.Chunk
Ref source.SourceRef
Value T
}
type TypedValidator[T any] interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, TypedValidationRequest[T]) (ValidationResult, error)
}
type ChunkValidationRequest struct {
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunks []source.Chunk
}
type ChunkValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, ChunkValidationRequest) (ValidationResult, error)
}
type SerializedValidationRequest struct {
Stage string
LaneID string
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunk *source.Chunk
Chunks []source.Chunk
Schema ArtifactSchema
MediaType string
Content []byte
}
type SerializedValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, SerializedValidationRequest) (ValidationResult, error)
}

11
internal/framework/pipeline/artifact_codec_registry.go
View File

@@ -136,6 +136,17 @@ func (r *ArtifactCodecRegistry) Spec(kind contracts.ArtifactKind) (ArtifactCodec
return cloneArtifactCodecSpec(entry.spec), true
}
func (r *ArtifactCodecRegistry) valueType(kind contracts.ArtifactKind) (reflect.Type, bool) {
if r == nil {
return nil, false
}
entry, ok := r.entries[normalizeArtifactKind(kind)]
if !ok {
return nil, false
}
return entry.valueType, true
}
func (r *ArtifactCodecRegistry) RegisteredKinds() []contracts.ArtifactKind {
if r == nil || len(r.entries) == 0 {
return nil

4
internal/framework/pipeline/default_modules_test.go
View File

@@ -71,12 +71,12 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
if err := units.Register(chunkers); err != nil {
t.Fatalf("register generic chunker: %v", err)
}
if err := extractors.RegisterWithSpec(pipeline.ModuleSpec{
if err := extractors.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
Key: "extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"records"},
}, func() (contracts.Extractor, error) {
}, func() (contracts.LegacyRawExtractor, error) {
return defaultExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)

95
internal/framework/pipeline/extractor_registry.go
View File

@@ -2,71 +2,113 @@ package pipeline
import (
"fmt"
"reflect"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type ExtractorConstructor func() (contracts.Extractor, error)
type LegacyRawExtractorConstructor func() (contracts.LegacyRawExtractor, error)
type ExtractorRegistry struct {
constructors map[string]ExtractorConstructor
specs map[string]ModuleSpec
legacyConstructors map[string]LegacyRawExtractorConstructor
typedEntries map[string]typedExtractorEntry
specs map[string]ModuleSpec
}
type typedExtractorEntry struct {
spec ModuleSpec
valueType reflect.Type
constructor func() (any, error)
}
func NewExtractorRegistry() *ExtractorRegistry {
return &ExtractorRegistry{
constructors: make(map[string]ExtractorConstructor),
specs: make(map[string]ModuleSpec),
legacyConstructors: make(map[string]LegacyRawExtractorConstructor),
typedEntries: make(map[string]typedExtractorEntry),
specs: make(map[string]ModuleSpec),
}
}
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageExtract), constructor)
func (r *ExtractorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawExtractorConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageExtract), constructor)
}
func (r *ExtractorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ExtractorConstructor) error {
func (r *ExtractorRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawExtractorConstructor) error {
if r == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw extractor %q must not declare an artifact kind", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
if _, ok := r.specs[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]ExtractorConstructor)
if r.legacyConstructors == nil {
r.legacyConstructors = make(map[string]LegacyRawExtractorConstructor)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.legacyConstructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error) {
func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error {
if registry == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind == "" {
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := registry.specs[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
}
entry := typedExtractorEntry{
spec: cloneModuleSpec(normalizedSpec),
valueType: reflect.TypeFor[T](),
constructor: func() (any, error) {
return constructor()
},
}
if registry.typedEntries == nil {
registry.typedEntries = make(map[string]typedExtractorEntry)
}
if registry.specs == nil {
registry.specs = make(map[string]ModuleSpec)
}
registry.typedEntries[normalizedSpec.Key] = entry
registry.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func (r *ExtractorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawExtractor, error) {
if r == nil {
return nil, fmt.Errorf("extractor registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("extractor key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
constructor, ok := r.legacyConstructors[normalizedKey]
if !ok {
return nil, fmt.Errorf("extractor %q is not registered", normalizedKey)
return nil, fmt.Errorf("legacy raw extractor %q is not registered", normalizedKey)
}
extractor, err := constructor()
if err != nil {
return nil, fmt.Errorf("build extractor %q: %w", normalizedKey, err)
@@ -77,7 +119,6 @@ func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error) {
if extractor.Key() != normalizedKey {
return nil, fmt.Errorf("extractor %q returned key %q", normalizedKey, extractor.Key())
}
return extractor, nil
}
@@ -85,7 +126,6 @@ func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
@@ -93,10 +133,17 @@ func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
return cloneModuleSpec(spec), true
}
func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
if r == nil {
return typedExtractorEntry{}, false
}
entry, ok := r.typedEntries[strings.TrimSpace(key)]
return entry, ok
}
func (r *ExtractorRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
return sortedRegistryKeys(r.constructors)
return sortedRegistryKeys(r.specs)
}

50
internal/framework/pipeline/extractor_registry_test.go
View File

@@ -13,11 +13,11 @@ import (
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.Build("generic-extractor")
extractor, err := registry.BuildLegacyRaw("generic-extractor")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
@@ -29,11 +29,11 @@ func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.Build("\tgeneric-extractor\n")
extractor, err := registry.BuildLegacyRaw("\tgeneric-extractor\n")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
@@ -65,7 +65,7 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
},
}
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
if err := registry.RegisterLegacyRawWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
@@ -112,7 +112,7 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
@@ -129,7 +129,7 @@ func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
@@ -168,7 +168,7 @@ func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
@@ -194,7 +194,7 @@ func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.Register(" \t", fakeExtractorConstructor("generic-extractor"))
err := registry.RegisterLegacyRaw(" \t", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -206,11 +206,11 @@ func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -223,7 +223,7 @@ func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.Register("generic-extractor", nil)
err := registry.RegisterLegacyRaw("generic-extractor", nil)
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -236,7 +236,7 @@ func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.Build("missing-extractor")
_, err := registry.BuildLegacyRaw("missing-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
@@ -249,13 +249,13 @@ func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
registry := NewExtractorRegistry()
constructorErr := errors.New("constructor failed")
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, constructorErr
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
@@ -270,13 +270,13 @@ func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
@@ -288,11 +288,11 @@ func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
@@ -305,7 +305,7 @@ func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
registry := NewExtractorRegistry()
for _, key := range []string{"zeta", "alpha", "middle"} {
if err := registry.Register(key, fakeExtractorConstructor(key)); err != nil {
if err := registry.RegisterLegacyRaw(key, fakeExtractorConstructor(key)); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
@@ -326,10 +326,10 @@ func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
var registry *ExtractorRegistry
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
t.Fatal("Register() error = nil, want error")
}
if _, err := registry.Build("generic-extractor"); err == nil {
if _, err := registry.BuildLegacyRaw("generic-extractor"); err == nil {
t.Fatal("Build() error = nil, want error")
}
if _, ok := registry.Spec("generic-extractor"); ok {
@@ -343,7 +343,7 @@ func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.Build(" \n")
_, err := registry.BuildLegacyRaw(" \n")
if err == nil {
t.Fatal("Build() error = nil, want error")
@@ -357,8 +357,8 @@ type registryFakeExtractor struct {
key string
}
func fakeExtractorConstructor(key string) ExtractorConstructor {
return func() (contracts.Extractor, error) {
func fakeExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}

128
internal/framework/pipeline/merger_registry.go
View File

@@ -2,71 +2,114 @@ package pipeline
import (
"fmt"
"reflect"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type MergerConstructor func() (contracts.Merger, error)
type LegacyRawMergerConstructor func() (contracts.LegacyRawMerger, error)
type artifactVariantKey struct {
module string
kind contracts.ArtifactKind
}
type MergerRegistry struct {
constructors map[string]MergerConstructor
specs map[string]ModuleSpec
legacyConstructors map[string]LegacyRawMergerConstructor
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedMergerEntry
}
type typedMergerEntry struct {
spec ModuleSpec
valueType reflect.Type
constructor func() (any, error)
}
func NewMergerRegistry() *MergerRegistry {
return &MergerRegistry{
constructors: make(map[string]MergerConstructor),
specs: make(map[string]ModuleSpec),
legacyConstructors: make(map[string]LegacyRawMergerConstructor),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
}
}
func (r *MergerRegistry) Register(key string, constructor MergerConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageMerge), constructor)
func (r *MergerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawMergerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageMerge), constructor)
}
func (r *MergerRegistry) RegisterWithSpec(spec ModuleSpec, constructor MergerConstructor) error {
func (r *MergerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawMergerConstructor) error {
if r == nil {
return fmt.Errorf("merger registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("merger", StageMerge, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw merger %q must not declare an artifact kind", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
return fmt.Errorf("merger %q is already registered", normalizedSpec.Key)
if _, ok := r.legacyConstructors[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw merger %q is already registered", normalizedSpec.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]MergerConstructor)
if r.legacyConstructors == nil {
r.legacyConstructors = make(map[string]LegacyRawMergerConstructor)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
r.legacyConstructors[normalizedSpec.Key] = constructor
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func (r *MergerRegistry) Build(key string) (contracts.Merger, error) {
func RegisterMerger[T any](registry *MergerRegistry, spec ModuleSpec, constructor func() (contracts.Merger[T], error)) error {
if registry == nil {
return fmt.Errorf("merger registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("merger", StageMerge, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind == "" {
return fmt.Errorf("typed merger %q artifact kind must not be empty", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", normalizedSpec.Key)
}
key := artifactVariantKey{module: normalizedSpec.Key, kind: normalizedSpec.ArtifactKind}
if _, ok := registry.typedEntries[key]; ok {
return fmt.Errorf("merger %q variant for artifact kind %q is already registered", key.module, key.kind)
}
if registry.typedEntries == nil {
registry.typedEntries = make(map[artifactVariantKey]typedMergerEntry)
}
registry.typedEntries[key] = typedMergerEntry{
spec: cloneModuleSpec(normalizedSpec),
valueType: reflect.TypeFor[T](),
constructor: func() (any, error) {
return constructor()
},
}
return nil
}
func (r *MergerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawMerger, error) {
if r == nil {
return nil, fmt.Errorf("merger registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("merger key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
constructor, ok := r.legacyConstructors[normalizedKey]
if !ok {
return nil, fmt.Errorf("merger %q is not registered", normalizedKey)
return nil, fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
}
merger, err := constructor()
if err != nil {
return nil, fmt.Errorf("build merger %q: %w", normalizedKey, err)
@@ -77,7 +120,6 @@ func (r *MergerRegistry) Build(key string) (contracts.Merger, error) {
if merger.Key() != normalizedKey {
return nil, fmt.Errorf("merger %q returned key %q", normalizedKey, merger.Key())
}
return merger, nil
}
@@ -85,18 +127,46 @@ func (r *MergerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
}
return cloneModuleSpec(spec), true
}
func (r *MergerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedMergerEntry, bool) {
if r == nil {
return typedMergerEntry{}, false
}
entry, ok := r.typedEntries[artifactVariantKey{module: strings.TrimSpace(key), kind: normalizeArtifactKind(kind)}]
return entry, ok
}
func (r *MergerRegistry) registeredKinds(key string) []contracts.ArtifactKind {
if r == nil {
return nil
}
module := strings.TrimSpace(key)
kinds := make([]contracts.ArtifactKind, 0)
for variant := range r.typedEntries {
if variant.module == module {
kinds = append(kinds, variant.kind)
}
}
sortArtifactKinds(kinds)
return kinds
}
func (r *MergerRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
return sortedRegistryKeys(r.constructors)
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}
return sortedRegistryKeys(keys)
}

28
internal/framework/pipeline/merger_registry_test.go
View File

@@ -7,7 +7,7 @@ import (
)
func TestMergerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Merger]{
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawMerger]{
name: "MergerRegistry",
key: "generic-merger",
stage: StageMerge,
@@ -15,14 +15,14 @@ func TestMergerRegistryBehavior(t *testing.T) {
newRegistry: func() any {
return NewMergerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.Merger, error)) error {
return registry.(*MergerRegistry).Register(key, constructor)
register: func(registry any, key string, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Merger, error)) error {
return registry.(*MergerRegistry).RegisterWithSpec(spec, constructor)
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.Merger, error) {
return registry.(*MergerRegistry).Build(key)
build: func(registry any, key string) (contracts.LegacyRawMerger, error) {
return registry.(*MergerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*MergerRegistry).Spec(key)
@@ -30,13 +30,13 @@ func TestMergerRegistryBehavior(t *testing.T) {
registeredKeys: func(registry any) []string {
return registry.(*MergerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.Merger, error)) error {
nilRegister: func(key string, constructor func() (contracts.LegacyRawMerger, error)) error {
var registry *MergerRegistry
return registry.Register(key, constructor)
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.Merger, error) {
nilBuild: func(key string) (contracts.LegacyRawMerger, error) {
var registry *MergerRegistry
return registry.Build(key)
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *MergerRegistry
@@ -46,12 +46,12 @@ func TestMergerRegistryBehavior(t *testing.T) {
var registry *MergerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.Merger, error) {
return func() (contracts.Merger, error) {
constructor: func(key string) func() (contracts.LegacyRawMerger, error) {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
},
moduleKey: func(module contracts.Merger) string {
moduleKey: func(module contracts.LegacyRawMerger) string {
return module.Key()
},
})

10
internal/framework/pipeline/module.go
View File

@@ -23,6 +23,7 @@ const (
type ModuleSpec struct {
Key string
Stage ModuleStage
ArtifactKind contracts.ArtifactKind
Provides []string
Requires []string
ReferenceSlots []contracts.ReferenceSlot
@@ -39,6 +40,7 @@ func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{
Key: strings.TrimSpace(spec.Key),
Stage: spec.Stage,
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires),
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
@@ -74,6 +76,7 @@ func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{
Key: spec.Key,
Stage: spec.Stage,
ArtifactKind: spec.ArtifactKind,
Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...),
ReferenceSlots: contracts.CloneReferenceSlots(spec.ReferenceSlots),
@@ -87,6 +90,9 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.Stage != expectedStage {
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
}
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
}
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
}
@@ -113,6 +119,10 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
return keys
}
func sortArtifactKinds(kinds []contracts.ArtifactKind) {
sort.Slice(kinds, func(i, j int) bool { return kinds[i] < kinds[j] })
}
func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil

123
internal/framework/pipeline/normalizer_registry.go
View File

@@ -2,71 +2,109 @@ package pipeline
import (
"fmt"
"reflect"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type NormalizerConstructor func() (contracts.Normalizer, error)
type LegacyRawNormalizerConstructor func() (contracts.LegacyRawNormalizer, error)
type NormalizerRegistry struct {
constructors map[string]NormalizerConstructor
specs map[string]ModuleSpec
legacyConstructors map[string]LegacyRawNormalizerConstructor
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedNormalizerEntry
}
type typedNormalizerEntry struct {
spec ModuleSpec
valueType reflect.Type
constructor func() (any, error)
}
func NewNormalizerRegistry() *NormalizerRegistry {
return &NormalizerRegistry{
constructors: make(map[string]NormalizerConstructor),
specs: make(map[string]ModuleSpec),
legacyConstructors: make(map[string]LegacyRawNormalizerConstructor),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
}
}
func (r *NormalizerRegistry) Register(key string, constructor NormalizerConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageNormalize), constructor)
func (r *NormalizerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawNormalizerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageNormalize), constructor)
}
func (r *NormalizerRegistry) RegisterWithSpec(spec ModuleSpec, constructor NormalizerConstructor) error {
func (r *NormalizerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawNormalizerConstructor) error {
if r == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("normalizer", StageNormalize, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw normalizer %q must not declare an artifact kind", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
return fmt.Errorf("normalizer %q is already registered", normalizedSpec.Key)
if _, ok := r.legacyConstructors[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw normalizer %q is already registered", normalizedSpec.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]NormalizerConstructor)
if r.legacyConstructors == nil {
r.legacyConstructors = make(map[string]LegacyRawNormalizerConstructor)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
r.legacyConstructors[normalizedSpec.Key] = constructor
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func (r *NormalizerRegistry) Build(key string) (contracts.Normalizer, error) {
func RegisterNormalizer[T any](registry *NormalizerRegistry, spec ModuleSpec, constructor func() (contracts.Normalizer[T], error)) error {
if registry == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("normalizer", StageNormalize, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind == "" {
return fmt.Errorf("typed normalizer %q artifact kind must not be empty", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", normalizedSpec.Key)
}
key := artifactVariantKey{module: normalizedSpec.Key, kind: normalizedSpec.ArtifactKind}
if _, ok := registry.typedEntries[key]; ok {
return fmt.Errorf("normalizer %q variant for artifact kind %q is already registered", key.module, key.kind)
}
if registry.typedEntries == nil {
registry.typedEntries = make(map[artifactVariantKey]typedNormalizerEntry)
}
registry.typedEntries[key] = typedNormalizerEntry{
spec: cloneModuleSpec(normalizedSpec),
valueType: reflect.TypeFor[T](),
constructor: func() (any, error) {
return constructor()
},
}
return nil
}
func (r *NormalizerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawNormalizer, error) {
if r == nil {
return nil, fmt.Errorf("normalizer registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("normalizer key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
constructor, ok := r.legacyConstructors[normalizedKey]
if !ok {
return nil, fmt.Errorf("normalizer %q is not registered", normalizedKey)
return nil, fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
}
normalizer, err := constructor()
if err != nil {
return nil, fmt.Errorf("build normalizer %q: %w", normalizedKey, err)
@@ -77,7 +115,6 @@ func (r *NormalizerRegistry) Build(key string) (contracts.Normalizer, error) {
if normalizer.Key() != normalizedKey {
return nil, fmt.Errorf("normalizer %q returned key %q", normalizedKey, normalizer.Key())
}
return normalizer, nil
}
@@ -85,18 +122,46 @@ func (r *NormalizerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
}
return cloneModuleSpec(spec), true
}
func (r *NormalizerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedNormalizerEntry, bool) {
if r == nil {
return typedNormalizerEntry{}, false
}
entry, ok := r.typedEntries[artifactVariantKey{module: strings.TrimSpace(key), kind: normalizeArtifactKind(kind)}]
return entry, ok
}
func (r *NormalizerRegistry) registeredKinds(key string) []contracts.ArtifactKind {
if r == nil {
return nil
}
module := strings.TrimSpace(key)
kinds := make([]contracts.ArtifactKind, 0)
for variant := range r.typedEntries {
if variant.module == module {
kinds = append(kinds, variant.kind)
}
}
sortArtifactKinds(kinds)
return kinds
}
func (r *NormalizerRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
return sortedRegistryKeys(r.constructors)
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}
return sortedRegistryKeys(keys)
}

28
internal/framework/pipeline/normalizer_registry_test.go
View File

@@ -7,7 +7,7 @@ import (
)
func TestNormalizerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Normalizer]{
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawNormalizer]{
name: "NormalizerRegistry",
key: "generic-normalizer",
stage: StageNormalize,
@@ -15,14 +15,14 @@ func TestNormalizerRegistryBehavior(t *testing.T) {
newRegistry: func() any {
return NewNormalizerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.Normalizer, error)) error {
return registry.(*NormalizerRegistry).Register(key, constructor)
register: func(registry any, key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Normalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterWithSpec(spec, constructor)
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.Normalizer, error) {
return registry.(*NormalizerRegistry).Build(key)
build: func(registry any, key string) (contracts.LegacyRawNormalizer, error) {
return registry.(*NormalizerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*NormalizerRegistry).Spec(key)
@@ -30,13 +30,13 @@ func TestNormalizerRegistryBehavior(t *testing.T) {
registeredKeys: func(registry any) []string {
return registry.(*NormalizerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.Normalizer, error)) error {
nilRegister: func(key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
var registry *NormalizerRegistry
return registry.Register(key, constructor)
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.Normalizer, error) {
nilBuild: func(key string) (contracts.LegacyRawNormalizer, error) {
var registry *NormalizerRegistry
return registry.Build(key)
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *NormalizerRegistry
@@ -46,12 +46,12 @@ func TestNormalizerRegistryBehavior(t *testing.T) {
var registry *NormalizerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.Normalizer, error) {
return func() (contracts.Normalizer, error) {
constructor: func(key string) func() (contracts.LegacyRawNormalizer, error) {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
},
moduleKey: func(module contracts.Normalizer) string {
moduleKey: func(module contracts.LegacyRawNormalizer) string {
return module.Key()
},
})

183
internal/framework/pipeline/profile.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
@@ -102,14 +103,19 @@ type ResolvedReferenceTarget struct {
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
ID string
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedValidatorChain struct {
@@ -122,6 +128,8 @@ type ResolvedValidatorChain struct {
type ResolvedValidator struct {
Binding ModuleBinding `json:"binding"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
Target ValidatorTarget `json:"target,omitempty"`
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
}
type ResolvedPipeline struct {
@@ -216,7 +224,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
@@ -279,6 +287,10 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
}
artifactType, err := resolveArtifactIdentity(pipelineID, laneID, &lane, extractSpec, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
@@ -296,7 +308,7 @@ func resolveArtifactLane(
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
}
@@ -319,7 +331,7 @@ func resolveArtifactLane(
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
}
@@ -346,15 +358,15 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
}
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, lane.ArtifactKind, artifactType, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, lane.ArtifactKind, artifactType, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, lane.ArtifactKind, artifactType, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
@@ -367,7 +379,128 @@ func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
}
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) {
if extractSpec.ArtifactKind == "" {
return nil, nil
}
if catalog.Extractors == nil {
return nil, fmt.Errorf("pipeline %q lane %q extractor registry must not be nil", pipelineID, laneID)
}
extractor, ok := catalog.Extractors.typedEntry(lane.Extract.Module)
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q declares artifact kind %q without a typed registration", pipelineID, laneID, lane.Extract.Module, extractSpec.ArtifactKind)
}
if catalog.ArtifactCodecs == nil {
return nil, fmt.Errorf("pipeline %q lane %q artifact codec registry must not be nil for kind %q", pipelineID, laneID, extractSpec.ArtifactKind)
}
codecSpec, ok := catalog.ArtifactCodecs.Spec(extractSpec.ArtifactKind)
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q artifact codec %q is not registered", pipelineID, laneID, extractSpec.ArtifactKind)
}
codecType, ok := catalog.ArtifactCodecs.valueType(extractSpec.ArtifactKind)
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q artifact codec %q has no Go type", pipelineID, laneID, extractSpec.ArtifactKind)
}
if extractor.valueType != codecType {
return nil, artifactTypeMismatchError(pipelineID, laneID, StageExtract, lane.Extract.Module, extractSpec.ArtifactKind, codecType, extractor.valueType)
}
lane.ArtifactKind = codecSpec.Kind
lane.ArtifactSchemaID = codecSpec.Schema.ID
lane.ArtifactSchemaName = codecSpec.Schema.Name
lane.ArtifactSchemaVersion = codecSpec.Schema.Version
lane.ArtifactSchemaDigest = codecSpec.SchemaDigest
return codecType, nil
}
func mergerSpecForArtifact(catalog ModuleCatalog, key string, kind contracts.ArtifactKind, expectedType reflect.Type) (ModuleSpec, error) {
if kind == "" {
return mergerSpec(catalog, key)
}
if catalog.Mergers == nil {
return ModuleSpec{}, fmt.Errorf("module %q is not registered", key)
}
entry, ok := catalog.Mergers.typedEntry(key, kind)
if !ok {
return ModuleSpec{}, missingArtifactVariantError("merger", key, kind, catalog.Mergers.registeredKinds(key))
}
if entry.valueType != expectedType {
return ModuleSpec{}, fmt.Errorf("artifact kind %q requires Go type %s, but merger %q variant uses %s", kind, typeName(expectedType), key, typeName(entry.valueType))
}
return cloneModuleSpec(entry.spec), nil
}
func normalizerSpecForArtifact(catalog ModuleCatalog, key string, kind contracts.ArtifactKind, expectedType reflect.Type) (ModuleSpec, error) {
if kind == "" {
return normalizerSpec(catalog, key)
}
if catalog.Normalizers == nil {
return ModuleSpec{}, fmt.Errorf("module %q is not registered", key)
}
entry, ok := catalog.Normalizers.typedEntry(key, kind)
if !ok {
return ModuleSpec{}, missingArtifactVariantError("normalizer", key, kind, catalog.Normalizers.registeredKinds(key))
}
if entry.valueType != expectedType {
return ModuleSpec{}, fmt.Errorf("artifact kind %q requires Go type %s, but normalizer %q variant uses %s", kind, typeName(expectedType), key, typeName(entry.valueType))
}
return cloneModuleSpec(entry.spec), nil
}
func validatorSpecForTarget(registry *ValidatorRegistry, stage ModuleStage, key string, kind contracts.ArtifactKind, expectedType reflect.Type) (ValidatorSpec, ValidatorTarget, error) {
key = strings.TrimSpace(key)
if stage == StageChunk {
if entry, ok := registry.chunkEntry(key); ok {
return entry.spec, ValidatorTargetChunk, nil
}
if entry, ok := registry.serializedEntry(key); ok && entry.spec.SupportsChunks {
return entry.spec.ValidatorSpec, ValidatorTargetSerialized, nil
}
if spec, ok := registry.Spec(key); ok {
return spec, "", nil
}
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q for chunk target", key)
}
if kind == "" {
if spec, ok := registry.Spec(key); ok {
return spec, "", nil
}
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q on legacy raw path", key)
}
if entry, ok := registry.typedEntry(key, kind); ok {
if entry.valueType != expectedType {
return ValidatorSpec{}, "", fmt.Errorf("artifact kind %q requires Go type %s, but validator %q variant uses %s", kind, typeName(expectedType), key, typeName(entry.valueType))
}
return entry.spec, ValidatorTargetTyped, nil
}
if entry, ok := registry.serializedEntry(key); ok && entry.spec.SupportsArtifacts {
return entry.spec.ValidatorSpec, ValidatorTargetSerialized, nil
}
return ValidatorSpec{}, "", missingArtifactVariantError("validator", key, kind, registry.registeredTypedKinds(key))
}
func missingArtifactVariantError(moduleType, key string, kind contracts.ArtifactKind, registered []contracts.ArtifactKind) error {
if len(registered) == 0 {
return fmt.Errorf("%s %q has no typed variant for artifact kind %q", moduleType, key, kind)
}
values := make([]string, len(registered))
for i, value := range registered {
values[i] = string(value)
}
return fmt.Errorf("%s %q has no typed variant for artifact kind %q; registered kinds: %s", moduleType, key, kind, strings.Join(values, ", "))
}
func artifactTypeMismatchError(pipelineID, laneID string, stage ModuleStage, module string, kind contracts.ArtifactKind, expected, actual reflect.Type) error {
return fmt.Errorf("pipeline %q lane %q %s module %q artifact kind %q requires Go type %s, got %s", pipelineID, laneID, stage, module, kind, typeName(expected), typeName(actual))
}
func typeName(value reflect.Type) string {
if value == nil {
return "<nil>"
}
return value.String()
}
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, artifactKind contracts.ArtifactKind, artifactType reflect.Type, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
@@ -396,9 +529,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
}
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
for _, validator := range bindings {
spec, ok := catalog.Validators.Spec(validator.Module)
if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
spec, target, err := validatorSpecForTarget(catalog.Validators, stage, validator.Module, artifactKind, artifactType)
if err != nil {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q: %w", pipelineID, stage, chain.ModuleKey, err)
}
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
@@ -406,6 +539,8 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass,
Target: target,
ArtifactKind: artifactKind,
})
}
return chain, nil
@@ -436,6 +571,8 @@ func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator
out[i] = ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
Target: validator.Target,
ArtifactKind: validator.ArtifactKind,
}
}
return out
@@ -498,7 +635,13 @@ func validatePipelineReferenceDefaults(
}
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
mergeSpec, err := mergerSpec(catalog, merge.Module)
var artifactType reflect.Type
if extractSpec.ArtifactKind != "" && catalog.Extractors != nil {
if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok {
artifactType = entry.valueType
}
}
mergeSpec, err := mergerSpecForArtifact(catalog, merge.Module, extractSpec.ArtifactKind, artifactType)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageMerge, merge.Module, err)
}
@@ -507,7 +650,7 @@ func validatePipelineReferenceDefaults(
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
normalizeSpec, err := normalizerSpecForArtifact(catalog, normalize.Module, extractSpec.ArtifactKind, artifactType)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
}

30
internal/framework/pipeline/profile_test.go
View File

@@ -771,7 +771,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
@@ -779,7 +779,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.Extractor, error) {
}, func() (contracts.LegacyRawExtractor, error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
@@ -1251,20 +1251,20 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
t.Fatalf("register chunk spec %#v: %v", spec, err)
}
case StageExtract:
if err := catalog.Extractors.RegisterWithSpec(spec, profileExtractorConstructor(spec.Key)); err != nil {
if err := catalog.Extractors.RegisterLegacyRawWithSpec(spec, profileExtractorConstructor(spec.Key)); err != nil {
t.Fatalf("register extractor spec %#v: %v", spec, err)
}
case StageMerge:
if err := catalog.Mergers.RegisterWithSpec(spec, profileMergerConstructor(spec.Key)); err != nil {
if err := catalog.Mergers.RegisterLegacyRawWithSpec(spec, profileMergerConstructor(spec.Key)); err != nil {
t.Fatalf("register merger spec %#v: %v", spec, err)
}
case StageNormalize:
if err := catalog.Normalizers.RegisterWithSpec(spec, profileNormalizerConstructor(spec.Key)); err != nil {
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(spec, profileNormalizerConstructor(spec.Key)); err != nil {
t.Fatalf("register normalizer spec %#v: %v", spec, err)
}
case StageValidate:
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
if err := catalog.Validators.RegisterLegacyRawWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
case StageOutput:
@@ -1279,7 +1279,7 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper()
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
if err := catalog.Validators.RegisterLegacyRawWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
}
@@ -1308,26 +1308,26 @@ func profileChunkerConstructor(key string) ChunkerConstructor {
}
}
func profileExtractorConstructor(key string) ExtractorConstructor {
return func() (contracts.Extractor, error) {
func profileExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func profileMergerConstructor(key string) MergerConstructor {
return func() (contracts.Merger, error) {
func profileMergerConstructor(key string) LegacyRawMergerConstructor {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
}
func profileNormalizerConstructor(key string) NormalizerConstructor {
return func() (contracts.Normalizer, error) {
func profileNormalizerConstructor(key string) LegacyRawNormalizerConstructor {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
}
func profileValidatorConstructor(key string) ValidatorConstructor {
return func() (contracts.Validator, error) {
func profileValidatorConstructor(key string) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return registryValidator{name: key}, nil
}
}

6
internal/framework/pipeline/registry_integration_test.go
View File

@@ -64,13 +64,13 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
}
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
if err := registries.Mergers.Register("merge", func() (contracts.Merger, error) {
if err := registries.Mergers.RegisterLegacyRaw("merge", func() (contracts.LegacyRawMerger, error) {
*built = append(*built, "merge")
return integrationMerger{}, nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := registries.Normalizers.Register("normalize", func() (contracts.Normalizer, error) {
if err := registries.Normalizers.RegisterLegacyRaw("normalize", func() (contracts.LegacyRawNormalizer, error) {
*built = append(*built, "normalize")
return integrationNormalizer{}, nil
}); err != nil {
@@ -88,7 +88,7 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
t.Helper()
if err := registry.Register(key, func() (contracts.Extractor, error) {
if err := registry.RegisterLegacyRaw(key, func() (contracts.LegacyRawExtractor, error) {
*built = append(*built, key)
return integrationExtractor{key: key, executed: executed}, nil
}); err != nil {

17
internal/framework/pipeline/runner.go
View File

@@ -390,15 +390,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
extractor, err := r.registries.Extractors.BuildLegacyRaw(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
}
merger, err := r.registries.Mergers.Build(lane.Merge.Module)
merger, err := r.registries.Mergers.BuildLegacyRaw(lane.Merge.Module)
if err != nil {
return fmt.Errorf("build merger %q for lane %q: %w", lane.Merge.Module, lane.ID, err)
}
normalizer, err := r.registries.Normalizers.Build(lane.Normalize.Module)
normalizer, err := r.registries.Normalizers.BuildLegacyRaw(lane.Normalize.Module)
if err != nil {
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
}
@@ -980,7 +980,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
var warnings []contracts.Warning
for index, validatorBinding := range chain.Validators {
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
validator, err := r.registries.Validators.BuildLegacyRaw(validatorBinding.Binding.Module)
if err != nil {
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
}
@@ -1133,6 +1133,9 @@ func validateRunInput(input RunInput) error {
if lane.ID == "" {
return fmt.Errorf("resolved pipeline artifact lane id must not be empty")
}
if lane.ArtifactKind != "" {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
if lane.Extract.Module == "" {
return fmt.Errorf("resolved pipeline lane %q extract module must not be empty", lane.ID)
}
@@ -1340,11 +1343,11 @@ func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module an
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.Extractor:
case contracts.LegacyRawExtractor:
return "extractor"
case contracts.Merger:
case contracts.LegacyRawMerger:
return "merger"
case contracts.Normalizer:
case contracts.LegacyRawNormalizer:
return "normalizer"
default:
return ""

14
internal/framework/pipeline/runner_test.go
View File

@@ -1948,7 +1948,7 @@ type runnerModules struct {
extractors map[string]*runnerExtractor
mergers map[string]*runnerMerger
normalizers map[string]*runnerNormalizer
validators map[string]contracts.Validator
validators map[string]contracts.LegacyRawValidator
output *runnerOutputEncoder
inputBuildErr error
chunkerBuildErr error
@@ -1967,7 +1967,7 @@ func defaultRunnerModules() *runnerModules {
normalizers: map[string]*runnerNormalizer{
"normalize": {key: "normalize"},
},
validators: map[string]contracts.Validator{
validators: map[string]contracts.LegacyRawValidator{
"configured": &runnerValidator{name: "configured"},
"second-validator": &runnerValidator{name: "second-validator"},
},
@@ -2015,26 +2015,26 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
}
for key, extractor := range modules.extractors {
extractor := extractor
if err := registries.Extractors.Register(key, func() (contracts.Extractor, error) { return extractor, nil }); err != nil {
if err := registries.Extractors.RegisterLegacyRaw(key, func() (contracts.LegacyRawExtractor, error) { return extractor, nil }); err != nil {
t.Fatalf("register extractor %q: %v", key, err)
}
}
for key, merger := range modules.mergers {
merger := merger
if err := registries.Mergers.Register(key, func() (contracts.Merger, error) { return merger, nil }); err != nil {
if err := registries.Mergers.RegisterLegacyRaw(key, func() (contracts.LegacyRawMerger, error) { return merger, nil }); err != nil {
t.Fatalf("register merger %q: %v", key, err)
}
}
for key, normalizer := range modules.normalizers {
normalizer := normalizer
if err := registries.Normalizers.Register(key, func() (contracts.Normalizer, error) { return normalizer, nil }); err != nil {
if err := registries.Normalizers.RegisterLegacyRaw(key, func() (contracts.LegacyRawNormalizer, error) { return normalizer, nil }); err != nil {
t.Fatalf("register normalizer %q: %v", key, err)
}
}
for key, validator := range modules.validators {
validator := validator
spec := ValidatorSpec{Key: key, ExecutionClass: validator.ExecutionClass()}
if err := registries.Validators.RegisterWithSpec(spec, func() (contracts.Validator, error) { return validator, nil }); err != nil {
if err := registries.Validators.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawValidator, error) { return validator, nil }); err != nil {
t.Fatalf("register validator %q: %v", key, err)
}
}
@@ -2653,7 +2653,7 @@ func assertRunError(t *testing.T, err error, want string) {
}
}
func resolvedValidatorForTest(validator contracts.Validator) ResolvedValidator {
func resolvedValidatorForTest(validator contracts.LegacyRawValidator) ResolvedValidator {
return ResolvedValidator{
Binding: Binding(validator.Name()),
ExecutionClass: validator.ExecutionClass(),

362
internal/framework/pipeline/typed_resolution_test.go Normal file
View File

@@ -0,0 +1,362 @@
package pipeline
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type typedTestExtractor[T any] struct{ key string }
func (e typedTestExtractor[T]) Key() string { return e.key }
func (typedTestExtractor[T]) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (typedTestExtractor[T]) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[T], error) {
return contracts.TypedExtractionResult[T]{}, nil
}
type typedTestMerger[T any] struct{ key string }
func (m typedTestMerger[T]) Key() string { return m.key }
func (typedTestMerger[T]) Merge(context.Context, contracts.TypedMergeRequest[T]) (contracts.TypedMergeResult[T], error) {
return contracts.TypedMergeResult[T]{}, nil
}
type typedTestNormalizer[T any] struct{ key string }
func (n typedTestNormalizer[T]) Key() string { return n.key }
func (typedTestNormalizer[T]) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (typedTestNormalizer[T]) Normalize(context.Context, contracts.TypedNormalizeRequest[T]) (contracts.TypedNormalizeResult[T], error) {
return contracts.TypedNormalizeResult[T]{}, nil
}
type typedTestValidator[T any] struct{ key string }
func (v typedTestValidator[T]) Name() string { return v.key }
func (typedTestValidator[T]) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (typedTestValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type typedTestChunkValidator struct{ key string }
func (v typedTestChunkValidator) Name() string { return v.key }
func (typedTestChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (typedTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type typedTestSerializedValidator struct{ key string }
func (v typedTestSerializedValidator) Name() string { return v.key }
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (typedTestSerializedValidator) Validate(context.Context, contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type typedCatalogOptions struct {
registerNotesCodec bool
registerScoreCodec bool
registerNotesMerger bool
registerScoreMerger bool
registerNotesNormalizer bool
registerScoreNormalizer bool
registerScoreValidator bool
scoreExtractorUsesNotes bool
notesCodec testArtifactCodec[codecNotes]
scoreCodec testArtifactCodec[codecScore]
}
func completeTypedCatalogOptions() typedCatalogOptions {
return typedCatalogOptions{
registerNotesCodec: true,
registerScoreCodec: true,
registerNotesMerger: true,
registerScoreMerger: true,
registerNotesNormalizer: true,
registerScoreNormalizer: true,
registerScoreValidator: true,
notesCodec: notesCodec(),
scoreCodec: scoreCodec(),
}
}
func TestResolveTypedHeterogeneousLanes(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
resolved, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got, want := resolvedLaneIDs(resolved.ArtifactLanes), []string{"notes", "score"}; !reflect.DeepEqual(got, want) {
t.Fatalf("lane order = %#v, want %#v", got, want)
}
assertResolvedArtifactIdentity(t, resolved.ArtifactLanes[0], "test/notes", "notes.v1")
assertResolvedArtifactIdentity(t, resolved.ArtifactLanes[1], "test/score", "score.v1")
chunkChain := resolved.ValidatorChains[0]
if got := resolvedValidatorTargets(chunkChain.Validators); !reflect.DeepEqual(got, []ValidatorTarget{ValidatorTargetChunk, ValidatorTargetSerialized}) {
t.Fatalf("chunk validator targets = %#v, want chunk then serialized", got)
}
notesExtract := resolved.ValidatorChains[1]
if got := resolvedValidatorTargets(notesExtract.Validators); !reflect.DeepEqual(got, []ValidatorTarget{ValidatorTargetTyped, ValidatorTargetSerialized}) {
t.Fatalf("notes extract validator targets = %#v, want typed then serialized", got)
}
if notesExtract.Validators[0].ArtifactKind != "test/notes" || resolved.ValidatorChains[4].Validators[0].ArtifactKind != "test/score" {
t.Fatalf("resolved validator kinds = %#v, want lane kinds", resolved.ValidatorChains)
}
}
func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {
tests := []struct {
name string
mutate func(*typedCatalogOptions)
want string
}{
{name: "missing codec", mutate: func(options *typedCatalogOptions) { options.registerScoreCodec = false }, want: `artifact codec "test/score" is not registered`},
{name: "missing merger variant", mutate: func(options *typedCatalogOptions) { options.registerScoreMerger = false }, want: `merger "typed/merge" has no typed variant for artifact kind "test/score"`},
{name: "missing normalizer variant", mutate: func(options *typedCatalogOptions) { options.registerScoreNormalizer = false }, want: `normalizer "typed/normalize" has no typed variant for artifact kind "test/score"`},
{name: "extractor Go type mismatch", mutate: func(options *typedCatalogOptions) { options.scoreExtractorUsesNotes = true }, want: `artifact kind "test/score" requires Go type pipeline.codecScore, got pipeline.codecNotes`},
{name: "wrong validator kind", mutate: func(options *typedCatalogOptions) { options.registerScoreValidator = false }, want: `validator "typed/check" has no typed variant for artifact kind "test/score"; registered kinds: test/notes`},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
options := completeTypedCatalogOptions()
test.mutate(&options)
_, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, options))
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
}
})
}
}
func TestLegacyRawRegistrationCannotSatisfyTypedLane(t *testing.T) {
options := completeTypedCatalogOptions()
options.registerScoreMerger = false
catalog := typedResolutionCatalog(t, options)
if err := catalog.Mergers.RegisterLegacyRaw("typed/merge", func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil {
t.Fatalf("RegisterLegacyRaw() error = %v, want nil", err)
}
_, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err == nil || !strings.Contains(err.Error(), `no typed variant for artifact kind "test/score"`) {
t.Fatalf("ResolvePipeline() error = %v, want typed variant error", err)
}
}
func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
registry := NewMergerRegistry()
spec := ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}
constructor := func() (contracts.Merger[codecNotes], error) {
return typedTestMerger[codecNotes]{key: "typed/merge"}, nil
}
if err := RegisterMerger(registry, spec, constructor); err != nil {
t.Fatalf("RegisterMerger() error = %v, want nil", err)
}
if err := RegisterMerger(registry, spec, constructor); err == nil || !strings.Contains(err.Error(), "already registered") {
t.Fatalf("duplicate RegisterMerger() error = %v, want duplicate variant error", err)
}
}
func TestResolvedPipelineDigestIncludesArtifactSchemaIdentity(t *testing.T) {
baseOptions := completeTypedCatalogOptions()
base, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, baseOptions))
if err != nil {
t.Fatalf("ResolvePipeline(base) error = %v, want nil", err)
}
identityOptions := completeTypedCatalogOptions()
identityOptions.notesCodec.schema.ID = "notes-renamed.v1"
identity, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, identityOptions))
if err != nil {
t.Fatalf("ResolvePipeline(identity) error = %v, want nil", err)
}
if base.Digest == identity.Digest {
t.Fatalf("pipeline digest = %q after schema identity change, want different digest", identity.Digest)
}
digestOptions := completeTypedCatalogOptions()
digestOptions.notesCodec.schema.JSONSchema = []byte(`{"additionalProperties":false,"description":"changed","properties":{"items":{"items":{"type":"string"},"type":"array"}},"required":["items"],"type":"object"}`)
changedSchema, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, digestOptions))
if err != nil {
t.Fatalf("ResolvePipeline(schema bytes) error = %v, want nil", err)
}
if base.Digest == changedSchema.Digest {
t.Fatalf("pipeline digest = %q after schema digest change, want different digest", changedSchema.Digest)
}
}
func typedResolutionCatalog(t *testing.T, options typedCatalogOptions) ModuleCatalog {
t.Helper()
catalog := ModuleCatalog{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(),
ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
mustRegisterTypedTestBase(t, catalog)
if options.registerNotesCodec {
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, options.notesCodec)
}
if options.registerScoreCodec {
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, options.scoreCodec)
}
mustRegisterTypedExtractor(t, catalog.Extractors, "typed/extract-notes", "test/notes", typedTestExtractor[codecNotes]{key: "typed/extract-notes"})
if options.scoreExtractorUsesNotes {
mustRegisterTypedExtractor(t, catalog.Extractors, "typed/extract-score", "test/score", typedTestExtractor[codecNotes]{key: "typed/extract-score"})
} else {
mustRegisterTypedExtractor(t, catalog.Extractors, "typed/extract-score", "test/score", typedTestExtractor[codecScore]{key: "typed/extract-score"})
}
if options.registerNotesMerger {
mustRegisterTypedMerger(t, catalog.Mergers, "test/notes", typedTestMerger[codecNotes]{key: "typed/merge"})
}
if options.registerScoreMerger {
mustRegisterTypedMerger(t, catalog.Mergers, "test/score", typedTestMerger[codecScore]{key: "typed/merge"})
}
if options.registerNotesNormalizer {
mustRegisterTypedNormalizer(t, catalog.Normalizers, "test/notes", typedTestNormalizer[codecNotes]{key: "typed/normalize"})
}
if options.registerScoreNormalizer {
mustRegisterTypedNormalizer(t, catalog.Normalizers, "test/score", typedTestNormalizer[codecScore]{key: "typed/normalize"})
}
mustRegisterTypedValidator(t, catalog.Validators, "test/notes", typedTestValidator[codecNotes]{key: "typed/check"})
if options.registerScoreValidator {
mustRegisterTypedValidator(t, catalog.Validators, "test/score", typedTestValidator[codecScore]{key: "typed/check"})
}
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk/check", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.ChunkValidator, error) {
return typedTestChunkValidator{key: "chunk/check"}, nil
}); err != nil {
t.Fatalf("RegisterChunkValidator() error = %v", err)
}
if err := RegisterSerializedValidator(catalog.Validators, SerializedValidatorSpec{
ValidatorSpec: ValidatorSpec{Key: "serialized/check", ExecutionClass: contracts.ExecutionClassDeterministic},
SupportsArtifacts: true,
}, func() (contracts.SerializedValidator, error) {
return typedTestSerializedValidator{key: "serialized/check"}, nil
}); err != nil {
t.Fatalf("RegisterSerializedValidator() error = %v", err)
}
if err := RegisterSerializedValidator(catalog.Validators, SerializedValidatorSpec{
ValidatorSpec: ValidatorSpec{Key: "serialized/chunks", ExecutionClass: contracts.ExecutionClassDeterministic},
SupportsChunks: true,
}, func() (contracts.SerializedValidator, error) {
return typedTestSerializedValidator{key: "serialized/chunks"}, nil
}); err != nil {
t.Fatalf("RegisterSerializedValidator(chunks) error = %v", err)
}
return catalog
}
func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
t.Helper()
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{Key: "typed/input", Stage: StageInput}, func() (contracts.InputAdapter, error) { return nil, nil }); err != nil {
t.Fatalf("register input: %v", err)
}
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) { return nil, nil }); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{Key: "typed/output", Stage: StageOutput}, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil {
t.Fatalf("register output: %v", err)
}
}
func mustRegisterArtifactCodec[T any](t *testing.T, registry *ArtifactCodecRegistry, codec contracts.ArtifactCodec[T]) {
t.Helper()
if err := RegisterArtifactCodec(registry, codec); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v", err)
}
}
func mustRegisterTypedExtractor[T any](t *testing.T, registry *ExtractorRegistry, key string, kind contracts.ArtifactKind, extractor contracts.Extractor[T]) {
t.Helper()
if err := RegisterExtractor(registry, ModuleSpec{Key: key, Stage: StageExtract, ArtifactKind: kind}, func() (contracts.Extractor[T], error) { return extractor, nil }); err != nil {
t.Fatalf("RegisterExtractor() error = %v", err)
}
}
func mustRegisterTypedMerger[T any](t *testing.T, registry *MergerRegistry, kind contracts.ArtifactKind, merger contracts.Merger[T]) {
t.Helper()
if err := RegisterMerger(registry, ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: kind}, func() (contracts.Merger[T], error) { return merger, nil }); err != nil {
t.Fatalf("RegisterMerger() error = %v", err)
}
}
func mustRegisterTypedNormalizer[T any](t *testing.T, registry *NormalizerRegistry, kind contracts.ArtifactKind, normalizer contracts.Normalizer[T]) {
t.Helper()
if err := RegisterNormalizer(registry, ModuleSpec{Key: "typed/normalize", Stage: StageNormalize, ArtifactKind: kind}, func() (contracts.Normalizer[T], error) { return normalizer, nil }); err != nil {
t.Fatalf("RegisterNormalizer() error = %v", err)
}
}
func mustRegisterTypedValidator[T any](t *testing.T, registry *ValidatorRegistry, kind contracts.ArtifactKind, validator contracts.TypedValidator[T]) {
t.Helper()
if err := RegisterTypedValidator(registry, kind, ValidatorSpec{Key: "typed/check", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[T], error) { return validator, nil }); err != nil {
t.Fatalf("RegisterTypedValidator() error = %v", err)
}
}
func typedResolutionProfile() PipelineProfile {
validatorOverride := ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("typed/check"), Binding("serialized/check")}}
return PipelineProfile{
ID: "typed-pipeline",
Input: Binding("typed/input"),
Chunk: ModuleBinding{Module: "typed/chunk", Validators: ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("chunk/check"), Binding("serialized/chunks")}}},
Artifacts: map[string]ArtifactLaneProfile{
"score": typedLaneProfile("typed/extract-score", validatorOverride),
"notes": typedLaneProfile("typed/extract-notes", validatorOverride),
},
Output: Binding("typed/output"),
}
}
func typedLaneProfile(extractor string, validators ValidatorOverride) ArtifactLaneProfile {
return ArtifactLaneProfile{
Extract: ModuleBinding{Module: extractor, Validators: validators},
Merge: Binding("typed/merge"),
Normalize: Binding("typed/normalize"),
}
}
func assertResolvedArtifactIdentity(t *testing.T, lane ResolvedArtifactLane, kind contracts.ArtifactKind, schemaID string) {
t.Helper()
if lane.ArtifactKind != kind || lane.ArtifactSchemaID != schemaID || lane.ArtifactSchemaName == "" || lane.ArtifactSchemaVersion == "" || lane.ArtifactSchemaDigest == "" {
t.Fatalf("resolved lane identity = %#v, want kind %q schema %q with complete metadata", lane, kind, schemaID)
}
}
func resolvedLaneIDs(lanes []ResolvedArtifactLane) []string {
ids := make([]string, len(lanes))
for i, lane := range lanes {
ids[i] = lane.ID
}
return ids
}
func resolvedValidatorTargets(validators []ResolvedValidator) []ValidatorTarget {
targets := make([]ValidatorTarget, len(validators))
for i, validator := range validators {
targets[i] = validator.Target
}
return targets
}
var _ contracts.Extractor[codecNotes] = typedTestExtractor[codecNotes]{}
var _ contracts.Merger[codecNotes] = typedTestMerger[codecNotes]{}
var _ contracts.Normalizer[codecNotes] = typedTestNormalizer[codecNotes]{}
var _ contracts.TypedValidator[codecNotes] = typedTestValidator[codecNotes]{}
var _ contracts.ChunkValidator = typedTestChunkValidator{}
var _ contracts.SerializedValidator = typedTestSerializedValidator{}

247
internal/framework/pipeline/validator_registry.go
View File

@@ -2,40 +2,78 @@ package pipeline
import (
"fmt"
"reflect"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type ValidatorConstructor func() (contracts.Validator, error)
type LegacyRawValidatorConstructor func() (contracts.LegacyRawValidator, error)
type ValidatorSpec struct {
Key string `json:"key"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}
type SerializedValidatorSpec struct {
ValidatorSpec
SupportsChunks bool `json:"supports_chunks,omitempty"`
SupportsArtifacts bool `json:"supports_artifacts,omitempty"`
}
type ValidatorTarget string
const (
ValidatorTargetLegacyRaw ValidatorTarget = "legacy_raw"
ValidatorTargetChunk ValidatorTarget = "chunk"
ValidatorTargetSerialized ValidatorTarget = "serialized"
ValidatorTargetTyped ValidatorTarget = "typed"
)
type ValidatorRegistry struct {
constructors map[string]ValidatorConstructor
specs map[string]ValidatorSpec
legacyConstructors map[string]LegacyRawValidatorConstructor
legacySpecs map[string]ValidatorSpec
typedEntries map[artifactVariantKey]typedValidatorEntry
chunkEntries map[string]chunkValidatorEntry
serializedEntries map[string]serializedValidatorEntry
}
type typedValidatorEntry struct {
spec ValidatorSpec
kind contracts.ArtifactKind
valueType reflect.Type
constructor func() (any, error)
}
type chunkValidatorEntry struct {
spec ValidatorSpec
constructor func() (contracts.ChunkValidator, error)
}
type serializedValidatorEntry struct {
spec SerializedValidatorSpec
constructor func() (contracts.SerializedValidator, error)
}
func NewValidatorRegistry() *ValidatorRegistry {
return &ValidatorRegistry{
constructors: make(map[string]ValidatorConstructor),
specs: make(map[string]ValidatorSpec),
legacyConstructors: make(map[string]LegacyRawValidatorConstructor),
legacySpecs: make(map[string]ValidatorSpec),
typedEntries: make(map[artifactVariantKey]typedValidatorEntry),
chunkEntries: make(map[string]chunkValidatorEntry),
serializedEntries: make(map[string]serializedValidatorEntry),
}
}
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
return r.RegisterWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
func (r *ValidatorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawValidatorConstructor) error {
return r.RegisterLegacyRawWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
}
func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor ValidatorConstructor) error {
func (r *ValidatorRegistry) RegisterLegacyRawWithSpec(spec ValidatorSpec, constructor LegacyRawValidatorConstructor) error {
if r == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
@@ -43,36 +81,111 @@ func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor Val
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
return fmt.Errorf("validator %q is already registered", normalizedSpec.Key)
if _, ok := r.legacyConstructors[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw validator %q is already registered", normalizedSpec.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]ValidatorConstructor)
if r.legacyConstructors == nil {
r.legacyConstructors = make(map[string]LegacyRawValidatorConstructor)
}
if r.specs == nil {
r.specs = make(map[string]ValidatorSpec)
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ValidatorSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = normalizedSpec
r.legacyConstructors[normalizedSpec.Key] = constructor
r.legacySpecs[normalizedSpec.Key] = normalizedSpec
return nil
}
func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
func RegisterTypedValidator[T any](registry *ValidatorRegistry, kind contracts.ArtifactKind, spec ValidatorSpec, constructor func() (contracts.TypedValidator[T], error)) error {
if registry == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
}
kind = normalizeArtifactKind(kind)
if kind == "" {
return fmt.Errorf("typed validator %q artifact kind must not be empty", normalizedSpec.Key)
}
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", normalizedSpec.Key)
}
key := artifactVariantKey{module: normalizedSpec.Key, kind: kind}
if _, ok := registry.typedEntries[key]; ok {
return fmt.Errorf("validator %q variant for artifact kind %q is already registered", key.module, key.kind)
}
if registry.typedEntries == nil {
registry.typedEntries = make(map[artifactVariantKey]typedValidatorEntry)
}
registry.typedEntries[key] = typedValidatorEntry{
spec: normalizedSpec,
kind: kind,
valueType: reflect.TypeFor[T](),
constructor: func() (any, error) {
return constructor()
},
}
return nil
}
func RegisterChunkValidator(registry *ValidatorRegistry, spec ValidatorSpec, constructor func() (contracts.ChunkValidator, error)) error {
if registry == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
}
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", normalizedSpec.Key)
}
if _, ok := registry.chunkEntries[normalizedSpec.Key]; ok {
return fmt.Errorf("chunk validator %q is already registered", normalizedSpec.Key)
}
if registry.chunkEntries == nil {
registry.chunkEntries = make(map[string]chunkValidatorEntry)
}
registry.chunkEntries[normalizedSpec.Key] = chunkValidatorEntry{spec: normalizedSpec, constructor: constructor}
return nil
}
func RegisterSerializedValidator(registry *ValidatorRegistry, spec SerializedValidatorSpec, constructor func() (contracts.SerializedValidator, error)) error {
if registry == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedValidatorSpec, err := normalizeValidatorSpec(spec.ValidatorSpec)
if err != nil {
return err
}
spec.ValidatorSpec = normalizedValidatorSpec
if !spec.SupportsChunks && !spec.SupportsArtifacts {
return fmt.Errorf("serialized validator %q must support chunks, artifacts, or both", spec.Key)
}
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", spec.Key)
}
if _, ok := registry.serializedEntries[spec.Key]; ok {
return fmt.Errorf("serialized validator %q is already registered", spec.Key)
}
if registry.serializedEntries == nil {
registry.serializedEntries = make(map[string]serializedValidatorEntry)
}
registry.serializedEntries[spec.Key] = serializedValidatorEntry{spec: spec, constructor: constructor}
return nil
}
func (r *ValidatorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawValidator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("validator key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
constructor, ok := r.legacyConstructors[normalizedKey]
if !ok {
return nil, fmt.Errorf("validator %q is not registered", normalizedKey)
return nil, fmt.Errorf("legacy raw validator %q is not registered", normalizedKey)
}
validator, err := constructor()
if err != nil {
return nil, fmt.Errorf("build validator %q: %w", normalizedKey, err)
@@ -83,14 +196,10 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
spec, ok := r.specs[normalizedKey]
if !ok {
return nil, fmt.Errorf("validator %q spec is not registered", normalizedKey)
}
spec := r.legacySpecs[normalizedKey]
if validator.ExecutionClass() != spec.ExecutionClass {
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
}
return validator, nil
}
@@ -98,28 +207,57 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
if r == nil {
return ValidatorSpec{}, false
}
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
return spec, ok
}
spec, ok := r.specs[strings.TrimSpace(key)]
if !ok {
return ValidatorSpec{}, false
func (r *ValidatorRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedValidatorEntry, bool) {
if r == nil {
return typedValidatorEntry{}, false
}
return spec, true
entry, ok := r.typedEntries[artifactVariantKey{module: strings.TrimSpace(key), kind: normalizeArtifactKind(kind)}]
return entry, ok
}
func (r *ValidatorRegistry) chunkEntry(key string) (chunkValidatorEntry, bool) {
if r == nil {
return chunkValidatorEntry{}, false
}
entry, ok := r.chunkEntries[strings.TrimSpace(key)]
return entry, ok
}
func (r *ValidatorRegistry) serializedEntry(key string) (serializedValidatorEntry, bool) {
if r == nil {
return serializedValidatorEntry{}, false
}
entry, ok := r.serializedEntries[strings.TrimSpace(key)]
return entry, ok
}
func (r *ValidatorRegistry) registeredTypedKinds(key string) []contracts.ArtifactKind {
if r == nil {
return nil
}
module := strings.TrimSpace(key)
kinds := make([]contracts.ArtifactKind, 0)
for variant := range r.typedEntries {
if variant.module == module {
kinds = append(kinds, variant.kind)
}
}
sortArtifactKinds(kinds)
return kinds
}
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
if r == nil || len(r.specs) == 0 {
if r == nil || len(r.legacySpecs) == 0 {
return nil
}
keys := make([]string, 0, len(r.specs))
for key := range r.specs {
keys = append(keys, key)
}
sort.Strings(keys)
keys := sortedRegistryKeys(r.legacySpecs)
specs := make([]ValidatorSpec, 0, len(keys))
for _, key := range keys {
specs = append(specs, r.specs[key])
specs = append(specs, r.legacySpecs[key])
}
return specs
}
@@ -128,15 +266,24 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
return sortedRegistryKeys(r.constructors)
keys := make(map[string]struct{})
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}
for key := range r.chunkEntries {
keys[key] = struct{}{}
}
for key := range r.serializedEntries {
keys[key] = struct{}{}
}
return sortedRegistryKeys(keys)
}
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
normalized := ValidatorSpec{
Key: strings.TrimSpace(spec.Key),
ExecutionClass: spec.ExecutionClass,
}
normalized := ValidatorSpec{Key: strings.TrimSpace(spec.Key), ExecutionClass: spec.ExecutionClass}
if normalized.Key == "" {
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
}
@@ -147,3 +294,7 @@ func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
}
return normalized, nil
}
func sortValidatorSpecs(specs []ValidatorSpec) {
sort.Slice(specs, func(i, j int) bool { return specs[i].Key < specs[j].Key })
}

18
internal/framework/pipeline/validator_registry_test.go
View File

@@ -11,11 +11,11 @@ import (
func TestValidatorRegistryBehavior(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.Register(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
if err := registry.RegisterLegacyRaw(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build("generic-validator")
validator, err := registry.BuildLegacyRaw("generic-validator")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
@@ -36,7 +36,7 @@ func TestValidatorRegistryBehavior(t *testing.T) {
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
registry := NewValidatorRegistry()
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
if err := registry.RegisterWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
if err := registry.RegisterLegacyRawWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
@@ -53,7 +53,7 @@ func TestValidatorRegistryRegistersSpecs(t *testing.T) {
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
registry := NewValidatorRegistry()
for _, key := range []string{"zeta", "alpha"} {
if err := registry.Register(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
if err := registry.RegisterLegacyRaw(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register(%q) error = %v", key, err)
}
}
@@ -66,7 +66,7 @@ func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
err := registry.RegisterWithSpec(
err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
)
@@ -77,14 +77,14 @@ func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterWithSpec(
if err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := registry.Build("validator")
_, err := registry.BuildLegacyRaw("validator")
if err == nil {
t.Fatal("Build() error = nil, want execution class mismatch")
}
@@ -98,8 +98,8 @@ type testValidator struct {
executionClass contracts.ExecutionClass
}
func validatorConstructor(name string, executionClass contracts.ExecutionClass) ValidatorConstructor {
return func() (contracts.Validator, error) {
func validatorConstructor(name string, executionClass contracts.ExecutionClass) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return testValidator{name: name, executionClass: executionClass}, nil
}
}

16
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -51,12 +51,12 @@ func TestWalkingSkeletonFixture(t *testing.T) {
func TestWalkingSkeletonResolutionRejectsMissingCapability(t *testing.T) {
catalog := walkingSkeletonCatalog(t)
catalog.Extractors = NewExtractorRegistry()
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"missing"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.Extractor, error) {
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
@@ -125,29 +125,29 @@ func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
}); err != nil {
t.Fatalf("register fake chunker: %v", err)
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"chunks"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.Extractor, error) {
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("register fake extractor: %v", err)
}
if err := catalog.Mergers.RegisterWithSpec(ModuleSpec{
if err := catalog.Mergers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultMergeModule,
Stage: StageMerge,
Requires: []string{"fake_artifacts"},
}, func() (contracts.Merger, error) {
}, func() (contracts.LegacyRawMerger, error) {
return walkingSkeletonMerger{}, nil
}); err != nil {
t.Fatalf("register append-order merger: %v", err)
}
if err := catalog.Normalizers.RegisterWithSpec(ModuleSpec{
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultNormalizeModule,
Stage: StageNormalize,
}, func() (contracts.Normalizer, error) {
}, func() (contracts.LegacyRawNormalizer, error) {
return walkingSkeletonNormalizer{}, nil
}); err != nil {
t.Fatalf("register no-op normalizer: %v", err)