eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy raw pipeline contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
internal/modules/generic/merge/appendorder/merger.go
View File

@@ -1,234 +1,15 @@
package appendorder
import (
"context"
"encoding/json"
"fmt"
"mime"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "appendorder"
var _ contracts.LegacyRawMerger = (*Merger)(nil)
type Merger struct{}
func New() *Merger {
return &Merger{}
}
func (m *Merger) Key() string {
return Key
}
func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
if m == nil {
return contracts.MergeResult{}, mergerErrorf("merger must not be nil")
}
if ctx == nil {
return contracts.MergeResult{}, mergerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
}
outputs, err := orderedOutputs(req.ExtractOutputs)
if err != nil {
return contracts.MergeResult{}, err
}
if len(outputs) == 1 {
payload := cloneRawPayload(outputs[0].Payload)
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: outputs[0].SourceID,
Schema: outputs[0].Schema,
Payload: payload,
},
}, nil
}
content, err := mergedContent(outputs)
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: sourceID(outputs),
Schema: commonSchema(outputs),
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
}
func Register(registry *pipeline.MergerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawMerger, error) {
return New(), nil
})
}
func orderedOutputs(outputs []contracts.ExtractOutput) ([]contracts.ExtractOutput, error) {
ordered := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
if !isJSONMediaType(output.Payload.MediaType) {
return nil, mergerErrorf("extract output for chunk %q has unsupported media type %q", output.ChunkID, output.Payload.MediaType)
}
if !json.Valid(output.Payload.Content) {
return nil, mergerErrorf("extract output for chunk %q contains invalid JSON", output.ChunkID)
}
ordered = append(ordered, cloneExtractOutput(output))
}
sort.SliceStable(ordered, func(i, j int) bool {
return ordered[i].ChunkIndex < ordered[j].ChunkIndex
})
return ordered, nil
}
func mergedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
values := make([]any, 0, len(outputs))
objects := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
var value any
if err := json.Unmarshal(output.Payload.Content, &value); err != nil {
return nil, mergerErrorf("decode extract output for chunk %q: %w", output.ChunkID, err)
}
values = append(values, value)
object, ok := value.(map[string]any)
if !ok {
continue
}
objects = append(objects, object)
}
if len(objects) == len(outputs) {
if field, ok := commonArrayField(objects); ok {
merged := make([]any, 0)
for _, object := range objects {
items := object[field].([]any)
merged = append(merged, items...)
}
return marshalMerged(map[string]any{field: merged})
}
}
return marshalMerged(values)
}
func commonArrayField(objects []map[string]any) (string, bool) {
if len(objects) == 0 {
return "", false
}
candidates := map[string]struct{}{}
for key, value := range objects[0] {
if _, ok := value.([]any); ok {
candidates[key] = struct{}{}
}
}
for _, object := range objects[1:] {
for key := range candidates {
if _, ok := object[key].([]any); !ok {
delete(candidates, key)
}
}
}
if len(candidates) != 1 {
return "", false
}
for key := range candidates {
return key, true
}
return "", false
}
func marshalMerged(value any) ([]byte, error) {
content, err := json.Marshal(value)
if err != nil {
return nil, mergerErrorf("encode merged output: %w", err)
}
return content, nil
}
func isJSONMediaType(mediaType string) bool {
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
if err != nil {
base = strings.TrimSpace(mediaType)
}
return strings.EqualFold(base, "application/json")
}
func sourceID(outputs []contracts.ExtractOutput) string {
for _, output := range outputs {
if output.SourceID != "" {
return output.SourceID
}
}
return ""
}
func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
if len(outputs) == 0 {
return contracts.ResponseSchema{}
}
schema := outputs[0].Schema
for _, output := range outputs[1:] {
if !sameResponseSchema(output.Schema, schema) {
return contracts.ResponseSchema{}
}
}
return schema
}
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageMerge, Provides: []string{"merged"}}
}
func mergerErrorf(format string, args ...any) error {

215
internal/modules/generic/merge/appendorder/merger_test.go
View File

@@ -1,215 +0,0 @@
package appendorder
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewMergerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
}
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestMergeConcatenatesCommonTopLevelArrayFieldInChunkOrder(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"events":[{"name":"second"}]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
var decoded struct {
Events []struct {
Name string `json:"name"`
} `json:"events"`
}
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded.Events) != 2 || decoded.Events[0].Name != "first" || decoded.Events[1].Name != "second" {
t.Fatalf("events = %#v, want concatenated chunk order", decoded.Events)
}
if result.Output.Schema.ID != "schema-id" {
t.Fatalf("schema = %#v, want common extract schema", result.Output.Schema)
}
}
func TestMergeFallsBackToOrderedJSONValueArrayWhenShapesDiffer(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"notes":["second"]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
var decoded []map[string]any
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded) != 2 {
t.Fatalf("len(decoded) = %d, want 2", len(decoded))
}
if _, ok := decoded[0]["events"]; !ok {
t.Fatalf("decoded[0] = %#v, want first chunk value", decoded[0])
}
if _, ok := decoded[1]["notes"]; !ok {
t.Fatalf("decoded[1] = %#v, want second chunk value", decoded[1])
}
}
func TestMergeRejectsInvalidJSONAndNonJSONMediaTypes(t *testing.T) {
tests := []struct {
name string
output contracts.ExtractOutput
want string
}{
{
name: "invalid JSON",
output: extractOutput("chunk-0", 0, `{"events":[`),
want: "invalid JSON",
},
{
name: "non JSON media type",
output: func() contracts.ExtractOutput {
output := extractOutput("chunk-0", 0, `{"events":[]}`)
output.Payload.MediaType = "text/plain"
return output
}(),
want: "unsupported media type",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{test.output},
})
if err == nil {
t.Fatal("Merge() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("Merge() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestTypedMergeUsesRequestOrderForReusableValueType(t *testing.T) {
type notes struct{ Values []string }
merger, err := NewTyped(func(values []notes) (notes, error) {
var combined notes
for _, value := range values {
combined.Values = append(combined.Values, value.Values...)
}
return combined, nil
})
if err != nil {
t.Fatalf("NewTyped() error = %v", err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[notes]{ExtractOutputs: []contracts.ExtractArtifact[notes]{
{ChunkIndex: 4, Value: notes{Values: []string{"first"}}},
{ChunkIndex: 1, Value: notes{Values: []string{"second"}}},
}})
if err != nil {
t.Fatalf("Merge() error = %v", err)
}
if got := result.Value.Values; !reflect.DeepEqual(got, []string{"first", "second"}) {
t.Fatalf("Values = %#v", got)
}
}
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
return contracts.ExtractOutput{
LaneID: "events",
ExtractorKey: "extract",
SourceID: "source-1",
ChunkID: chunkID,
ChunkIndex: chunkIndex,
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": chunkID},
},
}
}

25
internal/modules/generic/merge/appendorder/typed_test.go Normal file
View File

@@ -0,0 +1,25 @@
package appendorder
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedMergerCombinesValuesInFrameworkOrder(t *testing.T) {
merger, err := NewTyped(func(values []string) (string, error) {
if !reflect.DeepEqual(values, []string{"first", "second"}) {
t.Fatalf("values=%#v", values)
}
return values[0] + values[1], nil
})
if err != nil {
t.Fatal(err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[string]{ExtractOutputs: []contracts.ExtractArtifact[string]{{ChunkIndex: 0, Value: "first"}, {ChunkIndex: 1, Value: "second"}}})
if err != nil || result.Value != "firstsecond" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

72
internal/modules/generic/normalize/noop/normalizer.go
View File

@@ -1,85 +1,15 @@
package noop
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "noop"
var _ contracts.LegacyRawNormalizer = (*Normalizer)(nil)
type Normalizer struct{}
func New() *Normalizer {
return &Normalizer{}
}
func (n *Normalizer) Key() string {
return Key
}
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
}
if ctx == nil {
return contracts.NormalizeResult{}, normalizerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
}
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: Key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneRawPayload(req.MergeOutput.Payload),
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
}
func Register(registry *pipeline.NormalizerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawNormalizer, error) {
return New(), nil
})
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}
}
func normalizerErrorf(format string, args ...any) error {

112
internal/modules/generic/normalize/noop/normalizer_test.go
View File

@@ -1,112 +0,0 @@
package noop
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewNormalizerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
normalizer, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestTypedNormalizePassesThroughReusableValueType(t *testing.T) {
type score struct{ Value int }
result, err := NewTyped[score]().Normalize(context.Background(), contracts.TypedNormalizeRequest[score]{
MergeOutput: contracts.MergeArtifact[score]{Value: score{Value: 7}},
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if result.Value.Value != 7 {
t.Fatalf("Value = %d, want 7", result.Value.Value)
}
}
func mergeOutput(content string) contracts.MergeOutput {
return contracts.MergeOutput{
LaneID: "events",
MergerKey: "merge",
SourceID: "source-1",
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": "original"},
},
}
}

16
internal/modules/generic/normalize/noop/typed_test.go Normal file
View File

@@ -0,0 +1,16 @@
package noop
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedNormalizerPreservesValue(t *testing.T) {
normalizer := NewTyped[string]()
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
if err != nil || result.Value != "value" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

9
internal/modules/generic/output/json/encoder.go
View File

@@ -253,15 +253,6 @@ func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.Ser
return out
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return []contracts.RejectedOutput{}

4
internal/modules/generic/register/register.go
View File

@@ -7,8 +7,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
@@ -27,8 +25,6 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
register func() error
}{
{name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }},
{name: "appendorder merger", register: func() error { return appendorder.Register(registries.Mergers) }},
{name: "noop normalizer", register: func() error { return noop.Register(registries.Normalizers) }},
{name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }},
{name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }},
{name: "valid json validator", register: func() error { return validjson.Register(registries.Validators) }},

4
internal/modules/generic/register/register_test.go
View File

@@ -14,8 +14,8 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
t.Fatalf("Register() error = %v, want nil", err)
}
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), nil)
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), nil)
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
"generic/always_accept",
"generic/always_reject",

19
internal/modules/generic/validate/always_accept/validator.go
View File

@@ -12,7 +12,6 @@ const Key = "generic/always_accept"
type Options struct{}
type ChunkValidator struct{}
type TypedValidator[T any] struct{}
type legacyValidator struct{}
var _ contracts.ChunkValidator = (*ChunkValidator)(nil)
@@ -35,33 +34,17 @@ func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationR
return contracts.ValidationResult{Approved: true}, nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

8
internal/modules/generic/validate/always_accept/validator_test.go
View File

@@ -30,11 +30,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

19
internal/modules/generic/validate/always_reject/validator.go
View File

@@ -13,7 +13,6 @@ const ReasonCode = "always_reject"
type Options struct{}
type ChunkValidator struct{}
type TypedValidator[T any] struct{}
type legacyValidator struct{}
func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} }
func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} }
@@ -35,32 +34,16 @@ func (v *TypedValidator[T]) ExecutionClass() contracts.ExecutionClass {
func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) {
return rejection(), nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return rejection(), nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}
func RegisterTyped[T any](registry *pipeline.ValidatorRegistry, kind contracts.ArtifactKind) error {

8
internal/modules/generic/validate/always_reject/validator_test.go
View File

@@ -33,11 +33,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

23
internal/modules/generic/validate/valid_json/validator.go
View File

@@ -15,10 +15,7 @@ type Options struct{}
type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} }
@@ -32,16 +29,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content), nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content), nil
}
func validate(content []byte) contracts.ValidationResult {
if !json.Valid(content) {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}
@@ -54,7 +41,7 @@ func Spec() pipeline.ValidatorSpec {
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options)
@@ -62,14 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err
}
return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

8
internal/modules/generic/validate/valid_json/validator_test.go
View File

@@ -47,12 +47,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

20
internal/modules/generic/validate/valid_json_schema/validator.go
View File

@@ -17,10 +17,8 @@ const ReasonCodeSchemaInvalid = "json_schema_invalid"
type Options struct{}
type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key }
@@ -32,14 +30,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content, req.Schema.JSONSchema)
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content, req.Schema.JSONSchema)
}
func validate(content, schemaContent []byte) (contracts.ValidationResult, error) {
if len(schemaContent) == 0 {
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
@@ -71,7 +61,7 @@ func Spec() pipeline.ValidatorSpec {
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options)
@@ -79,14 +69,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err
}
return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

8
internal/modules/generic/validate/valid_json_schema/validator_test.go
View File

@@ -74,12 +74,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}