eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy raw pipeline contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
internal/modules/seriatim/input/transcript/config_test.go
View File

@@ -2,12 +2,14 @@ package transcript
import (
"context"
"encoding/json"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
@@ -131,29 +133,36 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
Provides: []string{"chunks"},
})
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
Key: "fake/extract",
Stage: pipeline.StageExtract,
ArtifactKind: seriatimArtifactKind,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
})
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
Requires: []string{"fake.artifacts"},
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
ArtifactKind: seriatimArtifactKind,
Requires: []string{"fake.artifacts"},
})
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
ArtifactKind: seriatimArtifactKind,
})
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
Key: pipeline.DefaultOutputModule,
Stage: pipeline.StageOutput,
})
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
@@ -173,7 +182,7 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) {
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) {
return fakeExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
@@ -182,8 +191,13 @@ func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, s
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) {
return appendorder.New(), nil
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) {
return appendorder.NewTyped(func(values []seriatimArtifact) (seriatimArtifact, error) {
if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
})
}); err != nil {
t.Fatalf("register merger: %v", err)
}
@@ -191,8 +205,8 @@ func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pi
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) {
return noop.New(), nil
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) {
return noop.NewTyped[seriatimArtifact](), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
@@ -223,8 +237,8 @@ func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
func (fakeExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
}
type fakeOutput struct{}
@@ -246,7 +260,30 @@ func withoutCapability(capabilities []string, capability string) []string {
}
var (
_ contracts.Chunker = fakeChunker{}
_ contracts.LegacyRawExtractor = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{}
_ contracts.Chunker = fakeChunker{}
_ contracts.Extractor[seriatimArtifact] = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{}
)
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
type seriatimArtifact struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
type seriatimArtifactCodec struct{}
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
return json.Marshal(value)
}
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
var value seriatimArtifact
err := json.Unmarshal(content, &value)
return value, err
}

73
internal/modules/seriatim/input/transcript/runner_test.go
View File

@@ -55,16 +55,16 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Artifact.Schema.ID != "fake.event" || rawOutput.Artifact.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
serializedOutput := output.NormalizeOutputs[0]
if serializedOutput.LaneID != "events" || serializedOutput.NormalizerKey != pipeline.DefaultNormalizeModule || serializedOutput.Artifact.Schema.ID != "fake.event" || serializedOutput.Artifact.Schema.Version != "v1" {
t.Fatalf("serialized output envelope = %#v, want fake extractor envelope", serializedOutput)
}
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := json.Unmarshal(rawOutput.Artifact.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
if err := json.Unmarshal(serializedOutput.Artifact.Content, &payload); err != nil {
t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
@@ -115,7 +115,7 @@ func configResolveInput(t *testing.T) config.ResolveInput {
}
}
func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtractor) pipeline.Registries {
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
@@ -133,19 +133,20 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := extractors.RegisterLegacyRaw("fake/extract", func() (contracts.LegacyRawExtractor, error) {
if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := mergers.RegisterLegacyRaw(pipeline.DefaultMergeModule, func() (contracts.LegacyRawMerger, error) {
return appendorder.New(), nil
if err := appendorder.RegisterTyped(mergers, seriatimArtifactKind, func(values []seriatimArtifact) (seriatimArtifact, error) {
if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := normalizers.RegisterLegacyRaw(pipeline.DefaultNormalizeModule, func() (contracts.LegacyRawNormalizer, error) {
return noop.New(), nil
}); err != nil {
if err := noop.RegisterTyped[seriatimArtifact](normalizers, seriatimArtifactKind); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
@@ -154,10 +155,14 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
t.Fatalf("register output: %v", err)
}
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.Registries{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
@@ -203,36 +208,33 @@ func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
e.calls++
if req.Source == nil {
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil")
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
}
if req.Chunk == nil {
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
}
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
}
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
}
for _, unit := range req.Chunk.Units {
if speaker, ok := Speaker(unit); !ok || speaker == "" {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
}
if _, ok := Start(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
}
if _, ok := End(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
}
}
payload, err := json.Marshal(struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
@@ -241,20 +243,7 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
}}, nil
}
type runnerSeriatimOutput struct{}
@@ -292,7 +281,7 @@ func equalInts(a, b []int) bool {
}
var (
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.LegacyRawExtractor = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{}
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{}
)