Organize generic and Seriatim modules by domain

This commit is contained in:
2026-07-17 04:57:47 +00:00
parent 0327659355
commit a81b9f1e1f
36 changed files with 83 additions and 78 deletions

View File

@@ -0,0 +1,205 @@
package transcript
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"math"
"math/big"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "seriatim"
const (
DocumentKind = "transcript"
UnitKind = "transcript_segment"
Format = "application/vnd.seriatim+json"
)
var providedCapabilities = []string{
"source.transcript",
"transcript.speaker",
"transcript.timestamps",
}
var _ contracts.InputAdapter = (*Adapter)(nil)
type Adapter struct{}
func New() *Adapter {
return &Adapter{}
}
func (a *Adapter) Key() string {
return Key
}
func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
if ctx == nil {
return nil, inputErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return nil, inputErrorf("context error before parsing: %w", err)
}
if len(req.Raw) == 0 {
return nil, inputErrorf("raw input must not be empty")
}
parsed, err := decodeTranscript(req.Raw)
if err != nil {
return nil, inputErrorf("parse JSON: %w", err)
}
if len(parsed.Segments) == 0 {
return nil, inputErrorf("segments must not be empty")
}
rawDigest := digest(req.Raw)
doc := &source.SourceDocument{
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
Kind: DocumentKind,
Format: Format,
Digest: rawDigest,
Metadata: copyMetadata(parsed.Metadata),
}
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
for i, segment := range parsed.Segments {
unit, err := sourceUnit(segment, i, seenSegmentIDs)
if err != nil {
return nil, err
}
doc.Units = append(doc.Units, unit)
}
if err := source.ValidateDocument(doc); err != nil {
return nil, inputErrorf("validate source document: %w", err)
}
return doc, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageInput,
Provides: append([]string(nil), providedCapabilities...),
}
}
func Register(registry *pipeline.InputAdapterRegistry) error {
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.InputAdapter, error) {
return New(), nil
})
}
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
segmentLabel := fmt.Sprintf("segment[%d]", index)
if segment.ID <= 0 {
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
}
if _, ok := seen[segment.ID]; ok {
return source.SourceUnit{}, inputErrorf("segment id %d is duplicated", segment.ID)
}
seen[segment.ID] = struct{}{}
speaker := strings.TrimSpace(segment.Speaker)
if speaker == "" {
return source.SourceUnit{}, inputErrorf("segment %d speaker must not be empty", segment.ID)
}
start, err := validTimestamp(segment.Start, fmt.Sprintf("segment %d start", segment.ID))
if err != nil {
return source.SourceUnit{}, err
}
end, err := validTimestamp(segment.End, fmt.Sprintf("segment %d end", segment.ID))
if err != nil {
return source.SourceUnit{}, err
}
if end.Cmp(start) < 0 {
return source.SourceUnit{}, inputErrorf("segment %d end must be greater than or equal to start", segment.ID)
}
if strings.TrimSpace(segment.Text) == "" {
return source.SourceUnit{}, inputErrorf("segment %d text must not be empty", segment.ID)
}
return source.SourceUnit{
ID: segment.ID,
Kind: UnitKind,
Text: segment.Text,
Metadata: map[string]any{
MetadataSpeaker: segment.Speaker,
MetadataStart: segment.Start,
MetadataEnd: segment.End,
},
}, nil
}
func validTimestamp(value fmt.Stringer, label string) (*big.Rat, error) {
raw := strings.TrimSpace(value.String())
if raw == "" {
return nil, inputErrorf("%s must not be empty", label)
}
parsed, err := strconv.ParseFloat(raw, 64)
if err != nil {
return nil, inputErrorf("%s must be a valid number: %w", label, err)
}
if math.IsInf(parsed, 0) || math.IsNaN(parsed) {
return nil, inputErrorf("%s must be finite", label)
}
if parsed < 0 {
return nil, inputErrorf("%s must not be negative", label)
}
rat, ok := new(big.Rat).SetString(raw)
if !ok {
return nil, inputErrorf("%s must be a valid number", label)
}
return rat, nil
}
func documentID(requestedID string, metadata map[string]any, rawDigest string) string {
if id := strings.TrimSpace(requestedID); id != "" {
return id
}
if id := stringMetadata(metadata, "id"); id != "" {
return id
}
if id := stringMetadata(metadata, "source_id"); id != "" {
return id
}
return "seriatim:" + strings.TrimPrefix(rawDigest, "sha256:")[:16]
}
func stringMetadata(metadata map[string]any, key string) string {
value, ok := metadata[key].(string)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func copyMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
copied := make(map[string]any, len(metadata))
for key, value := range metadata {
copied[key] = value
}
return copied
}
func digest(raw []byte) string {
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
func inputErrorf(format string, args ...any) error {
return fmt.Errorf("seriatim input: "+format, args...)
}

View File

@@ -0,0 +1,324 @@
package transcript
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestParseValidMinimalTranscript(t *testing.T) {
raw := readFixture(t, "testdata/valid_minimal.json")
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if doc.ID != "session-alpha" {
t.Fatalf("doc.ID = %q, want session-alpha", doc.ID)
}
if doc.Kind != DocumentKind {
t.Fatalf("doc.Kind = %q, want %q", doc.Kind, DocumentKind)
}
if doc.Format != Format {
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
}
if doc.Digest != testDigest(raw) {
t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw))
}
if got := doc.Metadata["title"]; got != "Synthetic session transcript" {
t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got)
}
if len(doc.Units) != 2 {
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
}
first := doc.Units[0]
if first.ID != 1 {
t.Fatalf("first.ID = %d, want 1", first.ID)
}
if first.Kind != UnitKind {
t.Fatalf("first.Kind = %q, want %q", first.Kind, UnitKind)
}
if first.Text != "The stone door opens." {
t.Fatalf("first.Text = %q, want fixture text", first.Text)
}
if speaker, ok := Speaker(first); !ok || speaker != "Narrator" {
t.Fatalf("Speaker(first) = %q, %v; want Narrator, true", speaker, ok)
}
if start, ok := Start(first); !ok || start != json.Number("0") {
t.Fatalf("Start(first) = %q, %v; want 0, true", start, ok)
}
if end, ok := End(first); !ok || end != json.Number("4.5") {
t.Fatalf("End(first) = %q, %v; want 4.5, true", end, ok)
}
ref := source.SourceRef{
SourceID: doc.ID,
StartUnitID: doc.Units[0].ID,
EndUnitID: doc.Units[len(doc.Units)-1].ID,
}
if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
}
func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
raw := []byte(`{"metadata":{"application":"seriatim","version":"v1.5.0","output_schema":"seriatim-intermediate"},"segments":[{"id":1,"start":451.821,"end":469.685,"speaker":"Narrator","text":"The stone door opens.","categories":["scene"]},{"id":2,"start":470,"end":471,"speaker":"Player","text":"I step inside."}]}`)
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if got, want := doc.Metadata["output_schema"], "seriatim-intermediate"; got != want {
t.Fatalf("doc.Metadata[output_schema] = %#v, want %q", got, want)
}
if doc.Format != Format {
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
}
if len(doc.Units) != 2 {
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
}
if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 {
t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID})
}
ref := source.SourceRef{
SourceID: doc.ID,
StartUnitID: 1,
EndUnitID: 2,
}
if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
}
func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) {
raw := readFixture(t, "testdata/valid_minimal.json")
doc, err := New().Parse(context.Background(), contracts.ParseRequest{
SourceID: " requested-source ",
Raw: raw,
})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if doc.ID != "requested-source" {
t.Fatalf("doc.ID = %q, want requested-source", doc.ID)
}
}
func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) {
raw := []byte(`{"metadata":{},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
first, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("first Parse() error = %v, want nil", err)
}
second, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("second Parse() error = %v, want nil", err)
}
if first.ID != second.ID {
t.Fatalf("fallback IDs differ: %q vs %q", first.ID, second.ID)
}
if !strings.HasPrefix(first.ID, "seriatim:") {
t.Fatalf("fallback ID = %q, want seriatim prefix", first.ID)
}
if first.ID != "seriatim:"+strings.TrimPrefix(testDigest(raw), "sha256:")[:16] {
t.Fatalf("fallback ID = %q, want digest-derived ID", first.ID)
}
}
func TestParseUsesMetadataSourceIDWhenMetadataIDIsAbsent(t *testing.T) {
raw := []byte(`{"metadata":{"source_id":" source-from-metadata "},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if doc.ID != "source-from-metadata" {
t.Fatalf("doc.ID = %q, want source-from-metadata", doc.ID)
}
}
func TestParseRejectsInvalidInput(t *testing.T) {
tests := []struct {
name string
raw []byte
wantErr []string
}{
{
name: "malformed JSON",
raw: []byte(`{"metadata":`),
wantErr: []string{"seriatim input", "parse JSON"},
},
{
name: "trailing JSON",
raw: []byte(`{"metadata":{},"segments":[]} {}`),
wantErr: []string{"seriatim input", "trailing"},
},
{
name: "missing metadata",
raw: []byte(`{"segments":[]}`),
wantErr: []string{"metadata"},
},
{
name: "null metadata",
raw: []byte(`{"metadata":null,"segments":[]}`),
wantErr: []string{"metadata", "object"},
},
{
name: "metadata wrong type",
raw: []byte(`{"metadata":[],"segments":[]}`),
wantErr: []string{"metadata", "object"},
},
{
name: "missing segments",
raw: []byte(`{"metadata":{}}`),
wantErr: []string{"segments"},
},
{
name: "null segments",
raw: []byte(`{"metadata":{},"segments":null}`),
wantErr: []string{"segments", "array"},
},
{
name: "segments wrong type",
raw: []byte(`{"metadata":{},"segments":{}}`),
wantErr: []string{"segments", "array"},
},
{
name: "empty segments",
raw: []byte(`{"metadata":{},"segments":[]}`),
wantErr: []string{"segments", "empty"},
},
{
name: "missing segment id",
raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"id", "positive"},
},
{
name: "invalid segment id type",
raw: validJSONWithSegment(`"id":{},"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"id", "string or number"},
},
{
name: "whitespace segment id",
raw: validJSONWithSegment(`"id":" 1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"id", "whitespace"},
},
{
name: "empty text",
raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"speaker":"Narrator","text":" "`),
wantErr: []string{"text", "empty"},
},
{
name: "missing speaker",
raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"text":"Synthetic text."`),
wantErr: []string{"speaker", "empty"},
},
{
name: "missing start",
raw: validJSONWithSegment(`"id":1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "empty"},
},
{
name: "missing end",
raw: validJSONWithSegment(`"id":1,"start":0,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "empty"},
},
{
name: "negative start",
raw: validJSONWithSegment(`"id":1,"start":-1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "negative"},
},
{
name: "non-numeric end",
raw: validJSONWithSegment(`"id":1,"start":0,"end":"late","speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"segment[0]", "end", "number"},
},
{
name: "non-finite timestamp",
raw: validJSONWithSegment(`"id":1,"start":1e10000,"end":1e10000,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "valid number"},
},
{
name: "end before start",
raw: validJSONWithSegment(`"id":1,"start":2,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "start"},
},
{
name: "end before start beyond float precision",
raw: validJSONWithSegment(`"id":1,"start":9007199254740993,"end":9007199254740992,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "start"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: tt.raw})
if err == nil {
t.Fatal("Parse() error = nil, want error")
}
for _, want := range tt.wantErr {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Parse() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestParseRejectsDuplicateSegmentIDs(t *testing.T) {
raw := readFixture(t, "testdata/duplicate_segment_id.json")
_, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err == nil {
t.Fatal("Parse() error = nil, want duplicate ID error")
}
if !strings.Contains(err.Error(), "duplicated") || !strings.Contains(err.Error(), "1") {
t.Fatalf("Parse() error = %q, want duplicate segment context", err.Error())
}
}
func TestParseRejectsInvalidContextOrEmptyInput(t *testing.T) {
if _, err := New().Parse(nil, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil {
t.Fatal("Parse(nil context) error = nil, want error")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := New().Parse(ctx, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil {
t.Fatal("Parse(canceled context) error = nil, want error")
}
if _, err := New().Parse(context.Background(), contracts.ParseRequest{}); err == nil {
t.Fatal("Parse(empty input) error = nil, want error")
}
}
func readFixture(t *testing.T, path string) []byte {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v, want nil", path, err)
}
return raw
}
func validJSONWithSegment(segmentFields string) []byte {
return []byte(`{"metadata":{"id":"fixture"},"segments":[{` + segmentFields + `}]}`)
}
func testDigest(raw []byte) string {
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -0,0 +1,251 @@
package transcript
import (
"context"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
)
func TestPipelineConfigLoadsAndResolvesWithSeriatimInput(t *testing.T) {
cfg := loadPipelineConfig(t)
resolved, err := cfg.Resolve(config.ResolveInput{
PipelineID: "seriatim-fixture",
Catalog: seriatimTestCatalog(t, ModuleSpec()),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if resolved.ResolvedPipeline.Input.Module != Key {
t.Fatalf("resolved input module = %q, want %q", resolved.ResolvedPipeline.Input.Module, Key)
}
if resolved.ResolvedPipeline.Digest == "" {
t.Fatal("resolved digest is empty")
}
again, err := cfg.Resolve(config.ResolveInput{
PipelineID: "seriatim-fixture",
Catalog: seriatimTestCatalog(t, ModuleSpec()),
})
if err != nil {
t.Fatalf("second Resolve() error = %v, want nil", err)
}
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
}
}
func TestPipelineConfigRejectsMissingSeriatimCapability(t *testing.T) {
spec := ModuleSpec()
spec.Provides = withoutCapability(spec.Provides, "transcript.timestamps")
cfg := loadPipelineConfig(t)
_, err := cfg.Resolve(config.ResolveInput{
PipelineID: "seriatim-fixture",
Catalog: seriatimTestCatalog(t, spec),
})
if err == nil {
t.Fatal("Resolve() error = nil, want missing capability error")
}
if !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), "transcript.timestamps") {
t.Fatalf("Resolve() error = %q, want missing transcript.timestamps capability", err.Error())
}
}
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
cfg := loadPipelineConfig(t)
_, err := cfg.Resolve(config.ResolveInput{
PipelineID: "seriatim-fixture",
Only: []string{"missing"},
Catalog: seriatimTestCatalog(t, ModuleSpec()),
})
if err == nil {
t.Fatal("Resolve() error = nil, want unknown lane error")
}
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
}
}
func loadPipelineConfig(t *testing.T) config.Config {
t.Helper()
data, err := os.ReadFile("testdata/pipeline.yml")
if err != nil {
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
}
fileCfg, err := config.ParseFileConfigYAML(data)
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
}
profile, ok := cfg.Pipelines["seriatim-fixture"]
if !ok {
t.Fatal("pipeline seriatim-fixture was not loaded")
}
if profile.Input.Module != Key {
t.Fatalf("loaded input module = %q, want %q", profile.Input.Module, Key)
}
return cfg
}
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if reflect.DeepEqual(inputSpec, ModuleSpec()) {
if err := Register(inputs); err != nil {
t.Fatalf("register seriatim input: %v", err)
}
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
return New(), nil
}); err != nil {
t.Fatalf("register seriatim input override: %v", err)
}
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{
Key: "fake/chunk",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks"},
})
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
})
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
Requires: []string{"fake.artifacts"},
})
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
})
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
Key: pipeline.DefaultOutputModule,
Stage: pipeline.StageOutput,
})
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) {
return fakeChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
}
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Extractor, error) {
return fakeExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
}
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Merger, error) {
return appendorder.New(), nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
}
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Normalizer, error) {
return noop.New(), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
}
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) {
return fakeOutput{}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
}
type fakeChunker struct{}
func (fakeChunker) Key() string { return "fake/chunk" }
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}
type fakeExtractor struct{}
func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}
type fakeOutput struct{}
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
func (fakeOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func withoutCapability(capabilities []string, capability string) []string {
filtered := make([]string, 0, len(capabilities))
for _, candidate := range capabilities {
if candidate != capability {
filtered = append(filtered, candidate)
}
}
return filtered
}
var (
_ contracts.Chunker = fakeChunker{}
_ contracts.Extractor = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{}
)

View File

@@ -0,0 +1,28 @@
package transcript
import (
"encoding/json"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
const (
MetadataSpeaker = "speaker"
MetadataStart = "start"
MetadataEnd = "end"
)
func Speaker(unit source.SourceUnit) (string, bool) {
value, ok := unit.Metadata[MetadataSpeaker].(string)
return value, ok
}
func Start(unit source.SourceUnit) (json.Number, bool) {
value, ok := unit.Metadata[MetadataStart].(json.Number)
return value, ok
}
func End(unit source.SourceUnit) (json.Number, bool) {
value, ok := unit.Metadata[MetadataEnd].(json.Number)
return value, ok
}

View File

@@ -0,0 +1,179 @@
package transcript
import (
"bytes"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
)
type transcript struct {
Metadata map[string]any `json:"metadata"`
Segments []segment `json:"segments"`
}
type segment struct {
ID int `json:"id"`
Start json.Number `json:"start"`
End json.Number `json:"end"`
Speaker string `json:"speaker"`
Text string `json:"text"`
}
func decodeTranscript(raw []byte) (transcript, error) {
var fields map[string]json.RawMessage
if err := decodeJSON(raw, &fields); err != nil {
return transcript{}, err
}
if fields == nil {
return transcript{}, fmt.Errorf("top-level value must be an object")
}
metadataRaw, ok := fields["metadata"]
if !ok {
return transcript{}, fmt.Errorf("metadata is required")
}
var metadata map[string]any
if err := decodeJSON(metadataRaw, &metadata); err != nil {
return transcript{}, fmt.Errorf("metadata must be an object: %w", err)
}
if metadata == nil {
return transcript{}, fmt.Errorf("metadata must be an object")
}
segmentsRaw, ok := fields["segments"]
if !ok {
return transcript{}, fmt.Errorf("segments are required")
}
var segmentValues []json.RawMessage
if err := decodeJSON(segmentsRaw, &segmentValues); err != nil {
return transcript{}, fmt.Errorf("segments must be an array: %w", err)
}
if segmentValues == nil {
return transcript{}, fmt.Errorf("segments must be an array")
}
segments := make([]segment, 0, len(segmentValues))
for i, rawSegment := range segmentValues {
segment, err := decodeSegment(rawSegment, i)
if err != nil {
return transcript{}, err
}
segments = append(segments, segment)
}
return transcript{
Metadata: metadata,
Segments: segments,
}, nil
}
func decodeSegment(raw []byte, index int) (segment, error) {
var fields map[string]json.RawMessage
if err := decodeJSON(raw, &fields); err != nil {
return segment{}, fmt.Errorf("segment[%d] must be an object: %w", index, err)
}
if fields == nil {
return segment{}, fmt.Errorf("segment[%d] must be an object", index)
}
var decoded segment
if err := decodeOptionalSegmentID(fields, "id", &decoded.ID); err != nil {
return segment{}, fmt.Errorf("segment[%d] id must be a positive integer string or number: %w", index, err)
}
if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil {
return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err)
}
if err := decodeOptionalNumber(fields, "end", &decoded.End); err != nil {
return segment{}, fmt.Errorf("segment[%d] end must be a number: %w", index, err)
}
if err := decodeOptionalString(fields, "speaker", &decoded.Speaker); err != nil {
return segment{}, fmt.Errorf("segment[%d] speaker must be a string: %w", index, err)
}
if err := decodeOptionalString(fields, "text", &decoded.Text); err != nil {
return segment{}, fmt.Errorf("segment[%d] text must be a string: %w", index, err)
}
return decoded, nil
}
func decodeOptionalString(fields map[string]json.RawMessage, key string, out *string) error {
raw, ok := fields[key]
if !ok {
return nil
}
return decodeJSON(raw, out)
}
func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out *int) error {
raw, ok := fields[key]
if !ok {
return nil
}
var text string
if err := decodeJSON(raw, &text); err == nil {
parsed, err := parsePositiveInt(text)
if err != nil {
return err
}
*out = parsed
return nil
}
var number json.Number
if err := decodeJSON(raw, &number); err == nil {
parsed, err := parsePositiveInt(number.String())
if err != nil {
return err
}
*out = parsed
return nil
}
return fmt.Errorf("must be a positive integer string or number")
}
func parsePositiveInt(value string) (int, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return 0, fmt.Errorf("must not be empty")
}
if trimmed != value {
return 0, fmt.Errorf("must not contain leading or trailing whitespace")
}
parsed, err := strconv.Atoi(value)
if err != nil {
return 0, fmt.Errorf("must be an integer")
}
if parsed <= 0 {
return 0, fmt.Errorf("must be positive")
}
if strconv.Itoa(parsed) != value {
return 0, fmt.Errorf("must be a canonical positive integer")
}
return parsed, nil
}
func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error {
raw, ok := fields[key]
if !ok {
return nil
}
return decodeJSON(raw, out)
}
func decodeJSON(raw []byte, out any) error {
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
if err := decoder.Decode(out); err != nil {
return err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
if err == nil {
return fmt.Errorf("unexpected trailing JSON value")
}
return err
}
return nil
}

View File

@@ -0,0 +1,106 @@
package transcript
import (
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestNewReturnsAdapterWithKey(t *testing.T) {
adapter := New()
if adapter == nil {
t.Fatal("New() = nil, want adapter")
}
if adapter.Key() != Key {
t.Fatalf("adapter.Key() = %q, want %q", adapter.Key(), Key)
}
}
func TestModuleSpec(t *testing.T) {
got := ModuleSpec()
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageInput,
Provides: []string{
"source.transcript",
"transcript.speaker",
"transcript.timestamps",
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got.Provides[0] = "changed"
again := ModuleSpec()
if !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
}
func TestRegisterMakesAdapterBuildable(t *testing.T) {
registry := pipeline.NewInputAdapterRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
adapter, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if adapter.Key() != Key {
t.Fatalf("adapter.Key() = %q, want %q", adapter.Key(), Key)
}
}
func TestRegisterStoresModuleSpec(t *testing.T) {
registry := pipeline.NewInputAdapterRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got, ok := registry.Spec(Key)
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec()
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {
t.Fatal("Register(nil) error = nil, want error")
}
if !strings.Contains(err.Error(), "input adapter registry") {
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
}
}
func TestMetadataHelpers(t *testing.T) {
unit := source.SourceUnit{
Metadata: map[string]any{
MetadataSpeaker: "Narrator",
MetadataStart: json.Number("1.25"),
MetadataEnd: json.Number("2.5"),
},
}
if got, ok := Speaker(unit); !ok || got != "Narrator" {
t.Fatalf("Speaker() = %q, %v; want Narrator, true", got, ok)
}
if got, ok := Start(unit); !ok || got != json.Number("1.25") {
t.Fatalf("Start() = %q, %v; want 1.25, true", got, ok)
}
if got, ok := End(unit); !ok || got != json.Number("2.5") {
t.Fatalf("End() = %q, %v; want 2.5, true", got, ok)
}
}

View File

@@ -0,0 +1,291 @@
package transcript
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
)
func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
raw := readFixture(t, "testdata/valid_minimal.json")
expectedDoc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
extractor := &runnerSeriatimExtractor{}
output, err := pipeline.New(seriatimRunnerRegistries(t, extractor)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.InputModule != Key {
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, Key)
}
if got := output.Manifest.SourceDigests; len(got) != 1 || got[0] != expectedDoc.Digest {
t.Fatalf("manifest source digests = %#v, want %q", got, expectedDoc.Digest)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Schema.ID != "fake.event" || rawOutput.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
}
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := json.Unmarshal(rawOutput.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
}
if extractor.calls != 1 {
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
}
func TestRunnerFailsOnInvalidSeriatimInput(t *testing.T) {
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
output, err := pipeline.New(seriatimRunnerRegistries(t, &runnerSeriatimExtractor{})).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: []byte(`{"metadata":{},"segments":[]}`),
SourceID: "invalid-source",
LLMClient: nil,
})
if err == nil {
t.Fatal("Run() error = nil, want invalid input error")
}
if !strings.Contains(err.Error(), "parse input with adapter") || !strings.Contains(err.Error(), "seriatim input") {
t.Fatalf("Run() error = %q, want Seriatim parse context", err.Error())
}
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
}
func configResolveInput(t *testing.T) config.ResolveInput {
t.Helper()
return config.ResolveInput{
PipelineID: "seriatim-fixture",
Catalog: seriatimTestCatalog(t, ModuleSpec()),
}
}
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor) pipeline.Registries {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if err := Register(inputs); err != nil {
t.Fatalf("register seriatim input: %v", err)
}
if err := chunkers.Register("fake/chunk", func() (contracts.Chunker, error) {
return runnerSeriatimChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := extractors.Register("fake/extract", func() (contracts.Extractor, error) {
return extractor, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := mergers.Register(pipeline.DefaultMergeModule, func() (contracts.Merger, error) {
return appendorder.New(), nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := normalizers.Register(pipeline.DefaultNormalizeModule, func() (contracts.Normalizer, error) {
return noop.New(), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
return runnerSeriatimOutput{}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
return pipeline.Registries{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
}
}
type runnerSeriatimChunker struct{}
func (runnerSeriatimChunker) Key() string {
return "fake/chunk"
}
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
},
},
}, nil
}
type runnerSeriatimExtractor struct {
calls int
}
func (e *runnerSeriatimExtractor) Key() string {
return "fake/extract"
}
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
e.calls++
if req.Source == nil {
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil")
}
if req.Chunk == nil {
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
}
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
}
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
}
for _, unit := range req.Chunk.Units {
if speaker, ok := Speaker(unit); !ok || speaker == "" {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
}
if _, ok := Start(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
}
if _, ok := End(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
}
}
payload, err := json.Marshal(struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
}
type runnerSeriatimOutput struct{}
func (runnerSeriatimOutput) Key() string {
return pipeline.DefaultOutputModule
}
func (runnerSeriatimOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
}, nil
}
func unitIDs(units []source.SourceUnit) []int {
ids := make([]int, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
func equalInts(a, b []int) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
var (
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.Extractor = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{}
)

View File

@@ -0,0 +1,21 @@
{
"metadata": {
"id": "duplicate-segment-fixture"
},
"segments": [
{
"id": 1,
"start": 0,
"end": 1,
"speaker": "Narrator",
"text": "First segment."
},
{
"id": 1,
"start": 1,
"end": 2,
"speaker": "Player",
"text": "Duplicate segment."
}
]
}

View File

@@ -0,0 +1,11 @@
version: 2
pipelines:
seriatim-fixture:
input: seriatim
chunk: fake/chunk
artifacts:
events:
extract: fake/extract
merge: appendorder
normalize: noop
output: json

View File

@@ -0,0 +1,23 @@
{
"metadata": {
"id": "session-alpha",
"source_id": "fallback-session",
"title": "Synthetic session transcript"
},
"segments": [
{
"id": 1,
"start": 0,
"end": 4.5,
"speaker": "Narrator",
"text": "The stone door opens."
},
{
"id": 2,
"start": 4.5,
"end": 8,
"speaker": "Player",
"text": "I cast light."
}
]
}