Organize generic and Seriatim modules by domain
This commit is contained in:
205
internal/modules/seriatim/input/transcript/adapter.go
Normal file
205
internal/modules/seriatim/input/transcript/adapter.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const Key = "seriatim"
|
||||
|
||||
const (
|
||||
DocumentKind = "transcript"
|
||||
UnitKind = "transcript_segment"
|
||||
Format = "application/vnd.seriatim+json"
|
||||
)
|
||||
|
||||
var providedCapabilities = []string{
|
||||
"source.transcript",
|
||||
"transcript.speaker",
|
||||
"transcript.timestamps",
|
||||
}
|
||||
|
||||
var _ contracts.InputAdapter = (*Adapter)(nil)
|
||||
|
||||
type Adapter struct{}
|
||||
|
||||
func New() *Adapter {
|
||||
return &Adapter{}
|
||||
}
|
||||
|
||||
func (a *Adapter) Key() string {
|
||||
return Key
|
||||
}
|
||||
|
||||
func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
if ctx == nil {
|
||||
return nil, inputErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, inputErrorf("context error before parsing: %w", err)
|
||||
}
|
||||
if len(req.Raw) == 0 {
|
||||
return nil, inputErrorf("raw input must not be empty")
|
||||
}
|
||||
|
||||
parsed, err := decodeTranscript(req.Raw)
|
||||
if err != nil {
|
||||
return nil, inputErrorf("parse JSON: %w", err)
|
||||
}
|
||||
if len(parsed.Segments) == 0 {
|
||||
return nil, inputErrorf("segments must not be empty")
|
||||
}
|
||||
|
||||
rawDigest := digest(req.Raw)
|
||||
doc := &source.SourceDocument{
|
||||
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
|
||||
Kind: DocumentKind,
|
||||
Format: Format,
|
||||
Digest: rawDigest,
|
||||
Metadata: copyMetadata(parsed.Metadata),
|
||||
}
|
||||
|
||||
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
|
||||
for i, segment := range parsed.Segments {
|
||||
unit, err := sourceUnit(segment, i, seenSegmentIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc.Units = append(doc.Units, unit)
|
||||
}
|
||||
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
return nil, inputErrorf("validate source document: %w", err)
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageInput,
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.InputAdapterRegistry) error {
|
||||
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.InputAdapter, error) {
|
||||
return New(), nil
|
||||
})
|
||||
}
|
||||
|
||||
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
|
||||
segmentLabel := fmt.Sprintf("segment[%d]", index)
|
||||
if segment.ID <= 0 {
|
||||
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
|
||||
}
|
||||
if _, ok := seen[segment.ID]; ok {
|
||||
return source.SourceUnit{}, inputErrorf("segment id %d is duplicated", segment.ID)
|
||||
}
|
||||
seen[segment.ID] = struct{}{}
|
||||
|
||||
speaker := strings.TrimSpace(segment.Speaker)
|
||||
if speaker == "" {
|
||||
return source.SourceUnit{}, inputErrorf("segment %d speaker must not be empty", segment.ID)
|
||||
}
|
||||
|
||||
start, err := validTimestamp(segment.Start, fmt.Sprintf("segment %d start", segment.ID))
|
||||
if err != nil {
|
||||
return source.SourceUnit{}, err
|
||||
}
|
||||
end, err := validTimestamp(segment.End, fmt.Sprintf("segment %d end", segment.ID))
|
||||
if err != nil {
|
||||
return source.SourceUnit{}, err
|
||||
}
|
||||
if end.Cmp(start) < 0 {
|
||||
return source.SourceUnit{}, inputErrorf("segment %d end must be greater than or equal to start", segment.ID)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(segment.Text) == "" {
|
||||
return source.SourceUnit{}, inputErrorf("segment %d text must not be empty", segment.ID)
|
||||
}
|
||||
|
||||
return source.SourceUnit{
|
||||
ID: segment.ID,
|
||||
Kind: UnitKind,
|
||||
Text: segment.Text,
|
||||
Metadata: map[string]any{
|
||||
MetadataSpeaker: segment.Speaker,
|
||||
MetadataStart: segment.Start,
|
||||
MetadataEnd: segment.End,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validTimestamp(value fmt.Stringer, label string) (*big.Rat, error) {
|
||||
raw := strings.TrimSpace(value.String())
|
||||
if raw == "" {
|
||||
return nil, inputErrorf("%s must not be empty", label)
|
||||
}
|
||||
parsed, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
return nil, inputErrorf("%s must be a valid number: %w", label, err)
|
||||
}
|
||||
if math.IsInf(parsed, 0) || math.IsNaN(parsed) {
|
||||
return nil, inputErrorf("%s must be finite", label)
|
||||
}
|
||||
if parsed < 0 {
|
||||
return nil, inputErrorf("%s must not be negative", label)
|
||||
}
|
||||
rat, ok := new(big.Rat).SetString(raw)
|
||||
if !ok {
|
||||
return nil, inputErrorf("%s must be a valid number", label)
|
||||
}
|
||||
return rat, nil
|
||||
}
|
||||
|
||||
func documentID(requestedID string, metadata map[string]any, rawDigest string) string {
|
||||
if id := strings.TrimSpace(requestedID); id != "" {
|
||||
return id
|
||||
}
|
||||
if id := stringMetadata(metadata, "id"); id != "" {
|
||||
return id
|
||||
}
|
||||
if id := stringMetadata(metadata, "source_id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return "seriatim:" + strings.TrimPrefix(rawDigest, "sha256:")[:16]
|
||||
}
|
||||
|
||||
func stringMetadata(metadata map[string]any, key string) string {
|
||||
value, ok := metadata[key].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func copyMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
copied := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
copied[key] = value
|
||||
}
|
||||
return copied
|
||||
}
|
||||
|
||||
func digest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func inputErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("seriatim input: "+format, args...)
|
||||
}
|
||||
324
internal/modules/seriatim/input/transcript/adapter_test.go
Normal file
324
internal/modules/seriatim/input/transcript/adapter_test.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestParseValidMinimalTranscript(t *testing.T) {
|
||||
raw := readFixture(t, "testdata/valid_minimal.json")
|
||||
|
||||
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if doc.ID != "session-alpha" {
|
||||
t.Fatalf("doc.ID = %q, want session-alpha", doc.ID)
|
||||
}
|
||||
if doc.Kind != DocumentKind {
|
||||
t.Fatalf("doc.Kind = %q, want %q", doc.Kind, DocumentKind)
|
||||
}
|
||||
if doc.Format != Format {
|
||||
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
|
||||
}
|
||||
if doc.Digest != testDigest(raw) {
|
||||
t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw))
|
||||
}
|
||||
if got := doc.Metadata["title"]; got != "Synthetic session transcript" {
|
||||
t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got)
|
||||
}
|
||||
if len(doc.Units) != 2 {
|
||||
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
|
||||
}
|
||||
|
||||
first := doc.Units[0]
|
||||
if first.ID != 1 {
|
||||
t.Fatalf("first.ID = %d, want 1", first.ID)
|
||||
}
|
||||
if first.Kind != UnitKind {
|
||||
t.Fatalf("first.Kind = %q, want %q", first.Kind, UnitKind)
|
||||
}
|
||||
if first.Text != "The stone door opens." {
|
||||
t.Fatalf("first.Text = %q, want fixture text", first.Text)
|
||||
}
|
||||
if speaker, ok := Speaker(first); !ok || speaker != "Narrator" {
|
||||
t.Fatalf("Speaker(first) = %q, %v; want Narrator, true", speaker, ok)
|
||||
}
|
||||
if start, ok := Start(first); !ok || start != json.Number("0") {
|
||||
t.Fatalf("Start(first) = %q, %v; want 0, true", start, ok)
|
||||
}
|
||||
if end, ok := End(first); !ok || end != json.Number("4.5") {
|
||||
t.Fatalf("End(first) = %q, %v; want 4.5, true", end, ok)
|
||||
}
|
||||
|
||||
ref := source.SourceRef{
|
||||
SourceID: doc.ID,
|
||||
StartUnitID: doc.Units[0].ID,
|
||||
EndUnitID: doc.Units[len(doc.Units)-1].ID,
|
||||
}
|
||||
if err := source.ValidateRef(doc, ref); err != nil {
|
||||
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
|
||||
raw := []byte(`{"metadata":{"application":"seriatim","version":"v1.5.0","output_schema":"seriatim-intermediate"},"segments":[{"id":1,"start":451.821,"end":469.685,"speaker":"Narrator","text":"The stone door opens.","categories":["scene"]},{"id":2,"start":470,"end":471,"speaker":"Player","text":"I step inside."}]}`)
|
||||
|
||||
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
if got, want := doc.Metadata["output_schema"], "seriatim-intermediate"; got != want {
|
||||
t.Fatalf("doc.Metadata[output_schema] = %#v, want %q", got, want)
|
||||
}
|
||||
if doc.Format != Format {
|
||||
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
|
||||
}
|
||||
if len(doc.Units) != 2 {
|
||||
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
|
||||
}
|
||||
if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 {
|
||||
t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID})
|
||||
}
|
||||
ref := source.SourceRef{
|
||||
SourceID: doc.ID,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 2,
|
||||
}
|
||||
if err := source.ValidateRef(doc, ref); err != nil {
|
||||
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) {
|
||||
raw := readFixture(t, "testdata/valid_minimal.json")
|
||||
|
||||
doc, err := New().Parse(context.Background(), contracts.ParseRequest{
|
||||
SourceID: " requested-source ",
|
||||
Raw: raw,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
if doc.ID != "requested-source" {
|
||||
t.Fatalf("doc.ID = %q, want requested-source", doc.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) {
|
||||
raw := []byte(`{"metadata":{},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
|
||||
|
||||
first, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("first Parse() error = %v, want nil", err)
|
||||
}
|
||||
second, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("second Parse() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if first.ID != second.ID {
|
||||
t.Fatalf("fallback IDs differ: %q vs %q", first.ID, second.ID)
|
||||
}
|
||||
if !strings.HasPrefix(first.ID, "seriatim:") {
|
||||
t.Fatalf("fallback ID = %q, want seriatim prefix", first.ID)
|
||||
}
|
||||
if first.ID != "seriatim:"+strings.TrimPrefix(testDigest(raw), "sha256:")[:16] {
|
||||
t.Fatalf("fallback ID = %q, want digest-derived ID", first.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUsesMetadataSourceIDWhenMetadataIDIsAbsent(t *testing.T) {
|
||||
raw := []byte(`{"metadata":{"source_id":" source-from-metadata "},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
|
||||
|
||||
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
if doc.ID != "source-from-metadata" {
|
||||
t.Fatalf("doc.ID = %q, want source-from-metadata", doc.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidInput(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw []byte
|
||||
wantErr []string
|
||||
}{
|
||||
{
|
||||
name: "malformed JSON",
|
||||
raw: []byte(`{"metadata":`),
|
||||
wantErr: []string{"seriatim input", "parse JSON"},
|
||||
},
|
||||
{
|
||||
name: "trailing JSON",
|
||||
raw: []byte(`{"metadata":{},"segments":[]} {}`),
|
||||
wantErr: []string{"seriatim input", "trailing"},
|
||||
},
|
||||
{
|
||||
name: "missing metadata",
|
||||
raw: []byte(`{"segments":[]}`),
|
||||
wantErr: []string{"metadata"},
|
||||
},
|
||||
{
|
||||
name: "null metadata",
|
||||
raw: []byte(`{"metadata":null,"segments":[]}`),
|
||||
wantErr: []string{"metadata", "object"},
|
||||
},
|
||||
{
|
||||
name: "metadata wrong type",
|
||||
raw: []byte(`{"metadata":[],"segments":[]}`),
|
||||
wantErr: []string{"metadata", "object"},
|
||||
},
|
||||
{
|
||||
name: "missing segments",
|
||||
raw: []byte(`{"metadata":{}}`),
|
||||
wantErr: []string{"segments"},
|
||||
},
|
||||
{
|
||||
name: "null segments",
|
||||
raw: []byte(`{"metadata":{},"segments":null}`),
|
||||
wantErr: []string{"segments", "array"},
|
||||
},
|
||||
{
|
||||
name: "segments wrong type",
|
||||
raw: []byte(`{"metadata":{},"segments":{}}`),
|
||||
wantErr: []string{"segments", "array"},
|
||||
},
|
||||
{
|
||||
name: "empty segments",
|
||||
raw: []byte(`{"metadata":{},"segments":[]}`),
|
||||
wantErr: []string{"segments", "empty"},
|
||||
},
|
||||
{
|
||||
name: "missing segment id",
|
||||
raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"id", "positive"},
|
||||
},
|
||||
{
|
||||
name: "invalid segment id type",
|
||||
raw: validJSONWithSegment(`"id":{},"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"id", "string or number"},
|
||||
},
|
||||
{
|
||||
name: "whitespace segment id",
|
||||
raw: validJSONWithSegment(`"id":" 1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"id", "whitespace"},
|
||||
},
|
||||
{
|
||||
name: "empty text",
|
||||
raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"speaker":"Narrator","text":" "`),
|
||||
wantErr: []string{"text", "empty"},
|
||||
},
|
||||
{
|
||||
name: "missing speaker",
|
||||
raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"text":"Synthetic text."`),
|
||||
wantErr: []string{"speaker", "empty"},
|
||||
},
|
||||
{
|
||||
name: "missing start",
|
||||
raw: validJSONWithSegment(`"id":1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"start", "empty"},
|
||||
},
|
||||
{
|
||||
name: "missing end",
|
||||
raw: validJSONWithSegment(`"id":1,"start":0,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"end", "empty"},
|
||||
},
|
||||
{
|
||||
name: "negative start",
|
||||
raw: validJSONWithSegment(`"id":1,"start":-1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"start", "negative"},
|
||||
},
|
||||
{
|
||||
name: "non-numeric end",
|
||||
raw: validJSONWithSegment(`"id":1,"start":0,"end":"late","speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"segment[0]", "end", "number"},
|
||||
},
|
||||
{
|
||||
name: "non-finite timestamp",
|
||||
raw: validJSONWithSegment(`"id":1,"start":1e10000,"end":1e10000,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"start", "valid number"},
|
||||
},
|
||||
{
|
||||
name: "end before start",
|
||||
raw: validJSONWithSegment(`"id":1,"start":2,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"end", "start"},
|
||||
},
|
||||
{
|
||||
name: "end before start beyond float precision",
|
||||
raw: validJSONWithSegment(`"id":1,"start":9007199254740993,"end":9007199254740992,"speaker":"Narrator","text":"Synthetic text."`),
|
||||
wantErr: []string{"end", "start"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: tt.raw})
|
||||
if err == nil {
|
||||
t.Fatal("Parse() error = nil, want error")
|
||||
}
|
||||
for _, want := range tt.wantErr {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Parse() error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsDuplicateSegmentIDs(t *testing.T) {
|
||||
raw := readFixture(t, "testdata/duplicate_segment_id.json")
|
||||
|
||||
_, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err == nil {
|
||||
t.Fatal("Parse() error = nil, want duplicate ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "duplicated") || !strings.Contains(err.Error(), "1") {
|
||||
t.Fatalf("Parse() error = %q, want duplicate segment context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidContextOrEmptyInput(t *testing.T) {
|
||||
if _, err := New().Parse(nil, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil {
|
||||
t.Fatal("Parse(nil context) error = nil, want error")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := New().Parse(ctx, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil {
|
||||
t.Fatal("Parse(canceled context) error = nil, want error")
|
||||
}
|
||||
|
||||
if _, err := New().Parse(context.Background(), contracts.ParseRequest{}); err == nil {
|
||||
t.Fatal("Parse(empty input) error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func readFixture(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v, want nil", path, err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func validJSONWithSegment(segmentFields string) []byte {
|
||||
return []byte(`{"metadata":{"id":"fixture"},"segments":[{` + segmentFields + `}]}`)
|
||||
}
|
||||
|
||||
func testDigest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
251
internal/modules/seriatim/input/transcript/config_test.go
Normal file
251
internal/modules/seriatim/input/transcript/config_test.go
Normal file
@@ -0,0 +1,251 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
)
|
||||
|
||||
func TestPipelineConfigLoadsAndResolvesWithSeriatimInput(t *testing.T) {
|
||||
cfg := loadPipelineConfig(t)
|
||||
|
||||
resolved, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if resolved.ResolvedPipeline.Input.Module != Key {
|
||||
t.Fatalf("resolved input module = %q, want %q", resolved.ResolvedPipeline.Input.Module, Key)
|
||||
}
|
||||
if resolved.ResolvedPipeline.Digest == "" {
|
||||
t.Fatal("resolved digest is empty")
|
||||
}
|
||||
|
||||
again, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
||||
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigRejectsMissingSeriatimCapability(t *testing.T) {
|
||||
spec := ModuleSpec()
|
||||
spec.Provides = withoutCapability(spec.Provides, "transcript.timestamps")
|
||||
cfg := loadPipelineConfig(t)
|
||||
|
||||
_, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Catalog: seriatimTestCatalog(t, spec),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), "transcript.timestamps") {
|
||||
t.Fatalf("Resolve() error = %q, want missing transcript.timestamps capability", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
||||
cfg := loadPipelineConfig(t)
|
||||
|
||||
_, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Only: []string{"missing"},
|
||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want unknown lane error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func loadPipelineConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
|
||||
data, err := os.ReadFile("testdata/pipeline.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
||||
}
|
||||
fileCfg, err := config.ParseFileConfigYAML(data)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
profile, ok := cfg.Pipelines["seriatim-fixture"]
|
||||
if !ok {
|
||||
t.Fatal("pipeline seriatim-fixture was not loaded")
|
||||
}
|
||||
if profile.Input.Module != Key {
|
||||
t.Fatalf("loaded input module = %q, want %q", profile.Input.Module, Key)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
extractors := pipeline.NewExtractorRegistry()
|
||||
mergers := pipeline.NewMergerRegistry()
|
||||
normalizers := pipeline.NewNormalizerRegistry()
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
|
||||
if reflect.DeepEqual(inputSpec, ModuleSpec()) {
|
||||
if err := Register(inputs); err != nil {
|
||||
t.Fatalf("register seriatim input: %v", err)
|
||||
}
|
||||
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
|
||||
return New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register seriatim input override: %v", err)
|
||||
}
|
||||
|
||||
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{
|
||||
Key: "fake/chunk",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks"},
|
||||
})
|
||||
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
|
||||
Provides: []string{"fake.artifacts"},
|
||||
})
|
||||
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
Requires: []string{"fake.artifacts"},
|
||||
})
|
||||
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
})
|
||||
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultOutputModule,
|
||||
Stage: pipeline.StageOutput,
|
||||
})
|
||||
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) {
|
||||
return fakeChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Extractor, error) {
|
||||
return fakeExtractor{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Merger, error) {
|
||||
return appendorder.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Normalizer, error) {
|
||||
return noop.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) {
|
||||
return fakeOutput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeChunker struct{}
|
||||
|
||||
func (fakeChunker) Key() string { return "fake/chunk" }
|
||||
|
||||
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{}, nil
|
||||
}
|
||||
|
||||
type fakeExtractor struct{}
|
||||
|
||||
func (fakeExtractor) Key() string { return "fake/extract" }
|
||||
|
||||
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
}
|
||||
|
||||
type fakeOutput struct{}
|
||||
|
||||
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
|
||||
|
||||
func (fakeOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
|
||||
func withoutCapability(capabilities []string, capability string) []string {
|
||||
filtered := make([]string, 0, len(capabilities))
|
||||
for _, candidate := range capabilities {
|
||||
if candidate != capability {
|
||||
filtered = append(filtered, candidate)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = fakeChunker{}
|
||||
_ contracts.Extractor = fakeExtractor{}
|
||||
_ contracts.OutputEncoder = fakeOutput{}
|
||||
)
|
||||
28
internal/modules/seriatim/input/transcript/metadata.go
Normal file
28
internal/modules/seriatim/input/transcript/metadata.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
const (
|
||||
MetadataSpeaker = "speaker"
|
||||
MetadataStart = "start"
|
||||
MetadataEnd = "end"
|
||||
)
|
||||
|
||||
func Speaker(unit source.SourceUnit) (string, bool) {
|
||||
value, ok := unit.Metadata[MetadataSpeaker].(string)
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func Start(unit source.SourceUnit) (json.Number, bool) {
|
||||
value, ok := unit.Metadata[MetadataStart].(json.Number)
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func End(unit source.SourceUnit) (json.Number, bool) {
|
||||
value, ok := unit.Metadata[MetadataEnd].(json.Number)
|
||||
return value, ok
|
||||
}
|
||||
179
internal/modules/seriatim/input/transcript/model.go
Normal file
179
internal/modules/seriatim/input/transcript/model.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type transcript struct {
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
Segments []segment `json:"segments"`
|
||||
}
|
||||
|
||||
type segment struct {
|
||||
ID int `json:"id"`
|
||||
Start json.Number `json:"start"`
|
||||
End json.Number `json:"end"`
|
||||
Speaker string `json:"speaker"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func decodeTranscript(raw []byte) (transcript, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := decodeJSON(raw, &fields); err != nil {
|
||||
return transcript{}, err
|
||||
}
|
||||
if fields == nil {
|
||||
return transcript{}, fmt.Errorf("top-level value must be an object")
|
||||
}
|
||||
|
||||
metadataRaw, ok := fields["metadata"]
|
||||
if !ok {
|
||||
return transcript{}, fmt.Errorf("metadata is required")
|
||||
}
|
||||
var metadata map[string]any
|
||||
if err := decodeJSON(metadataRaw, &metadata); err != nil {
|
||||
return transcript{}, fmt.Errorf("metadata must be an object: %w", err)
|
||||
}
|
||||
if metadata == nil {
|
||||
return transcript{}, fmt.Errorf("metadata must be an object")
|
||||
}
|
||||
|
||||
segmentsRaw, ok := fields["segments"]
|
||||
if !ok {
|
||||
return transcript{}, fmt.Errorf("segments are required")
|
||||
}
|
||||
var segmentValues []json.RawMessage
|
||||
if err := decodeJSON(segmentsRaw, &segmentValues); err != nil {
|
||||
return transcript{}, fmt.Errorf("segments must be an array: %w", err)
|
||||
}
|
||||
if segmentValues == nil {
|
||||
return transcript{}, fmt.Errorf("segments must be an array")
|
||||
}
|
||||
segments := make([]segment, 0, len(segmentValues))
|
||||
for i, rawSegment := range segmentValues {
|
||||
segment, err := decodeSegment(rawSegment, i)
|
||||
if err != nil {
|
||||
return transcript{}, err
|
||||
}
|
||||
segments = append(segments, segment)
|
||||
}
|
||||
|
||||
return transcript{
|
||||
Metadata: metadata,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeSegment(raw []byte, index int) (segment, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := decodeJSON(raw, &fields); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] must be an object: %w", index, err)
|
||||
}
|
||||
if fields == nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] must be an object", index)
|
||||
}
|
||||
|
||||
var decoded segment
|
||||
if err := decodeOptionalSegmentID(fields, "id", &decoded.ID); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] id must be a positive integer string or number: %w", index, err)
|
||||
}
|
||||
if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err)
|
||||
}
|
||||
if err := decodeOptionalNumber(fields, "end", &decoded.End); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] end must be a number: %w", index, err)
|
||||
}
|
||||
if err := decodeOptionalString(fields, "speaker", &decoded.Speaker); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] speaker must be a string: %w", index, err)
|
||||
}
|
||||
if err := decodeOptionalString(fields, "text", &decoded.Text); err != nil {
|
||||
return segment{}, fmt.Errorf("segment[%d] text must be a string: %w", index, err)
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
func decodeOptionalString(fields map[string]json.RawMessage, key string, out *string) error {
|
||||
raw, ok := fields[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return decodeJSON(raw, out)
|
||||
}
|
||||
|
||||
func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out *int) error {
|
||||
raw, ok := fields[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var text string
|
||||
if err := decodeJSON(raw, &text); err == nil {
|
||||
parsed, err := parsePositiveInt(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*out = parsed
|
||||
return nil
|
||||
}
|
||||
|
||||
var number json.Number
|
||||
if err := decodeJSON(raw, &number); err == nil {
|
||||
parsed, err := parsePositiveInt(number.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*out = parsed
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("must be a positive integer string or number")
|
||||
}
|
||||
|
||||
func parsePositiveInt(value string) (int, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return 0, fmt.Errorf("must not be empty")
|
||||
}
|
||||
if trimmed != value {
|
||||
return 0, fmt.Errorf("must not contain leading or trailing whitespace")
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be an integer")
|
||||
}
|
||||
if parsed <= 0 {
|
||||
return 0, fmt.Errorf("must be positive")
|
||||
}
|
||||
if strconv.Itoa(parsed) != value {
|
||||
return 0, fmt.Errorf("must be a canonical positive integer")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error {
|
||||
raw, ok := fields[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return decodeJSON(raw, out)
|
||||
}
|
||||
|
||||
func decodeJSON(raw []byte, out any) error {
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(out); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
if err == nil {
|
||||
return fmt.Errorf("unexpected trailing JSON value")
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
106
internal/modules/seriatim/input/transcript/registry_test.go
Normal file
106
internal/modules/seriatim/input/transcript/registry_test.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestNewReturnsAdapterWithKey(t *testing.T) {
|
||||
adapter := New()
|
||||
if adapter == nil {
|
||||
t.Fatal("New() = nil, want adapter")
|
||||
}
|
||||
if adapter.Key() != Key {
|
||||
t.Fatalf("adapter.Key() = %q, want %q", adapter.Key(), Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleSpec(t *testing.T) {
|
||||
got := ModuleSpec()
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageInput,
|
||||
Provides: []string{
|
||||
"source.transcript",
|
||||
"transcript.speaker",
|
||||
"transcript.timestamps",
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
got.Provides[0] = "changed"
|
||||
again := ModuleSpec()
|
||||
if !reflect.DeepEqual(again, want) {
|
||||
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterMakesAdapterBuildable(t *testing.T) {
|
||||
registry := pipeline.NewInputAdapterRegistry()
|
||||
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
adapter, err := registry.Build(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v, want nil", err)
|
||||
}
|
||||
if adapter.Key() != Key {
|
||||
t.Fatalf("adapter.Key() = %q, want %q", adapter.Key(), Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterStoresModuleSpec(t *testing.T) {
|
||||
registry := pipeline.NewInputAdapterRegistry()
|
||||
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got, ok := registry.Spec(Key)
|
||||
if !ok {
|
||||
t.Fatal("Spec() ok = false, want true")
|
||||
}
|
||||
want := ModuleSpec()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterNilRegistryReturnsError(t *testing.T) {
|
||||
err := Register(nil)
|
||||
if err == nil {
|
||||
t.Fatal("Register(nil) error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "input adapter registry") {
|
||||
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataHelpers(t *testing.T) {
|
||||
unit := source.SourceUnit{
|
||||
Metadata: map[string]any{
|
||||
MetadataSpeaker: "Narrator",
|
||||
MetadataStart: json.Number("1.25"),
|
||||
MetadataEnd: json.Number("2.5"),
|
||||
},
|
||||
}
|
||||
|
||||
if got, ok := Speaker(unit); !ok || got != "Narrator" {
|
||||
t.Fatalf("Speaker() = %q, %v; want Narrator, true", got, ok)
|
||||
}
|
||||
if got, ok := Start(unit); !ok || got != json.Number("1.25") {
|
||||
t.Fatalf("Start() = %q, %v; want 1.25, true", got, ok)
|
||||
}
|
||||
if got, ok := End(unit); !ok || got != json.Number("2.5") {
|
||||
t.Fatalf("End() = %q, %v; want 2.5, true", got, ok)
|
||||
}
|
||||
}
|
||||
291
internal/modules/seriatim/input/transcript/runner_test.go
Normal file
291
internal/modules/seriatim/input/transcript/runner_test.go
Normal file
@@ -0,0 +1,291 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
)
|
||||
|
||||
func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
|
||||
raw := readFixture(t, "testdata/valid_minimal.json")
|
||||
expectedDoc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractor := &runnerSeriatimExtractor{}
|
||||
output, err := pipeline.New(seriatimRunnerRegistries(t, extractor)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if output.Manifest.InputModule != Key {
|
||||
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, Key)
|
||||
}
|
||||
if got := output.Manifest.SourceDigests; len(got) != 1 || got[0] != expectedDoc.Digest {
|
||||
t.Fatalf("manifest source digests = %#v, want %q", got, expectedDoc.Digest)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
|
||||
rawOutput := output.NormalizeOutputs[0]
|
||||
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Schema.ID != "fake.event" || rawOutput.Schema.Version != "v1" {
|
||||
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
|
||||
}
|
||||
var payload struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}
|
||||
if err := json.Unmarshal(rawOutput.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
|
||||
}
|
||||
if len(payload.SourceRefs) != 1 {
|
||||
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
|
||||
}
|
||||
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
|
||||
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||
}
|
||||
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
|
||||
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
|
||||
}
|
||||
if extractor.calls != 1 {
|
||||
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
|
||||
}
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if output.OutputFiles[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerFailsOnInvalidSeriatimInput(t *testing.T) {
|
||||
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New(seriatimRunnerRegistries(t, &runnerSeriatimExtractor{})).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: []byte(`{"metadata":{},"segments":[]}`),
|
||||
SourceID: "invalid-source",
|
||||
LLMClient: nil,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want invalid input error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "parse input with adapter") || !strings.Contains(err.Error(), "seriatim input") {
|
||||
t.Fatalf("Run() error = %q, want Seriatim parse context", err.Error())
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "failed" {
|
||||
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func configResolveInput(t *testing.T) config.ResolveInput {
|
||||
t.Helper()
|
||||
return config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Catalog: seriatimTestCatalog(t, ModuleSpec()),
|
||||
}
|
||||
}
|
||||
|
||||
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor) pipeline.Registries {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
extractors := pipeline.NewExtractorRegistry()
|
||||
mergers := pipeline.NewMergerRegistry()
|
||||
normalizers := pipeline.NewNormalizerRegistry()
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
|
||||
if err := Register(inputs); err != nil {
|
||||
t.Fatalf("register seriatim input: %v", err)
|
||||
}
|
||||
if err := chunkers.Register("fake/chunk", func() (contracts.Chunker, error) {
|
||||
return runnerSeriatimChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
if err := extractors.Register("fake/extract", func() (contracts.Extractor, error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
if err := mergers.Register(pipeline.DefaultMergeModule, func() (contracts.Merger, error) {
|
||||
return appendorder.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := normalizers.Register(pipeline.DefaultNormalizeModule, func() (contracts.Normalizer, error) {
|
||||
return noop.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
|
||||
return runnerSeriatimOutput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
|
||||
return pipeline.Registries{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
type runnerSeriatimChunker struct{}
|
||||
|
||||
func (runnerSeriatimChunker) Key() string {
|
||||
return "fake/chunk"
|
||||
}
|
||||
|
||||
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []contracts.SourceChunk{
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
Content: []byte(`{"units":[1,2]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type runnerSeriatimExtractor struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) Key() string {
|
||||
return "fake/extract"
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
e.calls++
|
||||
if req.Source == nil {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
|
||||
}
|
||||
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
for _, unit := range req.Chunk.Units {
|
||||
if speaker, ok := Speaker(unit); !ok || speaker == "" {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
|
||||
}
|
||||
if _, ok := Start(unit); !ok {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
|
||||
}
|
||||
if _, ok := End(unit); !ok {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
|
||||
}
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}{
|
||||
Value: "seriatim-source-ref",
|
||||
SourceRefs: []source.SourceRef{
|
||||
{
|
||||
SourceID: req.Source.ID,
|
||||
StartUnitID: req.Chunk.Units[0].ID,
|
||||
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, err
|
||||
}
|
||||
|
||||
return contracts.ExtractionResult{
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: payload,
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type runnerSeriatimOutput struct{}
|
||||
|
||||
func (runnerSeriatimOutput) Key() string {
|
||||
return pipeline.DefaultOutputModule
|
||||
}
|
||||
|
||||
func (runnerSeriatimOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{
|
||||
Files: []contracts.OutputFile{
|
||||
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func unitIDs(units []source.SourceUnit) []int {
|
||||
ids := make([]int, 0, len(units))
|
||||
for _, unit := range units {
|
||||
ids = append(ids, unit.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func equalInts(a, b []int) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = runnerSeriatimChunker{}
|
||||
_ contracts.Extractor = (*runnerSeriatimExtractor)(nil)
|
||||
_ contracts.OutputEncoder = runnerSeriatimOutput{}
|
||||
)
|
||||
21
internal/modules/seriatim/input/transcript/testdata/duplicate_segment_id.json
vendored
Normal file
21
internal/modules/seriatim/input/transcript/testdata/duplicate_segment_id.json
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"metadata": {
|
||||
"id": "duplicate-segment-fixture"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"speaker": "Narrator",
|
||||
"text": "First segment."
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1,
|
||||
"end": 2,
|
||||
"speaker": "Player",
|
||||
"text": "Duplicate segment."
|
||||
}
|
||||
]
|
||||
}
|
||||
11
internal/modules/seriatim/input/transcript/testdata/pipeline.yml
vendored
Normal file
11
internal/modules/seriatim/input/transcript/testdata/pipeline.yml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
version: 2
|
||||
pipelines:
|
||||
seriatim-fixture:
|
||||
input: seriatim
|
||||
chunk: fake/chunk
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
merge: appendorder
|
||||
normalize: noop
|
||||
output: json
|
||||
23
internal/modules/seriatim/input/transcript/testdata/valid_minimal.json
vendored
Normal file
23
internal/modules/seriatim/input/transcript/testdata/valid_minimal.json
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"metadata": {
|
||||
"id": "session-alpha",
|
||||
"source_id": "fallback-session",
|
||||
"title": "Synthetic session transcript"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 4.5,
|
||||
"speaker": "Narrator",
|
||||
"text": "The stone door opens."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"start": 4.5,
|
||||
"end": 8,
|
||||
"speaker": "Player",
|
||||
"text": "I cast light."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user