eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add canonical source unit provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:19:57 +00:00
parent 15c369c509
commit 40709e4ad8
26 changed files with 254 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/cli/compatibility_test.go
View File

@@ -238,7 +238,7 @@ func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) {
if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount {
t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount)
}
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:346f0b0b0cf2d8081ca222dbea47ff231ae9d81c8120ac51a4fc9fa6dfb6cc07"}) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) {
t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests)
}
if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{
@@ -325,7 +325,10 @@ func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
client := frameworkllm.NewScheduledClient(underlying, scheduler)
doc := &source.SourceDocument{
ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds."}, {ID: 2, Kind: "segment", Text: "The spell takes effect."}},
Units: []source.SourceUnit{
{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
},
}
chunk := contracts.SourceChunk{
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, StartUnitID: 1, EndUnitID: 2,

2
internal/cli/run_test.go
View File

@@ -3594,7 +3594,7 @@ func (fakeRunInputAdapter) Parse(ctx context.Context, req contracts.ParseRequest
Format: "test",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "text", Text: string(req.Raw)},
{ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}},
},
}, nil
}

35
internal/core/source/digest.go Normal file
View File

@@ -0,0 +1,35 @@
package source
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
)
// DigestDocument returns a deterministic digest of the canonical source
// document content. The existing Digest field is excluded from its own digest.
func DigestDocument(doc *SourceDocument) (string, error) {
if doc == nil {
return "", fmt.Errorf("source document must not be nil")
}
payload := struct {
ID string `json:"id"`
Kind string `json:"kind"`
Format string `json:"format"`
Units []SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"`
}{
ID: doc.ID,
Kind: doc.Kind,
Format: doc.Format,
Units: doc.Units,
Metadata: doc.Metadata,
}
encoded, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("encode source document for digest: %w", err)
}
sum := sha256.Sum256(encoded)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}

1
internal/core/source/source.go
View File

@@ -13,6 +13,7 @@ type SourceUnit struct {
ID int `json:"id"`
Kind string `json:"kind"`
Text string `json:"text"`
Ref SourceRef `json:"ref"`
Metadata map[string]any `json:"metadata,omitempty"`
}

82
internal/core/source/source_test.go
View File

@@ -142,6 +142,86 @@ func TestValidateDocumentDuplicateUnitIDs(t *testing.T) {
}
}
func TestValidateDocumentUnitReferences(t *testing.T) {
tests := []struct {
name string
mutate func(*SourceDocument)
wantErr string
}{
{
name: "missing",
mutate: func(doc *SourceDocument) { doc.Units[0].Ref = SourceRef{} },
wantErr: "source unit[0].ref: source ref source_id must not be empty",
},
{
name: "foreign source",
mutate: func(doc *SourceDocument) { doc.Units[0].Ref.SourceID = "source-2" },
wantErr: "source unit[0].ref: source ref source_id \"source-2\" does not match document id \"source-1\"",
},
{
name: "non-self range",
mutate: func(doc *SourceDocument) {
doc.Units[0].Ref.StartUnitID = 2
doc.Units[0].Ref.EndUnitID = 2
},
wantErr: "source unit[0].ref must identify source unit id 1",
},
{
name: "reversed range",
mutate: func(doc *SourceDocument) {
doc.Units[0].Ref.StartUnitID = 2
doc.Units[0].Ref.EndUnitID = 1
},
wantErr: "source unit[0].ref: source ref start_unit_id 2 appears after end_unit_id 1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc := validDocument()
tt.mutate(doc)
err := ValidateDocument(doc)
if err == nil {
t.Fatal("ValidateDocument() error = nil, want unit reference error")
}
if err.Error() != tt.wantErr {
t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr)
}
})
}
}
func TestDigestDocumentIsDeterministicAndIncludesUnitReference(t *testing.T) {
doc := validDocument()
doc.Metadata = map[string]any{"second": "value", "first": true}
first, err := DigestDocument(doc)
if err != nil {
t.Fatalf("DigestDocument() error = %v, want nil", err)
}
reordered := validDocument()
reordered.Metadata = map[string]any{"first": true, "second": "value"}
second, err := DigestDocument(reordered)
if err != nil {
t.Fatalf("DigestDocument(reordered) error = %v, want nil", err)
}
if first != second {
t.Fatalf("digests = %q and %q, want deterministic map ordering", first, second)
}
changed := validDocument()
changed.Metadata = map[string]any{"first": true, "second": "value"}
changed.Units[0].Ref.SourceID = "different-source"
changedDigest, err := DigestDocument(changed)
if err != nil {
t.Fatalf("DigestDocument(changed) error = %v, want nil", err)
}
if first == changedDigest {
t.Fatalf("digest = %q after reference change, want different digest", changedDigest)
}
}
func TestValidateRefValid(t *testing.T) {
doc := validDocument()
ref := SourceRef{
@@ -274,11 +354,13 @@ func validDocument() *SourceDocument {
ID: 1,
Kind: "paragraph",
Text: "First unit.",
Ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
{
ID: 2,
Kind: "paragraph",
Text: "Second unit.",
Ref: SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2},
},
},
}

8
internal/core/source/validation.go
View File

@@ -44,6 +44,14 @@ func ValidateDocument(doc *SourceDocument) error {
}
seenUnitIDs[unit.ID] = struct{}{}
}
for i, unit := range doc.Units {
if err := ValidateRef(doc, unit.Ref); err != nil {
return fmt.Errorf("source unit[%d].ref: %w", i, err)
}
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
}
}
return nil
}

1
internal/framework/checkpoint/recorder.go
View File

@@ -414,6 +414,7 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

7
internal/framework/checkpoint/recorder_test.go
View File

@@ -22,7 +22,7 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []contracts.SourceChunk{
{
@@ -89,7 +89,7 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []contracts.SourceChunk{
{
@@ -155,6 +155,9 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
}
if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want {
t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want)
}
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)

4
internal/framework/contracts/composition_test.go
View File

@@ -116,8 +116,8 @@ func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.Parse
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "First source unit."},
{ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: 1, Kind: "unit", Text: "First source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 2, EndUnitID: 2}},
},
}, nil
}

1
internal/framework/pipeline/chunk_validation.go
View File

@@ -102,6 +102,7 @@ func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
}
}

30
internal/framework/pipeline/debug_test.go Normal file
View File

@@ -0,0 +1,30 @@
package pipeline
import (
"encoding/json"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
doc := validSourceDocument()
envelope := debugSourceDocumentEnvelope(doc)
encoded, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("Marshal(debug source document) error = %v, want nil", err)
}
var decoded debugSourceDocument
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("Unmarshal(debug source document) error = %v, want nil", err)
}
if got, want := decoded.Units[0].Ref, (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}); got != want {
t.Fatalf("debug source unit ref = %#v, want %#v", got, want)
}
envelope.Units[0].Ref.SourceID = "mutated"
if got := doc.Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("source document ref = %q after debug mutation, want source-1", got)
}
}

2
internal/framework/pipeline/input_registry_test.go
View File

@@ -301,7 +301,7 @@ func (adapter fakeAdapter) Parse(ctx context.Context, req contracts.ParseRequest
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
},
}, nil
}

2
internal/framework/pipeline/registry_integration_test.go
View File

@@ -247,7 +247,7 @@ func integrationSourceDocument() *source.SourceDocument {
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
},
}
}

24
internal/framework/pipeline/runner_test.go
View File

@@ -402,6 +402,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
if chunk.Units[0].ID != 1 || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Ref; got != modules.input.doc.Units[0].Ref {
t.Fatalf("chunk unit ref = %#v, want source document ref %#v", got, modules.input.doc.Units[0].Ref)
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
@@ -411,8 +414,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Ref.SourceID = "changed-source"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Ref.SourceID != "source-1" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
@@ -2464,9 +2468,9 @@ func validSourceDocument() *source.SourceDocument {
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: 3, Kind: "unit", Text: "Third source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
},
}
}
@@ -2482,6 +2486,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 1,
Kind: "source-kind",
Text: "source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
@@ -2491,6 +2496,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 2,
Kind: "source-kind",
Text: "second source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2},
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
@@ -2523,15 +2529,15 @@ func sourceChunkWithContent(id string, index int, content []byte, mediaType stri
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit."}
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}
case "u2":
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit."}
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}}
case "u3":
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit."}
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}}
case "u9":
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 9}}
default:
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 99, EndUnitID: 99}}
}
}

5
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -200,6 +200,11 @@ func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.Parse
ID: unit.ID,
Kind: "unit",
Text: unit.Text,
Ref: source.SourceRef{
SourceID: fixture.ID,
StartUnitID: unit.ID,
EndUnitID: unit.ID,
},
})
}
return &source.SourceDocument{

1
internal/modules/dnd/chunk/scenes/chunker.go
View File

@@ -332,6 +332,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

12
internal/modules/dnd/chunk/scenes/chunker_test.go
View File

@@ -318,12 +318,16 @@ func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
}
doc.Units[0].ID = 99
doc.Units[0].Ref.SourceID = "mutated"
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "session-alpha" {
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
@@ -535,10 +539,10 @@ func sceneSourceDocument() *source.SourceDocument {
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."},
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 3, EndUnitID: 3}},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 4, EndUnitID: 4}},
},
}
}

2
internal/modules/dnd/extract/spells/test_helpers_test.go
View File

@@ -39,6 +39,7 @@ func promptSourceDocument() *source.SourceDocument {
ID: 1,
Kind: "transcript_segment",
Text: "Aria raises her hand and casts Cure Wounds.",
Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "Alice",
"start": json.Number("1.25"),
@@ -50,6 +51,7 @@ func promptSourceDocument() *source.SourceDocument {
ID: 2,
Kind: "transcript_segment",
Text: "The fighter's wounds begin to close.",
Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2},
Metadata: map[string]any{"ignored": "not rendered"},
},
},

1
internal/modules/generic/chunk/units/chunker.go
View File

@@ -243,6 +243,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

5
internal/modules/generic/chunk/units/chunker_test.go
View File

@@ -169,11 +169,15 @@ func TestChunkDefensivelyCopiesUnits(t *testing.T) {
}
doc.Units[0].ID = 99
doc.Units[0].Ref.SourceID = "changed"
doc.Units[0].Metadata["speaker"] = "changed"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
@@ -186,6 +190,7 @@ func testSource(count int) *source.SourceDocument {
ID: i,
Kind: "unit",
Text: "Text for " + zeroPad3(i),
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: i, EndUnitID: i},
Metadata: map[string]any{
"speaker": "speaker-" + zeroPad3(i),
},

14
internal/modules/seriatim/input/transcript/adapter.go
View File

@@ -65,18 +65,21 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
Kind: DocumentKind,
Format: Format,
Digest: rawDigest,
Metadata: copyMetadata(parsed.Metadata),
}
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
for i, segment := range parsed.Segments {
unit, err := sourceUnit(segment, i, seenSegmentIDs)
unit, err := sourceUnit(doc.ID, segment, i, seenSegmentIDs)
if err != nil {
return nil, err
}
doc.Units = append(doc.Units, unit)
}
doc.Digest, err = source.DigestDocument(doc)
if err != nil {
return nil, inputErrorf("digest source document: %w", err)
}
if err := source.ValidateDocument(doc); err != nil {
return nil, inputErrorf("validate source document: %w", err)
@@ -98,7 +101,7 @@ func Register(registry *pipeline.InputAdapterRegistry) error {
})
}
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
func sourceUnit(sourceID string, segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
segmentLabel := fmt.Sprintf("segment[%d]", index)
if segment.ID <= 0 {
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
@@ -133,6 +136,11 @@ func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.Sourc
ID: segment.ID,
Kind: UnitKind,
Text: segment.Text,
Ref: source.SourceRef{
SourceID: sourceID,
StartUnitID: segment.ID,
EndUnitID: segment.ID,
},
Metadata: map[string]any{
MetadataSpeaker: segment.Speaker,
MetadataStart: segment.Start,

16
internal/modules/seriatim/input/transcript/adapter_test.go
View File

@@ -30,8 +30,8 @@ func TestParseValidMinimalTranscript(t *testing.T) {
if doc.Format != Format {
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
}
if doc.Digest != testDigest(raw) {
t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw))
if doc.Digest == testDigest(raw) || !strings.HasPrefix(doc.Digest, "sha256:") {
t.Fatalf("doc.Digest = %q, want canonical source digest distinct from raw input digest", doc.Digest)
}
if got := doc.Metadata["title"]; got != "Synthetic session transcript" {
t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got)
@@ -39,6 +39,12 @@ func TestParseValidMinimalTranscript(t *testing.T) {
if len(doc.Units) != 2 {
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
}
for i, unit := range doc.Units {
want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID}
if unit.Ref != want {
t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want)
}
}
first := doc.Units[0]
if first.ID != 1 {
@@ -89,6 +95,12 @@ func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 {
t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID})
}
for i, unit := range doc.Units {
want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID}
if unit.Ref != want {
t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want)
}
}
ref := source.SourceRef{
SourceID: doc.ID,
StartUnitID: 1,