eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add canonical source unit provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:19:57 +00:00
parent 15c369c509
commit 40709e4ad8
26 changed files with 254 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/modules/dnd/chunk/scenes/chunker.go
View File

@@ -332,6 +332,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

12
internal/modules/dnd/chunk/scenes/chunker_test.go
View File

@@ -318,12 +318,16 @@ func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
}
doc.Units[0].ID = 99
doc.Units[0].Ref.SourceID = "mutated"
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "session-alpha" {
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
@@ -535,10 +539,10 @@ func sceneSourceDocument() *source.SourceDocument {
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."},
{ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "Alice"}},
{ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 3, EndUnitID: 3}},
{ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 4, EndUnitID: 4}},
},
}
}

2
internal/modules/dnd/extract/spells/test_helpers_test.go
View File

@@ -39,6 +39,7 @@ func promptSourceDocument() *source.SourceDocument {
ID: 1,
Kind: "transcript_segment",
Text: "Aria raises her hand and casts Cure Wounds.",
Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "Alice",
"start": json.Number("1.25"),
@@ -50,6 +51,7 @@ func promptSourceDocument() *source.SourceDocument {
ID: 2,
Kind: "transcript_segment",
Text: "The fighter's wounds begin to close.",
Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2},
Metadata: map[string]any{"ignored": "not rendered"},
},
},

1
internal/modules/generic/chunk/units/chunker.go
View File

@@ -243,6 +243,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

5
internal/modules/generic/chunk/units/chunker_test.go
View File

@@ -169,11 +169,15 @@ func TestChunkDefensivelyCopiesUnits(t *testing.T) {
}
doc.Units[0].ID = 99
doc.Units[0].Ref.SourceID = "changed"
doc.Units[0].Metadata["speaker"] = "changed"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
@@ -186,6 +190,7 @@ func testSource(count int) *source.SourceDocument {
ID: i,
Kind: "unit",
Text: "Text for " + zeroPad3(i),
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: i, EndUnitID: i},
Metadata: map[string]any{
"speaker": "speaker-" + zeroPad3(i),
},

14
internal/modules/seriatim/input/transcript/adapter.go
View File

@@ -65,18 +65,21 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
Kind: DocumentKind,
Format: Format,
Digest: rawDigest,
Metadata: copyMetadata(parsed.Metadata),
}
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
for i, segment := range parsed.Segments {
unit, err := sourceUnit(segment, i, seenSegmentIDs)
unit, err := sourceUnit(doc.ID, segment, i, seenSegmentIDs)
if err != nil {
return nil, err
}
doc.Units = append(doc.Units, unit)
}
doc.Digest, err = source.DigestDocument(doc)
if err != nil {
return nil, inputErrorf("digest source document: %w", err)
}
if err := source.ValidateDocument(doc); err != nil {
return nil, inputErrorf("validate source document: %w", err)
@@ -98,7 +101,7 @@ func Register(registry *pipeline.InputAdapterRegistry) error {
})
}
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
func sourceUnit(sourceID string, segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
segmentLabel := fmt.Sprintf("segment[%d]", index)
if segment.ID <= 0 {
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
@@ -133,6 +136,11 @@ func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.Sourc
ID: segment.ID,
Kind: UnitKind,
Text: segment.Text,
Ref: source.SourceRef{
SourceID: sourceID,
StartUnitID: segment.ID,
EndUnitID: segment.ID,
},
Metadata: map[string]any{
MetadataSpeaker: segment.Speaker,
MetadataStart: segment.Start,

16
internal/modules/seriatim/input/transcript/adapter_test.go
View File

@@ -30,8 +30,8 @@ func TestParseValidMinimalTranscript(t *testing.T) {
if doc.Format != Format {
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
}
if doc.Digest != testDigest(raw) {
t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw))
if doc.Digest == testDigest(raw) || !strings.HasPrefix(doc.Digest, "sha256:") {
t.Fatalf("doc.Digest = %q, want canonical source digest distinct from raw input digest", doc.Digest)
}
if got := doc.Metadata["title"]; got != "Synthetic session transcript" {
t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got)
@@ -39,6 +39,12 @@ func TestParseValidMinimalTranscript(t *testing.T) {
if len(doc.Units) != 2 {
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
}
for i, unit := range doc.Units {
want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID}
if unit.Ref != want {
t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want)
}
}
first := doc.Units[0]
if first.ID != 1 {
@@ -89,6 +95,12 @@ func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 {
t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID})
}
for i, unit := range doc.Units {
want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID}
if unit.Ref != want {
t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want)
}
}
ref := source.SourceRef{
SourceID: doc.ID,
StartUnitID: 1,