eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add canonical source unit provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:19:57 +00:00
parent 15c369c509
commit 40709e4ad8
26 changed files with 254 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/framework/pipeline/runner_test.go
View File

@@ -402,6 +402,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
if chunk.Units[0].ID != 1 || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Ref; got != modules.input.doc.Units[0].Ref {
t.Fatalf("chunk unit ref = %#v, want source document ref %#v", got, modules.input.doc.Units[0].Ref)
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
@@ -411,8 +414,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Ref.SourceID = "changed-source"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Ref.SourceID != "source-1" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
@@ -2464,9 +2468,9 @@ func validSourceDocument() *source.SourceDocument {
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: 3, Kind: "unit", Text: "Third source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
},
}
}
@@ -2482,6 +2486,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 1,
Kind: "source-kind",
Text: "source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
@@ -2491,6 +2496,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 2,
Kind: "source-kind",
Text: "second source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2},
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
@@ -2523,15 +2529,15 @@ func sourceChunkWithContent(id string, index int, content []byte, mediaType stri
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit."}
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}
case "u2":
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit."}
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}}
case "u3":
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit."}
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}}
case "u9":
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 9}}
default:
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 99, EndUnitID: 99}}
}
}