eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add canonical source unit provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:19:57 +00:00
parent 15c369c509
commit 40709e4ad8
26 changed files with 254 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/framework/checkpoint/recorder.go
View File

@@ -414,6 +414,7 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}

7
internal/framework/checkpoint/recorder_test.go
View File

@@ -22,7 +22,7 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []contracts.SourceChunk{
{
@@ -89,7 +89,7 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []contracts.SourceChunk{
{
@@ -155,6 +155,9 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
}
if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want {
t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want)
}
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)

4
internal/framework/contracts/composition_test.go
View File

@@ -116,8 +116,8 @@ func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.Parse
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "First source unit."},
{ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: 1, Kind: "unit", Text: "First source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 2, EndUnitID: 2}},
},
}, nil
}

1
internal/framework/pipeline/chunk_validation.go
View File

@@ -102,6 +102,7 @@ func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
}
}

30
internal/framework/pipeline/debug_test.go Normal file
View File

@@ -0,0 +1,30 @@
package pipeline
import (
"encoding/json"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
doc := validSourceDocument()
envelope := debugSourceDocumentEnvelope(doc)
encoded, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("Marshal(debug source document) error = %v, want nil", err)
}
var decoded debugSourceDocument
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("Unmarshal(debug source document) error = %v, want nil", err)
}
if got, want := decoded.Units[0].Ref, (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}); got != want {
t.Fatalf("debug source unit ref = %#v, want %#v", got, want)
}
envelope.Units[0].Ref.SourceID = "mutated"
if got := doc.Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("source document ref = %q after debug mutation, want source-1", got)
}
}

2
internal/framework/pipeline/input_registry_test.go
View File

@@ -301,7 +301,7 @@ func (adapter fakeAdapter) Parse(ctx context.Context, req contracts.ParseRequest
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
},
}, nil
}

2
internal/framework/pipeline/registry_integration_test.go
View File

@@ -247,7 +247,7 @@ func integrationSourceDocument() *source.SourceDocument {
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
},
}
}

24
internal/framework/pipeline/runner_test.go
View File

@@ -402,6 +402,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
if chunk.Units[0].ID != 1 || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Ref; got != modules.input.doc.Units[0].Ref {
t.Fatalf("chunk unit ref = %#v, want source document ref %#v", got, modules.input.doc.Units[0].Ref)
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
@@ -411,8 +414,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Ref.SourceID = "changed-source"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Ref.SourceID != "source-1" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
@@ -2464,9 +2468,9 @@ func validSourceDocument() *source.SourceDocument {
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
{ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: 3, Kind: "unit", Text: "Third source unit."},
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
},
}
}
@@ -2482,6 +2486,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 1,
Kind: "source-kind",
Text: "source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
@@ -2491,6 +2496,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
ID: 2,
Kind: "source-kind",
Text: "second source text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2},
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
@@ -2523,15 +2529,15 @@ func sourceChunkWithContent(id string, index int, content []byte, mediaType stri
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit."}
return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}
case "u2":
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit."}
return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}}
case "u3":
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit."}
return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}}
case "u9":
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 9}}
default:
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit."}
return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 99, EndUnitID: 99}}
}
}

5
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -200,6 +200,11 @@ func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.Parse
ID: unit.ID,
Kind: "unit",
Text: unit.Text,
Ref: source.SourceRef{
SourceID: fixture.ID,
StartUnitID: unit.ID,
EndUnitID: unit.ID,
},
})
}
return &source.SourceDocument{