eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move chunks into the canonical source model

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:32:11 +00:00
parent 40709e4ad8
commit 075888c97f
33 changed files with 548 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
internal/framework/pipeline/checkpoint.go
View File

@@ -21,7 +21,7 @@ type CheckpointRecorder interface {
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
SourceFailed(moduleKey string, err error) error
ChunkRunning(moduleKey string, sourceDigest string) error
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []source.Chunk, warnings []contracts.Warning) error
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
ChunkFailed(moduleKey string, sourceDigest string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
@@ -55,7 +55,7 @@ type SourceCheckpoint struct {
}
type ChunkCheckpoint struct {
Chunks []contracts.SourceChunk
Chunks []source.Chunk
Warnings []contracts.Warning
}
@@ -94,7 +94,7 @@ func (noopCheckpointRecorder) SourceRunning(string) error
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil }
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []contracts.SourceChunk, []contracts.Warning) error {
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []source.Chunk, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error {
@@ -180,17 +180,21 @@ func digestFingerprints(name string, digest string) []CheckpointFingerprint {
return []CheckpointFingerprint{{Name: name, Value: digest}}
}
func joinedChunkDigest(chunks []contracts.SourceChunk) string {
func joinedChunkDigest(chunks []source.Chunk) (string, error) {
if len(chunks) == 0 {
return ""
return "", nil
}
values := make([]string, 0, len(chunks))
for _, chunk := range chunks {
values = append(values, chunk.ID+"="+checkpointContentDigest(chunk.Content))
digest, err := source.DigestChunk(chunk)
if err != nil {
return "", fmt.Errorf("digest chunk %q: %w", chunk.ID, err)
}
values = append(values, chunk.ID+"="+digest)
}
sort.Strings(values)
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
return "sha256:" + hex.EncodeToString(sum[:])
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {

53
internal/framework/pipeline/chunk_validation.go
View File

@@ -5,10 +5,9 @@ import (
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []source.Chunk) ([]source.Chunk, error) {
if len(chunks) == 0 {
return nil, fmt.Errorf("chunks must not be empty")
}
@@ -20,7 +19,7 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
sourceUnits[unit.ID] = unit
}
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
canonicalChunks := make([]source.Chunk, 0, len(chunks))
seenChunkIDs := make(map[string]struct{}, len(chunks))
for chunkIndex, chunk := range chunks {
if strings.TrimSpace(chunk.ID) == "" {
@@ -37,17 +36,6 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
}
startIndex, ok := sourceUnitIndexes[chunk.StartUnitID]
if !ok {
return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID)
}
endIndex, ok := sourceUnitIndexes[chunk.EndUnitID]
if !ok {
return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID)
}
if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
}
@@ -57,6 +45,9 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
if strings.TrimSpace(chunk.MediaType) == "" {
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
}
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
}
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
previousSourceIndex := -1
@@ -74,23 +65,33 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
if !ok {
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
}
if sourceIndex <= previousSourceIndex {
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
if previousSourceIndex >= 0 && sourceIndex != previousSourceIndex+1 {
return nil, fmt.Errorf("chunk %q source units must form a contiguous range in source document order", chunk.ID)
}
if unit.Ref != sourceUnits[unit.ID].Ref {
return nil, fmt.Errorf("chunk %q source unit %d ref does not match source document", chunk.ID, unit.ID)
}
previousSourceIndex = sourceIndex
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
}
expectedRef := source.SourceRef{
SourceID: doc.ID,
StartUnitID: canonicalUnits[0].Ref.StartUnitID,
EndUnitID: canonicalUnits[len(canonicalUnits)-1].Ref.EndUnitID,
}
if chunk.Ref != expectedRef {
return nil, fmt.Errorf("chunk %q ref %#v does not match unit span %#v", chunk.ID, chunk.Ref, expectedRef)
}
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
StartUnitID: chunk.StartUnitID,
EndUnitID: chunk.EndUnitID,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
canonicalChunks = append(canonicalChunks, source.Chunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: expectedRef,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
})
}

34
internal/framework/pipeline/debug.go
View File

@@ -104,14 +104,13 @@ type debugSourceDocument struct {
}
type debugSourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref source.SourceRef `json:"ref"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugExtractOutput struct {
@@ -448,20 +447,19 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
}
}
func debugSourceChunkEnvelope(chunk contracts.SourceChunk) debugSourceChunk {
func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
return debugSourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
StartUnitID: chunk.StartUnitID,
EndUnitID: chunk.EndUnitID,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
}
}
func debugSourceChunkEnvelopes(chunks []contracts.SourceChunk) []debugSourceChunk {
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil
}

21
internal/framework/pipeline/debug_test.go
View File

@@ -28,3 +28,24 @@ func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
t.Fatalf("source document ref = %q after debug mutation, want source-1", got)
}
}
func TestDebugSourceChunkPreservesReference(t *testing.T) {
doc := validSourceDocument()
chunk := source.Chunk{
ID: "chunk-1", SourceID: doc.ID, Index: 0,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"), MediaType: "text/plain", Units: doc.Units[:1],
}
envelope := debugSourceChunkEnvelope(chunk)
encoded, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("Marshal(debug source chunk) error = %v, want nil", err)
}
var decoded debugSourceChunk
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("Unmarshal(debug source chunk) error = %v, want nil", err)
}
if decoded.Ref != chunk.Ref {
t.Fatalf("debug chunk ref = %#v, want %#v", decoded.Ref, chunk.Ref)
}
}

21
internal/framework/pipeline/registry_integration_test.go
View File

@@ -117,16 +117,19 @@ func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
Chunks: []source.Chunk{
{
ID: "chunk-0",
SourceID: req.Source.ID,
Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: req.Source.Units,
ID: "chunk-0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: req.Source.Units,
},
},
}, nil

26
internal/framework/pipeline/runner.go
View File

@@ -173,7 +173,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
attachModuleManifestMetadata(&output, "chunker", chunker)
var canonicalChunks []contracts.SourceChunk
var canonicalChunks []source.Chunk
var chunkWarnings []contracts.Warning
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
@@ -388,7 +388,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
@@ -406,7 +406,11 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
extractWarnings := []contracts.Warning{}
extractRejectedStart := len(output.Rejected)
extractDependencies := digestFingerprints("chunks", joinedChunkDigest(chunks))
chunksDigest, err := joinedChunkDigest(chunks)
if err != nil {
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractDependencies := digestFingerprints("chunks", chunksDigest)
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
extractStarted := time.Now().UTC()
@@ -887,8 +891,8 @@ type rawValidationTarget struct {
llmClient contracts.StructuredLLMClient
chunkID string
chunkIndex int
chunk *contracts.SourceChunk
chunks []contracts.SourceChunk
chunk *source.Chunk
chunks []source.Chunk
schema contracts.ResponseSchema
payload contracts.RawPayload
extractOutputs []contracts.ExtractOutput
@@ -946,7 +950,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
return false, lastRejection, nil
}
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
return r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
moduleKey: moduleKey,
@@ -1463,7 +1467,7 @@ func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMat
)
}
func chunkInputMaterial(sourceInput contracts.LLMInputMaterial, chunk contracts.SourceChunk) contracts.LLMInputMaterial {
func chunkInputMaterial(sourceInput contracts.LLMInputMaterial, chunk source.Chunk) contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial(
"source",
chunk.MediaType,
@@ -1537,7 +1541,7 @@ func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSche
return schema
}
func cloneSourceChunkPtr(chunk *contracts.SourceChunk) *contracts.SourceChunk {
func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
if chunk == nil {
return nil
}
@@ -1545,18 +1549,18 @@ func cloneSourceChunkPtr(chunk *contracts.SourceChunk) *contracts.SourceChunk {
return &cloned
}
func cloneSourceChunk(chunk contracts.SourceChunk) contracts.SourceChunk {
func cloneSourceChunk(chunk source.Chunk) source.Chunk {
chunk.Content = append([]byte(nil), chunk.Content...)
chunk.Units = cloneSourceUnits(chunk.Units)
chunk.Metadata = cloneMetadata(chunk.Metadata)
return chunk
}
func cloneSourceChunks(chunks []contracts.SourceChunk) []contracts.SourceChunk {
func cloneSourceChunks(chunks []source.Chunk) []source.Chunk {
if len(chunks) == 0 {
return nil
}
out := make([]contracts.SourceChunk, 0, len(chunks))
out := make([]source.Chunk, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, cloneSourceChunk(chunk))
}

171
internal/framework/pipeline/runner_test.go
View File

@@ -249,17 +249,17 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
func TestRunRejectsInvalidChunks(t *testing.T) {
tests := []struct {
name string
chunks []contracts.SourceChunk
chunks []source.Chunk
want string
}{
{
name: "empty chunk id",
chunks: []contracts.SourceChunk{chunkWithUnits("", "source-1", 0, unitWithID("u1"))},
chunks: []source.Chunk{chunkWithUnits("", "source-1", 0, unitWithID("u1"))},
want: "id must not be empty",
},
{
name: "duplicate chunk id",
chunks: []contracts.SourceChunk{
chunks: []source.Chunk{
chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1")),
chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u2")),
},
@@ -267,58 +267,95 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
},
{
name: "wrong source id",
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "other-source", 0, unitWithID("u1"))},
chunks: []source.Chunk{chunkWithUnits("chunk-0", "other-source", 0, unitWithID("u1"))},
want: "source_id",
},
{
name: "wrong index",
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u1"))},
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u1"))},
want: "index",
},
{
name: "missing ref",
chunks: []source.Chunk{func() source.Chunk {
chunk := chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"))
chunk.Ref = source.SourceRef{}
return chunk
}()},
want: "source_id",
},
{
name: "foreign ref",
chunks: []source.Chunk{func() source.Chunk {
chunk := chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"))
chunk.Ref.SourceID = "other-source"
return chunk
}()},
want: "does not match document id",
},
{
name: "unknown start id",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 9, 1, unitWithID("u1"))},
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 9, 1, unitWithID("u1"))},
want: "start_unit_id",
},
{
name: "unknown end id",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 9, unitWithID("u1"))},
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 9, unitWithID("u1"))},
want: "end_unit_id",
},
{
name: "reversed bounds",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 2, 1, unitWithID("u1"), unitWithID("u2"))},
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 2, 1, unitWithID("u1"), unitWithID("u2"))},
want: "appears after",
},
{
name: "empty units",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[]}`), MediaType: "application/json"}},
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, Content: []byte(`{"units":[]}`), MediaType: "application/json"}},
want: "units must not be empty",
},
{
name: "empty content",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, MediaType: "application/json", Units: []source.SourceUnit{unitWithID("u1")}}},
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, MediaType: "application/json", Units: []source.SourceUnit{unitWithID("u1")}}},
want: "content must not be empty",
},
{
name: "empty media type",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[1]}`), Units: []source.SourceUnit{unitWithID("u1")}}},
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, Content: []byte(`{"units":[1]}`), Units: []source.SourceUnit{unitWithID("u1")}}},
want: "media_type must not be empty",
},
{
name: "repeated unit inside chunk",
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u1"))},
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u1"))},
want: "repeats source unit",
},
{
name: "unknown unit",
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u9"))},
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u9"))},
want: "was not found",
},
{
name: "units out of source order",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 2, unitWithID("u2"), unitWithID("u1"))},
want: "source document order",
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 2, unitWithID("u2"), unitWithID("u1"))},
want: "contiguous range",
},
{
name: "noncontiguous units",
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 3, unitWithID("u1"), unitWithID("u3"))},
want: "contiguous range",
},
{
name: "ref does not match unit span",
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 2, unitWithID("u1"))},
want: "does not match unit span",
},
{
name: "unit ref does not match source",
chunks: []source.Chunk{func() source.Chunk {
unit := unitWithID("u1")
unit.Ref.SourceID = "other-source"
return chunkWithRef("chunk-0", "source-1", 0, 1, 1, unit)
}()},
want: "ref does not match source document",
},
}
@@ -342,7 +379,7 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
modules.chunker.chunks = []source.Chunk{
chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u2")),
chunkWithUnits("chunk-1", "source-1", 1, unitWithID("u2")),
}
@@ -359,20 +396,20 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules := defaultRunnerModules()
modules.input.doc = sourceDocumentWithUnitMetadata()
modules.chunker.chunks = []contracts.SourceChunk{
modules.chunker.chunks = []source.Chunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
StartUnitID: 1,
EndUnitID: 1,
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{
{
ID: 1,
Kind: "mutated-kind",
Text: "mutated text",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{
"speaker": "chunker-speaker",
"note": "chunker note",
@@ -423,15 +460,14 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
modules.chunker.chunks = []source.Chunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
StartUnitID: 1,
EndUnitID: 1,
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{
unitWithID("u1"),
},
@@ -975,7 +1011,7 @@ func TestRunPassesPerChunkRawOutputsToMergeAndNormalize(t *testing.T) {
func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
modules.chunker.chunks = []source.Chunk{
sourceChunkWithContent("chunk-0", 0, []byte(`{"chunk":0}`), "application/vnd.test+json"),
}
@@ -1027,10 +1063,27 @@ func TestRunDoesNotPassCheckpointPathsToModules(t *testing.T) {
}
}
func TestCheckpointChunkDigestIncludesCanonicalReference(t *testing.T) {
chunk := sourceChunkWithID("chunk-0", 0)
first, err := joinedChunkDigest([]source.Chunk{chunk})
if err != nil {
t.Fatalf("joinedChunkDigest() error = %v, want nil", err)
}
chunk.Ref.EndUnitID = 2
second, err := joinedChunkDigest([]source.Chunk{chunk})
if err != nil {
t.Fatalf("joinedChunkDigest(changed ref) error = %v, want nil", err)
}
if first == second {
t.Fatalf("checkpoint chunk digests = %q and %q, want provenance change to alter dependency identity", first, second)
}
}
func TestRunReusesCheckpointedWorkflowOutputs(t *testing.T) {
modules := defaultRunnerModules()
doc := validSourceDocument()
chunks := []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0)}
chunks := []source.Chunk{sourceChunkWithID("chunk-0", 0)}
extractOutput := contracts.ExtractOutput{
LaneID: "alpha",
ExtractorKey: "extract-alpha",
@@ -1468,7 +1521,7 @@ func TestRunCollectsStageWarnings(t *testing.T) {
func TestRunCollectsChunkValidatorWarnings(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0)}
modules.chunker.chunks = []source.Chunk{sourceChunkWithID("chunk-0", 0)}
validator := &runnerChainValidator{
name: "chain-chunk",
warnings: []contracts.Warning{{ReasonCode: "chunk-validator-warning", Message: "chunk validator warning"}},
@@ -1904,7 +1957,7 @@ type runnerModules struct {
func defaultRunnerModules() *runnerModules {
return &runnerModules{
input: &runnerInputAdapter{key: "input", doc: validSourceDocument()},
chunker: &runnerChunker{key: "chunk", chunks: []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
chunker: &runnerChunker{key: "chunk", chunks: []source.Chunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
extractors: map[string]*runnerExtractor{
"extract-alpha": {key: "extract-alpha"},
},
@@ -2013,7 +2066,7 @@ func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
type runnerChunker struct {
key string
chunks []contracts.SourceChunk
chunks []source.Chunk
warnings []contracts.Warning
err error
failureErr error
@@ -2505,21 +2558,20 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
}
}
func sourceChunkWithID(id string, index int) contracts.SourceChunk {
func sourceChunkWithID(id string, index int) source.Chunk {
unit := unitWithID("u1")
return contracts.SourceChunk{
ID: id,
SourceID: "source-1",
Index: index,
StartUnitID: unit.ID,
EndUnitID: unit.ID,
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{unit},
return source.Chunk{
ID: id,
SourceID: "source-1",
Index: index,
Ref: unit.Ref,
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{unit},
}
}
func sourceChunkWithContent(id string, index int, content []byte, mediaType string) contracts.SourceChunk {
func sourceChunkWithContent(id string, index int, content []byte, mediaType string) source.Chunk {
chunk := sourceChunkWithID(id, index)
chunk.Content = append([]byte(nil), content...)
chunk.MediaType = mediaType
@@ -2541,25 +2593,24 @@ func unitWithID(id string) source.SourceUnit {
}
}
func chunkWithUnits(id string, sourceID string, index int, units ...source.SourceUnit) contracts.SourceChunk {
func chunkWithUnits(id string, sourceID string, index int, units ...source.SourceUnit) source.Chunk {
startUnitID, endUnitID := 1, 1
if len(units) > 0 {
startUnitID = units[0].ID
endUnitID = units[len(units)-1].ID
}
return chunkWithBounds(id, sourceID, index, startUnitID, endUnitID, units...)
return chunkWithRef(id, sourceID, index, startUnitID, endUnitID, units...)
}
func chunkWithBounds(id string, sourceID string, index int, startUnitID int, endUnitID int, units ...source.SourceUnit) contracts.SourceChunk {
return contracts.SourceChunk{
ID: id,
SourceID: sourceID,
Index: index,
StartUnitID: startUnitID,
EndUnitID: endUnitID,
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), units...),
func chunkWithRef(id string, sourceID string, index int, startUnitID int, endUnitID int, units ...source.SourceUnit) source.Chunk {
return source.Chunk{
ID: id,
SourceID: sourceID,
Index: index,
Ref: source.SourceRef{SourceID: sourceID, StartUnitID: startUnitID, EndUnitID: endUnitID},
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), units...),
}
}

32
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -231,26 +231,24 @@ func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.C
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
}
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[1].ID,
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[1].ID},
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
},
{
ID: req.Source.ID + ":chunk:1",
SourceID: req.Source.ID,
Index: 1,
StartUnitID: req.Source.Units[2].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
ID: req.Source.ID + ":chunk:1",
SourceID: req.Source.ID,
Index: 1,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[2].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
Content: []byte(`{"units":[3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
},
},
}, nil