Implement integer source units and chunk payloads

This commit is contained in:
2026-07-07 18:34:23 +00:00
parent 4f057b99ac
commit 9e3f8809b3
45 changed files with 618 additions and 451 deletions

View File

@@ -44,8 +44,8 @@ Approved artifacts use the generic artifact envelope documented in
"source_refs": [ "source_refs": [
{ {
"source_id": "session-alpha", "source_id": "session-alpha",
"start_unit_id": "seg-001", "start_unit_id": 1,
"end_unit_id": "seg-001" "end_unit_id": 1
} }
] ]
} }
@@ -78,7 +78,7 @@ Each source reference uses the generic source-reference shape:
Validation requires: Validation requires:
- at least one source reference; - at least one source reference;
- non-empty source ID and unit IDs; - non-empty source ID and positive unit IDs;
- source ID matching the source document ID; - source ID matching the source document ID;
- start and end unit IDs existing in the source document; - start and end unit IDs existing in the source document;
- start unit appearing before or at the same position as end unit. - start unit appearing before or at the same position as end unit.
@@ -98,8 +98,8 @@ The extractor asks the LLM for this top-level response shape:
"source_refs": [ "source_refs": [
{ {
"source_id": "session-alpha", "source_id": "session-alpha",
"start_unit_id": "seg-001", "start_unit_id": 1,
"end_unit_id": "seg-001" "end_unit_id": 1
} }
] ]
} }

View File

@@ -156,8 +156,8 @@ Each artifact file has this shape:
"source_refs": [ "source_refs": [
{ {
"source_id": "session-alpha", "source_id": "session-alpha",
"start_unit_id": "seg-001", "start_unit_id": 1,
"end_unit_id": "seg-001" "end_unit_id": 1
} }
] ]
} }

View File

@@ -28,7 +28,7 @@ output that provides the same required segment fields.
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 4, "end": 4,
"speaker": "Aria", "speaker": "Aria",
@@ -56,10 +56,9 @@ The adapter rejects:
- missing, null, or non-object `metadata`; - missing, null, or non-object `metadata`;
- missing, null, non-array, or empty `segments`; - missing, null, non-array, or empty `segments`;
- segment values that are not objects; - segment values that are not objects;
- segment `id` values that are neither strings nor numbers; - segment `id` values that are not positive integer JSON numbers or numeric
strings;
- non-string `speaker` or `text`; - non-string `speaker` or `text`;
- empty segment IDs;
- segment IDs with leading or trailing whitespace;
- duplicate segment IDs; - duplicate segment IDs;
- missing or empty `speaker`; - missing or empty `speaker`;
- missing, empty, invalid, non-finite, or negative `start`; - missing, empty, invalid, non-finite, or negative `start`;
@@ -87,8 +86,7 @@ The adapter maps input to `SourceDocument`:
Each segment becomes one `SourceUnit`: Each segment becomes one `SourceUnit`:
- `segment.id` becomes `SourceUnit.ID`; numeric IDs are converted to their JSON - `segment.id` becomes integer `SourceUnit.ID`;
number text, so `1` becomes `"1"`;
- `segment.text` becomes `SourceUnit.Text`; - `segment.text` becomes `SourceUnit.Text`;
- `SourceUnit.Kind` is `transcript_segment`; - `SourceUnit.Kind` is `transcript_segment`;
- `speaker`, `start`, and `end` are stored in source-unit metadata. - `speaker`, `start`, and `end` are stored in source-unit metadata.

View File

@@ -84,9 +84,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
source document, clones source units, assigns chunk IDs such as `chunk-000001`, source document, clones source units, assigns chunk IDs such as `chunk-000001`,
and records chunk metadata for start unit, end unit, and unit count. and records chunk metadata for start unit, end unit, and unit count.
The pipeline runner canonicalizes chunk units from the source document by ID The pipeline runner canonicalizes chunk units from the source document by
before extractors and mergers run. Chunker-owned context should stay in integer ID before extractors and mergers run. Chunkers also populate chunk
`SourceChunk.Metadata`. start and end unit IDs, content bytes, and media type. Chunker-owned context
should stay in `SourceChunk.Metadata`.
Options: Options:
@@ -126,12 +127,11 @@ Options: none. Non-empty options are rejected.
The chunker enforces full source-unit coverage from the first source unit to the The chunker enforces full source-unit coverage from the first source unit to the
last, sequential contiguous scenes, and no overlap. Its LLM-facing schema uses last, sequential contiguous scenes, and no overlap. Its LLM-facing schema uses
integer `start_unit_id` and `end_unit_id` values as 1-based source-unit numbers; integer `start_unit_id` and `end_unit_id` values matching source-unit IDs. It
the module canonicalizes valid integer references to source-unit IDs before assigns chunk IDs such as `scene-000001`, emits JSON chunk content, and stores
producing chunks. It assigns chunk IDs such as `scene-000001` and stores scene scene metadata including title, primary mode, participants, summary, boundary
metadata including title, primary mode, participants, summary, boundary note, note, confidence, boundary unit IDs, and unit count. Boundary caveats become
confidence, boundary unit IDs, and unit count. Boundary caveats become warnings warnings with reason code
with reason code
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed `scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
structured output rather than silently dropped. structured output rather than silently dropped.
@@ -150,8 +150,7 @@ input materials, response schema, and session ID to the runtime; converts
spell-cast responses into artifact candidates; and supplies deterministic spell-cast responses into artifact candidates; and supplies deterministic
validators. validators.
Its LLM-facing source-reference schema uses integer `start_unit_id` and Its LLM-facing source-reference schema uses integer `start_unit_id` and
`end_unit_id` values as 1-based source-unit numbers; the module canonicalizes `end_unit_id` values matching source-unit IDs.
valid integer references to source-unit IDs before validation and output.
Its prompt definition lives under `assets/prompts` and its schema under Its prompt definition lives under `assets/prompts` and its schema under
`assets/schemas`. Shared reusable D&D prompt fragments are provided by `assets/schemas`. Shared reusable D&D prompt fragments are provided by

View File

@@ -136,15 +136,19 @@ invariants before running extractors:
- chunk IDs must be non-empty and unique in the chunk result; - chunk IDs must be non-empty and unique in the chunk result;
- each chunk `SourceID` must match the source document ID; - each chunk `SourceID` must match the source document ID;
- each chunk `Index` must match its zero-based returned order; - each chunk `Index` must match its zero-based returned order;
- each chunk start and end unit ID must exist in the source document, with the
start unit at or before the end unit;
- each chunk must include non-empty extraction content and media type;
- each chunk must contain at least one source unit; - each chunk must contain at least one source unit;
- a chunk must not repeat a source unit; - a chunk must not repeat a source unit;
- every chunk source unit must exist in the source document; - every chunk source unit must exist in the source document;
- source units inside each chunk must appear in source-document order. - source units inside each chunk must appear in source-document order.
After validation, the runner rebuilds each chunk from source-document units by After validation, the runner rebuilds each chunk from source-document units by
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors integer ID, preserving chunk boundaries, content bytes, media type, and cloned
and downstream stages therefore see canonical source units, while chunk metadata. Extractors and downstream stages therefore see canonical source
`SourceChunk.Metadata` remains the supported place for chunker-owned context. units, while `SourceChunk.Metadata` remains the supported place for
chunker-owned context.
The framework does not require complete source-unit coverage and does not reject The framework does not require complete source-unit coverage and does not reject
overlap between different chunks. Stricter policies, such as full coverage or overlap between different chunks. Stricter policies, such as full coverage or

View File

@@ -5,14 +5,14 @@
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 4, "end": 4,
"speaker": "Aria", "speaker": "Aria",
"text": "Aria raises her holy symbol and casts Cure Wounds." "text": "Aria raises her holy symbol and casts Cure Wounds."
}, },
{ {
"id": "seg-002", "id": 2,
"start": 4, "start": 4,
"end": 8, "end": 8,
"speaker": "DM", "speaker": "DM",

View File

@@ -2012,7 +2012,7 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
t.Fatalf("source refs = %#v, want one source ref", artifactFile.Artifacts[0].SourceRefs) t.Fatalf("source refs = %#v, want one source ref", artifactFile.Artifacts[0].SourceRefs)
} }
ref := artifactFile.Artifacts[0].SourceRefs[0] ref := artifactFile.Artifacts[0].SourceRefs[0]
if ref.SourceID != "session-alpha" || ref.StartUnitID != "seg-001" || ref.EndUnitID != "seg-001" { if ref.SourceID != "session-alpha" || ref.StartUnitID != 1 || ref.EndUnitID != 1 {
t.Fatalf("source ref = %#v, want fixture source ref", ref) t.Fatalf("source ref = %#v, want fixture source ref", ref)
} }
@@ -2411,7 +2411,7 @@ func writeSeriatimInput(t *testing.T) string {
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 1, "end": 1,
"speaker": "Aria", "speaker": "Aria",
@@ -2454,8 +2454,8 @@ func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req cont
payload := map[string]any{ payload := map[string]any{
"scenes": []map[string]any{ "scenes": []map[string]any{
{ {
"start_unit_id": "seg-001", "start_unit_id": 1,
"end_unit_id": "seg-002", "end_unit_id": 2,
"short_title": "Opening spell", "short_title": "Opening spell",
"primary_mode": "Narrative", "primary_mode": "Narrative",
"main_participants": []string{"Aria"}, "main_participants": []string{"Aria"},
@@ -2478,9 +2478,9 @@ func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req cont
} }
return contracts.StructuredCompletionResponse{Content: encoded}, nil return contracts.StructuredCompletionResponse{Content: encoded}, nil
} }
startUnitID := "seg-001" startUnitID := 1
if client.invalidSourceRef { if client.invalidSourceRef {
startUnitID = "missing-segment" startUnitID = 999
} }
payload := client.payload payload := client.payload
if payload == nil { if payload == nil {
@@ -2491,11 +2491,11 @@ func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req cont
"spell": "Cure Wounds", "spell": "Cure Wounds",
"effect": "Heals a wounded ally.", "effect": "Heals a wounded ally.",
"narrative_description": "Aria casts Cure Wounds.", "narrative_description": "Aria casts Cure Wounds.",
"source_refs": []map[string]string{ "source_refs": []map[string]any{
{ {
"source_id": "session-alpha", "source_id": "session-alpha",
"start_unit_id": startUnitID, "start_unit_id": startUnitID,
"end_unit_id": "seg-001", "end_unit_id": 1,
}, },
}, },
}, },
@@ -2653,7 +2653,7 @@ func (fakeRunInputAdapter) Parse(ctx context.Context, req contracts.ParseRequest
Format: "test", Format: "test",
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "unit-1", Kind: "text", Text: string(req.Raw)}, {ID: 1, Kind: "text", Text: string(req.Raw)},
}, },
}, nil }, nil
} }
@@ -2671,7 +2671,7 @@ func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot {
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{
{ID: "chunk-1", SourceID: req.Source.ID, Index: 0, Units: req.Source.Units}, {ID: "chunk-1", SourceID: req.Source.ID, Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[1]}`), MediaType: "application/json", Units: req.Source.Units},
}, },
}, nil }, nil
} }
@@ -2704,7 +2704,7 @@ func (fakeRunExtractor) Extract(ctx context.Context, req contracts.ExtractionReq
{ {
Payload: []byte(`{"value":true}`), Payload: []byte(`{"value":true}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source", StartUnitID: "unit-1", EndUnitID: "unit-1"}, {SourceID: "source", StartUnitID: 1, EndUnitID: 1},
}, },
}, },
}, },

View File

@@ -16,7 +16,7 @@ func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"example"}`), Payload: json.RawMessage(`{"name":"example"}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u2"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 2},
}, },
Metadata: map[string]any{ Metadata: map[string]any{
"confidence": 0.75, "confidence": 0.75,
@@ -45,13 +45,13 @@ func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
} }
candidate.Payload[0] = '[' candidate.Payload[0] = '['
candidate.SourceRefs[0].StartUnitID = "changed" candidate.SourceRefs[0].StartUnitID = 99
candidate.Metadata["confidence"] = 0.5 candidate.Metadata["confidence"] = 0.5
if string(artifact.Payload) != `{"name":"example"}` { if string(artifact.Payload) != `{"name":"example"}` {
t.Fatalf("Payload changed after candidate mutation: %s", artifact.Payload) t.Fatalf("Payload changed after candidate mutation: %s", artifact.Payload)
} }
if artifact.SourceRefs[0].StartUnitID != "u1" { if artifact.SourceRefs[0].StartUnitID != 1 {
t.Fatalf("SourceRefs changed after candidate mutation: %#v", artifact.SourceRefs) t.Fatalf("SourceRefs changed after candidate mutation: %#v", artifact.SourceRefs)
} }
if artifact.Metadata["confidence"] != 0.75 { if artifact.Metadata["confidence"] != 0.75 {
@@ -67,7 +67,7 @@ func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: json.RawMessage(`{"value":true}`), Payload: json.RawMessage(`{"value":true}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}, },
Metadata: map[string]any{ Metadata: map[string]any{
"reviewed": true, "reviewed": true,

View File

@@ -10,7 +10,7 @@ type SourceDocument struct {
} }
type SourceUnit struct { type SourceUnit struct {
ID string `json:"id"` ID int `json:"id"`
Kind string `json:"kind"` Kind string `json:"kind"`
Text string `json:"text"` Text string `json:"text"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
@@ -18,6 +18,6 @@ type SourceUnit struct {
type SourceRef struct { type SourceRef struct {
SourceID string `json:"source_id"` SourceID string `json:"source_id"`
StartUnitID string `json:"start_unit_id"` StartUnitID int `json:"start_unit_id"`
EndUnitID string `json:"end_unit_id"` EndUnitID int `json:"end_unit_id"`
} }

View File

@@ -96,13 +96,8 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
}{ }{
{ {
name: "id", name: "id",
mutate: func(doc *SourceDocument) { doc.Units[1].ID = "" }, mutate: func(doc *SourceDocument) { doc.Units[1].ID = 0 },
wantErr: "source unit[1].id must not be empty", wantErr: "source unit[1].id must be positive",
},
{
name: "id surrounding whitespace",
mutate: func(doc *SourceDocument) { doc.Units[1].ID = " u2 " },
wantErr: "source unit[1].id \" u2 \" must not contain leading or trailing whitespace",
}, },
{ {
name: "kind", name: "kind",
@@ -135,14 +130,14 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
func TestValidateDocumentDuplicateUnitIDs(t *testing.T) { func TestValidateDocumentDuplicateUnitIDs(t *testing.T) {
doc := validDocument() doc := validDocument()
doc.Units[1].ID = "u1" doc.Units[1].ID = 1
err := ValidateDocument(doc) err := ValidateDocument(doc)
if err == nil { if err == nil {
t.Fatal("ValidateDocument() error = nil, want error") t.Fatal("ValidateDocument() error = nil, want error")
} }
if err.Error() != "source unit id \"u1\" is duplicated" { if err.Error() != "source unit id 1 is duplicated" {
t.Fatalf("ValidateDocument() error = %q", err.Error()) t.Fatalf("ValidateDocument() error = %q", err.Error())
} }
} }
@@ -151,8 +146,8 @@ func TestValidateRefValid(t *testing.T) {
doc := validDocument() doc := validDocument()
ref := SourceRef{ ref := SourceRef{
SourceID: "source-1", SourceID: "source-1",
StartUnitID: "u1", StartUnitID: 1,
EndUnitID: "u2", EndUnitID: 2,
} }
if err := ValidateRef(doc, ref); err != nil { if err := ValidateRef(doc, ref); err != nil {
@@ -164,8 +159,8 @@ func TestValidateRefSourceIDMismatch(t *testing.T) {
doc := validDocument() doc := validDocument()
ref := SourceRef{ ref := SourceRef{
SourceID: "source-2", SourceID: "source-2",
StartUnitID: "u1", StartUnitID: 1,
EndUnitID: "u2", EndUnitID: 2,
} }
err := ValidateRef(doc, ref) err := ValidateRef(doc, ref)
@@ -186,43 +181,33 @@ func TestValidateRefMissingUnitIDs(t *testing.T) {
}{ }{
{ {
name: "missing source id", name: "missing source id",
ref: SourceRef{StartUnitID: "u1", EndUnitID: "u2"}, ref: SourceRef{StartUnitID: 1, EndUnitID: 2},
wantErr: "source ref source_id must not be empty", wantErr: "source ref source_id must not be empty",
}, },
{ {
name: "source id surrounding whitespace", name: "source id surrounding whitespace",
ref: SourceRef{SourceID: " source-1 ", StartUnitID: "u1", EndUnitID: "u2"}, ref: SourceRef{SourceID: " source-1 ", StartUnitID: 1, EndUnitID: 2},
wantErr: "source ref source_id \" source-1 \" must not contain leading or trailing whitespace", wantErr: "source ref source_id \" source-1 \" must not contain leading or trailing whitespace",
}, },
{ {
name: "missing start id", name: "missing start id",
ref: SourceRef{SourceID: "source-1", EndUnitID: "u2"}, ref: SourceRef{SourceID: "source-1", EndUnitID: 2},
wantErr: "source ref start_unit_id must not be empty", wantErr: "source ref start_unit_id must be positive",
},
{
name: "start id surrounding whitespace",
ref: SourceRef{SourceID: "source-1", StartUnitID: " u1 ", EndUnitID: "u2"},
wantErr: "source ref start_unit_id \" u1 \" must not contain leading or trailing whitespace",
}, },
{ {
name: "missing end id", name: "missing end id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1"}, ref: SourceRef{SourceID: "source-1", StartUnitID: 1},
wantErr: "source ref end_unit_id must not be empty", wantErr: "source ref end_unit_id must be positive",
},
{
name: "end id surrounding whitespace",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: " u2 "},
wantErr: "source ref end_unit_id \" u2 \" must not contain leading or trailing whitespace",
}, },
{ {
name: "unknown start id", name: "unknown start id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u9", EndUnitID: "u2"}, ref: SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 2},
wantErr: "source ref start_unit_id \"u9\" was not found", wantErr: "source ref start_unit_id 9 was not found",
}, },
{ {
name: "unknown end id", name: "unknown end id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u9"}, ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 9},
wantErr: "source ref end_unit_id \"u9\" was not found", wantErr: "source ref end_unit_id 9 was not found",
}, },
} }
@@ -244,8 +229,8 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
doc := validDocument() doc := validDocument()
ref := SourceRef{ ref := SourceRef{
SourceID: "source-1", SourceID: "source-1",
StartUnitID: "u2", StartUnitID: 2,
EndUnitID: "u1", EndUnitID: 1,
} }
err := ValidateRef(doc, ref) err := ValidateRef(doc, ref)
@@ -261,7 +246,7 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
func TestUnitIndex(t *testing.T) { func TestUnitIndex(t *testing.T) {
doc := validDocument() doc := validDocument()
index, ok := UnitIndex(doc, "u2") index, ok := UnitIndex(doc, 2)
if !ok { if !ok {
t.Fatal("UnitIndex() ok = false, want true") t.Fatal("UnitIndex() ok = false, want true")
} }
@@ -269,7 +254,7 @@ func TestUnitIndex(t *testing.T) {
t.Fatalf("UnitIndex() index = %d, want 1", index) t.Fatalf("UnitIndex() index = %d, want 1", index)
} }
index, ok = UnitIndex(doc, "u9") index, ok = UnitIndex(doc, 9)
if ok { if ok {
t.Fatal("UnitIndex() ok = true, want false") t.Fatal("UnitIndex() ok = true, want false")
} }
@@ -286,12 +271,12 @@ func validDocument() *SourceDocument {
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []SourceUnit{ Units: []SourceUnit{
{ {
ID: "u1", ID: 1,
Kind: "paragraph", Kind: "paragraph",
Text: "First unit.", Text: "First unit.",
}, },
{ {
ID: "u2", ID: 2,
Kind: "paragraph", Kind: "paragraph",
Text: "Second unit.", Text: "Second unit.",
}, },

View File

@@ -28,13 +28,10 @@ func ValidateDocument(doc *SourceDocument) error {
return fmt.Errorf("source document units must not be empty") return fmt.Errorf("source document units must not be empty")
} }
seenUnitIDs := make(map[string]struct{}, len(doc.Units)) seenUnitIDs := make(map[int]struct{}, len(doc.Units))
for i, unit := range doc.Units { for i, unit := range doc.Units {
if isBlank(unit.ID) { if unit.ID <= 0 {
return fmt.Errorf("source unit[%d].id must not be empty", i) return fmt.Errorf("source unit[%d].id must be positive", i)
}
if hasSurroundingWhitespace(unit.ID) {
return fmt.Errorf("source unit[%d].id %q must not contain leading or trailing whitespace", i, unit.ID)
} }
if isBlank(unit.Kind) { if isBlank(unit.Kind) {
return fmt.Errorf("source unit[%d].kind must not be empty", i) return fmt.Errorf("source unit[%d].kind must not be empty", i)
@@ -43,7 +40,7 @@ func ValidateDocument(doc *SourceDocument) error {
return fmt.Errorf("source unit[%d].text must not be empty", i) return fmt.Errorf("source unit[%d].text must not be empty", i)
} }
if _, ok := seenUnitIDs[unit.ID]; ok { if _, ok := seenUnitIDs[unit.ID]; ok {
return fmt.Errorf("source unit id %q is duplicated", unit.ID) return fmt.Errorf("source unit id %d is duplicated", unit.ID)
} }
seenUnitIDs[unit.ID] = struct{}{} seenUnitIDs[unit.ID] = struct{}{}
} }
@@ -61,17 +58,11 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
if hasSurroundingWhitespace(ref.SourceID) { if hasSurroundingWhitespace(ref.SourceID) {
return fmt.Errorf("source ref source_id %q must not contain leading or trailing whitespace", ref.SourceID) return fmt.Errorf("source ref source_id %q must not contain leading or trailing whitespace", ref.SourceID)
} }
if isBlank(ref.StartUnitID) { if ref.StartUnitID <= 0 {
return fmt.Errorf("source ref start_unit_id must not be empty") return fmt.Errorf("source ref start_unit_id must be positive")
} }
if hasSurroundingWhitespace(ref.StartUnitID) { if ref.EndUnitID <= 0 {
return fmt.Errorf("source ref start_unit_id %q must not contain leading or trailing whitespace", ref.StartUnitID) return fmt.Errorf("source ref end_unit_id must be positive")
}
if isBlank(ref.EndUnitID) {
return fmt.Errorf("source ref end_unit_id must not be empty")
}
if hasSurroundingWhitespace(ref.EndUnitID) {
return fmt.Errorf("source ref end_unit_id %q must not contain leading or trailing whitespace", ref.EndUnitID)
} }
if ref.SourceID != doc.ID { if ref.SourceID != doc.ID {
return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, doc.ID) return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, doc.ID)
@@ -79,20 +70,20 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
startIndex, ok := UnitIndex(doc, ref.StartUnitID) startIndex, ok := UnitIndex(doc, ref.StartUnitID)
if !ok { if !ok {
return fmt.Errorf("source ref start_unit_id %q was not found", ref.StartUnitID) return fmt.Errorf("source ref start_unit_id %d was not found", ref.StartUnitID)
} }
endIndex, ok := UnitIndex(doc, ref.EndUnitID) endIndex, ok := UnitIndex(doc, ref.EndUnitID)
if !ok { if !ok {
return fmt.Errorf("source ref end_unit_id %q was not found", ref.EndUnitID) return fmt.Errorf("source ref end_unit_id %d was not found", ref.EndUnitID)
} }
if startIndex > endIndex { if startIndex > endIndex {
return fmt.Errorf("source ref start_unit_id %q appears after end_unit_id %q", ref.StartUnitID, ref.EndUnitID) return fmt.Errorf("source ref start_unit_id %d appears after end_unit_id %d", ref.StartUnitID, ref.EndUnitID)
} }
return nil return nil
} }
func UnitIndex(doc *SourceDocument, unitID string) (int, bool) { func UnitIndex(doc *SourceDocument, unitID int) (int, bool) {
if doc == nil { if doc == nil {
return 0, false return 0, false
} }

View File

@@ -150,8 +150,8 @@ func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.Parse
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "First source unit."}, {ID: 1, Kind: "unit", Text: "First source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."}, {ID: 2, Kind: "unit", Text: "Second source unit."},
}, },
}, nil }, nil
} }
@@ -180,6 +180,10 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
ID: req.Source.ID + ":chunk:0", ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"First source unit."},{"id":2,"kind":"unit","text":"Second source unit."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...), Units: append([]source.SourceUnit(nil), req.Source.Units...),
Metadata: map[string]any{"strategy": "whole-document"}, Metadata: map[string]any{"strategy": "whole-document"},
}, },

View File

@@ -92,6 +92,10 @@ type SourceChunk struct {
ID string `json:"id"` ID string `json:"id"`
SourceID string `json:"source_id"` SourceID string `json:"source_id"`
Index int `json:"index"` Index int `json:"index"`
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Units []source.SourceUnit `json:"units"` Units []source.SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }

View File

@@ -33,7 +33,7 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."}, {ID: 1, Kind: "section", Text: "Source text."},
}, },
} }
@@ -86,7 +86,7 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."}, {ID: 1, Kind: "section", Text: "Source text."},
}, },
} }
chunker := fakeChunker{key: "generic-chunker"} chunker := fakeChunker{key: "generic-chunker"}
@@ -113,6 +113,12 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
if chunk.Index != 0 { if chunk.Index != 0 {
t.Fatalf("SourceChunk.Index = %d, want 0", chunk.Index) t.Fatalf("SourceChunk.Index = %d, want 0", chunk.Index)
} }
if chunk.StartUnitID != 1 || chunk.EndUnitID != 1 {
t.Fatalf("SourceChunk boundaries = %d-%d, want 1-1", chunk.StartUnitID, chunk.EndUnitID)
}
if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
t.Fatalf("SourceChunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
}
if len(chunk.Units) != 1 { if len(chunk.Units) != 1 {
t.Fatalf("len(SourceChunk.Units) = %d, want 1", len(chunk.Units)) t.Fatalf("len(SourceChunk.Units) = %d, want 1", len(chunk.Units))
} }
@@ -125,7 +131,7 @@ func TestFakeChunkerReceivesLLMClient(t *testing.T) {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."}, {ID: 1, Kind: "section", Text: "Source text."},
}, },
} }
client := fakeLLMClient{} client := fakeLLMClient{}
@@ -151,14 +157,18 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "First source text."}, {ID: 1, Kind: "section", Text: "First source text."},
{ID: "u2", Kind: "section", Text: "Second source text."}, {ID: 2, Kind: "section", Text: "Second source text."},
}, },
} }
chunk := SourceChunk{ chunk := SourceChunk{
ID: "source-1:chunk:1", ID: "source-1:chunk:1",
SourceID: doc.ID, SourceID: doc.ID,
Index: 1, Index: 1,
StartUnitID: 2,
EndUnitID: 2,
Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{doc.Units[1]}, Units: []source.SourceUnit{doc.Units[1]},
} }
@@ -182,8 +192,8 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs)) t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs))
} }
ref := candidate.SourceRefs[0] ref := candidate.SourceRefs[0]
if ref.StartUnitID != "u2" || ref.EndUnitID != "u2" { if ref.StartUnitID != 2 || ref.EndUnitID != 2 {
t.Fatalf("SourceRef = %+v, want u2 range", ref) t.Fatalf("SourceRef = %+v, want unit 2 range", ref)
} }
} }
@@ -365,7 +375,7 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
SourceID: "source-1", SourceID: "source-1",
Index: 0, Index: 0,
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."}, {ID: 1, Kind: "section", Text: "Source text."},
}, },
} }
merger := fakeMerger{key: "generic-merger"} merger := fakeMerger{key: "generic-merger"}
@@ -490,6 +500,10 @@ func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkRe
ID: req.Source.ID + ":chunk:0", ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...), Units: append([]source.SourceUnit(nil), req.Source.Units...),
}, },
}, },

View File

@@ -9,8 +9,12 @@ import (
) )
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) { func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
sourceUnitIndexes := make(map[string]int, len(doc.Units)) if len(chunks) == 0 {
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units)) return nil, fmt.Errorf("chunks must not be empty")
}
sourceUnitIndexes := make(map[int]int, len(doc.Units))
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
for index, unit := range doc.Units { for index, unit := range doc.Units {
sourceUnitIndexes[unit.ID] = index sourceUnitIndexes[unit.ID] = index
sourceUnits[unit.ID] = unit sourceUnits[unit.ID] = unit
@@ -33,25 +37,42 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
if chunk.Index != chunkIndex { if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex) return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
} }
startIndex, ok := sourceUnitIndexes[chunk.StartUnitID]
if !ok {
return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID)
}
endIndex, ok := sourceUnitIndexes[chunk.EndUnitID]
if !ok {
return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID)
}
if len(chunk.Units) == 0 { if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID) return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
} }
if len(chunk.Content) == 0 {
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
}
if strings.TrimSpace(chunk.MediaType) == "" {
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
}
seenUnitIDs := make(map[string]struct{}, len(chunk.Units)) seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
previousSourceIndex := -1 previousSourceIndex := -1
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units)) canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
for unitIndex, unit := range chunk.Units { for unitIndex, unit := range chunk.Units {
if strings.TrimSpace(unit.ID) == "" { if unit.ID <= 0 {
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex) return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
} }
if _, ok := seenUnitIDs[unit.ID]; ok { if _, ok := seenUnitIDs[unit.ID]; ok {
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID) return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
} }
seenUnitIDs[unit.ID] = struct{}{} seenUnitIDs[unit.ID] = struct{}{}
sourceIndex, ok := sourceUnitIndexes[unit.ID] sourceIndex, ok := sourceUnitIndexes[unit.ID]
if !ok { if !ok {
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID) return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
} }
if sourceIndex <= previousSourceIndex { if sourceIndex <= previousSourceIndex {
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID) return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
@@ -64,6 +85,10 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
ID: chunk.ID, ID: chunk.ID,
SourceID: chunk.SourceID, SourceID: chunk.SourceID,
Index: chunk.Index, Index: chunk.Index,
StartUnitID: chunk.StartUnitID,
EndUnitID: chunk.EndUnitID,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits, Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata), Metadata: cloneMetadata(chunk.Metadata),
}) })

View File

@@ -301,7 +301,7 @@ func (adapter fakeAdapter) Parse(ctx context.Context, req contracts.ParseRequest
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: 1, Kind: "unit", Text: "Source unit."},
}, },
}, nil }, nil
} }

View File

@@ -129,6 +129,10 @@ func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.Chunk
ID: "chunk-0", ID: "chunk-0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: req.Source.Units, Units: req.Source.Units,
}, },
}, },
@@ -267,7 +271,7 @@ func integrationSourceDocument() *source.SourceDocument {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:abc123", Digest: "sha256:abc123",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: 1, Kind: "unit", Text: "Source unit."},
}, },
} }
} }

View File

@@ -264,45 +264,70 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
}{ }{
{ {
name: "empty chunk id", name: "empty chunk id",
chunks: []contracts.SourceChunk{{ID: "", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}}, chunks: []contracts.SourceChunk{chunkWithUnits("", "source-1", 0, unitWithID("u1"))},
want: "id must not be empty", want: "id must not be empty",
}, },
{ {
name: "duplicate chunk id", name: "duplicate chunk id",
chunks: []contracts.SourceChunk{ chunks: []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}, chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1")),
{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}}, chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u2")),
}, },
want: "duplicated", want: "duplicated",
}, },
{ {
name: "wrong source id", name: "wrong source id",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "other-source", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}}, chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "other-source", 0, unitWithID("u1"))},
want: "source_id", want: "source_id",
}, },
{ {
name: "wrong index", name: "wrong index",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u1")}}}, chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u1"))},
want: "index", want: "index",
}, },
{
name: "unknown start id",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 9, 1, unitWithID("u1"))},
want: "start_unit_id",
},
{
name: "unknown end id",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 9, unitWithID("u1"))},
want: "end_unit_id",
},
{
name: "reversed bounds",
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 2, 1, unitWithID("u1"), unitWithID("u2"))},
want: "appears after",
},
{ {
name: "empty units", name: "empty units",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0}}, chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[]}`), MediaType: "application/json"}},
want: "units must not be empty", want: "units must not be empty",
}, },
{
name: "empty content",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, MediaType: "application/json", Units: []source.SourceUnit{unitWithID("u1")}}},
want: "content must not be empty",
},
{
name: "empty media type",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[1]}`), Units: []source.SourceUnit{unitWithID("u1")}}},
want: "media_type must not be empty",
},
{ {
name: "repeated unit inside chunk", name: "repeated unit inside chunk",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u1")}}}, chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u1"))},
want: "repeats source unit", want: "repeats source unit",
}, },
{ {
name: "unknown unit", name: "unknown unit",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u9")}}}, chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u9"))},
want: "was not found", want: "was not found",
}, },
{ {
name: "units out of source order", name: "units out of source order",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u2"), unitWithID("u1")}}}, chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 2, unitWithID("u2"), unitWithID("u1"))},
want: "source document order", want: "source document order",
}, },
} }
@@ -328,8 +353,8 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) { func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{ modules.chunker.chunks = []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u2")}}, chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u2")),
{ID: "chunk-1", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}}, chunkWithUnits("chunk-1", "source-1", 1, unitWithID("u2")),
} }
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -349,9 +374,13 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
ID: "chunk-0", ID: "chunk-0",
SourceID: "source-1", SourceID: "source-1",
Index: 0, Index: 0,
StartUnitID: 1,
EndUnitID: 1,
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ {
ID: "u1", ID: 1,
Kind: "mutated-kind", Kind: "mutated-kind",
Text: "mutated text", Text: "mutated text",
Metadata: map[string]any{ Metadata: map[string]any{
@@ -380,7 +409,7 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
if chunk == nil { if chunk == nil {
t.Fatal("extractor chunk = nil, want canonical chunk") t.Fatal("extractor chunk = nil, want canonical chunk")
} }
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" { if chunk.Units[0].ID != 1 || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0]) t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
} }
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" { if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
@@ -405,6 +434,10 @@ func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
ID: "chunk-0", ID: "chunk-0",
SourceID: "source-1", SourceID: "source-1",
Index: 0, Index: 0,
StartUnitID: 1,
EndUnitID: 1,
Content: []byte(`{"units":[{"id":1}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
unitWithID("u1"), unitWithID("u1"),
}, },
@@ -1739,9 +1772,9 @@ func validSourceDocument() *source.SourceDocument {
Format: "text/plain", Format: "text/plain",
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: 1, Kind: "unit", Text: "Source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."}, {ID: 2, Kind: "unit", Text: "Second source unit."},
{ID: "u3", Kind: "unit", Text: "Third source unit."}, {ID: 3, Kind: "unit", Text: "Third source unit."},
}, },
} }
} }
@@ -1754,7 +1787,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ {
ID: "u1", ID: 1,
Kind: "source-kind", Kind: "source-kind",
Text: "source text", Text: "source text",
Metadata: map[string]any{ Metadata: map[string]any{
@@ -1763,7 +1796,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
}, },
}, },
{ {
ID: "u2", ID: 2,
Kind: "source-kind", Kind: "source-kind",
Text: "second source text", Text: "second source text",
Metadata: map[string]any{ Metadata: map[string]any{
@@ -1775,26 +1808,53 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
} }
func sourceChunkWithID(id string, index int) contracts.SourceChunk { func sourceChunkWithID(id string, index int) contracts.SourceChunk {
unit := unitWithID("u1")
return contracts.SourceChunk{ return contracts.SourceChunk{
ID: id, ID: id,
SourceID: "source-1", SourceID: "source-1",
Index: index, Index: index,
Units: []source.SourceUnit{ StartUnitID: unit.ID,
{ID: "u1", Kind: "unit", Text: "Source unit."}, EndUnitID: unit.ID,
}, Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{unit},
} }
} }
func unitWithID(id string) source.SourceUnit { func unitWithID(id string) source.SourceUnit {
switch id { switch id {
case "u1": case "u1":
return source.SourceUnit{ID: "u1", Kind: "unit", Text: "Source unit."} return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit."}
case "u2": case "u2":
return source.SourceUnit{ID: "u2", Kind: "unit", Text: "Second source unit."} return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit."}
case "u3": case "u3":
return source.SourceUnit{ID: "u3", Kind: "unit", Text: "Third source unit."} return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit."}
case "u9":
return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit."}
default: default:
return source.SourceUnit{ID: id, Kind: "unit", Text: "Unknown source unit."} return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit."}
}
}
func chunkWithUnits(id string, sourceID string, index int, units ...source.SourceUnit) contracts.SourceChunk {
startUnitID, endUnitID := 1, 1
if len(units) > 0 {
startUnitID = units[0].ID
endUnitID = units[len(units)-1].ID
}
return chunkWithBounds(id, sourceID, index, startUnitID, endUnitID, units...)
}
func chunkWithBounds(id string, sourceID string, index int, startUnitID int, endUnitID int, units ...source.SourceUnit) contracts.SourceChunk {
return contracts.SourceChunk{
ID: id,
SourceID: sourceID,
Index: index,
StartUnitID: startUnitID,
EndUnitID: endUnitID,
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), units...),
} }
} }

View File

@@ -2,15 +2,15 @@
"id": "fixture-source", "id": "fixture-source",
"units": [ "units": [
{ {
"id": "u1", "id": 1,
"text": "First event." "text": "First event."
}, },
{ {
"id": "u2", "id": 2,
"text": "Second event." "text": "Second event."
}, },
{ {
"id": "u3", "id": 3,
"text": "Third event." "text": "Third event."
} }
] ]

View File

@@ -25,8 +25,8 @@
"source_refs": [ "source_refs": [
{ {
"source_id": "fixture-source", "source_id": "fixture-source",
"start_unit_id": "u1", "start_unit_id": 1,
"end_unit_id": "u2" "end_unit_id": 2
} }
] ]
}, },
@@ -42,8 +42,8 @@
"source_refs": [ "source_refs": [
{ {
"source_id": "fixture-source", "source_id": "fixture-source",
"start_unit_id": "u3", "start_unit_id": 3,
"end_unit_id": "u3" "end_unit_id": 3
} }
] ]
} }

View File

@@ -185,7 +185,7 @@ func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.Parse
var fixture struct { var fixture struct {
ID string `json:"id"` ID string `json:"id"`
Units []struct { Units []struct {
ID string `json:"id"` ID int `json:"id"`
Text string `json:"text"` Text string `json:"text"`
} `json:"units"` } `json:"units"`
} }
@@ -230,12 +230,20 @@ func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.C
ID: req.Source.ID + ":chunk:0", ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[1].ID,
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...), Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
}, },
{ {
ID: req.Source.ID + ":chunk:1", ID: req.Source.ID + ":chunk:1",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 1, Index: 1,
StartUnitID: req.Source.Units[2].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...), Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
}, },
}, },

View File

@@ -22,8 +22,7 @@ dnd/scenes boundary policy:
- return sequential scenes with no gaps; - return sequential scenes with no gaps;
- do not overlap scenes; - do not overlap scenes;
- preserve source-unit order; - preserve source-unit order;
- use 1-based integer source-unit numbers from the transcript, where 1 is the - use integer source-unit IDs from the transcript;
first provided source unit;
- each scene must have start_unit_id and end_unit_id; - each scene must have start_unit_id and end_unit_id;
- do not include final chunk IDs or chunk indexes. - do not include final chunk IDs or chunk indexes.

View File

@@ -2,6 +2,7 @@ package scenes
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"strings" "strings"
@@ -142,7 +143,7 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
return nil, fmt.Errorf("scenes must not be empty") return nil, fmt.Errorf("scenes must not be empty")
} }
unitIndexes := make(map[string]int, len(doc.Units)) unitIndexes := make(map[int]int, len(doc.Units))
for i, unit := range doc.Units { for i, unit := range doc.Units {
unitIndexes[unit.ID] = i unitIndexes[unit.ID] = i
} }
@@ -157,18 +158,18 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
startIndex, ok := unitIndexes[normalized.StartUnitID] startIndex, ok := unitIndexes[normalized.StartUnitID]
if !ok { if !ok {
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID) return nil, fmt.Errorf("scene[%d] start_unit_id %d was not found", i, normalized.StartUnitID)
} }
endIndex, ok := unitIndexes[normalized.EndUnitID] endIndex, ok := unitIndexes[normalized.EndUnitID]
if !ok { if !ok {
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID) return nil, fmt.Errorf("scene[%d] end_unit_id %d was not found", i, normalized.EndUnitID)
} }
if startIndex > endIndex { if startIndex > endIndex {
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID) return nil, fmt.Errorf("scene[%d] start_unit_id %d appears after end_unit_id %d", i, normalized.StartUnitID, normalized.EndUnitID)
} }
if i == 0 && startIndex != 0 { if i == 0 && startIndex != 0 {
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID) return nil, fmt.Errorf("first scene must start at first source unit %d", doc.Units[0].ID)
} }
if i > 0 { if i > 0 {
if startIndex <= previousEnd { if startIndex <= previousEnd {
@@ -181,10 +182,18 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
previousEnd = endIndex previousEnd = endIndex
units := cloneUnits(doc.Units[startIndex : endIndex+1]) units := cloneUnits(doc.Units[startIndex : endIndex+1])
content, err := chunkContent(units)
if err != nil {
return nil, err
}
chunks = append(chunks, contracts.SourceChunk{ chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1), ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID, SourceID: doc.ID,
Index: i, Index: i,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
Content: content,
MediaType: "application/json",
Units: units, Units: units,
Metadata: map[string]any{ Metadata: map[string]any{
"scene_title": normalized.ShortTitle, "scene_title": normalized.ShortTitle,
@@ -201,11 +210,23 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
} }
if previousEnd != len(doc.Units)-1 { if previousEnd != len(doc.Units)-1 {
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID) return nil, fmt.Errorf("final scene must end at final source unit %d", doc.Units[len(doc.Units)-1].ID)
} }
return chunks, nil return chunks, nil
} }
func chunkContent(units []source.SourceUnit) ([]byte, error) {
content, err := json.Marshal(struct {
Units []source.SourceUnit `json:"units"`
}{
Units: units,
})
if err != nil {
return nil, fmt.Errorf("encode chunk content: %w", err)
}
return content, nil
}
func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) { func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) {
startUnitID, err := dnd.ResolveUnitID(doc, "start_unit_id", scene.StartUnitID) startUnitID, err := dnd.ResolveUnitID(doc, "start_unit_id", scene.StartUnitID)
if err != nil { if err != nil {
@@ -226,9 +247,16 @@ func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse)
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence), BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
} }
required := map[string]string{ requiredInts := map[string]int{
"start_unit_id": out.StartUnitID, "start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID, "end_unit_id": out.EndUnitID,
}
for field, value := range requiredInts {
if value <= 0 {
return normalizedScene{}, fmt.Errorf("scene[%d] %s must be positive", index, field)
}
}
required := map[string]string{
"short_title": out.ShortTitle, "short_title": out.ShortTitle,
"primary_mode": out.PrimaryMode, "primary_mode": out.PrimaryMode,
"summary": out.Summary, "summary": out.Summary,

View File

@@ -170,8 +170,8 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) { if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got) t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
} }
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)} gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}} wantUnits := [][]int{{1, 2}, {3, 4}}
if !reflect.DeepEqual(gotUnits, wantUnits) { if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits) t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
} }
@@ -179,13 +179,19 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if first.SourceID != "session-alpha" || first.Index != 0 { if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first) t.Fatalf("first chunk = %#v, want source and index fields", first)
} }
if first.StartUnitID != 1 || first.EndUnitID != 2 {
t.Fatalf("first boundaries = %d-%d, want 1-2", first.StartUnitID, first.EndUnitID)
}
if first.MediaType != "application/json" || len(first.Content) == 0 {
t.Fatalf("first payload = media type %q length %d, want JSON content", first.MediaType, len(first.Content))
}
if first.Metadata["scene_title"] != "Goblin parley" || if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" || first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." || first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." || first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" || first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != "seg-001" || first.Metadata["start_unit_id"] != 1 ||
first.Metadata["end_unit_id"] != "seg-002" || first.Metadata["end_unit_id"] != 2 ||
first.Metadata["unit_count"] != 2 { first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata) t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
} }
@@ -311,11 +317,11 @@ func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
t.Fatalf("Chunk() error = %v, want nil", err) t.Fatalf("Chunk() error = %v, want nil", err)
} }
doc.Units[0].ID = "mutated" doc.Units[0].ID = 99
doc.Units[0].Metadata["speaker"] = "mutated" doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated" client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != "seg-001" { if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0]) t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
} }
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" { if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
@@ -362,7 +368,7 @@ func TestChunkRejectsInvalidRequests(t *testing.T) {
canceledCtx, cancel := context.WithCancel(context.Background()) canceledCtx, cancel := context.WithCancel(context.Background())
cancel() cancel()
invalidDoc := sceneSourceDocument() invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = "" invalidDoc.Units[0].ID = 0
emptyDoc := sceneSourceDocument() emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil emptyDoc.Units = nil
@@ -407,37 +413,37 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
{ {
name: "unknown boundary id", name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-999"), scene(1, 999),
}), }),
want: "was not found", want: "was not found",
}, },
{ {
name: "out of order boundaries", name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-003", "seg-002"), scene(3, 2),
}), }),
want: "appears after", want: "appears after",
}, },
{ {
name: "gap", name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-001"), scene(1, 1),
scene("seg-003", "seg-004"), scene(3, 4),
}), }),
want: "gap", want: "gap",
}, },
{ {
name: "overlap", name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-002"), scene(1, 2),
scene("seg-002", "seg-004"), scene(2, 4),
}), }),
want: "overlap", want: "overlap",
}, },
{ {
name: "incomplete coverage", name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-003"), scene(1, 3),
}), }),
want: "final scene", want: "final scene",
}, },
@@ -445,8 +451,8 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
name: "empty metadata field", name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
{ {
StartUnitID: dnd.UnitRefFromString("seg-001"), StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromString("seg-004"), EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: " ", ShortTitle: " ",
PrimaryMode: "Narrative", PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"}, MainParticipants: []string{"Aria"},
@@ -461,8 +467,8 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
name: "empty participant", name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{ response: replaceScenes(validSceneResponse(), []sceneResponse{
{ {
StartUnitID: dnd.UnitRefFromString("seg-001"), StartUnitID: dnd.UnitRefFromInt(1),
EndUnitID: dnd.UnitRefFromString("seg-004"), EndUnitID: dnd.UnitRefFromInt(4),
ShortTitle: "Title", ShortTitle: "Title",
PrimaryMode: "Narrative", PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "}, MainParticipants: []string{"Aria", " "},
@@ -511,7 +517,7 @@ func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.Chun
} }
} }
const sceneTranscriptJSON = `{"id":"session-alpha","segments":[{"id":"seg-001","text":"Aria asks whether the goblin will parley."}]}` const sceneTranscriptJSON = `{"id":"session-alpha","segments":[{"id":1,"text":"Aria asks whether the goblin will parley."}]}`
func sceneSourceInput() contracts.LLMInputMaterial { func sceneSourceInput() contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json") return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
@@ -529,10 +535,10 @@ func sceneSourceDocument() *source.SourceDocument {
Format: "application/vnd.seriatim.minimal+json", Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}}, {ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."}, {ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."}, {ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."}, {ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."},
}, },
} }
} }
@@ -540,7 +546,7 @@ func sceneSourceDocument() *source.SourceDocument {
func validSceneResponse() chunkResponse { func validSceneResponse() chunkResponse {
return chunkResponse{ return chunkResponse{
Scenes: []sceneResponse{ Scenes: []sceneResponse{
scene("seg-001", "seg-004"), scene(1, 4),
}, },
BoundaryCaveats: []string{}, BoundaryCaveats: []string{},
} }
@@ -551,10 +557,10 @@ func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse
return response return response
} }
func scene(startUnitID string, endUnitID string) sceneResponse { func scene(startUnitID int, endUnitID int) sceneResponse {
return sceneResponse{ return sceneResponse{
StartUnitID: dnd.UnitRefFromString(startUnitID), StartUnitID: dnd.UnitRefFromInt(startUnitID),
EndUnitID: dnd.UnitRefFromString(endUnitID), EndUnitID: dnd.UnitRefFromInt(endUnitID),
ShortTitle: "Scene title", ShortTitle: "Scene title",
PrimaryMode: "Narrative", PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"}, MainParticipants: []string{"Aria"},
@@ -572,8 +578,8 @@ func chunkIDs(chunks []contracts.SourceChunk) []string {
return ids return ids
} }
func unitIDs(units []source.SourceUnit) []string { func unitIDs(units []source.SourceUnit) []int {
ids := make([]string, 0, len(units)) ids := make([]int, 0, len(units))
for _, unit := range units { for _, unit := range units {
ids = append(ids, unit.ID) ids = append(ids, unit.ID)
} }

View File

@@ -19,8 +19,8 @@ type sceneResponse struct {
} }
type normalizedScene struct { type normalizedScene struct {
StartUnitID string StartUnitID int
EndUnitID string EndUnitID int
ShortTitle string ShortTitle string
PrimaryMode string PrimaryMode string
MainParticipants []string MainParticipants []string

View File

@@ -68,10 +68,18 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
end = len(req.Source.Units) end = len(req.Source.Units)
} }
units := cloneUnits(req.Source.Units[start:end]) units := cloneUnits(req.Source.Units[start:end])
content, err := chunkContent(units)
if err != nil {
return contracts.ChunkResult{}, err
}
chunks = append(chunks, contracts.SourceChunk{ chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("chunk-%06d", len(chunks)+1), ID: fmt.Sprintf("chunk-%06d", len(chunks)+1),
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: len(chunks), Index: len(chunks),
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
Content: content,
MediaType: "application/json",
Units: units, Units: units,
Metadata: map[string]any{ Metadata: map[string]any{
"start_unit_id": units[0].ID, "start_unit_id": units[0].ID,
@@ -87,6 +95,18 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
return contracts.ChunkResult{Chunks: chunks}, nil return contracts.ChunkResult{Chunks: chunks}, nil
} }
func chunkContent(units []source.SourceUnit) ([]byte, error) {
content, err := json.Marshal(struct {
Units []source.SourceUnit `json:"units"`
}{
Units: units,
})
if err != nil {
return nil, chunkerErrorf("encode chunk content: %w", err)
}
return content, nil
}
func ModuleSpec() pipeline.ModuleSpec { func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{ return pipeline.ModuleSpec{
Key: Key, Key: Key,

View File

@@ -59,10 +59,16 @@ func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {
if chunk.Index != 0 || chunk.SourceID != "source-1" { if chunk.Index != 0 || chunk.SourceID != "source-1" {
t.Fatalf("chunk = %#v, want source and index fields", chunk) t.Fatalf("chunk = %#v, want source and index fields", chunk)
} }
if got := unitIDs(chunk.Units); !reflect.DeepEqual(got, []string{"u001", "u002", "u003"}) { if got := unitIDs(chunk.Units); !reflect.DeepEqual(got, []int{1, 2, 3}) {
t.Fatalf("unit IDs = %#v, want all units", got) t.Fatalf("unit IDs = %#v, want all units", got)
} }
if chunk.Metadata["start_unit_id"] != "u001" || chunk.Metadata["end_unit_id"] != "u003" || chunk.Metadata["unit_count"] != 3 { if chunk.StartUnitID != 1 || chunk.EndUnitID != 3 {
t.Fatalf("chunk boundaries = %d-%d, want 1-3", chunk.StartUnitID, chunk.EndUnitID)
}
if chunk.MediaType != "application/json" || len(chunk.Content) == 0 {
t.Fatalf("chunk payload = media type %q length %d, want JSON content", chunk.MediaType, len(chunk.Content))
}
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 3 || chunk.Metadata["unit_count"] != 3 {
t.Fatalf("metadata = %#v, want chunk bounds", chunk.Metadata) t.Fatalf("metadata = %#v, want chunk bounds", chunk.Metadata)
} }
} }
@@ -79,8 +85,8 @@ func TestChunkExactBoundaries(t *testing.T) {
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001", "chunk-000002", "chunk-000003"}) { if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001", "chunk-000002", "chunk-000003"}) {
t.Fatalf("chunk IDs = %#v, want stable IDs", got) t.Fatalf("chunk IDs = %#v, want stable IDs", got)
} }
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units), unitIDs(result.Chunks[2].Units)} gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units), unitIDs(result.Chunks[2].Units)}
wantUnits := [][]string{{"u001", "u002"}, {"u003", "u004"}, {"u005", "u006"}} wantUnits := [][]int{{1, 2}, {3, 4}, {5, 6}}
if !reflect.DeepEqual(gotUnits, wantUnits) { if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits) t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
} }
@@ -95,11 +101,11 @@ func TestChunkOverlap(t *testing.T) {
t.Fatalf("Chunk() error = %v, want nil", err) t.Fatalf("Chunk() error = %v, want nil", err)
} }
gotUnits := make([][]string, 0, len(result.Chunks)) gotUnits := make([][]int, 0, len(result.Chunks))
for _, chunk := range result.Chunks { for _, chunk := range result.Chunks {
gotUnits = append(gotUnits, unitIDs(chunk.Units)) gotUnits = append(gotUnits, unitIDs(chunk.Units))
} }
wantUnits := [][]string{{"u001", "u002", "u003"}, {"u003", "u004", "u005"}, {"u005", "u006", "u007"}} wantUnits := [][]int{{1, 2, 3}, {3, 4, 5}, {5, 6, 7}}
if !reflect.DeepEqual(gotUnits, wantUnits) { if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits) t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
} }
@@ -162,10 +168,10 @@ func TestChunkDefensivelyCopiesUnits(t *testing.T) {
t.Fatalf("len(Chunks) = %d, want 2", len(result.Chunks)) t.Fatalf("len(Chunks) = %d, want 2", len(result.Chunks))
} }
doc.Units[0].ID = "changed" doc.Units[0].ID = 99
doc.Units[0].Metadata["speaker"] = "changed" doc.Units[0].Metadata["speaker"] = "changed"
if result.Chunks[0].Units[0].ID != "u001" { if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0]) t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
} }
if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" { if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" {
@@ -176,11 +182,10 @@ func TestChunkDefensivelyCopiesUnits(t *testing.T) {
func testSource(count int) *source.SourceDocument { func testSource(count int) *source.SourceDocument {
units := make([]source.SourceUnit, 0, count) units := make([]source.SourceUnit, 0, count)
for i := 1; i <= count; i++ { for i := 1; i <= count; i++ {
id := "u" + zeroPad3(i)
units = append(units, source.SourceUnit{ units = append(units, source.SourceUnit{
ID: id, ID: i,
Kind: "unit", Kind: "unit",
Text: "Text for " + id, Text: "Text for " + zeroPad3(i),
Metadata: map[string]any{ Metadata: map[string]any{
"speaker": "speaker-" + zeroPad3(i), "speaker": "speaker-" + zeroPad3(i),
}, },
@@ -207,8 +212,8 @@ func chunkIDs(chunks []contracts.SourceChunk) []string {
return ids return ids
} }
func unitIDs(units []source.SourceUnit) []string { func unitIDs(units []source.SourceUnit) []int {
ids := make([]string, 0, len(units)) ids := make([]int, 0, len(units))
for _, unit := range units { for _, unit := range units {
ids = append(ids, unit.ID) ids = append(ids, unit.ID)
} }

View File

@@ -1,5 +1,4 @@
Source references must use 1-based integer source-unit numbers from the Source references must use integer source-unit IDs from the transcript.
transcript, where 1 is the first provided source unit.
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
caster, spell name, effect, narrative description, and source references using caster, spell name, effect, narrative description, and source references using

View File

@@ -247,6 +247,10 @@ func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (
ID: req.Source.ID + ":chunk:0", ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[1,2,3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...), Units: append([]source.SourceUnit(nil), req.Source.Units...),
}, },
}, },

View File

@@ -73,7 +73,7 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
if payload != wantPayload { if payload != wantPayload {
t.Fatalf("payload = %#v, want %#v", payload, wantPayload) t.Fatalf("payload = %#v, want %#v", payload, wantPayload)
} }
wantRef := source.SourceRef{SourceID: "session-alpha", StartUnitID: "seg-001", EndUnitID: "seg-002"} wantRef := source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}
if len(candidate.SourceRefs) != 1 || candidate.SourceRefs[0] != wantRef { if len(candidate.SourceRefs) != 1 || candidate.SourceRefs[0] != wantRef {
t.Fatalf("SourceRefs = %#v, want %#v", candidate.SourceRefs, []source.SourceRef{wantRef}) t.Fatalf("SourceRefs = %#v, want %#v", candidate.SourceRefs, []source.SourceRef{wantRef})
} }
@@ -254,14 +254,14 @@ func TestExtractPreservesResponseOrder(t *testing.T) {
Spell: "Cure Wounds", Spell: "Cure Wounds",
Effect: "Heals.", Effect: "Heals.",
NarrativeDescription: "First spell.", NarrativeDescription: "First spell.",
SourceRefs: responseSourceRefs("session-alpha", "seg-001", "seg-001"), SourceRefs: responseSourceRefs("session-alpha", 1, 1),
}, },
{ {
Caster: "Bandit Shaman", Caster: "Bandit Shaman",
Spell: "Fire Bolt", Spell: "Fire Bolt",
Effect: "Burns.", Effect: "Burns.",
NarrativeDescription: "Second spell.", NarrativeDescription: "Second spell.",
SourceRefs: responseSourceRefs("session-alpha", "seg-002", "seg-002"), SourceRefs: responseSourceRefs("session-alpha", 2, 2),
}, },
}, },
}, },
@@ -296,7 +296,7 @@ func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
Spell: "Cure Wounds", Spell: "Cure Wounds",
Effect: "Heals.", Effect: "Heals.",
NarrativeDescription: "Aria heals.", NarrativeDescription: "Aria heals.",
SourceRefs: responseSourceRefs("session-alpha", "seg-001", "seg-002"), SourceRefs: responseSourceRefs("session-alpha", 1, 2),
}, },
}, },
}, },
@@ -306,10 +306,10 @@ func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Extract() error = %v, want nil", err) t.Fatalf("Extract() error = %v, want nil", err)
} }
client.response.SpellCasts[0].SourceRefs[0].StartUnitID = dnd.UnitRefFromString("mutated") client.response.SpellCasts[0].SourceRefs[0].StartUnitID = dnd.UnitRefFromInt(99)
if got := result.Candidates[0].SourceRefs[0].StartUnitID; got != "seg-001" { if got := result.Candidates[0].SourceRefs[0].StartUnitID; got != 1 {
t.Fatalf("candidate source ref start = %q, want copied seg-001", got) t.Fatalf("candidate source ref start = %d, want copied 1", got)
} }
} }
@@ -322,7 +322,7 @@ func extractionRequestWithClient(client contracts.StructuredLLMClient) contracts
return req return req
} }
const spellTranscriptJSON = `{"id":"session-alpha","segments":[{"id":"seg-001","text":"Aria raises her hand and casts Cure Wounds."}]}` const spellTranscriptJSON = `{"id":"session-alpha","segments":[{"id":1,"text":"Aria raises her hand and casts Cure Wounds."}]}`
func spellSourceInput() contracts.LLMInputMaterial { func spellSourceInput() contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", "application/json", []byte(spellTranscriptJSON), "sha256:transcript", "file:///session-alpha.json") return contracts.NewLLMInputMaterial("source", "application/json", []byte(spellTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")

View File

@@ -26,14 +26,14 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
Spell: "Cure Wounds", Spell: "Cure Wounds",
Effect: "Heals an injured ally.", Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.", NarrativeDescription: "Aria restores the fighter after the fight.",
SourceRefs: responseSourceRefs(expectedDoc.ID, "seg-001", "seg-001"), SourceRefs: responseSourceRefs(expectedDoc.ID, 1, 1),
}, },
{ {
Caster: "Borin", Caster: "Borin",
Spell: "Fire Bolt", Spell: "Fire Bolt",
Effect: "Scorches the wight.", Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.", NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: responseSourceRefs(expectedDoc.ID, "seg-003", "seg-003"), SourceRefs: responseSourceRefs(expectedDoc.ID, 3, 3),
}, },
}, },
}, },
@@ -122,7 +122,7 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
Spell: "Fire Bolt", Spell: "Fire Bolt",
Effect: "Scorches the wight.", Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.", NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: responseSourceRefs(expectedDoc.ID, "seg-003", "seg-003"), SourceRefs: responseSourceRefs(expectedDoc.ID, 3, 3),
}, },
}, },
}, },
@@ -207,7 +207,7 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
Spell: "Cure Wounds", Spell: "Cure Wounds",
Effect: "Heals an injured ally.", Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.", NarrativeDescription: "Aria restores the fighter after the fight.",
SourceRefs: responseSourceRefs("spell-session", "seg-999", "seg-999"), SourceRefs: responseSourceRefs("spell-session", 999, 999),
}, },
}, },
}, },

View File

@@ -15,6 +15,10 @@ func promptExtractionRequest() contracts.ExtractionRequest {
ID: "session-alpha:chunk:0", ID: "session-alpha:chunk:0",
SourceID: doc.ID, SourceID: doc.ID,
Index: 0, Index: 0,
StartUnitID: doc.Units[0].ID,
EndUnitID: doc.Units[len(doc.Units)-1].ID,
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...), Units: append([]source.SourceUnit(nil), doc.Units...),
Metadata: map[string]any{"ignored": "chunk metadata"}, Metadata: map[string]any{"ignored": "chunk metadata"},
} }
@@ -32,7 +36,7 @@ func promptSourceDocument() *source.SourceDocument {
Digest: "sha256:test", Digest: "sha256:test",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ {
ID: "seg-001", ID: 1,
Kind: "transcript_segment", Kind: "transcript_segment",
Text: "Aria raises her hand and casts Cure Wounds.", Text: "Aria raises her hand and casts Cure Wounds.",
Metadata: map[string]any{ Metadata: map[string]any{
@@ -43,7 +47,7 @@ func promptSourceDocument() *source.SourceDocument {
}, },
}, },
{ {
ID: "seg-002", ID: 2,
Kind: "transcript_segment", Kind: "transcript_segment",
Text: "The fighter's wounds begin to close.", Text: "The fighter's wounds begin to close.",
Metadata: map[string]any{"ignored": "not rendered"}, Metadata: map[string]any{"ignored": "not rendered"},
@@ -61,12 +65,12 @@ func mustJSON(t *testing.T, value any) string {
return string(encoded) return string(encoded)
} }
func responseSourceRefs(sourceID string, startUnitID string, endUnitID string) []dnd.SourceRefResponse { func responseSourceRefs(sourceID string, startUnitID int, endUnitID int) []dnd.SourceRefResponse {
return []dnd.SourceRefResponse{ return []dnd.SourceRefResponse{
{ {
SourceID: sourceID, SourceID: sourceID,
StartUnitID: dnd.UnitRefFromString(startUnitID), StartUnitID: dnd.UnitRefFromInt(startUnitID),
EndUnitID: dnd.UnitRefFromString(endUnitID), EndUnitID: dnd.UnitRefFromInt(endUnitID),
}, },
} }
} }

View File

@@ -5,21 +5,21 @@
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 4, "end": 4,
"speaker": "Alice", "speaker": "Alice",
"text": "Aria raises her holy symbol and casts Cure Wounds." "text": "Aria raises her holy symbol and casts Cure Wounds."
}, },
{ {
"id": "seg-002", "id": 2,
"start": 4, "start": 4,
"end": 8, "end": 8,
"speaker": "DM", "speaker": "DM",
"text": "The bandit mage casts Shield as the blow lands." "text": "The bandit mage casts Shield as the blow lands."
}, },
{ {
"id": "seg-003", "id": 3,
"start": 8, "start": 8,
"end": 12, "end": 12,
"speaker": "Bob", "speaker": "Bob",

View File

@@ -130,22 +130,22 @@ func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
}{ }{
{ {
name: "unknown source id", name: "unknown source id",
ref: source.SourceRef{SourceID: "session-beta", StartUnitID: "seg-001", EndUnitID: "seg-002"}, ref: source.SourceRef{SourceID: "session-beta", StartUnitID: 1, EndUnitID: 2},
want: "does not match", want: "does not match",
}, },
{ {
name: "unknown start unit", name: "unknown start unit",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: "seg-999", EndUnitID: "seg-002"}, ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 999, EndUnitID: 2},
want: "start_unit_id", want: "start_unit_id",
}, },
{ {
name: "unknown end unit", name: "unknown end unit",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: "seg-001", EndUnitID: "seg-999"}, ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 999},
want: "end_unit_id", want: "end_unit_id",
}, },
{ {
name: "reversed unit range", name: "reversed unit range",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: "seg-002", EndUnitID: "seg-001"}, ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 1},
want: "appears after", want: "appears after",
}, },
} }
@@ -234,7 +234,7 @@ func validSpellCandidate(index int) artifacts.ArtifactCandidate {
Index: index, Index: index,
Payload: spellPayload(validSpellPayload()), Payload: spellPayload(validSpellPayload()),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "session-alpha", StartUnitID: "seg-001", EndUnitID: "seg-002"}, {SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2},
}, },
} }
} }

View File

@@ -69,7 +69,7 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc
Metadata: copyMetadata(parsed.Metadata), Metadata: copyMetadata(parsed.Metadata),
} }
seenSegmentIDs := make(map[string]struct{}, len(parsed.Segments)) seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
for i, segment := range parsed.Segments { for i, segment := range parsed.Segments {
unit, err := sourceUnit(segment, i, seenSegmentIDs) unit, err := sourceUnit(segment, i, seenSegmentIDs)
if err != nil { if err != nil {
@@ -98,39 +98,35 @@ func Register(registry *pipeline.InputAdapterRegistry) error {
}) })
} }
func sourceUnit(segment segment, index int, seen map[string]struct{}) (source.SourceUnit, error) { func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
segmentLabel := fmt.Sprintf("segment[%d]", index) segmentLabel := fmt.Sprintf("segment[%d]", index)
segmentID := strings.TrimSpace(segment.ID) if segment.ID <= 0 {
if segmentID == "" { return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
return source.SourceUnit{}, inputErrorf("%s id must not be empty", segmentLabel)
}
if segmentID != segment.ID {
return source.SourceUnit{}, inputErrorf("%s id %q must not contain leading or trailing whitespace", segmentLabel, segment.ID)
} }
if _, ok := seen[segment.ID]; ok { if _, ok := seen[segment.ID]; ok {
return source.SourceUnit{}, inputErrorf("segment id %q is duplicated", segment.ID) return source.SourceUnit{}, inputErrorf("segment id %d is duplicated", segment.ID)
} }
seen[segment.ID] = struct{}{} seen[segment.ID] = struct{}{}
speaker := strings.TrimSpace(segment.Speaker) speaker := strings.TrimSpace(segment.Speaker)
if speaker == "" { if speaker == "" {
return source.SourceUnit{}, inputErrorf("segment %q speaker must not be empty", segment.ID) return source.SourceUnit{}, inputErrorf("segment %d speaker must not be empty", segment.ID)
} }
start, err := validTimestamp(segment.Start, fmt.Sprintf("segment %q start", segment.ID)) start, err := validTimestamp(segment.Start, fmt.Sprintf("segment %d start", segment.ID))
if err != nil { if err != nil {
return source.SourceUnit{}, err return source.SourceUnit{}, err
} }
end, err := validTimestamp(segment.End, fmt.Sprintf("segment %q end", segment.ID)) end, err := validTimestamp(segment.End, fmt.Sprintf("segment %d end", segment.ID))
if err != nil { if err != nil {
return source.SourceUnit{}, err return source.SourceUnit{}, err
} }
if end.Cmp(start) < 0 { if end.Cmp(start) < 0 {
return source.SourceUnit{}, inputErrorf("segment %q end must be greater than or equal to start", segment.ID) return source.SourceUnit{}, inputErrorf("segment %d end must be greater than or equal to start", segment.ID)
} }
if strings.TrimSpace(segment.Text) == "" { if strings.TrimSpace(segment.Text) == "" {
return source.SourceUnit{}, inputErrorf("segment %q text must not be empty", segment.ID) return source.SourceUnit{}, inputErrorf("segment %d text must not be empty", segment.ID)
} }
return source.SourceUnit{ return source.SourceUnit{

View File

@@ -41,8 +41,8 @@ func TestParseValidMinimalTranscript(t *testing.T) {
} }
first := doc.Units[0] first := doc.Units[0]
if first.ID != "seg-001" { if first.ID != 1 {
t.Fatalf("first.ID = %q, want seg-001", first.ID) t.Fatalf("first.ID = %d, want 1", first.ID)
} }
if first.Kind != UnitKind { if first.Kind != UnitKind {
t.Fatalf("first.Kind = %q, want %q", first.Kind, UnitKind) t.Fatalf("first.Kind = %q, want %q", first.Kind, UnitKind)
@@ -86,13 +86,13 @@ func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
if len(doc.Units) != 2 { if len(doc.Units) != 2 {
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units)) t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
} }
if doc.Units[0].ID != "1" || doc.Units[1].ID != "2" { if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 {
t.Fatalf("unit IDs = %#v, want numeric IDs normalized to strings", []string{doc.Units[0].ID, doc.Units[1].ID}) t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID})
} }
ref := source.SourceRef{ ref := source.SourceRef{
SourceID: doc.ID, SourceID: doc.ID,
StartUnitID: "1", StartUnitID: 1,
EndUnitID: "2", EndUnitID: 2,
} }
if err := source.ValidateRef(doc, ref); err != nil { if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err) t.Fatalf("ValidateRef() error = %v, want nil", err)
@@ -115,7 +115,7 @@ func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) {
} }
func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) { func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) {
raw := []byte(`{"metadata":{},"segments":[{"id":"s1","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`) raw := []byte(`{"metadata":{},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
first, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) first, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil { if err != nil {
@@ -138,7 +138,7 @@ func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) {
} }
func TestParseUsesMetadataSourceIDWhenMetadataIDIsAbsent(t *testing.T) { func TestParseUsesMetadataSourceIDWhenMetadataIDIsAbsent(t *testing.T) {
raw := []byte(`{"metadata":{"source_id":" source-from-metadata "},"segments":[{"id":"s1","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`) raw := []byte(`{"metadata":{"source_id":" source-from-metadata "},"segments":[{"id":1,"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`)
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil { if err != nil {
@@ -203,7 +203,7 @@ func TestParseRejectsInvalidInput(t *testing.T) {
{ {
name: "missing segment id", name: "missing segment id",
raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"id", "empty"}, wantErr: []string{"id", "positive"},
}, },
{ {
name: "invalid segment id type", name: "invalid segment id type",
@@ -212,52 +212,52 @@ func TestParseRejectsInvalidInput(t *testing.T) {
}, },
{ {
name: "whitespace segment id", name: "whitespace segment id",
raw: validJSONWithSegment(`"id":" s1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":" 1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"id", "whitespace"}, wantErr: []string{"id", "whitespace"},
}, },
{ {
name: "empty text", name: "empty text",
raw: validJSONWithSegment(`"id":"s1","start":0,"end":1,"speaker":"Narrator","text":" "`), raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"speaker":"Narrator","text":" "`),
wantErr: []string{"text", "empty"}, wantErr: []string{"text", "empty"},
}, },
{ {
name: "missing speaker", name: "missing speaker",
raw: validJSONWithSegment(`"id":"s1","start":0,"end":1,"text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":0,"end":1,"text":"Synthetic text."`),
wantErr: []string{"speaker", "empty"}, wantErr: []string{"speaker", "empty"},
}, },
{ {
name: "missing start", name: "missing start",
raw: validJSONWithSegment(`"id":"s1","end":1,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "empty"}, wantErr: []string{"start", "empty"},
}, },
{ {
name: "missing end", name: "missing end",
raw: validJSONWithSegment(`"id":"s1","start":0,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":0,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "empty"}, wantErr: []string{"end", "empty"},
}, },
{ {
name: "negative start", name: "negative start",
raw: validJSONWithSegment(`"id":"s1","start":-1,"end":1,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":-1,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "negative"}, wantErr: []string{"start", "negative"},
}, },
{ {
name: "non-numeric end", name: "non-numeric end",
raw: validJSONWithSegment(`"id":"s1","start":0,"end":"late","speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":0,"end":"late","speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"segment[0]", "end", "number"}, wantErr: []string{"segment[0]", "end", "number"},
}, },
{ {
name: "non-finite timestamp", name: "non-finite timestamp",
raw: validJSONWithSegment(`"id":"s1","start":1e10000,"end":1e10000,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":1e10000,"end":1e10000,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"start", "valid number"}, wantErr: []string{"start", "valid number"},
}, },
{ {
name: "end before start", name: "end before start",
raw: validJSONWithSegment(`"id":"s1","start":2,"end":1,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":2,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "start"}, wantErr: []string{"end", "start"},
}, },
{ {
name: "end before start beyond float precision", name: "end before start beyond float precision",
raw: validJSONWithSegment(`"id":"s1","start":9007199254740993,"end":9007199254740992,"speaker":"Narrator","text":"Synthetic text."`), raw: validJSONWithSegment(`"id":1,"start":9007199254740993,"end":9007199254740992,"speaker":"Narrator","text":"Synthetic text."`),
wantErr: []string{"end", "start"}, wantErr: []string{"end", "start"},
}, },
} }
@@ -284,7 +284,7 @@ func TestParseRejectsDuplicateSegmentIDs(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("Parse() error = nil, want duplicate ID error") t.Fatal("Parse() error = nil, want duplicate ID error")
} }
if !strings.Contains(err.Error(), "duplicated") || !strings.Contains(err.Error(), "seg-001") { if !strings.Contains(err.Error(), "duplicated") || !strings.Contains(err.Error(), "1") {
t.Fatalf("Parse() error = %q, want duplicate segment context", err.Error()) t.Fatalf("Parse() error = %q, want duplicate segment context", err.Error())
} }
} }

View File

@@ -5,6 +5,8 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"strconv"
"strings"
) )
type transcript struct { type transcript struct {
@@ -13,7 +15,7 @@ type transcript struct {
} }
type segment struct { type segment struct {
ID string `json:"id"` ID int `json:"id"`
Start json.Number `json:"start"` Start json.Number `json:"start"`
End json.Number `json:"end"` End json.Number `json:"end"`
Speaker string `json:"speaker"` Speaker string `json:"speaker"`
@@ -78,7 +80,7 @@ func decodeSegment(raw []byte, index int) (segment, error) {
var decoded segment var decoded segment
if err := decodeOptionalSegmentID(fields, "id", &decoded.ID); err != nil { if err := decodeOptionalSegmentID(fields, "id", &decoded.ID); err != nil {
return segment{}, fmt.Errorf("segment[%d] id must be a string or number: %w", index, err) return segment{}, fmt.Errorf("segment[%d] id must be a positive integer string or number: %w", index, err)
} }
if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil { if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil {
return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err) return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err)
@@ -103,7 +105,7 @@ func decodeOptionalString(fields map[string]json.RawMessage, key string, out *st
return decodeJSON(raw, out) return decodeJSON(raw, out)
} }
func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out *string) error { func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out *int) error {
raw, ok := fields[key] raw, ok := fields[key]
if !ok { if !ok {
return nil return nil
@@ -111,17 +113,46 @@ func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out
var text string var text string
if err := decodeJSON(raw, &text); err == nil { if err := decodeJSON(raw, &text); err == nil {
*out = text parsed, err := parsePositiveInt(text)
if err != nil {
return err
}
*out = parsed
return nil return nil
} }
var number json.Number var number json.Number
if err := decodeJSON(raw, &number); err == nil { if err := decodeJSON(raw, &number); err == nil {
*out = number.String() parsed, err := parsePositiveInt(number.String())
if err != nil {
return err
}
*out = parsed
return nil return nil
} }
return fmt.Errorf("must be a string or number") return fmt.Errorf("must be a positive integer string or number")
}
func parsePositiveInt(value string) (int, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return 0, fmt.Errorf("must not be empty")
}
if trimmed != value {
return 0, fmt.Errorf("must not contain leading or trailing whitespace")
}
parsed, err := strconv.Atoi(value)
if err != nil {
return 0, fmt.Errorf("must be an integer")
}
if parsed <= 0 {
return 0, fmt.Errorf("must be positive")
}
if strconv.Itoa(parsed) != value {
return 0, fmt.Errorf("must be a canonical positive integer")
}
return parsed, nil
} }
func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error { func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error {

View File

@@ -57,7 +57,7 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil { if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err) t.Fatalf("ValidateRef() error = %v, want nil", err)
} }
if artifact.SourceRefs[0].StartUnitID != "seg-001" || artifact.SourceRefs[0].EndUnitID != "seg-002" { if artifact.SourceRefs[0].StartUnitID != 1 || artifact.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", artifact.SourceRefs[0]) t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", artifact.SourceRefs[0])
} }
if extractor.calls != 1 { if extractor.calls != 1 {
@@ -168,6 +168,10 @@ func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkReque
ID: req.Source.ID + ":chunk:0", ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID, SourceID: req.Source.ID,
Index: 0, Index: 0,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...), Units: append([]source.SourceUnit(nil), req.Source.Units...),
}, },
}, },
@@ -206,21 +210,21 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
if req.Chunk == nil { if req.Chunk == nil {
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil") return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
} }
if got := unitIDs(req.Source.Units); !equalStrings(got, []string{"seg-001", "seg-002"}) { if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got) return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
} }
if got := unitIDs(req.Chunk.Units); !equalStrings(got, []string{"seg-001", "seg-002"}) { if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got) return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
} }
for _, unit := range req.Chunk.Units { for _, unit := range req.Chunk.Units {
if speaker, ok := Speaker(unit); !ok || speaker == "" { if speaker, ok := Speaker(unit); !ok || speaker == "" {
return contracts.ExtractionResult{}, fmt.Errorf("unit %q missing speaker metadata", unit.ID) return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
} }
if _, ok := Start(unit); !ok { if _, ok := Start(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %q missing start metadata", unit.ID) return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
} }
if _, ok := End(unit); !ok { if _, ok := End(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %q missing end metadata", unit.ID) return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
} }
} }
@@ -254,15 +258,15 @@ func (runnerSeriatimOutput) Encode(ctx context.Context, req contracts.OutputRequ
}, nil }, nil
} }
func unitIDs(units []source.SourceUnit) []string { func unitIDs(units []source.SourceUnit) []int {
ids := make([]string, 0, len(units)) ids := make([]int, 0, len(units))
for _, unit := range units { for _, unit := range units {
ids = append(ids, unit.ID) ids = append(ids, unit.ID)
} }
return ids return ids
} }
func equalStrings(a, b []string) bool { func equalInts(a, b []int) bool {
if len(a) != len(b) { if len(a) != len(b) {
return false return false
} }

View File

@@ -4,14 +4,14 @@
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 1, "end": 1,
"speaker": "Narrator", "speaker": "Narrator",
"text": "First segment." "text": "First segment."
}, },
{ {
"id": "seg-001", "id": 1,
"start": 1, "start": 1,
"end": 2, "end": 2,
"speaker": "Player", "speaker": "Player",

View File

@@ -6,14 +6,14 @@
}, },
"segments": [ "segments": [
{ {
"id": "seg-001", "id": 1,
"start": 0, "start": 0,
"end": 4.5, "end": 4.5,
"speaker": "Narrator", "speaker": "Narrator",
"text": "The stone door opens." "text": "The stone door opens."
}, },
{ {
"id": "seg-002", "id": 2,
"start": 4.5, "start": 4.5,
"end": 8, "end": 8,
"speaker": "Player", "speaker": "Player",

View File

@@ -80,7 +80,7 @@ func TestMergeDefensivelyCopiesCandidates(t *testing.T) {
input[0].Candidates[0].Index = 99 input[0].Candidates[0].Index = 99
input[0].Candidates[0].Payload[0] = '[' input[0].Candidates[0].Payload[0] = '['
input[0].Candidates[0].SourceRefs[0].StartUnitID = "changed" input[0].Candidates[0].SourceRefs[0].StartUnitID = 99
input[0].Candidates[0].Metadata["name"] = "changed" input[0].Candidates[0].Metadata["name"] = "changed"
got := result.Candidates[0] got := result.Candidates[0]
@@ -90,7 +90,7 @@ func TestMergeDefensivelyCopiesCandidates(t *testing.T) {
if string(got.Payload) != `{"name":"original"}` { if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload) t.Fatalf("Payload = %s, want original payload", got.Payload)
} }
if got.SourceRefs[0].StartUnitID != "u1" { if got.SourceRefs[0].StartUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs) t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
} }
if got.Metadata["name"] != "original" { if got.Metadata["name"] != "original" {
@@ -119,7 +119,7 @@ func candidate(index int, name string) artifacts.ArtifactCandidate {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`), Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}, },
Metadata: map[string]any{ Metadata: map[string]any{
"name": name, "name": name,
@@ -141,7 +141,7 @@ func sourceChunk(index int) contracts.SourceChunk {
SourceID: "source-1", SourceID: "source-1",
Index: index, Index: index,
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: 1, Kind: "unit", Text: "Source unit."},
}, },
} }
} }

View File

@@ -84,7 +84,7 @@ func TestNormalizeDefensivelyCopiesCandidates(t *testing.T) {
input[0].Index = 99 input[0].Index = 99
input[0].Payload[0] = '[' input[0].Payload[0] = '['
input[0].SourceRefs[0].EndUnitID = "changed" input[0].SourceRefs[0].EndUnitID = 99
input[0].Metadata["name"] = "changed" input[0].Metadata["name"] = "changed"
got := result.Candidates[0] got := result.Candidates[0]
@@ -94,7 +94,7 @@ func TestNormalizeDefensivelyCopiesCandidates(t *testing.T) {
if string(got.Payload) != `{"name":"original"}` { if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload) t.Fatalf("Payload = %s, want original payload", got.Payload)
} }
if got.SourceRefs[0].EndUnitID != "u1" { if got.SourceRefs[0].EndUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs) t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
} }
if got.Metadata["name"] != "original" { if got.Metadata["name"] != "original" {
@@ -123,7 +123,7 @@ func candidate(index int, name string) artifacts.ArtifactCandidate {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`), Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}, },
Metadata: map[string]any{ Metadata: map[string]any{
"name": name, "name": name,

View File

@@ -247,7 +247,7 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
} }
req.Approved[0].Payload[0] = '[' req.Approved[0].Payload[0] = '['
req.Approved[0].SourceRefs[0].StartUnitID = "changed" req.Approved[0].SourceRefs[0].StartUnitID = 99
req.Approved[0].Metadata["name"] = "changed" req.Approved[0].Metadata["name"] = "changed"
req.Rejected[0].Candidate.Payload[0] = '[' req.Rejected[0].Candidate.Payload[0] = '['
req.Warnings[0].Message = "changed" req.Warnings[0].Message = "changed"
@@ -286,7 +286,7 @@ func artifact(artifactType, name string) artifacts.Artifact {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`), Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}, },
Metadata: map[string]any{"name": name}, Metadata: map[string]any{"name": name},
} }
@@ -300,7 +300,7 @@ func candidate(artifactType, name string) artifacts.ArtifactCandidate {
SchemaVersion: "v1", SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`), Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"}, {SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}, },
Metadata: map[string]any{"name": name}, Metadata: map[string]any{"name": name},
} }

View File

@@ -11,9 +11,8 @@ import (
) )
type UnitRef struct { type UnitRef struct {
value string value int
fromNumber bool fromNumber bool
number int
} }
type SourceRefResponse struct { type SourceRefResponse struct {
@@ -23,19 +22,22 @@ type SourceRefResponse struct {
} }
func UnitRefFromString(value string) UnitRef { func UnitRefFromString(value string) UnitRef {
return UnitRef{value: value} parsed, _ := parseUnitRefNumber(value)
return UnitRef{value: parsed}
} }
func UnitRefFromInt(value int) UnitRef { func UnitRefFromInt(value int) UnitRef {
return UnitRef{ return UnitRef{
value: strconv.Itoa(value), value: value,
fromNumber: true, fromNumber: true,
number: value,
} }
} }
func (ref UnitRef) String() string { func (ref UnitRef) String() string {
return ref.value if ref.value == 0 {
return ""
}
return strconv.Itoa(ref.value)
} }
func (ref *UnitRef) UnmarshalJSON(raw []byte) error { func (ref *UnitRef) UnmarshalJSON(raw []byte) error {
@@ -48,13 +50,17 @@ func (ref *UnitRef) UnmarshalJSON(raw []byte) error {
if err := json.Unmarshal(raw, &value); err != nil { if err := json.Unmarshal(raw, &value); err != nil {
return err return err
} }
*ref = UnitRefFromString(value) number, err := parseUnitRefNumber(value)
if err != nil {
return err
}
*ref = UnitRef{value: number}
return nil return nil
} }
number, err := strconv.Atoi(string(raw)) number, err := parseUnitRefNumber(string(raw))
if err != nil { if err != nil {
return fmt.Errorf("unit ref must be a string or integer") return err
} }
*ref = UnitRefFromInt(number) *ref = UnitRefFromInt(number)
return nil return nil
@@ -62,72 +68,47 @@ func (ref *UnitRef) UnmarshalJSON(raw []byte) error {
func (ref UnitRef) MarshalJSON() ([]byte, error) { func (ref UnitRef) MarshalJSON() ([]byte, error) {
if ref.fromNumber { if ref.fromNumber {
return []byte(strconv.Itoa(ref.number)), nil return []byte(strconv.Itoa(ref.value)), nil
} }
return json.Marshal(ref.value) return json.Marshal(ref.String())
} }
func ResolveUnitID(doc *source.SourceDocument, field string, ref UnitRef) (string, error) { func ResolveUnitID(doc *source.SourceDocument, field string, ref UnitRef) (int, error) {
value := strings.TrimSpace(ref.value) if ref.value <= 0 {
if value == "" { return 0, fmt.Errorf("%s must be positive", field)
return "", fmt.Errorf("%s must not be empty", field)
} }
if id, ok := canonicalUnitID(doc, value); ok { if _, ok := source.UnitIndex(doc, ref.value); !ok {
return id, nil return 0, fmt.Errorf("%s %d was not found", field, ref.value)
} }
if number, ok := unitNumber(value); ok { return ref.value, nil
if id, ok := unitIDByNumber(doc, number); ok {
return id, nil
}
return "", fmt.Errorf("%s %d was not found as a source-unit ID or 1-based unit number", field, number)
}
return "", fmt.Errorf("%s %q was not found", field, value)
} }
func SourceRefCandidate(doc *source.SourceDocument, ref SourceRefResponse) source.SourceRef { func SourceRefCandidate(doc *source.SourceDocument, ref SourceRefResponse) source.SourceRef {
return source.SourceRef{ return source.SourceRef{
SourceID: strings.TrimSpace(ref.SourceID), SourceID: strings.TrimSpace(ref.SourceID),
StartUnitID: unitIDCandidate(doc, ref.StartUnitID), StartUnitID: unitIDCandidate(ref.StartUnitID),
EndUnitID: unitIDCandidate(doc, ref.EndUnitID), EndUnitID: unitIDCandidate(ref.EndUnitID),
} }
} }
func unitIDCandidate(doc *source.SourceDocument, ref UnitRef) string { func unitIDCandidate(ref UnitRef) int {
value := strings.TrimSpace(ref.value) return ref.value
if id, ok := canonicalUnitID(doc, value); ok {
return id
}
if number, ok := unitNumber(value); ok {
if id, ok := unitIDByNumber(doc, number); ok {
return id
}
}
return value
} }
func canonicalUnitID(doc *source.SourceDocument, value string) (string, bool) { func parseUnitRefNumber(value string) (int, error) {
if doc == nil { trimmed := strings.TrimSpace(value)
return "", false if trimmed == "" {
return 0, fmt.Errorf("unit ref must not be empty")
} }
for _, unit := range doc.Units { if trimmed != value {
if unit.ID == value { return 0, fmt.Errorf("unit ref must not contain leading or trailing whitespace")
return unit.ID, true
} }
}
return "", false
}
func unitIDByNumber(doc *source.SourceDocument, number int) (string, bool) {
if doc == nil || number < 1 || number > len(doc.Units) {
return "", false
}
return doc.Units[number-1].ID, true
}
func unitNumber(value string) (int, bool) {
number, err := strconv.Atoi(value) number, err := strconv.Atoi(value)
if err != nil { if err != nil {
return 0, false return 0, fmt.Errorf("unit ref must be an integer")
} }
return number, true if number <= 0 {
return 0, fmt.Errorf("unit ref must be positive")
}
return number, nil
} }

View File

@@ -8,7 +8,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
) )
func TestUnitRefUnmarshalAcceptsIntegerAndString(t *testing.T) { func TestUnitRefUnmarshalAcceptsIntegerAndNumericString(t *testing.T) {
var integerRef UnitRef var integerRef UnitRef
if err := json.Unmarshal([]byte(`12`), &integerRef); err != nil { if err := json.Unmarshal([]byte(`12`), &integerRef); err != nil {
t.Fatalf("Unmarshal(integer) error = %v, want nil", err) t.Fatalf("Unmarshal(integer) error = %v, want nil", err)
@@ -18,55 +18,49 @@ func TestUnitRefUnmarshalAcceptsIntegerAndString(t *testing.T) {
} }
var stringRef UnitRef var stringRef UnitRef
if err := json.Unmarshal([]byte(`"seg-001"`), &stringRef); err != nil { if err := json.Unmarshal([]byte(`"12"`), &stringRef); err != nil {
t.Fatalf("Unmarshal(string) error = %v, want nil", err) t.Fatalf("Unmarshal(string) error = %v, want nil", err)
} }
if got := stringRef.String(); got != "seg-001" { if got := stringRef.String(); got != "12" {
t.Fatalf("string ref = %q, want seg-001", got) t.Fatalf("string ref = %q, want 12", got)
} }
} }
func TestUnitRefUnmarshalRejectsNonIntegerTypes(t *testing.T) { func TestUnitRefUnmarshalRejectsNonIntegerValues(t *testing.T) {
for _, raw := range []string{`true`, `null`, `1.5`, `{}`} { for _, raw := range []string{`true`, `null`, `1.5`, `{}`, `"seg-001"`, `" 1 "`, `0`, `-1`} {
t.Run(raw, func(t *testing.T) { t.Run(raw, func(t *testing.T) {
var ref UnitRef var ref UnitRef
err := json.Unmarshal([]byte(raw), &ref) err := json.Unmarshal([]byte(raw), &ref)
if err == nil { if err == nil {
t.Fatal("Unmarshal() error = nil, want error") t.Fatal("Unmarshal() error = nil, want error")
} }
if !strings.Contains(err.Error(), "string or integer") {
t.Fatalf("Unmarshal() error = %q, want type context", err.Error())
}
}) })
} }
} }
func TestResolveUnitIDPrefersExactSourceUnitID(t *testing.T) { func TestResolveUnitIDReturnsExistingIntegerSourceUnitID(t *testing.T) {
doc := unitRefSourceDocument("2", "10") doc := unitRefSourceDocument(2, 10)
got, err := ResolveUnitID(doc, "start_unit_id", UnitRefFromInt(2)) got, err := ResolveUnitID(doc, "start_unit_id", UnitRefFromInt(2))
if err != nil { if err != nil {
t.Fatalf("ResolveUnitID() error = %v, want nil", err) t.Fatalf("ResolveUnitID() error = %v, want nil", err)
} }
if got != "2" { if got != 2 {
t.Fatalf("ResolveUnitID() = %q, want exact source unit ID", got) t.Fatalf("ResolveUnitID() = %d, want exact source unit ID", got)
} }
} }
func TestResolveUnitIDFallsBackToOneBasedUnitNumber(t *testing.T) { func TestResolveUnitIDDoesNotFallbackToOneBasedUnitNumber(t *testing.T) {
doc := unitRefSourceDocument("seg-001", "seg-002") doc := unitRefSourceDocument(10, 20)
got, err := ResolveUnitID(doc, "end_unit_id", UnitRefFromInt(2)) _, err := ResolveUnitID(doc, "end_unit_id", UnitRefFromInt(2))
if err != nil { if err == nil {
t.Fatalf("ResolveUnitID() error = %v, want nil", err) t.Fatal("ResolveUnitID() error = nil, want missing source-unit ID")
}
if got != "seg-002" {
t.Fatalf("ResolveUnitID() = %q, want second source unit ID", got)
} }
} }
func TestResolveUnitIDRejectsMissingUnit(t *testing.T) { func TestResolveUnitIDRejectsMissingUnit(t *testing.T) {
doc := unitRefSourceDocument("seg-001") doc := unitRefSourceDocument(1)
_, err := ResolveUnitID(doc, "start_unit_id", UnitRefFromInt(9)) _, err := ResolveUnitID(doc, "start_unit_id", UnitRefFromInt(9))
if err == nil { if err == nil {
@@ -78,14 +72,14 @@ func TestResolveUnitIDRejectsMissingUnit(t *testing.T) {
} }
func TestSourceRefCandidateCanonicalizesValidRefsAndPreservesInvalidRefs(t *testing.T) { func TestSourceRefCandidateCanonicalizesValidRefsAndPreservesInvalidRefs(t *testing.T) {
doc := unitRefSourceDocument("seg-001", "seg-002") doc := unitRefSourceDocument(1, 2)
valid := SourceRefCandidate(doc, SourceRefResponse{ valid := SourceRefCandidate(doc, SourceRefResponse{
SourceID: " session-alpha ", SourceID: " session-alpha ",
StartUnitID: UnitRefFromInt(1), StartUnitID: UnitRefFromInt(1),
EndUnitID: UnitRefFromInt(2), EndUnitID: UnitRefFromInt(2),
}) })
if valid != (source.SourceRef{SourceID: "session-alpha", StartUnitID: "seg-001", EndUnitID: "seg-002"}) { if valid != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
t.Fatalf("valid candidate = %#v, want canonical source ref", valid) t.Fatalf("valid candidate = %#v, want canonical source ref", valid)
} }
@@ -94,12 +88,12 @@ func TestSourceRefCandidateCanonicalizesValidRefsAndPreservesInvalidRefs(t *test
StartUnitID: UnitRefFromInt(9), StartUnitID: UnitRefFromInt(9),
EndUnitID: UnitRefFromString("missing"), EndUnitID: UnitRefFromString("missing"),
}) })
if invalid != (source.SourceRef{SourceID: "session-alpha", StartUnitID: "9", EndUnitID: "missing"}) { if invalid != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 9, EndUnitID: 0}) {
t.Fatalf("invalid candidate = %#v, want unresolved values for validator", invalid) t.Fatalf("invalid candidate = %#v, want unresolved values for validator", invalid)
} }
} }
func unitRefSourceDocument(ids ...string) *source.SourceDocument { func unitRefSourceDocument(ids ...int) *source.SourceDocument {
doc := &source.SourceDocument{ doc := &source.SourceDocument{
ID: "session-alpha", ID: "session-alpha",
Kind: "transcript", Kind: "transcript",