Trim and validate scene caveats
This commit is contained in:
@@ -88,7 +88,8 @@ last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
|
|||||||
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
||||||
title, primary mode, participants, summary, boundary note, confidence, boundary
|
title, primary mode, participants, summary, boundary note, confidence, boundary
|
||||||
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
||||||
`scene_boundary_caveat`.
|
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
|
||||||
|
structured output rather than silently dropped.
|
||||||
|
|
||||||
Malformed model output fails explicitly rather than falling back to another
|
Malformed model output fails explicitly rather than falling back to another
|
||||||
chunker. The chunker exposes prompt and response-schema provenance through
|
chunker. The chunker exposes prompt and response-schema provenance through
|
||||||
|
|||||||
@@ -183,6 +183,7 @@ Symptoms include:
|
|||||||
|
|
||||||
- `dnd scenes chunker`
|
- `dnd scenes chunker`
|
||||||
- `malformed structured output`
|
- `malformed structured output`
|
||||||
|
- `boundary_caveats`
|
||||||
- `start_unit_id`
|
- `start_unit_id`
|
||||||
- `end_unit_id`
|
- `end_unit_id`
|
||||||
- `gap`
|
- `gap`
|
||||||
@@ -199,6 +200,8 @@ Fix:
|
|||||||
- Inspect retained diagnostics for the run error and resolved pipeline.
|
- Inspect retained diagnostics for the run error and resolved pipeline.
|
||||||
- If the error names malformed structured output, retry with a model that
|
- If the error names malformed structured output, retry with a model that
|
||||||
follows structured response schemas reliably.
|
follows structured response schemas reliably.
|
||||||
|
- If the error names `boundary_caveats`, check for blank or whitespace-only
|
||||||
|
caveat text in the scene response.
|
||||||
- Scene boundaries must use exact source-unit IDs, cover the full source
|
- Scene boundaries must use exact source-unit IDs, cover the full source
|
||||||
document, be contiguous, and not overlap.
|
document, be contiguous, and not overlap.
|
||||||
|
|
||||||
|
|||||||
@@ -76,7 +76,8 @@
|
|||||||
"boundary_caveats": {
|
"boundary_caveats": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"type": "string"
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -99,13 +99,17 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
|
|||||||
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
|
}
|
||||||
chunks, err := chunksFromResponse(req.Source, response)
|
chunks, err := chunksFromResponse(req.Source, response)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
}
|
}
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: chunks,
|
Chunks: chunks,
|
||||||
Warnings: warningsFromCaveats(response.BoundaryCaveats),
|
Warnings: warnings,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,19 +263,23 @@ func validBoundaryConfidence(value string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func warningsFromCaveats(caveats []string) []contracts.Warning {
|
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
|
||||||
if len(caveats) == 0 {
|
if len(caveats) == 0 {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
warnings := make([]contracts.Warning, 0, len(caveats))
|
warnings := make([]contracts.Warning, 0, len(caveats))
|
||||||
for _, caveat := range caveats {
|
for i, caveat := range caveats {
|
||||||
|
trimmed := strings.TrimSpace(caveat)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
|
||||||
|
}
|
||||||
warnings = append(warnings, contracts.Warning{
|
warnings = append(warnings, contracts.Warning{
|
||||||
Scope: Key,
|
Scope: Key,
|
||||||
ReasonCode: "scene_boundary_caveat",
|
ReasonCode: "scene_boundary_caveat",
|
||||||
Message: caveat,
|
Message: trimmed,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return warnings
|
return warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
BoundaryConfidence: "Medium",
|
BoundaryConfidence: "Medium",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
BoundaryCaveats: []string{"The transition into combat is gradual."},
|
BoundaryCaveats: []string{" The transition into combat is gradual. "},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,6 +162,25 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{
|
||||||
|
response: chunkResponse{
|
||||||
|
Scenes: validSceneResponse().Scenes,
|
||||||
|
BoundaryCaveats: []string{
|
||||||
|
" ",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want malformed structured output error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
||||||
|
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
||||||
doc := sceneSourceDocument()
|
doc := sceneSourceDocument()
|
||||||
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
|
|||||||
@@ -77,6 +77,14 @@ func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
|
|||||||
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
||||||
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
|
||||||
|
if boundaryCaveatItems["type"] != "string" {
|
||||||
|
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
|
||||||
|
}
|
||||||
|
if boundaryCaveatItems["minLength"] != float64(1) {
|
||||||
|
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
|
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user