Compare commits
3 Commits
e19cc02c4d
...
d3f790095e
| Author | SHA1 | Date | |
|---|---|---|---|
| d3f790095e | |||
| 95218218e2 | |||
| e700df82d8 |
@@ -58,6 +58,18 @@ approved.
|
|||||||
"pipeline_digest": "sha256:...",
|
"pipeline_digest": "sha256:...",
|
||||||
"input_module": "seriatim",
|
"input_module": "seriatim",
|
||||||
"chunker": "generic",
|
"chunker": "generic",
|
||||||
|
"module_metadata": {
|
||||||
|
"chunker": {
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:...",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_id": "schema-dnd-scenes",
|
||||||
|
"response_schema_name": "dnd_scenes",
|
||||||
|
"response_schema_version": "v1",
|
||||||
|
"response_schema_sha256": "sha256:..."
|
||||||
|
}
|
||||||
|
},
|
||||||
"source_digests": ["sha256:..."],
|
"source_digests": ["sha256:..."],
|
||||||
"extractors": ["dnd/spells"],
|
"extractors": ["dnd/spells"],
|
||||||
"merger": "appendorder",
|
"merger": "appendorder",
|
||||||
@@ -89,6 +101,10 @@ Fields with empty values may be omitted by JSON encoding.
|
|||||||
`validation_status` is `approved` when no candidates were rejected and
|
`validation_status` is `approved` when no candidates were rejected and
|
||||||
`rejected` when one or more candidates were rejected.
|
`rejected` when one or more candidates were rejected.
|
||||||
|
|
||||||
|
Top-level `module_metadata` is reserved for singleton pipeline modules
|
||||||
|
(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
|
||||||
|
`artifact_lanes[].metadata`.
|
||||||
|
|
||||||
## Artifact Files
|
## Artifact Files
|
||||||
|
|
||||||
Each artifact file has this shape:
|
Each artifact file has this shape:
|
||||||
|
|||||||
@@ -49,6 +49,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
|
|||||||
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
|
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
|
||||||
and records chunk metadata for start unit, end unit, and unit count.
|
and records chunk metadata for start unit, end unit, and unit count.
|
||||||
|
|
||||||
|
The pipeline runner canonicalizes chunk units from the source document by ID
|
||||||
|
before extractors and mergers run. Chunker-owned context should stay in
|
||||||
|
`SourceChunk.Metadata`.
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
|
||||||
- `max_units`: positive integer, default `50`;
|
- `max_units`: positive integer, default `50`;
|
||||||
@@ -84,11 +88,13 @@ last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
|
|||||||
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
||||||
title, primary mode, participants, summary, boundary note, confidence, boundary
|
title, primary mode, participants, summary, boundary note, confidence, boundary
|
||||||
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
||||||
`scene_boundary_caveat`.
|
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
|
||||||
|
structured output rather than silently dropped.
|
||||||
|
|
||||||
Malformed model output fails explicitly rather than falling back to another
|
Malformed model output fails explicitly rather than falling back to another
|
||||||
chunker. The chunker exposes prompt and response-schema provenance through its
|
chunker. The chunker exposes prompt and response-schema provenance through
|
||||||
metadata provider without raw prompts, raw schemas, source text, or secrets.
|
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
|
||||||
|
text, or secrets.
|
||||||
|
|
||||||
## `dnd/spells` Extractor
|
## `dnd/spells` Extractor
|
||||||
|
|
||||||
@@ -114,7 +120,8 @@ Artifact type and schema version:
|
|||||||
- schema version: `v1`
|
- schema version: `v1`
|
||||||
|
|
||||||
The extractor adds prompt and response-schema provenance to lane manifest
|
The extractor adds prompt and response-schema provenance to lane manifest
|
||||||
metadata. Durable artifact payload details belong in the
|
metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
|
||||||
|
details belong in the
|
||||||
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
||||||
|
|
||||||
## D&D Spell Validators
|
## D&D Spell Validators
|
||||||
|
|||||||
@@ -97,6 +97,11 @@ invariants before running extractors:
|
|||||||
- every chunk source unit must exist in the source document;
|
- every chunk source unit must exist in the source document;
|
||||||
- source units inside each chunk must appear in source-document order.
|
- source units inside each chunk must appear in source-document order.
|
||||||
|
|
||||||
|
After validation, the runner rebuilds each chunk from source-document units by
|
||||||
|
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors
|
||||||
|
and downstream stages therefore see canonical source units, while
|
||||||
|
`SourceChunk.Metadata` remains the supported place for chunker-owned context.
|
||||||
|
|
||||||
The framework does not require complete source-unit coverage and does not reject
|
The framework does not require complete source-unit coverage and does not reject
|
||||||
overlap between different chunks. Stricter policies, such as full coverage or
|
overlap between different chunks. Stricter policies, such as full coverage or
|
||||||
non-overlap, belong to individual chunk modules when they are part of that
|
non-overlap, belong to individual chunk modules when they are part of that
|
||||||
@@ -145,9 +150,15 @@ On successful execution, the manifest validation status is:
|
|||||||
|
|
||||||
## Manifest Population
|
## Manifest Population
|
||||||
|
|
||||||
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
|
The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
|
||||||
lanes, LLM profile metadata, source digest, validation status, and timing.
|
module metadata, artifact lanes, LLM profile metadata, source digest,
|
||||||
|
validation status, and timing.
|
||||||
|
|
||||||
Modules can add non-secret manifest metadata by implementing
|
Singleton pipeline modules may add non-secret metadata by implementing
|
||||||
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
|
`contracts.ManifestMetadataProvider`. The runner records that metadata under
|
||||||
|
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
|
||||||
|
|
||||||
|
Lane-owned modules may add non-secret metadata through
|
||||||
|
`artifact_lanes[].metadata`. The runner records extractor, merger, and
|
||||||
|
normalizer metadata there. The D&D spell extractor uses lane metadata for
|
||||||
prompt and response-schema provenance.
|
prompt and response-schema provenance.
|
||||||
|
|||||||
@@ -34,8 +34,8 @@ The `json` output module writes these files:
|
|||||||
|
|
||||||
- `index.json`: file index with paths to the manifest, artifact files,
|
- `index.json`: file index with paths to the manifest, artifact files,
|
||||||
rejected artifacts, and warnings.
|
rejected artifacts, and warnings.
|
||||||
- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
|
- `manifest.json`: run manifest with resolved pipeline provenance, top-level
|
||||||
validation status, and timing.
|
module metadata, module keys, validation status, and timing.
|
||||||
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
|
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
|
||||||
type. For the current D&D spell extractor, this includes
|
type. For the current D&D spell extractor, this includes
|
||||||
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
|
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
|
||||||
@@ -63,7 +63,7 @@ Implemented diagnostics artifacts:
|
|||||||
- `effective-config.json`: resolved config with API keys redacted.
|
- `effective-config.json`: resolved config with API keys redacted.
|
||||||
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
||||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||||
is available.
|
is available, including top-level module metadata when present.
|
||||||
- `warnings.json`: warning list.
|
- `warnings.json`: warning list.
|
||||||
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
||||||
- `error.log`: failure message, written after diagnostics directory creation
|
- `error.log`: failure message, written after diagnostics directory creation
|
||||||
|
|||||||
@@ -183,6 +183,7 @@ Symptoms include:
|
|||||||
|
|
||||||
- `dnd scenes chunker`
|
- `dnd scenes chunker`
|
||||||
- `malformed structured output`
|
- `malformed structured output`
|
||||||
|
- `boundary_caveats`
|
||||||
- `start_unit_id`
|
- `start_unit_id`
|
||||||
- `end_unit_id`
|
- `end_unit_id`
|
||||||
- `gap`
|
- `gap`
|
||||||
@@ -199,6 +200,8 @@ Fix:
|
|||||||
- Inspect retained diagnostics for the run error and resolved pipeline.
|
- Inspect retained diagnostics for the run error and resolved pipeline.
|
||||||
- If the error names malformed structured output, retry with a model that
|
- If the error names malformed structured output, retry with a model that
|
||||||
follows structured response schemas reliably.
|
follows structured response schemas reliably.
|
||||||
|
- If the error names `boundary_caveats`, check for blank or whitespace-only
|
||||||
|
caveat text in the scene response.
|
||||||
- Scene boundaries must use exact source-unit IDs, cover the full source
|
- Scene boundaries must use exact source-unit IDs, cover the full source
|
||||||
document, be contiguous, and not overlap.
|
document, be contiguous, and not overlap.
|
||||||
|
|
||||||
|
|||||||
@@ -1094,6 +1094,37 @@ func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
|
|||||||
if manifest.Chunker != scenes.Key {
|
if manifest.Chunker != scenes.Key {
|
||||||
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
|
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
|
||||||
}
|
}
|
||||||
|
chunkerMetadata := manifest.ModuleMetadata["chunker"]
|
||||||
|
if chunkerMetadata == nil {
|
||||||
|
t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
wantMetadataKeys := []string{
|
||||||
|
"prompt_id",
|
||||||
|
"prompt_version",
|
||||||
|
"prompt_sha256",
|
||||||
|
"response_schema_key",
|
||||||
|
"response_schema_id",
|
||||||
|
"response_schema_name",
|
||||||
|
"response_schema_version",
|
||||||
|
"response_schema_sha256",
|
||||||
|
}
|
||||||
|
if len(chunkerMetadata) != len(wantMetadataKeys) {
|
||||||
|
t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys))
|
||||||
|
}
|
||||||
|
for _, key := range wantMetadataKeys {
|
||||||
|
value, ok := chunkerMetadata[key]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata)
|
||||||
|
}
|
||||||
|
if _, ok := value.(string); !ok {
|
||||||
|
t.Fatalf("chunker metadata[%q] = %#v, want string", key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} {
|
||||||
|
if _, ok := chunkerMetadata[forbidden]; ok {
|
||||||
|
t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
|
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
|
||||||
if strings.Contains(string(manifestBytes), forbidden) {
|
if strings.Contains(string(manifestBytes), forbidden) {
|
||||||
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
|
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
|
||||||
|
|||||||
@@ -49,22 +49,23 @@ type LLMProfileManifest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
PipelineID string `json:"pipeline_id,omitempty"`
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||||
InputModule string `json:"input_module,omitempty"`
|
InputModule string `json:"input_module,omitempty"`
|
||||||
Chunker string `json:"chunker,omitempty"`
|
Chunker string `json:"chunker,omitempty"`
|
||||||
SourceDigests []string `json:"source_digests,omitempty"`
|
SourceDigests []string `json:"source_digests,omitempty"`
|
||||||
Extractors []string `json:"extractors,omitempty"`
|
Extractors []string `json:"extractors,omitempty"`
|
||||||
Merger string `json:"merger,omitempty"`
|
Merger string `json:"merger,omitempty"`
|
||||||
Normalizer string `json:"normalizer,omitempty"`
|
Normalizer string `json:"normalizer,omitempty"`
|
||||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||||
SchemaVersion string `json:"schema_version,omitempty"`
|
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||||
ValidationStatus string `json:"validation_status,omitempty"`
|
SchemaVersion string `json:"schema_version,omitempty"`
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
ValidationStatus string `json:"validation_status,omitempty"`
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
||||||
|
|||||||
@@ -183,6 +183,42 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
|
||||||
|
manifest := RunManifest{
|
||||||
|
ModuleMetadata: map[string]map[string]any{
|
||||||
|
"chunker": {
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:abc123",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_name": "dnd_scenes",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
gotJSON, err := json.Marshal(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleMetadata, ok := got["module_metadata"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, moduleMetadata, "chunker")
|
||||||
|
|
||||||
|
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
|
||||||
|
}
|
||||||
|
|
||||||
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -8,53 +8,75 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func validateChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) error {
|
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
|
||||||
sourceUnitIndexes := make(map[string]int, len(doc.Units))
|
sourceUnitIndexes := make(map[string]int, len(doc.Units))
|
||||||
|
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
|
||||||
for index, unit := range doc.Units {
|
for index, unit := range doc.Units {
|
||||||
sourceUnitIndexes[unit.ID] = index
|
sourceUnitIndexes[unit.ID] = index
|
||||||
|
sourceUnits[unit.ID] = unit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
|
||||||
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
||||||
for chunkIndex, chunk := range chunks {
|
for chunkIndex, chunk := range chunks {
|
||||||
if strings.TrimSpace(chunk.ID) == "" {
|
if strings.TrimSpace(chunk.ID) == "" {
|
||||||
return fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
||||||
}
|
}
|
||||||
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
||||||
return fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
||||||
}
|
}
|
||||||
seenChunkIDs[chunk.ID] = struct{}{}
|
seenChunkIDs[chunk.ID] = struct{}{}
|
||||||
|
|
||||||
if chunk.SourceID != doc.ID {
|
if chunk.SourceID != doc.ID {
|
||||||
return fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
||||||
}
|
}
|
||||||
if chunk.Index != chunkIndex {
|
if chunk.Index != chunkIndex {
|
||||||
return fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
||||||
}
|
}
|
||||||
if len(chunk.Units) == 0 {
|
if len(chunk.Units) == 0 {
|
||||||
return fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
|
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
|
||||||
previousSourceIndex := -1
|
previousSourceIndex := -1
|
||||||
|
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
||||||
for unitIndex, unit := range chunk.Units {
|
for unitIndex, unit := range chunk.Units {
|
||||||
if strings.TrimSpace(unit.ID) == "" {
|
if strings.TrimSpace(unit.ID) == "" {
|
||||||
return fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
|
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
|
||||||
}
|
}
|
||||||
if _, ok := seenUnitIDs[unit.ID]; ok {
|
if _, ok := seenUnitIDs[unit.ID]; ok {
|
||||||
return fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
|
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
|
||||||
}
|
}
|
||||||
seenUnitIDs[unit.ID] = struct{}{}
|
seenUnitIDs[unit.ID] = struct{}{}
|
||||||
|
|
||||||
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
||||||
}
|
}
|
||||||
if sourceIndex <= previousSourceIndex {
|
if sourceIndex <= previousSourceIndex {
|
||||||
return fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
||||||
}
|
}
|
||||||
previousSourceIndex = sourceIndex
|
previousSourceIndex = sourceIndex
|
||||||
|
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
|
||||||
|
ID: chunk.ID,
|
||||||
|
SourceID: chunk.SourceID,
|
||||||
|
Index: chunk.Index,
|
||||||
|
Units: canonicalUnits,
|
||||||
|
Metadata: cloneMetadata(chunk.Metadata),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return canonicalChunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
||||||
|
return source.SourceUnit{
|
||||||
|
ID: unit.ID,
|
||||||
|
Kind: unit.Kind,
|
||||||
|
Text: unit.Text,
|
||||||
|
Metadata: cloneMetadata(unit.Metadata),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "input", adapter)
|
||||||
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
||||||
SourceID: input.SourceID,
|
SourceID: input.SourceID,
|
||||||
Path: input.Path,
|
Path: input.Path,
|
||||||
@@ -89,6 +90,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
|
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
LLMClient: input.LLMClient,
|
LLMClient: input.LLMClient,
|
||||||
@@ -103,13 +105,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if len(chunkResult.Chunks) == 0 {
|
if len(chunkResult.Chunks) == 0 {
|
||||||
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||||
}
|
}
|
||||||
if err := validateChunkResult(doc, chunkResult.Chunks); err != nil {
|
canonicalChunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||||
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
nextCandidateIndex := 0
|
nextCandidateIndex := 0
|
||||||
for _, lane := range input.Pipeline.ArtifactLanes {
|
for _, lane := range input.Pipeline.ArtifactLanes {
|
||||||
if err := r.runLane(ctx, input, doc, chunkResult.Chunks, lane, &output, &nextCandidateIndex); err != nil {
|
if err := r.runLane(ctx, input, doc, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
|
||||||
return failOutput(output), err
|
return failOutput(output), err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -125,6 +128,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
|
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "output", encoder)
|
||||||
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||||
Manifest: output.Manifest,
|
Manifest: output.Manifest,
|
||||||
Approved: output.Approved,
|
Approved: output.Approved,
|
||||||
@@ -386,14 +390,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
|||||||
|
|
||||||
metadata := make(map[string]any)
|
metadata := make(map[string]any)
|
||||||
for _, module := range modules {
|
for _, module := range modules {
|
||||||
provider, ok := module.(contracts.ManifestMetadataProvider)
|
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
|
|
||||||
if len(moduleMetadata) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := manifestMetadataKey(module)
|
key := manifestMetadataKey(module)
|
||||||
if key == "" {
|
if key == "" {
|
||||||
continue
|
continue
|
||||||
@@ -407,6 +407,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
|
||||||
|
if output == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if output.Manifest.ModuleMetadata == nil {
|
||||||
|
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
|
||||||
|
}
|
||||||
|
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
|
||||||
|
}
|
||||||
|
|
||||||
func manifestMetadataKey(module any) string {
|
func manifestMetadataKey(module any) string {
|
||||||
switch module.(type) {
|
switch module.(type) {
|
||||||
case contracts.Extractor:
|
case contracts.Extractor:
|
||||||
@@ -420,6 +434,19 @@ func manifestMetadataKey(module any) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func moduleManifestMetadata(module any) (map[string]any, bool) {
|
||||||
|
provider, ok := module.(contracts.ManifestMetadataProvider)
|
||||||
|
if !ok {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
|
||||||
|
if len(moduleMetadata) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return moduleMetadata, true
|
||||||
|
}
|
||||||
|
|
||||||
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
|
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
|
||||||
out := make([]contracts.OutputFile, 0, len(result.Files))
|
out := make([]contracts.OutputFile, 0, len(result.Files))
|
||||||
for _, file := range result.Files {
|
for _, file := range result.Files {
|
||||||
|
|||||||
@@ -341,6 +341,106 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.input.doc = sourceDocumentWithUnitMetadata()
|
||||||
|
modules.chunker.chunks = []contracts.SourceChunk{
|
||||||
|
{
|
||||||
|
ID: "chunk-0",
|
||||||
|
SourceID: "source-1",
|
||||||
|
Index: 0,
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{
|
||||||
|
ID: "u1",
|
||||||
|
Kind: "mutated-kind",
|
||||||
|
Text: "mutated text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "chunker-speaker",
|
||||||
|
"note": "chunker note",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(output.Approved) != 1 {
|
||||||
|
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor := modules.extractors["extract-alpha"]
|
||||||
|
if len(extractor.requests) != 1 {
|
||||||
|
t.Fatalf("len(extractor requests) = %d, want 1", len(extractor.requests))
|
||||||
|
}
|
||||||
|
chunk := extractor.requests[0].Chunk
|
||||||
|
if chunk == nil {
|
||||||
|
t.Fatal("extractor chunk = nil, want canonical chunk")
|
||||||
|
}
|
||||||
|
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
|
||||||
|
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
|
||||||
|
}
|
||||||
|
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
|
||||||
|
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
|
||||||
|
}
|
||||||
|
if got := chunk.Units[0].Metadata["topic"]; got != "source-topic" {
|
||||||
|
t.Fatalf("chunk unit metadata = %#v, want cloned source document metadata", chunk.Units[0].Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.input.doc.Units[0].Kind = "changed-kind"
|
||||||
|
modules.input.doc.Units[0].Text = "changed text"
|
||||||
|
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
|
||||||
|
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
|
||||||
|
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.chunker.chunks = []contracts.SourceChunk{
|
||||||
|
{
|
||||||
|
ID: "chunk-0",
|
||||||
|
SourceID: "source-1",
|
||||||
|
Index: 0,
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
unitWithID("u1"),
|
||||||
|
},
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"scene_title": "Original scene",
|
||||||
|
"boundary_note": "Chunker note",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor := modules.extractors["extract-alpha"]
|
||||||
|
if len(extractor.requests) != 1 || extractor.requests[0].Chunk == nil {
|
||||||
|
t.Fatalf("extractor requests = %#v, want one canonical chunk", extractor.requests)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
|
||||||
|
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
|
||||||
|
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.chunker.chunks[0].Metadata["scene_title"] = "changed"
|
||||||
|
modules.chunker.chunks[0].Metadata["boundary_note"] = "changed"
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
|
||||||
|
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
|
||||||
|
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
llmClient := fakeLLMClient{}
|
llmClient := fakeLLMClient{}
|
||||||
@@ -459,6 +559,47 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.input.manifestMetadata = map[string]any{
|
||||||
|
"input_profile": "input-metadata",
|
||||||
|
}
|
||||||
|
modules.chunker.manifestMetadata = map[string]any{
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:chunker-prompt",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_id": "schema-dnd-scenes",
|
||||||
|
"response_schema_name": "dnd_scenes",
|
||||||
|
}
|
||||||
|
modules.output.manifestMetadata = map[string]any{
|
||||||
|
"output_profile": "output-metadata",
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Manifest.ModuleMetadata == nil {
|
||||||
|
t.Fatal("ModuleMetadata = nil, want module metadata map")
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
|
||||||
|
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
|
||||||
|
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
|
||||||
|
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.chunker.manifestMetadata["prompt_id"] = "changed"
|
||||||
|
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
|
||||||
|
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
|
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
|
|
||||||
@@ -863,6 +1004,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
|
|||||||
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
|
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
|
||||||
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
|
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
|
||||||
}
|
}
|
||||||
|
if output.Manifest.ModuleMetadata != nil {
|
||||||
|
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
|
||||||
|
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
||||||
@@ -1051,10 +1197,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type runnerInputAdapter struct {
|
type runnerInputAdapter struct {
|
||||||
key string
|
key string
|
||||||
doc *source.SourceDocument
|
doc *source.SourceDocument
|
||||||
err error
|
err error
|
||||||
requests []contracts.ParseRequest
|
manifestMetadata map[string]any
|
||||||
|
requests []contracts.ParseRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (adapter *runnerInputAdapter) Key() string {
|
func (adapter *runnerInputAdapter) Key() string {
|
||||||
@@ -1066,12 +1213,17 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
|
|||||||
return adapter.doc, adapter.err
|
return adapter.doc, adapter.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
|
||||||
|
return adapter.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type runnerChunker struct {
|
type runnerChunker struct {
|
||||||
key string
|
key string
|
||||||
chunks []contracts.SourceChunk
|
chunks []contracts.SourceChunk
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
requests []contracts.ChunkRequest
|
manifestMetadata map[string]any
|
||||||
|
requests []contracts.ChunkRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (chunker *runnerChunker) Key() string {
|
func (chunker *runnerChunker) Key() string {
|
||||||
@@ -1086,6 +1238,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
|
|||||||
}, chunker.err
|
}, chunker.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
|
||||||
|
return chunker.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type runnerExtractor struct {
|
type runnerExtractor struct {
|
||||||
key string
|
key string
|
||||||
artifactType string
|
artifactType string
|
||||||
@@ -1226,11 +1382,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
|
|||||||
}
|
}
|
||||||
|
|
||||||
type runnerOutputEncoder struct {
|
type runnerOutputEncoder struct {
|
||||||
key string
|
key string
|
||||||
files []contracts.OutputFile
|
files []contracts.OutputFile
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
requests []contracts.OutputRequest
|
manifestMetadata map[string]any
|
||||||
|
requests []contracts.OutputRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (encoder *runnerOutputEncoder) Key() string {
|
func (encoder *runnerOutputEncoder) Key() string {
|
||||||
@@ -1245,6 +1402,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
|
|||||||
}, encoder.err
|
}, encoder.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
|
||||||
|
return encoder.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type fakeLLMClient struct{}
|
type fakeLLMClient struct{}
|
||||||
|
|
||||||
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
@@ -1273,6 +1434,34 @@ func validSourceDocument() *source.SourceDocument {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sourceDocumentWithUnitMetadata() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "source-1",
|
||||||
|
Kind: "document",
|
||||||
|
Format: "text/plain",
|
||||||
|
Digest: "sha256:source",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{
|
||||||
|
ID: "u1",
|
||||||
|
Kind: "source-kind",
|
||||||
|
Text: "source text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "source-speaker",
|
||||||
|
"topic": "source-topic",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "u2",
|
||||||
|
Kind: "source-kind",
|
||||||
|
Text: "second source text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "source-speaker-2",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func sourceChunkWithID(id string, index int) contracts.SourceChunk {
|
func sourceChunkWithID(id string, index int) contracts.SourceChunk {
|
||||||
return contracts.SourceChunk{
|
return contracts.SourceChunk{
|
||||||
ID: id,
|
ID: id,
|
||||||
|
|||||||
@@ -76,7 +76,8 @@
|
|||||||
"boundary_caveats": {
|
"boundary_caveats": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"type": "string"
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -99,13 +99,17 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
|
|||||||
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
|
}
|
||||||
chunks, err := chunksFromResponse(req.Source, response)
|
chunks, err := chunksFromResponse(req.Source, response)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
}
|
}
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: chunks,
|
Chunks: chunks,
|
||||||
Warnings: warningsFromCaveats(response.BoundaryCaveats),
|
Warnings: warnings,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,19 +263,23 @@ func validBoundaryConfidence(value string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func warningsFromCaveats(caveats []string) []contracts.Warning {
|
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
|
||||||
if len(caveats) == 0 {
|
if len(caveats) == 0 {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
warnings := make([]contracts.Warning, 0, len(caveats))
|
warnings := make([]contracts.Warning, 0, len(caveats))
|
||||||
for _, caveat := range caveats {
|
for i, caveat := range caveats {
|
||||||
|
trimmed := strings.TrimSpace(caveat)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
|
||||||
|
}
|
||||||
warnings = append(warnings, contracts.Warning{
|
warnings = append(warnings, contracts.Warning{
|
||||||
Scope: Key,
|
Scope: Key,
|
||||||
ReasonCode: "scene_boundary_caveat",
|
ReasonCode: "scene_boundary_caveat",
|
||||||
Message: caveat,
|
Message: trimmed,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return warnings
|
return warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
BoundaryConfidence: "Medium",
|
BoundaryConfidence: "Medium",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
BoundaryCaveats: []string{"The transition into combat is gradual."},
|
BoundaryCaveats: []string{" The transition into combat is gradual. "},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,6 +162,25 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{
|
||||||
|
response: chunkResponse{
|
||||||
|
Scenes: validSceneResponse().Scenes,
|
||||||
|
BoundaryCaveats: []string{
|
||||||
|
" ",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want malformed structured output error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
||||||
|
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
||||||
doc := sceneSourceDocument()
|
doc := sceneSourceDocument()
|
||||||
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
|
|||||||
@@ -77,6 +77,14 @@ func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
|
|||||||
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
||||||
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
|
||||||
|
if boundaryCaveatItems["type"] != "string" {
|
||||||
|
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
|
||||||
|
}
|
||||||
|
if boundaryCaveatItems["minLength"] != float64(1) {
|
||||||
|
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
|
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user