Add development pipeline inheritance example
This commit is contained in:
@@ -40,6 +40,14 @@ have no effect; `NOTARIUS_CONFIG` remains a source-discovery input.
|
|||||||
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
|
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
|
||||||
pipeline from [a fragment](../examples/pipelines/dnd-session.yml).
|
pipeline from [a fragment](../examples/pipelines/dnd-session.yml).
|
||||||
|
|
||||||
|
The complete fragment also defines `dnd-session-development`, which inherits
|
||||||
|
the complete pipeline and profile while setting
|
||||||
|
`structured_output_repair_attempts: 0`. Select it with
|
||||||
|
`notarius run dnd-session-development --config /path/to/dnd-complete.config.yml`;
|
||||||
|
it disables PromptKit structural repairs without changing configured producer
|
||||||
|
retry budgets. When using another profile, operators can instead select
|
||||||
|
`dnd-session` and set its `llm_profile` to that profile.
|
||||||
|
|
||||||
Use these complete files as starting points rather than combining the
|
Use these complete files as starting points rather than combining the
|
||||||
illustrative fragments in this reference.
|
illustrative fragments in this reference.
|
||||||
|
|
||||||
|
|||||||
@@ -392,6 +392,8 @@ Prompt size: command-boundary behavior and representative error propagation.
|
|||||||
|
|
||||||
## Stage 7 — Publish And Test The Development Pipeline Example
|
## Stage 7 — Publish And Test The Development Pipeline Example
|
||||||
|
|
||||||
|
> Completion: complete.
|
||||||
|
|
||||||
Goal: give operators one complete, runnable demonstration without duplicating
|
Goal: give operators one complete, runnable demonstration without duplicating
|
||||||
the full pipeline.
|
the full pipeline.
|
||||||
|
|
||||||
|
|||||||
@@ -162,3 +162,9 @@ pipelines:
|
|||||||
module: appendorder
|
module: appendorder
|
||||||
normalize:
|
normalize:
|
||||||
module: dnd/enemy-events
|
module: dnd/enemy-events
|
||||||
|
|
||||||
|
# Development variant: disable PromptKit structural repairs while retaining
|
||||||
|
# the inherited steps and configured producer retry budgets.
|
||||||
|
dnd-session-development:
|
||||||
|
extends: dnd-session
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -25,6 +26,7 @@ import (
|
|||||||
|
|
||||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
|
completePipelines := make(map[string]pipeline.ResolvedPipeline)
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
t.Run(example.name, func(t *testing.T) {
|
t.Run(example.name, func(t *testing.T) {
|
||||||
cfg := loadMaintainedExample(t, example.path)
|
cfg := loadMaintainedExample(t, example.path)
|
||||||
@@ -58,9 +60,45 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|||||||
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
||||||
}
|
}
|
||||||
if example.name == "complete" {
|
if example.name == "complete" {
|
||||||
|
if pipelineID == "dnd-session-development" {
|
||||||
|
parent, found := completePipelines["dnd-session"]
|
||||||
|
if !found {
|
||||||
|
t.Fatal("complete parent pipeline was not resolved before its development variant")
|
||||||
|
}
|
||||||
|
if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") {
|
||||||
|
t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent))
|
||||||
|
}
|
||||||
|
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
|
||||||
|
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
|
||||||
|
}
|
||||||
|
eligibleBindings := 0
|
||||||
|
for _, step := range materialized.Steps {
|
||||||
|
for _, lane := range step.ArtifactLanes {
|
||||||
|
for _, binding := range []struct {
|
||||||
|
module pipeline.ModuleBinding
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
|
}{
|
||||||
|
{module: lane.Extract, executionClass: lane.ExtractExecutionClass},
|
||||||
|
{module: lane.Normalize, executionClass: lane.NormalizeExecutionClass},
|
||||||
|
} {
|
||||||
|
if binding.executionClass != "llm_backed" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
eligibleBindings++
|
||||||
|
if got := binding.module.StructuredOutputRepairAttempts; got == nil || *got != 0 {
|
||||||
|
t.Fatalf("development %s repair attempts = %v, want explicit zero", binding.module.Module, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if eligibleBindings == 0 {
|
||||||
|
t.Fatal("development variant had no eligible LLM bindings for repair policy")
|
||||||
|
}
|
||||||
|
}
|
||||||
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
||||||
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
||||||
}
|
}
|
||||||
|
completePipelines[pipelineID] = materialized
|
||||||
locationLane := referenceContractLane(t, materialized, "location-registry")
|
locationLane := referenceContractLane(t, materialized, "location-registry")
|
||||||
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
||||||
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
||||||
@@ -164,12 +202,16 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
|||||||
t.Setenv("OPENROUTER_API_KEY", "")
|
t.Setenv("OPENROUTER_API_KEY", "")
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
t.Run(example.name, func(t *testing.T) {
|
t.Run(example.name, func(t *testing.T) {
|
||||||
var stdout, stderr strings.Builder
|
for _, pipelineID := range example.pipelineIDs {
|
||||||
code := RunWithOptions([]string{
|
t.Run(pipelineID, func(t *testing.T) {
|
||||||
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
|
var stdout, stderr strings.Builder
|
||||||
}, &stdout, &stderr, Options{})
|
code := RunWithOptions([]string{
|
||||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
|
"config", "validate", "--config", example.path, "--pipeline", pipelineID,
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
}, &stdout, &stderr, Options{})
|
||||||
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "`+pipelineID+`"`) {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -188,6 +230,31 @@ func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string {
|
||||||
|
var descriptions []string
|
||||||
|
for _, step := range resolved.Steps {
|
||||||
|
for _, lane := range step.ArtifactLanes {
|
||||||
|
for _, target := range []struct {
|
||||||
|
name string
|
||||||
|
target pipeline.ResolvedReferenceTarget
|
||||||
|
}{
|
||||||
|
{name: "extract", target: lane.ExtractReferences},
|
||||||
|
{name: "merge", target: lane.MergeReferences},
|
||||||
|
{name: "normalize", target: lane.NormalizeReferences},
|
||||||
|
} {
|
||||||
|
for _, binding := range target.target.Bindings {
|
||||||
|
if binding.Artifact == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(descriptions)
|
||||||
|
return descriptions
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
fake := &productionFakeLLMClient{}
|
fake := &productionFakeLLMClient{}
|
||||||
|
|||||||
@@ -824,7 +824,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
return []maintainedExample{
|
return []maintainedExample{
|
||||||
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
||||||
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session", "dnd-session-development"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user