|
|
|
|
@@ -3,6 +3,7 @@ package cli
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"sort"
|
|
|
|
|
@@ -25,6 +26,7 @@ import (
|
|
|
|
|
|
|
|
|
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
|
|
|
components := productionTestComponents(t)
|
|
|
|
|
completePipelines := make(map[string]pipeline.ResolvedPipeline)
|
|
|
|
|
for _, example := range maintainedExampleFiles(t) {
|
|
|
|
|
t.Run(example.name, func(t *testing.T) {
|
|
|
|
|
cfg := loadMaintainedExample(t, example.path)
|
|
|
|
|
@@ -58,9 +60,45 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
|
|
|
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
|
|
|
|
}
|
|
|
|
|
if example.name == "complete" {
|
|
|
|
|
if pipelineID == "dnd-session-development" {
|
|
|
|
|
parent, found := completePipelines["dnd-session"]
|
|
|
|
|
if !found {
|
|
|
|
|
t.Fatal("complete parent pipeline was not resolved before its development variant")
|
|
|
|
|
}
|
|
|
|
|
if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") {
|
|
|
|
|
t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent))
|
|
|
|
|
}
|
|
|
|
|
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
|
|
|
|
|
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
|
|
|
|
|
}
|
|
|
|
|
eligibleBindings := 0
|
|
|
|
|
for _, step := range materialized.Steps {
|
|
|
|
|
for _, lane := range step.ArtifactLanes {
|
|
|
|
|
for _, binding := range []struct {
|
|
|
|
|
module pipeline.ModuleBinding
|
|
|
|
|
executionClass contracts.ExecutionClass
|
|
|
|
|
}{
|
|
|
|
|
{module: lane.Extract, executionClass: lane.ExtractExecutionClass},
|
|
|
|
|
{module: lane.Normalize, executionClass: lane.NormalizeExecutionClass},
|
|
|
|
|
} {
|
|
|
|
|
if binding.executionClass != "llm_backed" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
eligibleBindings++
|
|
|
|
|
if got := binding.module.StructuredOutputRepairAttempts; got == nil || *got != 0 {
|
|
|
|
|
t.Fatalf("development %s repair attempts = %v, want explicit zero", binding.module.Module, got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if eligibleBindings == 0 {
|
|
|
|
|
t.Fatal("development variant had no eligible LLM bindings for repair policy")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
|
|
|
|
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
|
|
|
|
}
|
|
|
|
|
completePipelines[pipelineID] = materialized
|
|
|
|
|
locationLane := referenceContractLane(t, materialized, "location-registry")
|
|
|
|
|
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
|
|
|
|
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
|
|
|
|
@@ -164,12 +202,16 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
|
|
|
|
t.Setenv("OPENROUTER_API_KEY", "")
|
|
|
|
|
for _, example := range maintainedExampleFiles(t) {
|
|
|
|
|
t.Run(example.name, func(t *testing.T) {
|
|
|
|
|
var stdout, stderr strings.Builder
|
|
|
|
|
code := RunWithOptions([]string{
|
|
|
|
|
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
|
|
|
|
|
}, &stdout, &stderr, Options{})
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
|
|
|
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
for _, pipelineID := range example.pipelineIDs {
|
|
|
|
|
t.Run(pipelineID, func(t *testing.T) {
|
|
|
|
|
var stdout, stderr strings.Builder
|
|
|
|
|
code := RunWithOptions([]string{
|
|
|
|
|
"config", "validate", "--config", example.path, "--pipeline", pipelineID,
|
|
|
|
|
}, &stdout, &stderr, Options{})
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "`+pipelineID+`"`) {
|
|
|
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
@@ -188,6 +230,31 @@ func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
|
|
|
|
return result
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string {
|
|
|
|
|
var descriptions []string
|
|
|
|
|
for _, step := range resolved.Steps {
|
|
|
|
|
for _, lane := range step.ArtifactLanes {
|
|
|
|
|
for _, target := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
target pipeline.ResolvedReferenceTarget
|
|
|
|
|
}{
|
|
|
|
|
{name: "extract", target: lane.ExtractReferences},
|
|
|
|
|
{name: "merge", target: lane.MergeReferences},
|
|
|
|
|
{name: "normalize", target: lane.NormalizeReferences},
|
|
|
|
|
} {
|
|
|
|
|
for _, binding := range target.target.Bindings {
|
|
|
|
|
if binding.Artifact == nil {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(descriptions)
|
|
|
|
|
return descriptions
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|
|
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
|
|
|
fake := &productionFakeLLMClient{}
|
|
|
|
|
|