Resolve structured output repair configuration
This commit is contained in:
@@ -408,6 +408,148 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
|
||||
pipelineAttempts := 2
|
||||
chunkAttempts := 1
|
||||
extractAttempts := 0
|
||||
validatorAttempts := 3
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = &pipelineAttempts
|
||||
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
encoded, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(resolved) error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
|
||||
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
|
||||
}
|
||||
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
got *int
|
||||
want int
|
||||
}{
|
||||
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
|
||||
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
|
||||
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.got == nil || *test.got != test.want {
|
||||
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
|
||||
if extractChain == nil || len(extractChain.Validators) != 1 {
|
||||
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
|
||||
}
|
||||
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
|
||||
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
|
||||
}
|
||||
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
|
||||
if chunkChain == nil || len(chunkChain.Validators) != 1 {
|
||||
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
|
||||
}
|
||||
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
|
||||
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
|
||||
}
|
||||
|
||||
pipelineAttempts = 1
|
||||
chunkAttempts = 2
|
||||
extractAttempts = 3
|
||||
validatorAttempts = 0
|
||||
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
|
||||
}
|
||||
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
|
||||
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
|
||||
}
|
||||
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
|
||||
}
|
||||
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
|
||||
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*PipelineProfile)
|
||||
}{
|
||||
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
|
||||
{name: "validator", mutate: func(profile *PipelineProfile) {
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
test.mutate(&profile)
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.StructuredOutputRepairAttempts = repairAttempts(2)
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
|
||||
}
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
|
||||
digests := make(map[string]string)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
attempts *int
|
||||
}{
|
||||
{name: "prompt owned", attempts: nil},
|
||||
{name: "disabled", attempts: repairAttempts(0)},
|
||||
{name: "configured", attempts: repairAttempts(1)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = test.attempts
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
digests[test.name] = resolved.Digest
|
||||
})
|
||||
}
|
||||
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
|
||||
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
@@ -1843,6 +1985,10 @@ func llmProfilePipeline() PipelineProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func repairAttempts(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
func llmProfileValues(profile string) map[string]string {
|
||||
return map[string]string{
|
||||
"input": profile,
|
||||
|
||||
Reference in New Issue
Block a user