eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve pipeline LLM profile defaults

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 17:09:16 +00:00
parent 58815aaf33
commit bf3fadf9ae
6 changed files with 342 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
internal/framework/pipeline/profile_test.go
View File

@@ -24,13 +24,13 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
resolved, err := ResolvePipeline(PipelineProfile{
ID: " campaign ",
Input: ModuleBinding{Module: " text ", LLMProfile: " fast "},
Input: Binding(" text "),
Chunk: ModuleBinding{Module: " window ", Options: map[string]any{
"size": 10,
}},
Artifacts: map[string]ArtifactLaneProfile{
" records ": {
Extract: ModuleBinding{Module: " record-extractor ", LLMProfile: " careful "},
Extract: Binding(" record-extractor "),
Merge: Binding(" dedupe "),
Normalize: Binding(" canonical "),
},
@@ -44,7 +44,7 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
if resolved.ID != "campaign" {
t.Fatalf("ID = %q, want campaign", resolved.ID)
}
if !reflect.DeepEqual(resolved.Input, ModuleBinding{Module: "text", LLMProfile: "fast"}) {
if !reflect.DeepEqual(resolved.Input, ModuleBinding{Module: "text"}) {
t.Fatalf("Input = %#v, want trimmed explicit input", resolved.Input)
}
if resolved.InputExecutionClass != contracts.ExecutionClassDeterministic {
@@ -66,7 +66,7 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
if lane.ID != "records" {
t.Fatalf("lane.ID = %q, want records", lane.ID)
}
if !reflect.DeepEqual(lane.Extract, ModuleBinding{Module: "record-extractor", LLMProfile: "careful"}) {
if !reflect.DeepEqual(lane.Extract, ModuleBinding{Module: "record-extractor"}) {
t.Fatalf("lane.Extract = %#v, want explicit extractor", lane.Extract)
}
if lane.ExtractExecutionClass != contracts.ExecutionClassDeterministic || lane.MergeExecutionClass != contracts.ExecutionClassDeterministic || lane.NormalizeExecutionClass != contracts.ExecutionClassDeterministic {
@@ -159,6 +159,176 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
}
}
func TestResolvePipelineAppliesEffectiveLLMProfiles(t *testing.T) {
for _, test := range []struct {
name string
profile PipelineProfile
options ResolveOptions
want map[string]string
}{
{
name: "runtime override",
profile: func() PipelineProfile {
profile := llmProfilePipeline()
profile.LLMProfile = " pipeline "
profile.Chunk.LLMProfile = "binding"
return profile
}(),
options: ResolveOptions{LLMProfileOverride: " runtime "},
want: llmProfileValues("runtime"),
},
{
name: "binding exception",
profile: func() PipelineProfile {
profile := llmProfilePipeline()
profile.LLMProfile = "pipeline"
lane := profile.Artifacts["events"]
lane.Extract.LLMProfile = "extract"
lane.Extract.Validators = ValidatorOverride{
Set: true,
Validators: []ModuleBinding{{Module: "llm-validator", LLMProfile: "validator"}},
}
profile.Artifacts["events"] = lane
return profile
}(),
want: func() map[string]string {
values := llmProfileValues("pipeline")
values["extract"] = "extract"
values["validator:extract:events"] = "validator"
return values
}(),
},
{
name: "pipeline default",
profile: func() PipelineProfile {
profile := llmProfilePipeline()
profile.LLMProfile = "pipeline"
return profile
}(),
want: llmProfileValues("pipeline"),
},
{
name: "prompt fallback",
profile: llmProfilePipeline(),
want: llmProfileValues(""),
},
} {
t.Run(test.name, func(t *testing.T) {
resolved, err := ResolvePipeline(test.profile, test.options, llmProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolvedLLMProfileValues(resolved); !reflect.DeepEqual(got, test.want) {
t.Fatalf("resolved profiles = %#v, want %#v", got, test.want)
}
})
}
}
func TestResolvePipelineAppliesProfilesOnlyToSelectedLLMBackedBindings(t *testing.T) {
profile := llmProfilePipeline()
profile.LLMProfile = "pipeline"
profile.Artifacts["notes"] = ArtifactLaneProfile{Extract: Binding("llm-extractor")}
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, llmProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := laneIDs(resolved.Steps[0].ArtifactLanes); !reflect.DeepEqual(got, []string{"events"}) {
t.Fatalf("selected lanes = %#v, want events only", got)
}
if got := resolvedLLMProfileValues(resolved); !reflect.DeepEqual(got, llmProfileValues("pipeline")) {
t.Fatalf("resolved profiles = %#v, want selected LLM bindings only", got)
}
}
func TestResolvePipelineLeavesUnusedProfilesOffDeterministicBindings(t *testing.T) {
profile := baselineProfile()
profile.LLMProfile = "unused"
resolved, err := ResolvePipeline(profile, ResolveOptions{LLMProfileOverride: "also-unused"}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.Input.LLMProfile != "" || resolved.Chunk.LLMProfile != "" || resolved.Output.LLMProfile != "" {
t.Fatalf("deterministic pipeline profiles = input %q chunk %q output %q, want empty", resolved.Input.LLMProfile, resolved.Chunk.LLMProfile, resolved.Output.LLMProfile)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.Extract.LLMProfile != "" || lane.Merge.LLMProfile != "" || lane.Normalize.LLMProfile != "" {
t.Fatalf("deterministic lane profiles = extract %q merge %q normalize %q, want empty", lane.Extract.LLMProfile, lane.Merge.LLMProfile, lane.Normalize.LLMProfile)
}
}
func TestResolvePipelineRejectsLLMProfileForDeterministicModule(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*PipelineProfile)
}{
{name: "input", mutate: func(profile *PipelineProfile) { profile.Input.LLMProfile = "invalid" }},
{name: "chunk", mutate: func(profile *PipelineProfile) { profile.Chunk.LLMProfile = "invalid" }},
{name: "extract", mutate: func(profile *PipelineProfile) {
lane := profile.Artifacts["events"]
lane.Extract.LLMProfile = "invalid"
profile.Artifacts["events"] = lane
}},
{name: "merge", mutate: func(profile *PipelineProfile) {
lane := profile.Artifacts["events"]
lane.Merge.LLMProfile = "invalid"
profile.Artifacts["events"] = lane
}},
{name: "normalize", mutate: func(profile *PipelineProfile) {
lane := profile.Artifacts["events"]
lane.Normalize.LLMProfile = "invalid"
profile.Artifacts["events"] = lane
}},
{name: "output", mutate: func(profile *PipelineProfile) { profile.Output.LLMProfile = "invalid" }},
} {
t.Run(test.name, func(t *testing.T) {
profile := baselineProfile()
test.mutate(&profile)
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "deterministic") {
t.Fatalf("ResolvePipeline() error = %v, want deterministic profile rejection", err)
}
})
}
}
func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
inherited := llmProfilePipeline()
inherited.LLMProfile = "shared"
explicit := llmProfilePipeline()
explicit.Input.LLMProfile = "shared"
explicit.Chunk.LLMProfile = "shared"
explicit.Output.LLMProfile = "shared"
lane := explicit.Artifacts["events"]
lane.Extract.LLMProfile = "shared"
lane.Merge.LLMProfile = "shared"
lane.Normalize.LLMProfile = "shared"
explicit.Artifacts["events"] = lane
inheritedResolved, err := ResolvePipeline(inherited, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
if err != nil {
t.Fatalf("ResolvePipeline(inherited) error = %v, want nil", err)
}
explicitResolved, err := ResolvePipeline(explicit, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
if err != nil {
t.Fatalf("ResolvePipeline(explicit) error = %v, want nil", err)
}
if inheritedResolved.Digest != explicitResolved.Digest {
t.Fatalf("effective profile digests differ: %q != %q", inheritedResolved.Digest, explicitResolved.Digest)
}
explicit.Chunk.LLMProfile = "different"
changedResolved, err := ResolvePipeline(explicit, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
if err != nil {
t.Fatalf("ResolvePipeline(changed) error = %v, want nil", err)
}
if explicitResolved.Digest == changedResolved.Digest {
t.Fatalf("digest = %q after effective profile change, want different", explicitResolved.Digest)
}
}
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
@@ -1543,6 +1713,92 @@ func defaultProfileSpecs() []ModuleSpec {
}
}
func llmProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper()
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "llm-chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) {
return typedTestChunkValidator{key: "llm-chunk-validator"}, nil
}); err != nil {
t.Fatalf("register chunk validator: %v", err)
}
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
for _, mapping := range []ValidatorChainMapping{
{Stage: StageChunk, Module: "llm-chunk", Validators: []ModuleBinding{Binding("llm-chunk-validator")}},
{Stage: StageExtract, Module: "llm-extractor", Validators: []ModuleBinding{Binding("llm-validator")}},
{Stage: StageMerge, Module: "llm-merge", Validators: []ModuleBinding{Binding("llm-validator")}},
{Stage: StageNormalize, Module: "llm-normalize", Validators: []ModuleBinding{Binding("llm-validator")}},
} {
if err := catalog.ValidatorChains.Register(mapping); err != nil {
t.Fatalf("register validator chain %#v: %v", mapping, err)
}
}
return catalog
}
func llmProfileCatalogWithoutValidatorChains(t *testing.T) ModuleCatalog {
catalog := llmProfileCatalog(t)
catalog.ValidatorChains = NewValidatorChainRegistry()
return catalog
}
func llmProfilePipeline() PipelineProfile {
return PipelineProfile{
ID: "llm-profile",
Input: Binding("llm-input"),
Chunk: Binding("llm-chunk"),
Output: Binding("llm-output"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: Binding("llm-extractor"),
Merge: Binding("llm-merge"),
Normalize: Binding("llm-normalize"),
},
},
}
}
func llmProfileValues(profile string) map[string]string {
return map[string]string{
"input": profile,
"chunk": profile,
"extract": profile,
"merge": profile,
"normalize": profile,
"output": profile,
"validator:chunk:": profile,
"validator:extract:events": profile,
"validator:merge:events": profile,
"validator:normalize:events": profile,
}
}
func resolvedLLMProfileValues(resolved ResolvedPipeline) map[string]string {
values := map[string]string{
"input": resolved.Input.LLMProfile,
"chunk": resolved.Chunk.LLMProfile,
"output": resolved.Output.LLMProfile,
}
lane := resolved.Steps[0].ArtifactLanes[0]
values["extract"] = lane.Extract.LLMProfile
values["merge"] = lane.Merge.LLMProfile
values["normalize"] = lane.Normalize.LLMProfile
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
values["validator:"+string(chain.Stage)+":"+chain.LaneID] = validator.Binding.LLMProfile
}
}
}
return values
}
func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSpec) {
t.Helper()