eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Declare producer correction capability

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-26 23:42:11 +00:00
parent 9d0faabf61
commit 85a5b52be7
12 changed files with 297 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
internal/framework/pipeline/profile_test.go
View File

@@ -126,6 +126,109 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) {
}
}
func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) {
correction := contracts.CorrectionProtocolSingleResponseV1
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.ChunkCorrectionProtocol != correction {
t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction {
t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction)
}
withoutCapability := cloneResolvedPipeline(resolved)
withoutCapability.ChunkCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = ""
withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability)
if err != nil {
t.Fatalf("resolvedPipelineDigest() error = %v", err)
}
if withoutCapabilityDigest == resolved.Digest {
t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if prepared.ChunkCorrectionProtocol != correction {
t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction)
}
preparedLane := prepared.Steps[0].ArtifactLanes[0]
if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction {
t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction)
}
}
func TestPrepareAllowsLLMProducerWithoutCorrectionCapability(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
profile := llmProfilePipeline()
profile.Chunk.Retries = 1
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.ChunkCorrectionProtocol != "" {
t.Fatalf("ChunkCorrectionProtocol = %q, want empty unsupported value", resolved.ChunkCorrectionProtocol)
}
if _, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{}); err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
}
func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) {
for _, test := range []struct {
name string
class contracts.ExecutionClass
want string
}{
{name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"},
{name: "LLM validator", class: contracts.ExecutionClassLLMBacked},
} {
t.Run(test.name, func(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class})
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}}
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if test.want != "" {
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
}
return
}
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 {
t.Fatalf("validator retries = %d, want 1", got)
}
})
}
}
func TestResolvePipelineAppliesDefaults(t *testing.T) {
resolved, err := ResolvePipeline(PipelineProfile{
ID: "defaulted",