eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare pipelines before source execution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 06:18:46 +00:00
parent 1c84d19e5f
commit ce3a07512f
26 changed files with 1562 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
internal/framework/pipeline/runner_test.go
View File

@@ -14,17 +14,17 @@ import (
)
func TestNewAndDataTypes(t *testing.T) {
runner := New(Registries{})
runner := New()
if runner == nil {
t.Fatal("New() = nil, want runner")
}
input := RunInput{
Pipeline: resolvedPipeline(),
pipeline: resolvedPipeline(),
SourceID: "source-1",
Path: "input.txt",
RawInput: []byte("source text"),
LLMClient: fakeLLMClient{},
llmClient: fakeLLMClient{},
Metadata: map[string]any{"request": "test"},
}
output := RunOutput{
@@ -35,7 +35,7 @@ func TestNewAndDataTypes(t *testing.T) {
OutputFiles: []contracts.OutputFile{{Name: "outputs/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
}
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
if input.pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
t.Fatalf("RunInput = %#v, want constructed fields", input)
}
if output.Manifest.PipelineID != "pipeline-1" || len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
@@ -43,6 +43,26 @@ func TestNewAndDataTypes(t *testing.T) {
}
}
type preparedRunnerHarness struct {
t *testing.T
registries Registries
}
func newPreparedRunner(t *testing.T, registries Registries) preparedRunnerHarness {
t.Helper()
return preparedRunnerHarness{t: t, registries: registries}
}
func (h preparedRunnerHarness) Run(ctx context.Context, input RunInput) (RunOutput, error) {
h.t.Helper()
prepared, err := Prepare(input.pipeline, h.registries, ModuleDependencies{LLM: input.llmClient})
if err != nil {
return RunOutput{}, err
}
input.Prepared = prepared
return New().Run(ctx, input)
}
func TestRunRejectsInvalidSetup(t *testing.T) {
tests := []struct {
name string
@@ -54,10 +74,15 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) { return (*Runner)(nil).Run(context.Background(), RunInput{}) },
error: "runner must not be nil",
},
{
name: "nil prepared pipeline",
run: func() (RunOutput, error) { return New().Run(context.Background(), RunInput{}) },
error: "prepared pipeline must not be nil",
},
{
name: "empty pipeline id",
run: func() (RunOutput, error) {
return New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: ResolvedPipeline{Digest: "sha256:pipeline"}})
return newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{pipeline: ResolvedPipeline{Digest: "sha256:pipeline"}})
},
error: "pipeline id",
},
@@ -66,7 +91,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
pipeline := resolvedPipeline()
pipeline.Digest = ""
return New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: pipeline})
return newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{pipeline: pipeline})
},
error: "pipeline digest",
},
@@ -75,7 +100,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
pipeline := resolvedPipeline()
pipeline.ArtifactLanes = nil
return New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: pipeline})
return newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{pipeline: pipeline})
},
error: "artifact lanes",
},
@@ -84,7 +109,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Inputs = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "input registry",
},
@@ -93,7 +118,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Chunkers = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "chunker registry",
},
@@ -102,7 +127,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Extractors = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "extractor registry",
},
@@ -111,7 +136,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Mergers = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "merger registry",
},
@@ -120,7 +145,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Normalizers = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "normalizer registry",
},
@@ -129,7 +154,7 @@ func TestRunRejectsInvalidSetup(t *testing.T) {
run: func() (RunOutput, error) {
registries := newRunnerRegistries(t, nil)
registries.Outputs = nil
return New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
return newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
},
error: "output encoder registry",
},
@@ -147,7 +172,7 @@ func TestRunAllowsNilValidatorRegistryWithoutConfiguredValidators(t *testing.T)
registries := newRunnerRegistries(t, nil)
registries.Validators = nil
_, err := New(registries).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
_, err := newPreparedRunner(t, registries).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -190,9 +215,15 @@ func TestRunRejectsInputBuildParseAndInvalidSourceErrors(t *testing.T) {
modules := defaultRunnerModules()
test.configure(modules)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if test.name == "input build" {
if output.Manifest.PipelineID != "" {
t.Fatalf("PipelineID = %q, want no run manifest for preparation failure", output.Manifest.PipelineID)
}
return
}
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
@@ -236,9 +267,15 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
modules := defaultRunnerModules()
test.configure(modules)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if test.name == "chunker build" {
if output.Manifest.PipelineID != "" {
t.Fatalf("PipelineID = %q, want no run manifest for preparation failure", output.Manifest.PipelineID)
}
return
}
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
@@ -364,7 +401,7 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = test.chunks
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
@@ -384,7 +421,7 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
chunkWithUnits("chunk-1", "source-1", 1, unitWithID("u2")),
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -419,7 +456,7 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -478,7 +515,7 @@ func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
},
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -508,9 +545,9 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
modules := defaultRunnerModules()
llmClient := fakeLLMClient{}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: llmClient,
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
llmClient: llmClient,
Metadata: map[string]any{"request": "test"},
})
if err != nil {
@@ -546,8 +583,8 @@ func TestRunPassesSourceInputAndSessionIDToPromptCapableStages(t *testing.T) {
modules := defaultRunnerModules()
rawInput := []byte("{\"source\":\"exact bytes\"}")
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Path: "session.json",
RawInput: rawInput,
SessionID: " explicit-session ",
@@ -619,8 +656,8 @@ func TestRunPassesSourceInputAndSessionIDToPromptCapableStages(t *testing.T) {
func TestRunDefaultsSessionIDFromParsedSourceDocumentID(t *testing.T) {
modules := defaultRunnerModules()
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Path: "notes.unknown",
RawInput: []byte("notes"),
})
@@ -643,8 +680,8 @@ func TestRunPassesInputRequestFields(t *testing.T) {
modules := defaultRunnerModules()
metadata := map[string]any{"request": "test"}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
SourceID: "source-1",
Path: "input.txt",
RawInput: []byte("source text"),
@@ -676,7 +713,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
pipeline.ArtifactLanes[0].Merge = ModuleBinding{Module: "merge", LLMProfile: "merge-profile", Options: map[string]any{"merge_option": "merge-value"}}
pipeline.ArtifactLanes[0].Normalize = ModuleBinding{Module: "normalize", LLMProfile: "normalize-profile", Options: map[string]any{"normalize_option": "normalize-value"}}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -724,7 +761,7 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -745,7 +782,7 @@ func TestRunPassesMergeReferencesToMergerRequest(t *testing.T) {
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].MergeReferences.ReferenceSet = testReferenceSet("merge_notes", "merge reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -766,7 +803,7 @@ func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
pipeline := resolvedPipeline()
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -787,7 +824,7 @@ func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -829,11 +866,11 @@ func TestRunPassesValidationRequestContextToValidators(t *testing.T) {
rawInput := []byte("{\"source\":\"exact bytes\"}")
llmClient := fakeLLMClient{}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: pipeline,
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: pipeline,
Path: "session.json",
RawInput: rawInput,
LLMClient: llmClient,
llmClient: llmClient,
SessionID: "session-123",
Metadata: map[string]any{"request": "test"},
})
@@ -911,7 +948,7 @@ func TestRunPassesValidationRequestContextToValidators(t *testing.T) {
}
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
_, err := newPreparedRunner(t, newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil with nil LLM client when modules do not use it", err)
}
@@ -921,8 +958,8 @@ func TestRunIncludesInputWarnings(t *testing.T) {
modules := defaultRunnerModules()
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Warnings: []contracts.Warning{warning},
})
if err != nil {
@@ -950,7 +987,7 @@ func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
"output_profile": "output-metadata",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -977,7 +1014,7 @@ func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
func TestRunPassesPerChunkRawOutputsToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1015,7 +1052,7 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
sourceChunkWithContent("chunk-0", 0, []byte(`{"chunk":0}`), "application/vnd.test+json"),
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1035,8 +1072,8 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
func TestRunDoesNotPassCheckpointPathsToModules(t *testing.T) {
modules := defaultRunnerModules()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Checkpoints: NoopCheckpointRecorder(),
})
if err != nil {
@@ -1128,8 +1165,8 @@ func TestRunReusesCheckpointedWorkflowOutputs(t *testing.T) {
},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Checkpoint: loader,
})
if err != nil {
@@ -1181,8 +1218,8 @@ func TestRunPreservesCheckpointedExtractRejections(t *testing.T) {
reuse: map[string]bool{"extract": true},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
Checkpoint: loader,
})
if err != nil {
@@ -1208,7 +1245,7 @@ func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
pipeline := resolvedPipeline()
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1232,7 +1269,7 @@ func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
pipeline := resolvedPipeline()
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1258,7 +1295,7 @@ func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
pipeline := resolvedPipeline()
setResolvedValidatorChain(t, &pipeline, StageMerge, "alpha", "merge", resolvedValidatorForTest(validator))
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1281,7 +1318,7 @@ func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
pipeline := resolvedPipeline()
setResolvedValidatorChain(t, &pipeline, StageNormalize, "alpha", "normalize", resolvedValidatorForTest(validator))
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1304,7 +1341,7 @@ func TestRunRetriesSameModuleInputAfterFrameworkError(t *testing.T) {
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].Extract.Retries = 1
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1329,7 +1366,7 @@ func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
pipeline.ArtifactLanes[0].Extract.Retries = 1
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1353,9 +1390,9 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
modules.chunker.err = errors.New("malformed structured output")
recorder := newMemoryDebugRecorder()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: debugResponseLLMClient{content: []byte(`{"raw":true}`), profileID: "debug-profile"},
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
llmClient: debugResponseLLMClient{content: []byte(`{"raw":true}`), profileID: "debug-profile"},
Debug: recorder,
})
if err == nil || !strings.Contains(err.Error(), "malformed structured output") {
@@ -1412,9 +1449,9 @@ func TestRunDebugWritesNonJSONLLMResponseContentAsText(t *testing.T) {
modules.chunker.err = errors.New("malformed structured output")
recorder := newMemoryDebugRecorder()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: debugResponseLLMClient{content: []byte("plain text response"), profileID: "debug-profile"},
_, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
llmClient: debugResponseLLMClient{content: []byte("plain text response"), profileID: "debug-profile"},
Debug: recorder,
})
if err == nil || !strings.Contains(err.Error(), "malformed structured output") {
@@ -1450,7 +1487,7 @@ func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T
setResolvedValidatorChain(t, &pipeline, StageExtract, "alpha", "extract-alpha", resolvedValidatorForTest(validator))
pipeline.ArtifactLanes[0].Extract.Retries = 1
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1480,7 +1517,7 @@ func TestRunContextCancellationStopsRetries(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
output, err := New(newRunnerRegistries(t, modules)).Run(ctx, RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(ctx, RunInput{pipeline: pipeline})
if !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context.Canceled", err)
@@ -1494,8 +1531,8 @@ func TestRunContextCancellationStopsRetries(t *testing.T) {
}
func TestRunRejectsConfiguredValidators(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
_, err := newPreparedRunner(t, newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
})
assertRunError(t, err, "extract.validators")
}
@@ -1508,7 +1545,7 @@ func TestRunCollectsStageWarnings(t *testing.T) {
modules.normalizers["normalize"].warnings = []contracts.Warning{{ReasonCode: "normalize-warning", Message: "normalize warning"}}
modules.output.warnings = []contracts.Warning{{ReasonCode: "output-warning", Message: "output warning"}}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1530,7 +1567,7 @@ func TestRunCollectsChunkValidatorWarnings(t *testing.T) {
pipeline := resolvedPipeline()
setResolvedValidatorChain(t, &pipeline, StageChunk, "", "chunk", resolvedValidatorForTest(validator))
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1543,7 +1580,7 @@ func TestRunCollectsChunkValidatorWarnings(t *testing.T) {
func TestRunOutputEncoderReceivesManifestAndRawOutputs(t *testing.T) {
modules := defaultRunnerModules()
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1595,7 +1632,7 @@ func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
{Name: test.fileName, ContentType: "application/json", Bytes: []byte(`{}`)},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
assertRunError(t, err, "output file name")
if output.Manifest.ValidationStatus != "failed" {
@@ -1609,7 +1646,7 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
modules := defaultRunnerModules()
modules.output.err = errors.New("encode failed")
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
assertRunError(t, err, "encode failed")
if output.Manifest.ValidationStatus != "failed" {
@@ -1698,7 +1735,7 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
output, err := newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{pipeline: resolved})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1769,8 +1806,8 @@ func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
{ID: "default", Provider: "scriptorium", Model: "model-a"},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
output, err := newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
RunID: "run-test",
StartedAt: startedAt,
LLMProfiles: profiles,
@@ -1795,9 +1832,9 @@ func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
}
func TestRunManifestIncludesProfilesReportedByLLMClient(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: manifestReportingLLMClient{profiles: []artifacts.LLMProfileManifest{
output, err := newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
pipeline: resolvedPipeline(),
llmClient: manifestReportingLLMClient{profiles: []artifacts.LLMProfileManifest{
{ID: "profile-b", Provider: "scriptorium", Model: "model-b"},
{ID: "profile-a", Provider: "scriptorium", Model: "model-a"},
{ID: "profile-b", Provider: "scriptorium", Model: "model-b"},
@@ -1817,7 +1854,7 @@ func TestRunManifestIncludesProfilesReportedByLLMClient(t *testing.T) {
}
func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1840,7 +1877,7 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
"response_schema_name": "test_schema",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -1871,7 +1908,7 @@ func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
Normalize: Binding("normalize"),
})
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
output, err := newPreparedRunner(t, newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{pipeline: pipeline})
assertRunError(t, err, "extract failed")
if output.Manifest.ValidationStatus != "failed" {