eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final cleanup after checkpoint 3 and remove the completed implementation plan

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-03 11:26:49 -05:00
parent 25fbd791c8
commit d6dadb6c70
5 changed files with 235 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
internal/framework/pipeline/runner_test.go
View File

@@ -311,6 +311,66 @@ func TestRunPassesInputRequestFields(t *testing.T) {
}
}
func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipelineWithValidators("configured")
pipeline.Input = ModuleBinding{Module: "input", LLMProfile: "input-profile", Options: map[string]any{"input_option": "input-value"}}
pipeline.Chunk = ModuleBinding{Module: "chunk", LLMProfile: "chunk-profile", Options: map[string]any{"chunk_option": "chunk-value"}}
pipeline.Output = ModuleBinding{Module: "output", LLMProfile: "output-profile", Options: map[string]any{"output_option": "output-value"}}
pipeline.ArtifactLanes[0].Extract = ModuleBinding{Module: "extract-alpha", LLMProfile: "extract-profile", Options: map[string]any{"extract_option": "extract-value"}}
pipeline.ArtifactLanes[0].Merge = ModuleBinding{Module: "merge", LLMProfile: "merge-profile", Options: map[string]any{"merge_option": "merge-value"}}
pipeline.ArtifactLanes[0].Normalize = ModuleBinding{Module: "normalize", LLMProfile: "normalize-profile", Options: map[string]any{"normalize_option": "normalize-value"}}
pipeline.ArtifactLanes[0].Validators[0] = ModuleBinding{Module: "configured", LLMProfile: "validator-profile", Options: map[string]any{"validator_option": "validator-value"}}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if got := modules.input.requests[0].LLMProfile; got != "input-profile" {
t.Fatalf("input LLMProfile = %q, want input-profile", got)
}
if got := modules.input.requests[0].Options["input_option"]; got != "input-value" {
t.Fatalf("input Options = %#v, want input option", modules.input.requests[0].Options)
}
if got := modules.chunker.requests[0].LLMProfile; got != "chunk-profile" {
t.Fatalf("chunk LLMProfile = %q, want chunk-profile", got)
}
if got := modules.chunker.requests[0].Options["chunk_option"]; got != "chunk-value" {
t.Fatalf("chunk Options = %#v, want chunk option", modules.chunker.requests[0].Options)
}
if got := modules.extractors["extract-alpha"].requests[0].LLMProfile; got != "extract-profile" {
t.Fatalf("extract LLMProfile = %q, want extract-profile", got)
}
if got := modules.extractors["extract-alpha"].requests[0].Options["extract_option"]; got != "extract-value" {
t.Fatalf("extract Options = %#v, want extract option", modules.extractors["extract-alpha"].requests[0].Options)
}
if got := modules.mergers["merge"].requests[0].LLMProfile; got != "merge-profile" {
t.Fatalf("merge LLMProfile = %q, want merge-profile", got)
}
if got := modules.mergers["merge"].requests[0].Options["merge_option"]; got != "merge-value" {
t.Fatalf("merge Options = %#v, want merge option", modules.mergers["merge"].requests[0].Options)
}
if got := modules.normalizers["normalize"].requests[0].LLMProfile; got != "normalize-profile" {
t.Fatalf("normalize LLMProfile = %q, want normalize-profile", got)
}
if got := modules.normalizers["normalize"].requests[0].Options["normalize_option"]; got != "normalize-value" {
t.Fatalf("normalize Options = %#v, want normalize option", modules.normalizers["normalize"].requests[0].Options)
}
if got := modules.validators["configured"].requests[0].LLMProfile; got != "validator-profile" {
t.Fatalf("validator LLMProfile = %q, want validator-profile", got)
}
if got := modules.validators["configured"].requests[0].Options["validator_option"]; got != "validator-value" {
t.Fatalf("validator Options = %#v, want validator option", modules.validators["configured"].requests[0].Options)
}
if got := modules.output.requests[0].LLMProfile; got != "output-profile" {
t.Fatalf("output LLMProfile = %q, want output-profile", got)
}
if got := modules.output.requests[0].Options["output_option"]; got != "output-value" {
t.Fatalf("output Options = %#v, want output option", modules.output.requests[0].Options)
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
@@ -346,6 +406,54 @@ func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
}
}
func TestRunRejectsInvalidPostNormalizeCandidateEnvelope(t *testing.T) {
tests := []struct {
name string
candidates []artifacts.ArtifactCandidate
want string
}{
{
name: "duplicate index",
candidates: []artifacts.ArtifactCandidate{
runnerCandidate(0),
runnerCandidate(0),
},
want: "duplicated",
},
{
name: "missing extractor key",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ArtifactType: "artifact", SchemaVersion: "v1", Payload: []byte(`{"value":true}`)},
},
want: "extractor_key",
},
{
name: "mismatched schema version",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ExtractorKey: "extract-alpha", ArtifactType: "artifact", SchemaVersion: "other", Payload: []byte(`{"value":true}`)},
},
want: "schema_version",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.normalizers["normalize"].result = test.candidates
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want no approved artifacts", len(output.Approved))
}
})
}
}
func TestRunValidatorApprovalAndRejection(t *testing.T) {
modules := defaultRunnerModules()
rejectFirst := &runnerValidator{
@@ -784,6 +892,7 @@ type runnerExtractor struct {
validators []contracts.Validator
warnings []contracts.Warning
err error
requests []contracts.ExtractionRequest
seenChunkIDs []string
seenLLMClients []contracts.StructuredLLMClient
seenMetadata []map[string]any
@@ -806,6 +915,7 @@ func (extractor *runnerExtractor) Validators() []contracts.Validator {
}
func (extractor *runnerExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
extractor.requests = append(extractor.requests, req)
if req.Chunk != nil {
extractor.seenChunkIDs = append(extractor.seenChunkIDs, req.Chunk.ID)
}
@@ -880,6 +990,7 @@ type runnerValidator struct {
err error
order *[]string
calls int
requests []contracts.ValidationRequest
}
func (validator *runnerValidator) Name() string {
@@ -888,6 +999,7 @@ func (validator *runnerValidator) Name() string {
func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.order != nil {
*validator.order = append(*validator.order, validator.name)
}
@@ -981,6 +1093,16 @@ func candidateIndices(candidates []artifacts.ArtifactCandidate) []int {
return indices
}
func runnerCandidate(index int) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "extract-alpha",
ArtifactType: "artifact",
SchemaVersion: "v1",
Payload: []byte(`{"value":true}`),
}
}
func assertRunError(t *testing.T, err error, want string) {
t.Helper()