eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final cleanup after checkpoint 3 and remove the completed implementation plan

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-03 11:26:49 -05:00
parent 25fbd791c8
commit d6dadb6c70
5 changed files with 235 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
internal/framework/pipeline/runner.go
View File

@@ -65,10 +65,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
Metadata: input.Metadata,
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
LLMProfile: input.Pipeline.Input.LLMProfile,
Options: cloneOptions(input.Pipeline.Input.Options),
Metadata: input.Metadata,
})
if err != nil {
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
@@ -83,8 +85,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
Metadata: input.Metadata,
Source: doc,
LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, chunkResult.Warnings...)
if err != nil {
@@ -112,11 +116,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
}
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
Approved: output.Approved,
Rejected: output.Rejected,
Warnings: output.Warnings,
Metadata: input.Metadata,
Manifest: output.Manifest,
Approved: output.Approved,
Rejected: output.Rejected,
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, encoded.Warnings...)
if err != nil {
@@ -142,24 +148,28 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
}
var validators []contracts.Validator
var validators []validatorExecution
if len(lane.Validators) > 0 {
validators, err = r.buildConfiguredValidators(lane)
if err != nil {
return err
}
} else {
validators = extractor.Validators()
for _, validator := range extractor.Validators() {
validators = append(validators, validatorExecution{validator: validator})
}
}
chunkArtifacts := make([]contracts.ChunkArtifacts, 0, len(chunks))
for index := range chunks {
chunk := chunks[index]
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
LLMClient: input.LLMClient,
Metadata: input.Metadata,
Source: doc,
Chunk: &chunk,
LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, result.Warnings...)
if err != nil {
@@ -180,6 +190,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc,
LaneID: lane.ID,
ChunkArtifacts: chunkArtifacts,
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, mergeResult.Warnings...)
@@ -191,6 +203,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc,
LaneID: lane.ID,
Candidates: mergeResult.Candidates,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, normalizeResult.Warnings...)
@@ -198,6 +212,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
if err := validateCandidateEnvelope(extractor, normalizeResult.Candidates); err != nil {
return fmt.Errorf("validate normalized candidates for lane %q: %w", lane.ID, err)
}
approved, rejected, warnings, err := runValidators(ctx, extractor.Key(), validators, doc, normalizeResult.Candidates, input.Metadata)
output.Warnings = append(output.Warnings, warnings...)
output.Rejected = append(output.Rejected, rejected...)
@@ -211,14 +229,22 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return nil
}
func (r *Runner) buildConfiguredValidators(lane ResolvedArtifactLane) ([]contracts.Validator, error) {
validators := make([]contracts.Validator, 0, len(lane.Validators))
type validatorExecution struct {
validator contracts.Validator
binding ModuleBinding
}
func (r *Runner) buildConfiguredValidators(lane ResolvedArtifactLane) ([]validatorExecution, error) {
validators := make([]validatorExecution, 0, len(lane.Validators))
for _, binding := range lane.Validators {
validator, err := r.registries.Validators.Build(binding.Module)
if err != nil {
return nil, fmt.Errorf("build validator %q for lane %q: %w", binding.Module, lane.ID, err)
}
validators = append(validators, validator)
validators = append(validators, validatorExecution{
validator: validator,
binding: binding,
})
}
return validators, nil
}
@@ -359,18 +385,51 @@ func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.A
return normalized, nil
}
func runValidators(ctx context.Context, extractorKey string, validators []contracts.Validator, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
func validateCandidateEnvelope(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate) error {
seen := make(map[int]struct{}, len(candidates))
for _, candidate := range candidates {
if _, ok := seen[candidate.Index]; ok {
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
}
seen[candidate.Index] = struct{}{}
if candidate.ExtractorKey == "" {
return fmt.Errorf("candidate index %d extractor_key must not be empty", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
return fmt.Errorf("candidate index %d extractor_key %q does not match extractor %q", candidate.Index, candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType == "" {
return fmt.Errorf("candidate index %d artifact_type must not be empty", candidate.Index)
}
if candidate.ArtifactType != extractor.ArtifactType() {
return fmt.Errorf("candidate index %d artifact_type %q does not match extractor %q artifact type %q", candidate.Index, candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
}
if candidate.SchemaVersion == "" {
return fmt.Errorf("candidate index %d schema_version must not be empty", candidate.Index)
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
return fmt.Errorf("candidate index %d schema_version %q does not match extractor %q schema version %q", candidate.Index, candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
}
}
return nil
}
func runValidators(ctx context.Context, extractorKey string, validators []validatorExecution, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
eligible := candidates
var rejected []artifacts.RejectedArtifact
var warnings []contracts.Warning
for validatorIndex, validator := range validators {
for validatorIndex, execution := range validators {
validator := execution.validator
if validator == nil {
return nil, rejected, warnings, fmt.Errorf("extractor %q validator[%d] must not be nil", extractorKey, validatorIndex)
}
result, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: eligible,
LLMProfile: execution.binding.LLMProfile,
Options: cloneOptions(execution.binding.Options),
Metadata: metadata,
})
warnings = append(warnings, result.Warnings...)