eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy raw pipeline contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
internal/framework/checkpoint/loader.go
View File

@@ -88,147 +88,77 @@ func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if decision := l.readJSON(laneManifestPath("extract", laneID), &manifest); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
var payload extractOutputsEnvelope
if decision := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
outputs, err := extractOutputsFromEnvelope(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests(extractPayloads(outputs))) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint output digests do not match payload")
}
return pipeline.ExtractCheckpoint{
Outputs: outputs,
Rejected: cloneRejectedOutputs(payload.Rejected),
Warnings: cloneWarnings(payload.Warnings),
}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactExtract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactExtractCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
}
return pipeline.ArtifactExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
var payload mergeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
output, err := mergeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint output digest does not match payload")
}
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactMerge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactMergeCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactMergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
var payload normalizeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
output, err := normalizeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint output digest does not match payload")
}
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactNormalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactNormalizeCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactNormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.ArtifactCheckpointOutput, error) {
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.CheckpointArtifact, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]pipeline.ArtifactCheckpointOutput, 0, len(values))
out := make([]pipeline.CheckpointArtifact, 0, len(values))
for _, v := range values {
content, err := contentFromEnvelope(v.Content)
if err != nil {
@@ -237,7 +167,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
}
out = append(out, pipeline.ArtifactCheckpointOutput{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
return out, nil
}
@@ -326,70 +256,6 @@ func sourceChunksFromEnvelope(values []chunkEnvelope) ([]source.Chunk, error) {
return out, nil
}
func extractOutputsFromEnvelope(values []extractOutputEnvelope) ([]contracts.ExtractOutput, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]contracts.ExtractOutput, 0, len(values))
for _, value := range values {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return nil, err
}
out = append(out, contracts.ExtractOutput{
LaneID: value.LaneID,
ExtractorKey: value.ExtractorKey,
SourceID: value.SourceID,
ChunkID: value.ChunkID,
ChunkIndex: value.ChunkIndex,
Schema: value.Schema,
Payload: payload,
})
}
return out, nil
}
func mergeOutputFromEnvelope(value mergeOutputPayload) (contracts.MergeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.MergeOutput{}, err
}
return contracts.MergeOutput{
LaneID: value.LaneID,
MergerKey: value.MergerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func normalizeOutputFromEnvelope(value normalizeOutputPayload) (contracts.NormalizeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.NormalizeOutput{}, err
}
return contracts.NormalizeOutput{
LaneID: value.LaneID,
NormalizerKey: value.NormalizerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func rawPayloadFromEnvelope(value binaryEnvelope) (contracts.RawPayload, error) {
content, err := contentFromEnvelope(value)
if err != nil {
return contracts.RawPayload{}, err
}
return contracts.RawPayload{
Content: content,
MediaType: value.MediaType,
Metadata: cloneMetadata(value.Metadata),
Warnings: cloneWarnings(value.Warnings),
}, nil
}
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil {

176
internal/framework/checkpoint/recorder.go
View File

@@ -119,28 +119,7 @@ func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, depe
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := extractOutputsEnvelope{
Outputs: extractOutputEnvelopes(outputs),
Rejected: cloneRejectedOutputs(rejected),
Warnings: cloneWarnings(warnings),
}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
StageManifest: manifest,
ChunkCount: len(outputs) + len(rejected),
OutputCount: len(outputs),
})
}
func (r *WorkspaceRecorder) ArtifactExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
@@ -166,27 +145,12 @@ func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, depend
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error {
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
StageManifest: manifest,
InputCount: len(dependencies),
})
}
func (r *WorkspaceRecorder) ArtifactMergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -213,24 +177,12 @@ func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, de
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error {
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) ArtifactNormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -308,48 +260,6 @@ type chunkEnvelope struct {
Metadata map[string]any `json:"metadata,omitempty"`
}
type extractOutputsEnvelope struct {
Outputs []extractOutputEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type extractOutputEnvelope struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type mergeOutputEnvelope struct {
Output mergeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type mergeOutputPayload struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type normalizeOutputEnvelope struct {
Output normalizeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type normalizeOutputPayload struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type binaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
@@ -380,12 +290,12 @@ type artifactSingleEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
func artifactCheckpointEnvelopeFromOutput(output pipeline.ArtifactCheckpointOutput) artifactCheckpointEnvelope {
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
}
func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []artifactCheckpointEnvelope {
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
if len(outputs) == 0 {
return nil
}
@@ -395,7 +305,7 @@ func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []
}
return out
}
func artifactOutputDigests(outputs []pipeline.ArtifactCheckpointOutput) []pipeline.CheckpointFingerprint {
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs {
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
@@ -422,54 +332,6 @@ func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
return out
}
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
if len(outputs) == 0 {
return nil
}
out := make([]extractOutputEnvelope, 0, len(outputs))
for _, output := range outputs {
out = append(out, extractOutputEnvelope{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
})
}
return out
}
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
return mergeOutputPayload{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
return normalizeOutputPayload{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = nil
return schema
}
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
return binaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
@@ -528,28 +390,6 @@ func cloneMetadata(metadata map[string]any) map[string]any {
return out
}
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
for i, payload := range payloads {
values = append(values, pipeline.CheckpointFingerprint{
Name: fmt.Sprintf("payload[%d]", i),
Value: contentDigest(payload.Content),
})
}
return normalizeFingerprints(values)
}
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
if len(outputs) == 0 {
return nil
}
payloads := make([]contracts.RawPayload, 0, len(outputs))
for _, output := range outputs {
payloads = append(payloads, output.Payload)
}
return payloads
}
func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) {
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
for _, chunk := range chunks {

139
internal/framework/checkpoint/recorder_test.go
View File

@@ -79,117 +79,19 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
}
}
func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []source.Chunk{
{
ID: "chunk-1",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"),
MediaType: "text/plain",
Units: doc.Units,
},
}
extractOutput := contracts.ExtractOutput{
LaneID: "spells",
ExtractorKey: "dnd/spells",
SourceID: doc.ID,
ChunkID: "chunk-1",
Payload: contracts.RawPayload{
Content: []byte(`{"spell":"cure wounds"}`),
MediaType: "application/json",
},
}
mergeOutput := contracts.MergeOutput{
LaneID: "spells",
MergerKey: "appendorder",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
normalizeOutput := contracts.NormalizeOutput{
LaneID: "spells",
NormalizerKey: "noop",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"normalized":true}`),
MediaType: "application/json",
},
}
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
t.Fatalf("SourceSucceeded: %v", err)
}
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []contracts.ExtractOutput{extractOutput}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
mergeDeps := rawOutputDigests([]contracts.RawPayload{extractOutput.Payload})
if err := recorder.MergeSucceeded("spells", "appendorder", mergeDeps, mergeOutput, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
normalizeDeps := rawOutputDigests([]contracts.RawPayload{mergeOutput.Payload})
if err := recorder.NormalizeSucceeded("spells", "noop", normalizeDeps, normalizeOutput, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
sourceCheckpoint, decision := loader.Source("seriatim")
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
}
if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want {
t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want)
}
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)
}
if got, want := chunkCheckpoint.Chunks[0].Ref, chunks[0].Ref; got != want {
t.Fatalf("checkpoint chunk ref = %#v, want %#v", got, want)
}
extractCheckpoint, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extractCheckpoint.Outputs) != 1 || string(extractCheckpoint.Outputs[0].Payload.Content) != `{"spell":"cure wounds"}` {
t.Fatalf("extract decision = %#v checkpoint=%#v, want reused", decision, extractCheckpoint)
}
mergeCheckpoint, decision := loader.Merge("spells", "appendorder", mergeDeps)
if !decision.Reused || string(mergeCheckpoint.Output.Payload.Content) != `{"merged":true}` {
t.Fatalf("merge decision = %#v checkpoint=%#v, want reused", decision, mergeCheckpoint)
}
normalizeCheckpoint, decision := loader.Normalize("spells", "noop", normalizeDeps)
if !decision.Reused || string(normalizeCheckpoint.Output.Payload.Content) != `{"normalized":true}` {
t.Fatalf("normalize decision = %#v checkpoint=%#v, want reused", decision, normalizeCheckpoint)
}
}
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}}
stored := pipeline.ArtifactCheckpointOutput{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
stored := pipeline.CheckpointArtifact{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ArtifactExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.ArtifactCheckpointOutput{stored}, nil, nil); err != nil {
t.Fatalf("ArtifactExtractSucceeded: %v", err)
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.CheckpointArtifact{stored}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
extracted, decision := loader.ArtifactExtract("spells", "dnd/spells", extractDeps)
extracted, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extracted.Outputs) != 1 {
t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted)
}
@@ -198,20 +100,20 @@ func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T
t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got)
}
mergeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactMergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactMergeSucceeded: %v", err)
mergeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.MergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
merged, decision := loader.ArtifactMerge("spells", "merge", mergeDeps)
merged, decision := loader.Merge("spells", "merge", mergeDeps)
if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) {
t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged)
}
normalizeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactNormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactNormalizeSucceeded: %v", err)
normalizeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.NormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
normalized, decision := loader.ArtifactNormalize("spells", "normalize", normalizeDeps)
normalized, decision := loader.Normalize("spells", "normalize", normalizeDeps)
if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest {
t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized)
}
@@ -361,23 +263,20 @@ func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
output := contracts.NormalizeOutput{
LaneID: "spells",
NormalizerKey: "noop",
SourceID: "source-1",
Payload: contracts.RawPayload{
Content: []byte(`{"ok":true}`),
MediaType: "application/json",
},
schema := contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
output := pipeline.CheckpointArtifact{
LaneID: "events", ModuleKey: "noop", SourceID: "source-1",
Artifact: contracts.SerializedArtifact{Kind: "test/artifact", Schema: schema, MediaType: "application/json", Content: []byte(`{"ok":true}`)},
SchemaDigest: contracts.DigestArtifactSchema(schema),
}
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil {
if err := recorder.NormalizeSucceeded("events", "noop", nil, output, warnings); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
var manifest coreworkspace.NormalizeLaneManifest
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest)
readJSON(t, filepath.Join(root, "normalize", "events", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
}

289
internal/framework/contracts/composition_test.go
View File

@@ -1,289 +0,0 @@
package contracts_test
import (
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
var _ contracts.InputAdapter = compositionAdapter{}
var _ contracts.Chunker = compositionChunker{}
var _ contracts.LegacyRawExtractor = compositionExtractor{}
var _ contracts.LegacyRawMerger = compositionMerger{}
var _ contracts.LegacyRawNormalizer = compositionNormalizer{}
var _ contracts.LegacyRawValidator = compositionValidator{}
var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) {
ctx := context.Background()
adapter := compositionAdapter{}
chunker := compositionChunker{}
extractor := compositionExtractor{}
merger := compositionMerger{}
normalizer := compositionNormalizer{}
encoder := compositionOutputEncoder{}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if err := source.ValidateDocument(doc); err != nil {
t.Fatalf("ValidateDocument() error = %v, want nil", err)
}
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
Metadata: map[string]any{"request": "test"},
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(chunking.Chunks) != 1 {
t.Fatalf("len(Chunks) = %d, want 1", len(chunking.Chunks))
}
extraction, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunking.Chunks[0],
AmbientContext: map[string]any{"synopsis": "example synopsis"},
})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if extraction.Output.Payload.MediaType != "application/json" {
t.Fatalf("extract media type = %q, want application/json", extraction.Output.Payload.MediaType)
}
merge, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: "generic-lane",
ExtractOutputs: []contracts.ExtractOutput{extraction.Output},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if string(merge.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merge output = %s, want extract payload", merge.Output.Payload.Content)
}
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: "generic-lane",
MergeOutput: merge.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if string(normalize.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalize output = %s, want merge payload", normalize.Output.Payload.Content)
}
output, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.SerializedOutput{{LaneID: normalize.Output.LaneID, NormalizerKey: normalize.Output.NormalizerKey, SourceID: normalize.Output.SourceID, Artifact: contracts.SerializedArtifact{Schema: contracts.ArtifactSchema{ID: normalize.Output.Schema.ID, Name: normalize.Output.Schema.Name, Version: normalize.Output.Schema.Version}, MediaType: normalize.Output.Payload.MediaType, Content: append([]byte(nil), normalize.Output.Payload.Content...)}}},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if len(output.Files) != 1 {
t.Fatalf("len(Files) = %d, want 1", len(output.Files))
}
if output.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType)
}
if len(output.Files[0].Bytes) == 0 {
t.Fatal("len(Bytes) = 0, want encoded bytes")
}
}
type compositionAdapter struct{}
func (adapter compositionAdapter) Key() string {
return "generic-input"
}
func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return &source.SourceDocument{
ID: req.SourceID,
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "First source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 2, EndUnitID: 2}},
},
}, nil
}
type compositionChunker struct{}
func (chunker compositionChunker) Key() string {
return "generic-chunker"
}
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required")
}
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"First source unit."},{"id":2,"kind":"unit","text":"Second source unit."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
Metadata: map[string]any{"strategy": "whole-document"},
},
},
}, nil
}
type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string {
return "generic-extractor"
}
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if req.Source == nil {
return contracts.ExtractionResult{}, errors.New("source document is required")
}
if req.AmbientContext["synopsis"] == "" {
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
},
},
}, nil
}
type compositionMerger struct{}
func (merger compositionMerger) Key() string {
return "generic-merger"
}
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
output := req.ExtractOutputs[0]
return contracts.MergeResult{Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.Key(),
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneCompositionPayload(output.Payload),
}}, nil
}
type compositionNormalizer struct{}
func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer"
}
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.Key(),
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneCompositionPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneCompositionPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneCompositionMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneCompositionMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type compositionValidator struct{}
func (validator compositionValidator) Name() string {
return "generic-validator"
}
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}
type compositionOutputEncoder struct{}
func (encoder compositionOutputEncoder) Key() string {
return "generic-output"
}
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
payload := struct {
RunID string `json:"run_id"`
OutputCount int `json:"output_count"`
}{
RunID: req.Manifest.RunID,
OutputCount: len(req.NormalizeOutputs),
}
encoded, err := json.Marshal(payload)
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{
Files: []contracts.OutputFile{
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: encoded,
},
},
}, nil
}

140
internal/framework/contracts/contracts.go
View File

@@ -207,37 +207,6 @@ type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
}
type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"`
Chunk *source.Chunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ExtractionResult struct {
Output ExtractOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type LegacyRawExtractor interface {
Key() string
ReferenceSlots() []ReferenceSlot
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
}
type RawPayload struct {
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
}
type ExecutionClass string
const (
@@ -245,29 +214,6 @@ const (
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
type ValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *source.Chunk `json:"chunk,omitempty"`
Chunks []source.Chunk `json:"chunks,omitempty"`
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput MergeOutput `json:"merge_output,omitempty"`
}
type ValidationResult struct {
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"`
@@ -276,92 +222,6 @@ type ValidationResult struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type LegacyRawValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type ResponseSchema struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
JSONSchema []byte `json:"-"`
}
type ExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type MergeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
ExtractOutputs []ExtractOutput `json:"extract_outputs"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type MergeResult struct {
Output MergeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type MergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type LegacyRawMerger interface {
Key() string
Merge(ctx context.Context, req MergeRequest) (MergeResult, error)
}
type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
MergeOutput MergeOutput `json:"merge_output"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type NormalizeResult struct {
Output NormalizeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type NormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type LegacyRawNormalizer interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}
type Warning struct {
Scope string `json:"scope,omitempty"`
ReasonCode string `json:"reason_code"`

150
internal/framework/contracts/contracts_test.go
View File

@@ -12,14 +12,13 @@ import (
var _ InputAdapter = fakeAdapter{}
var _ Chunker = fakeChunker{}
var _ LegacyRawExtractor = fakeExtractor{}
var _ LegacyRawMerger = fakeMerger{}
var _ LegacyRawNormalizer = fakeNormalizer{}
var _ LegacyRawValidator = fakeValidator{}
var _ Extractor[fakeArtifact] = fakeExtractor{}
var _ Merger[fakeArtifact] = fakeMerger{}
var _ Normalizer[fakeArtifact] = fakeNormalizer{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsRawOutput(t *testing.T) {
func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
}
@@ -33,7 +32,7 @@ func TestFakeExtractorReturnsRawOutput(t *testing.T) {
},
}
result, err := extractor.Extract(context.Background(), ExtractionRequest{Source: doc})
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
@@ -41,14 +40,8 @@ func TestFakeExtractorReturnsRawOutput(t *testing.T) {
if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
}
if result.Output.ExtractorKey != "" {
t.Fatalf("ExtractorKey = %q, want runner-owned empty value", result.Output.ExtractorKey)
}
if result.Output.Schema.Version != "v1" {
t.Fatalf("Schema.Version = %q, want v1", result.Output.Schema.Version)
}
if result.Output.Payload.MediaType != "application/json" || string(result.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("payload = %q %s, want JSON raw output", result.Output.Payload.MediaType, result.Output.Payload.Content)
if result.Value.Value != "example" {
t.Fatalf("Value = %q, want example", result.Value.Value)
}
}
@@ -139,7 +132,7 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
Units: []source.SourceUnit{doc.Units[1]},
}
result, err := extractor.Extract(context.Background(), ExtractionRequest{
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{
Source: doc,
Chunk: &chunk,
AmbientContext: map[string]any{"mode": "chunked"},
@@ -147,11 +140,8 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if result.Output.ChunkID != "" || result.Output.ChunkIndex != 0 {
t.Fatalf("chunk provenance = %q/%d, want runner-owned zero values", result.Output.ChunkID, result.Output.ChunkIndex)
}
if string(result.Output.Payload.Content) != `{"value":"chunked"}` {
t.Fatalf("Payload.Content = %s, want chunked payload", result.Output.Payload.Content)
if result.Value.Value != "chunked" {
t.Fatalf("Value = %q, want chunked", result.Value.Value)
}
}
@@ -320,8 +310,8 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
}
}
func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
schema := ResponseSchema{
func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) {
schema := ArtifactSchema{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
@@ -347,26 +337,14 @@ func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
extractOutput := ExtractOutput{
LaneID: "generic-lane",
ExtractorKey: "generic-extractor",
SourceID: "source-1",
ChunkID: "source-1:chunk:0",
ChunkIndex: 0,
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
Metadata: map[string]any{"confidence": 0.75},
},
}
extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}}
merger := fakeMerger{key: "generic-merger"}
normalizer := fakeNormalizer{key: "generic-normalizer"}
encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), MergeRequest{
merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{
LaneID: "generic-lane",
ExtractOutputs: []ExtractOutput{extractOutput},
ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
@@ -374,13 +352,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
}
if string(merged.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merged content = %s, want raw extract content", merged.Output.Payload.Content)
if merged.Value.Value != "example" {
t.Fatalf("merged value = %q, want example", merged.Value.Value)
}
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{
normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{
LaneID: "generic-lane",
MergeOutput: merged.Output,
MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value},
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
@@ -388,13 +366,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
}
if string(normalized.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalized content = %s, want raw merge content", normalized.Output.Payload.Content)
if normalized.Value.Value != "example" {
t.Fatalf("normalized value = %q, want example", normalized.Value.Value)
}
encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []SerializedOutput{serializedTestOutput(normalized.Output)},
NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -413,10 +391,6 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
}
}
func serializedTestOutput(output NormalizeOutput) SerializedOutput {
return SerializedOutput{LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID, Artifact: SerializedArtifact{Schema: ArtifactSchema{ID: output.Schema.ID, Name: output.Schema.Name, Version: output.Schema.Version, JSONSchema: append([]byte(nil), output.Schema.JSONSchema...)}, MediaType: output.Payload.MediaType, Content: append([]byte(nil), output.Payload.Content...), Metadata: cloneArtifactMetadata(output.Payload.Metadata)}}
}
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
file := OutputFile{
Name: "artifacts/events.json",
@@ -514,6 +488,8 @@ type fakeExtractor struct {
key string
}
type fakeArtifact struct{ Value string }
func (extractor fakeExtractor) Key() string {
return extractor.key
}
@@ -522,21 +498,12 @@ func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
payload := json.RawMessage(`{"value":"example"}`)
func (extractor fakeExtractor) Extract(ctx context.Context, req TypedExtractionRequest) (TypedExtractionResult[fakeArtifact], error) {
value := "example"
if req.AmbientContext["mode"] == "chunked" {
payload = json.RawMessage(`{"value":"chunked"}`)
value = "chunked"
}
return ExtractionResult{
Output: ExtractOutput{
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: append([]byte(nil), payload...),
MediaType: "application/json",
},
},
}, nil
return TypedExtractionResult[fakeArtifact]{Value: fakeArtifact{Value: value}}, nil
}
type fakeMerger struct {
@@ -547,15 +514,8 @@ func (merger fakeMerger) Key() string {
return merger.key
}
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) {
output := req.ExtractOutputs[0]
return MergeResult{Output: MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.key,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneTestRawPayload(output.Payload),
}}, nil
func (merger fakeMerger) Merge(ctx context.Context, req TypedMergeRequest[fakeArtifact]) (TypedMergeResult[fakeArtifact], error) {
return TypedMergeResult[fakeArtifact]{Value: req.ExtractOutputs[0].Value}, nil
}
type fakeNormalizer struct {
@@ -570,54 +530,8 @@ func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Output: NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneTestRawPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneTestRawPayload(payload RawPayload) RawPayload {
return RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneTestMetadata(payload.Metadata),
Warnings: append([]Warning(nil), payload.Warnings...),
}
}
func cloneTestMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type fakeValidator struct {
name string
}
func (validator fakeValidator) Name() string {
return validator.name
}
func (validator fakeValidator) ExecutionClass() ExecutionClass {
return ExecutionClassDeterministic
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
return ValidationResult{
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req TypedNormalizeRequest[fakeArtifact]) (TypedNormalizeResult[fakeArtifact], error) {
return TypedNormalizeResult[fakeArtifact]{Value: req.MergeOutput.Value}, nil
}
type fakeLLMClient struct{}

80
internal/framework/pipeline/checkpoint.go
View File

@@ -25,14 +25,14 @@ type CheckpointRecorder interface {
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
ChunkFailed(moduleKey string, sourceDigest string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
@@ -59,25 +59,9 @@ type ChunkCheckpoint struct {
Warnings []contracts.Warning
}
type ExtractCheckpoint struct {
Outputs []contracts.ExtractOutput
Rejected []contracts.RejectedOutput
Warnings []contracts.Warning
}
type MergeCheckpoint struct {
Output contracts.MergeOutput
Warnings []contracts.Warning
}
type NormalizeCheckpoint struct {
Output contracts.NormalizeOutput
Warnings []contracts.Warning
}
// ArtifactCheckpointOutput is the durable, domain-neutral value stored at a
// typed lane checkpoint boundary.
type ArtifactCheckpointOutput struct {
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
// checkpoint boundary.
type CheckpointArtifact struct {
LaneID string
ModuleKey string
SourceID string
@@ -88,33 +72,21 @@ type ArtifactCheckpointOutput struct {
SchemaDigest string
}
type ArtifactExtractCheckpoint struct {
Outputs []ArtifactCheckpointOutput
type ExtractCheckpoint struct {
Outputs []CheckpointArtifact
Rejected []contracts.RejectedOutput
Warnings []contracts.Warning
}
type ArtifactMergeCheckpoint struct {
Output ArtifactCheckpointOutput
type MergeCheckpoint struct {
Output CheckpointArtifact
Warnings []contracts.Warning
}
type ArtifactNormalizeCheckpoint struct {
Output ArtifactCheckpointOutput
type NormalizeCheckpoint struct {
Output CheckpointArtifact
Warnings []contracts.Warning
}
type ArtifactCheckpointRecorder interface {
ArtifactExtractSucceeded(string, string, []CheckpointFingerprint, []ArtifactCheckpointOutput, []contracts.RejectedOutput, []contracts.Warning) error
ArtifactMergeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
ArtifactNormalizeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
}
type ArtifactCheckpointLoader interface {
ArtifactExtract(string, string, []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision)
ArtifactMerge(string, string, []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision)
ArtifactNormalize(string, string, []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision)
}
type CheckpointLoader interface {
Enabled() bool
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
@@ -144,14 +116,14 @@ func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error {
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error {
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
@@ -163,7 +135,7 @@ func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprin
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error {
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
@@ -190,28 +162,6 @@ func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(payloads))
for i, payload := range payloads {
values = append(values, CheckpointFingerprint{
Name: fmt.Sprintf("payload[%d]", i),
Value: checkpointContentDigest(payload.Content),
})
}
return normalizeCheckpointFingerprints(values)
}
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
if len(outputs) == 0 {
return nil
}
payloads := make([]contracts.RawPayload, 0, len(outputs))
for _, output := range outputs {
payloads = append(payloads, output.Payload)
}
return payloads
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
digest = strings.TrimSpace(digest)
if digest == "" {

2
internal/framework/pipeline/chunker_registry.go
View File

@@ -32,7 +32,7 @@ func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerC
if constructor == nil {
return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.Chunker, error) {
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Chunker, error) {
return constructor()
})
}

44
internal/framework/pipeline/chunker_registry_test.go
View File

@@ -338,34 +338,6 @@ func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkReq
return contracts.ChunkResult{}, nil
}
type registryMerger struct {
key string
}
func (merger registryMerger) Key() string {
return merger.key
}
func (merger registryMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
return contracts.MergeResult{}, nil
}
type registryNormalizer struct {
key string
}
func (normalizer registryNormalizer) Key() string {
return normalizer.key
}
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil
}
type registryOutputEncoder struct {
key string
}
@@ -377,19 +349,3 @@ func (encoder registryOutputEncoder) Key() string {
func (encoder registryOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
type registryValidator struct {
name string
}
func (validator registryValidator) Name() string {
return validator.name
}
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

4
internal/framework/pipeline/construction.go
View File

@@ -24,7 +24,9 @@ type BuildRequest struct {
// OptionValidator validates one module binding without constructing it.
type OptionValidator func(map[string]any) error
func allowLegacyOptions(map[string]any) error { return nil }
func rejectUnconfiguredOptions(options map[string]any) error {
return RejectUnknownOptions(options)
}
func validateRegisteredOptions(validator OptionValidator, options map[string]any) error {
if validator == nil {

140
internal/framework/pipeline/debug.go
View File

@@ -82,10 +82,6 @@ type debugBinaryEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type debugRawPayload struct {
Content debugBinaryEnvelope `json:"content"`
}
type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
@@ -113,32 +109,6 @@ type debugSourceChunk struct {
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugMergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugNormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugSerializedOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
@@ -202,25 +172,6 @@ type debugLLMCallReference struct {
Error bool `json:"error,omitempty"`
}
type debugValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *debugSourceChunk `json:"chunk,omitempty"`
Chunks []debugSourceChunk `json:"chunks,omitempty"`
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
}
type debugValidationCall struct {
ValidatorName string `json:"validator_name"`
Request any `json:"request"`
@@ -448,10 +399,6 @@ func debugContentEnvelope(content []byte, mediaType string, metadata map[string]
}
}
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
if doc == nil {
return nil
@@ -489,63 +436,6 @@ func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
return out
}
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
output.Schema.JSONSchema = nil
return debugExtractOutput{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugExtractOutputEnvelope(output))
}
return out
}
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
output.Schema.JSONSchema = nil
return debugMergeOutput{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
output.Schema.JSONSchema = nil
return debugNormalizeOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugNormalizeOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugNormalizeOutputEnvelope(output))
}
return out
}
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema)
@@ -714,36 +604,6 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
return response.Debug.Response.ModelName
}
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
req.Schema.JSONSchema = nil
out := debugValidationRequest{
Stage: req.Stage,
LaneID: req.LaneID,
ModuleKey: req.ModuleKey,
SourceID: req.SourceID,
SessionID: req.SessionID,
LLMProfile: req.LLMProfile,
Options: redactSensitiveMap(req.Options),
Metadata: redactSensitiveMap(req.Metadata),
Schema: req.Schema,
ChunkID: req.ChunkID,
ChunkIndex: req.ChunkIndex,
}
payload := debugPayloadEnvelope(req.Payload)
out.Payload = &payload
if req.Chunk != nil {
chunk := debugSourceChunkEnvelope(*req.Chunk)
out.Chunk = &chunk
}
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
merge := debugMergeOutputEnvelope(req.MergeOutput)
out.MergeOutput = &merge
}
return out
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message)))
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))

52
internal/framework/pipeline/default_modules_test.go
View File

@@ -2,6 +2,7 @@ package pipeline_test
import (
"context"
"encoding/json"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
@@ -71,30 +72,40 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
if err := units.Register(chunkers); err != nil {
t.Fatalf("register generic chunker: %v", err)
}
if err := extractors.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
Key: "extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"records"},
}, func() (contracts.LegacyRawExtractor, error) {
if err := pipeline.RegisterExtractor[defaultArtifact](extractors, pipeline.ModuleSpec{
Key: "extract",
Stage: pipeline.StageExtract,
ArtifactKind: defaultArtifactKind,
Requires: []string{"chunks"},
Provides: []string{"records"},
}, func() (contracts.Extractor[defaultArtifact], error) {
return defaultExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := appendorder.Register(mergers); err != nil {
if err := appendorder.RegisterTyped(mergers, defaultArtifactKind, func(values []defaultArtifact) (defaultArtifact, error) {
if len(values) == 0 {
return defaultArtifact{}, nil
}
return values[0], nil
}); err != nil {
t.Fatalf("register appendorder merger: %v", err)
}
if err := noop.Register(normalizers); err != nil {
if err := noop.RegisterTyped[defaultArtifact](normalizers, defaultArtifactKind); err != nil {
t.Fatalf("register noop normalizer: %v", err)
}
if err := jsonoutput.Register(outputs); err != nil {
t.Fatalf("register json output: %v", err)
}
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, defaultArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
@@ -117,6 +128,25 @@ func (defaultExtractor) Key() string { return "extract" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
func (defaultExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[defaultArtifact], error) {
return contracts.TypedExtractionResult[defaultArtifact]{}, nil
}
const defaultArtifactKind contracts.ArtifactKind = "test/default"
type defaultArtifact struct {
Value string `json:"value"`
}
type defaultArtifactCodec struct{}
func (defaultArtifactCodec) Kind() contracts.ArtifactKind { return defaultArtifactKind }
func (defaultArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "urn:notarius:test:default", Name: "default", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (defaultArtifactCodec) MediaType() string { return "application/json" }
func (defaultArtifactCodec) Encode(value defaultArtifact) ([]byte, error) { return json.Marshal(value) }
func (defaultArtifactCodec) Decode(content []byte) (defaultArtifact, error) {
var value defaultArtifact
err := json.Unmarshal(content, &value)
return value, err
}

197
internal/framework/pipeline/extractor_registry.go
View File

@@ -9,14 +9,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type LegacyRawExtractorConstructor func() (contracts.LegacyRawExtractor, error)
type LegacyRawExtractorBuilder func(BuildRequest) (contracts.LegacyRawExtractor, error)
type ExtractorRegistry struct {
legacyBuilders map[string]LegacyRawExtractorBuilder
legacyValidators map[string]OptionValidator
typedEntries map[string]typedExtractorEntry
specs map[string]ModuleSpec
typedEntries map[string]typedExtractorEntry
specs map[string]ModuleSpec
}
type typedExtractorEntry struct {
@@ -25,201 +20,79 @@ type typedExtractorEntry struct {
validateOptions OptionValidator
builder func(BuildRequest) (any, error)
extract typedExtractOperation
rawBuilder LegacyRawExtractorBuilder
}
func NewExtractorRegistry() *ExtractorRegistry {
return &ExtractorRegistry{
legacyBuilders: make(map[string]LegacyRawExtractorBuilder),
legacyValidators: make(map[string]OptionValidator),
typedEntries: make(map[string]typedExtractorEntry),
specs: make(map[string]ModuleSpec),
}
}
func (r *ExtractorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawExtractorConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageExtract), constructor)
}
func (r *ExtractorRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawExtractorConstructor) error {
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawExtractor, error) {
return constructor()
})
}
func (r *ExtractorRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawExtractorBuilder) error {
if r == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw extractor %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.specs[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawExtractorBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
return &ExtractorRegistry{typedEntries: map[string]typedExtractorEntry{}, specs: map[string]ModuleSpec{}}
}
func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error {
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterExtractorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Extractor[T], error) {
return constructor()
})
return RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Extractor[T], error) { return constructor() })
}
func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error)) error {
return registerExtractorBuilder(registry, spec, validateOptions, builder, nil)
}
// RegisterExtractorBuilderWithRawAdapter registers a typed extractor while a
// raw downstream remains in use. Resolution selects the adapter until the
// registration is replaced with the typed-only builder.
func RegisterExtractorBuilderWithRawAdapter[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
if rawBuilder == nil {
return fmt.Errorf("extractor raw adapter builder for %q must not be nil", strings.TrimSpace(spec.Key))
}
return registerExtractorBuilder(registry, spec, validateOptions, builder, rawBuilder)
}
func registerExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
if registry == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
normalized := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalized); err != nil {
return err
}
if normalizedSpec.ArtifactKind == "" {
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalizedSpec.Key)
if normalized.ArtifactKind == "" {
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalized.Key)
}
if validateOptions == nil {
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key)
return fmt.Errorf("extractor option validator for %q must not be nil", normalized.Key)
}
if builder == nil {
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key)
return fmt.Errorf("extractor builder for %q must not be nil", normalized.Key)
}
if _, ok := registry.specs[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
}
entry := typedExtractorEntry{
spec: cloneModuleSpec(normalizedSpec),
valueType: reflect.TypeFor[T](),
validateOptions: validateOptions,
builder: func(request BuildRequest) (any, error) {
return builder(cloneBuildRequest(request))
},
extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractor, ok := implementation.(contracts.Extractor[T])
if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalizedSpec.Key, implementation)
}
result, err := extractor.Extract(ctx, request)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
},
rawBuilder: rawBuilder,
if _, ok := registry.specs[normalized.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalized.Key)
}
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractor, ok := implementation.(contracts.Extractor[T])
if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
}
result, err := extractor.Extract(ctx, request)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
}}
if registry.typedEntries == nil {
registry.typedEntries = make(map[string]typedExtractorEntry)
registry.typedEntries = map[string]typedExtractorEntry{}
}
if registry.specs == nil {
registry.specs = make(map[string]ModuleSpec)
registry.specs = map[string]ModuleSpec{}
}
registry.typedEntries[normalizedSpec.Key] = entry
registry.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
registry.typedEntries[normalized.Key] = entry
registry.specs[normalized.Key] = cloneModuleSpec(normalized)
return nil
}
func (r *ExtractorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawExtractor, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *ExtractorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawExtractor, error) {
if r == nil {
return nil, fmt.Errorf("extractor registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("extractor key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
if entry, typedOK := r.typedEntries[normalizedKey]; typedOK && entry.rawBuilder != nil {
builder = entry.rawBuilder
ok = true
}
}
if !ok {
return nil, fmt.Errorf("legacy raw extractor %q is not registered", normalizedKey)
}
extractor, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build extractor %q: %w", normalizedKey, err)
}
if extractor == nil {
return nil, fmt.Errorf("extractor %q constructor returned nil", normalizedKey)
}
if extractor.Key() != normalizedKey {
return nil, fmt.Errorf("extractor %q returned key %q", normalizedKey, extractor.Key())
}
return extractor, nil
}
func (r *ExtractorRegistry) validateOptions(key string, options map[string]any) error {
if r == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if entry, ok := r.typedEntries[normalizedKey]; ok {
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
normalized := strings.TrimSpace(key)
entry, ok := r.typedEntries[normalized]
if !ok {
return fmt.Errorf("extractor %q is not registered", normalizedKey)
return fmt.Errorf("extractor %q is not registered", normalized)
}
return validateRegisteredOptions(validator, options)
return validateRegisteredOptions(entry.validateOptions, options)
}
func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
}
return cloneModuleSpec(spec), true
return cloneModuleSpec(spec), ok
}
func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
if r == nil {
return typedExtractorEntry{}, false
@@ -227,12 +100,6 @@ func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
entry, ok := r.typedEntries[strings.TrimSpace(key)]
return entry, ok
}
func (r *ExtractorRegistry) usesRawAdapter(key string) bool {
entry, ok := r.typedEntry(key)
return ok && entry.rawBuilder != nil
}
func (r *ExtractorRegistry) RegisteredKeys() []string {
if r == nil {
return nil

414
internal/framework/pipeline/extractor_registry_test.go
View File

@@ -1,414 +0,0 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.BuildLegacyRaw("generic-extractor")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.BuildLegacyRaw("\tgeneric-extractor\n")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
registry := NewExtractorRegistry()
spec := ModuleSpec{
Key: " generic-extractor ",
Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: " glossary ",
Description: " Supporting terms ",
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
MaxBytes: 1024,
},
{
Name: " roster ",
Description: " Characters ",
Required: true,
Multiple: true,
},
},
}
if err := registry.RegisterLegacyRawWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("\tgeneric-extractor\n")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Supporting terms",
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
MaxBytes: 1024,
},
{
Name: "roster",
Description: "Characters",
Required: true,
Multiple: true,
},
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
got.Provides[0] = "changed"
got.ReferenceSlots[0].Name = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true")
}
if !reflect.DeepEqual(again, want) {
t.Fatalf("Spec() after caller mutation = %#v, want %#v", again, want)
}
}
func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{Key: "generic-extractor", Stage: StageExtract}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "stage") {
t.Fatalf("RegisterWithSpec() error = %q, want stage error", err.Error())
}
}
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
tests := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
if _, ok := registry.Spec("missing-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
}
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRaw(" \t", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Register() error = %q, want empty key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "already registered") {
t.Fatalf("Register() error = %q, want duplicate key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRaw("generic-extractor", nil)
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "constructor") {
t.Fatalf("Register() error = %q, want constructor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.BuildLegacyRaw("missing-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "not registered") {
t.Fatalf("Build() error = %q, want unknown key error", err.Error())
}
}
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
registry := NewExtractorRegistry()
constructorErr := errors.New("constructor failed")
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, constructorErr
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !errors.Is(err, constructorErr) {
t.Fatalf("Build() error = %v, want wrapped constructor error", err)
}
if !strings.Contains(err.Error(), "generic-extractor") {
t.Fatalf("Build() error = %q, want key context", err.Error())
}
}
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned nil") {
t.Fatalf("Build() error = %q, want nil extractor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned key") {
t.Fatalf("Build() error = %q, want key mismatch error", err.Error())
}
}
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
registry := NewExtractorRegistry()
for _, key := range []string{"zeta", "alpha", "middle"} {
if err := registry.RegisterLegacyRaw(key, fakeExtractorConstructor(key)); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
keys := registry.RegisteredKeys()
want := []string{"alpha", "middle", "zeta"}
if !reflect.DeepEqual(keys, want) {
t.Fatalf("RegisteredKeys() = %#v, want %#v", keys, want)
}
keys[0] = "changed"
if got := registry.RegisteredKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredKeys() after caller mutation = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
var registry *ExtractorRegistry
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
t.Fatal("Register() error = nil, want error")
}
if _, err := registry.BuildLegacyRaw("generic-extractor"); err == nil {
t.Fatal("Build() error = nil, want error")
}
if _, ok := registry.Spec("generic-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
if keys := registry.RegisteredKeys(); keys != nil {
t.Fatalf("RegisteredKeys() = %#v, want nil", keys)
}
}
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.BuildLegacyRaw(" \n")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Build() error = %q, want empty key error", err.Error())
}
}
func TestExtractorRegistryTypedRegistrationCanProvideRawAdapter(t *testing.T) {
registry := NewExtractorRegistry()
spec := ModuleSpec{Key: "typed-extractor", Stage: StageExtract, ArtifactKind: "test/value"}
if err := RegisterExtractorBuilderWithRawAdapter(registry, spec, func(map[string]any) error { return nil },
func(BuildRequest) (contracts.Extractor[registryTypedValue], error) {
return registryTypedExtractor{key: spec.Key}, nil
},
func(BuildRequest) (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: spec.Key}, nil
},
); err != nil {
t.Fatalf("RegisterExtractorBuilderWithRawAdapter() error = %v", err)
}
if !registry.usesRawAdapter(spec.Key) {
t.Fatal("usesRawAdapter() = false, want true")
}
if _, ok := registry.typedEntry(spec.Key); !ok {
t.Fatal("typedEntry() ok = false, want true")
}
adapter, err := registry.BuildLegacyRaw(spec.Key)
if err != nil || adapter.Key() != spec.Key {
t.Fatalf("BuildLegacyRaw() = %#v, %v", adapter, err)
}
if err := RegisterExtractorBuilderWithRawAdapter[registryTypedValue](NewExtractorRegistry(), spec, func(map[string]any) error { return nil }, nil, nil); err == nil || !strings.Contains(err.Error(), "raw adapter") {
t.Fatalf("nil raw builder error = %v, want raw adapter context", err)
}
}
type registryFakeExtractor struct {
key string
}
type registryTypedValue struct{ Value string }
type registryTypedExtractor struct{ key string }
func (extractor registryTypedExtractor) Key() string { return extractor.key }
func (registryTypedExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (registryTypedExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[registryTypedValue], error) {
return contracts.TypedExtractionResult[registryTypedValue]{}, nil
}
func fakeExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func (extractor registryFakeExtractor) Key() string {
return extractor.key
}
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

2
internal/framework/pipeline/input_registry.go
View File

@@ -32,7 +32,7 @@ func (r *InputAdapterRegistry) RegisterWithSpec(spec ModuleSpec, constructor Inp
if constructor == nil {
return fmt.Errorf("input adapter constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.InputAdapter, error) {
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.InputAdapter, error) {
return constructor()
})
}

120
internal/framework/pipeline/merger_registry.go
View File

@@ -9,19 +9,13 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type LegacyRawMergerConstructor func() (contracts.LegacyRawMerger, error)
type LegacyRawMergerBuilder func(BuildRequest) (contracts.LegacyRawMerger, error)
type artifactVariantKey struct {
module string
kind contracts.ArtifactKind
}
type MergerRegistry struct {
legacyBuilders map[string]LegacyRawMergerBuilder
legacyValidators map[string]OptionValidator
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedMergerEntry
typedEntries map[artifactVariantKey]typedMergerEntry
}
type typedMergerEntry struct {
@@ -34,66 +28,15 @@ type typedMergerEntry struct {
func NewMergerRegistry() *MergerRegistry {
return &MergerRegistry{
legacyBuilders: make(map[string]LegacyRawMergerBuilder),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
}
}
func (r *MergerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawMergerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageMerge), constructor)
}
func (r *MergerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawMergerConstructor) error {
if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawMerger, error) {
return constructor()
})
}
func (r *MergerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawMergerBuilder) error {
if r == nil {
return fmt.Errorf("merger registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("merger", StageMerge, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw merger %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("merger option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("merger builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw merger %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawMergerBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func RegisterMerger[T any](registry *MergerRegistry, spec ModuleSpec, constructor func() (contracts.Merger[T], error)) error {
if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterMergerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Merger[T], error) {
return RegisterMergerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Merger[T], error) {
return constructor()
})
}
@@ -152,63 +95,29 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
return nil
}
func (r *MergerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawMerger, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *MergerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawMerger, error) {
if r == nil {
return nil, fmt.Errorf("merger registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("merger key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
}
merger, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build merger %q: %w", normalizedKey, err)
}
if merger == nil {
return nil, fmt.Errorf("merger %q constructor returned nil", normalizedKey)
}
if merger.Key() != normalizedKey {
return nil, fmt.Errorf("merger %q returned key %q", normalizedKey, merger.Key())
}
return merger, nil
}
func (r *MergerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
if r == nil {
return fmt.Errorf("merger registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if kind != "" {
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(validator, options)
return validateRegisteredOptions(entry.validateOptions, options)
}
func (r *MergerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
module := strings.TrimSpace(key)
for variant, entry := range r.typedEntries {
if variant.module == module {
return cloneModuleSpec(entry.spec), true
}
}
return cloneModuleSpec(spec), true
return ModuleSpec{}, false
}
func (r *MergerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedMergerEntry, bool) {
@@ -238,10 +147,7 @@ func (r *MergerRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
keys := make(map[string]struct{}, len(r.typedEntries))
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}

58
internal/framework/pipeline/merger_registry_test.go
View File

@@ -1,58 +0,0 @@
package pipeline
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestMergerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawMerger]{
name: "MergerRegistry",
key: "generic-merger",
stage: StageMerge,
wrongStage: StageExtract,
newRegistry: func() any {
return NewMergerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.LegacyRawMerger, error) {
return registry.(*MergerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*MergerRegistry).Spec(key)
},
registeredKeys: func(registry any) []string {
return registry.(*MergerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.LegacyRawMerger, error)) error {
var registry *MergerRegistry
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.LegacyRawMerger, error) {
var registry *MergerRegistry
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *MergerRegistry
return registry.Spec(key)
},
nilRegisteredKey: func() []string {
var registry *MergerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.LegacyRawMerger, error) {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
},
moduleKey: func(module contracts.LegacyRawMerger) string {
return module.Key()
},
})
}

120
internal/framework/pipeline/normalizer_registry.go
View File

@@ -9,14 +9,8 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type LegacyRawNormalizerConstructor func() (contracts.LegacyRawNormalizer, error)
type LegacyRawNormalizerBuilder func(BuildRequest) (contracts.LegacyRawNormalizer, error)
type NormalizerRegistry struct {
legacyBuilders map[string]LegacyRawNormalizerBuilder
legacyValidators map[string]OptionValidator
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedNormalizerEntry
typedEntries map[artifactVariantKey]typedNormalizerEntry
}
type typedNormalizerEntry struct {
@@ -29,66 +23,15 @@ type typedNormalizerEntry struct {
func NewNormalizerRegistry() *NormalizerRegistry {
return &NormalizerRegistry{
legacyBuilders: make(map[string]LegacyRawNormalizerBuilder),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
}
}
func (r *NormalizerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawNormalizerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageNormalize), constructor)
}
func (r *NormalizerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawNormalizerConstructor) error {
if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawNormalizer, error) {
return constructor()
})
}
func (r *NormalizerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawNormalizerBuilder) error {
if r == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("normalizer", StageNormalize, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw normalizer %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("normalizer option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("normalizer builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw normalizer %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawNormalizerBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func RegisterNormalizer[T any](registry *NormalizerRegistry, spec ModuleSpec, constructor func() (contracts.Normalizer[T], error)) error {
if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterNormalizerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Normalizer[T], error) {
return RegisterNormalizerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Normalizer[T], error) {
return constructor()
})
}
@@ -143,63 +86,29 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
return nil
}
func (r *NormalizerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawNormalizer, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *NormalizerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawNormalizer, error) {
if r == nil {
return nil, fmt.Errorf("normalizer registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("normalizer key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
}
normalizer, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build normalizer %q: %w", normalizedKey, err)
}
if normalizer == nil {
return nil, fmt.Errorf("normalizer %q constructor returned nil", normalizedKey)
}
if normalizer.Key() != normalizedKey {
return nil, fmt.Errorf("normalizer %q returned key %q", normalizedKey, normalizer.Key())
}
return normalizer, nil
}
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
if r == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if kind != "" {
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(validator, options)
return validateRegisteredOptions(entry.validateOptions, options)
}
func (r *NormalizerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
module := strings.TrimSpace(key)
for variant, entry := range r.typedEntries {
if variant.module == module {
return cloneModuleSpec(entry.spec), true
}
}
return cloneModuleSpec(spec), true
return ModuleSpec{}, false
}
func (r *NormalizerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedNormalizerEntry, bool) {
@@ -229,10 +138,7 @@ func (r *NormalizerRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
keys := make(map[string]struct{}, len(r.typedEntries))
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}

58
internal/framework/pipeline/normalizer_registry_test.go
View File

@@ -1,58 +0,0 @@
package pipeline
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestNormalizerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawNormalizer]{
name: "NormalizerRegistry",
key: "generic-normalizer",
stage: StageNormalize,
wrongStage: StageExtract,
newRegistry: func() any {
return NewNormalizerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.LegacyRawNormalizer, error) {
return registry.(*NormalizerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*NormalizerRegistry).Spec(key)
},
registeredKeys: func(registry any) []string {
return registry.(*NormalizerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
var registry *NormalizerRegistry
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.LegacyRawNormalizer, error) {
var registry *NormalizerRegistry
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *NormalizerRegistry
return registry.Spec(key)
},
nilRegisteredKey: func() []string {
var registry *NormalizerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.LegacyRawNormalizer, error) {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
},
moduleKey: func(module contracts.LegacyRawNormalizer) string {
return module.Key()
},
})
}

2
internal/framework/pipeline/output_registry.go
View File

@@ -32,7 +32,7 @@ func (r *OutputEncoderRegistry) RegisterWithSpec(spec ModuleSpec, constructor Ou
if constructor == nil {
return fmt.Errorf("output encoder constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.OutputEncoder, error) {
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.OutputEncoder, error) {
return constructor()
})
}

57
internal/framework/pipeline/preparation_test.go
View File

@@ -1,11 +1,13 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -143,7 +145,7 @@ func constructionProfile() PipelineProfile {
}
}
func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *runnerInputAdapter) {
func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *constructionInput) {
t.Helper()
if built == nil {
built = &[]string{}
@@ -153,48 +155,64 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
}
record := func(name string) { *built = append(*built, name) }
strict := func(options map[string]any) error { return RejectUnknownOptions(options, "known") }
modules := defaultRunnerModules()
input := &constructionInput{key: "input"}
registries := Registries{
Inputs: NewInputAdapterRegistry(), Chunkers: NewChunkerRegistry(), ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(), Mergers: NewMergerRegistry(), Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(), ValidatorChains: NewValidatorChainRegistry(), Outputs: NewOutputEncoderRegistry(),
}
if err := RegisterArtifactCodec(registries.ArtifactCodecs, notesCodec()); err != nil {
t.Fatal(err)
}
if err := registries.Inputs.RegisterBuilderWithSpec(defaultModuleSpec("input", StageInput), strict, func(BuildRequest) (contracts.InputAdapter, error) {
record("input")
return modules.input, nil
return input, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Chunkers.RegisterBuilderWithSpec(defaultModuleSpec("chunk", StageChunk), strict, func(BuildRequest) (contracts.Chunker, error) {
record("chunk")
return modules.chunker, nil
return &typedTestChunker{key: "chunk"}, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Extractors.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("extract", StageExtract), strict, func(request BuildRequest) (contracts.LegacyRawExtractor, error) {
extractSpec := defaultModuleSpec("extract", StageExtract)
extractSpec.ArtifactKind = "test/notes"
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
record("extract")
if failure.requireExtractorLLM && request.Dependencies.LLM == nil {
return nil, errors.New("structured LLM client is required")
}
return &runnerExtractor{key: "extract"}, nil
return typedTestExtractor[codecNotes]{key: "extract"}, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Mergers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("merge", StageMerge), strict, func(BuildRequest) (contracts.LegacyRawMerger, error) {
mergeSpec := defaultModuleSpec("merge", StageMerge)
mergeSpec.ArtifactKind = "test/notes"
if err := RegisterMergerBuilder(registries.Mergers, mergeSpec, strict, func(BuildRequest) (contracts.Merger[codecNotes], error) {
record("merge")
return modules.mergers["merge"], nil
return typedTestMerger[codecNotes]{key: "merge"}, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Normalizers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("normalize", StageNormalize), strict, func(BuildRequest) (contracts.LegacyRawNormalizer, error) {
normalizeSpec := defaultModuleSpec("normalize", StageNormalize)
normalizeSpec.ArtifactKind = "test/notes"
if err := RegisterNormalizerBuilder(registries.Normalizers, normalizeSpec, strict, func(BuildRequest) (contracts.Normalizer[codecNotes], error) {
record("normalize")
return modules.normalizers["normalize"], nil
return typedTestNormalizer[codecNotes]{key: "normalize"}, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Validators.RegisterLegacyRawBuilderWithSpec(ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}, strict, func(BuildRequest) (contracts.LegacyRawValidator, error) {
validatorSpec := ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}
if err := RegisterChunkValidatorBuilder(registries.Validators, validatorSpec, strict, func(BuildRequest) (contracts.ChunkValidator, error) {
record("validator")
return modules.validators["configured"], nil
return typedTestChunkValidator{key: "configured"}, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterTypedValidatorBuilder(registries.Validators, "test/notes", validatorSpec, strict, func(BuildRequest) (contracts.TypedValidator[codecNotes], error) {
record("validator")
return typedTestValidator[codecNotes]{key: "configured"}, nil
}); err != nil {
t.Fatal(err)
}
@@ -203,9 +221,20 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
if failure.output != nil {
return nil, failure.output
}
return modules.output, nil
return &typedTestOutput{key: "output"}, nil
}); err != nil {
t.Fatal(err)
}
return registries, modules.input
return registries, input
}
type constructionInput struct {
key string
requests []contracts.ParseRequest
}
func (input *constructionInput) Key() string { return input.key }
func (input *constructionInput) Parse(_ context.Context, request contracts.ParseRequest) (*source.SourceDocument, error) {
input.requests = append(input.requests, request)
return typedTestDocument(), nil
}

96
internal/framework/pipeline/prepare.go
View File

@@ -32,19 +32,12 @@ type PreparedArtifactLane struct {
type preparedLaneExecutor struct {
resolved ResolvedArtifactLane
legacy *preparedLegacyLane
typed *preparedTypedLane
extractValidators preparedValidatorChain
mergeValidators preparedValidatorChain
normalizeValidators preparedValidatorChain
}
type preparedLegacyLane struct {
extractor contracts.LegacyRawExtractor
merger contracts.LegacyRawMerger
normalizer contracts.LegacyRawNormalizer
}
type preparedTypedLane struct {
extractor any
merger any
@@ -62,7 +55,6 @@ type preparedValidatorChain struct {
type preparedValidator struct {
resolved ResolvedValidator
legacy contracts.LegacyRawValidator
typed any
typedValidate typedValidateOperation
chunk contracts.ChunkValidator
@@ -130,75 +122,50 @@ func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registrie
request := func(binding ModuleBinding) BuildRequest {
return BuildRequest{Dependencies: deps, Options: cloneOptions(binding.Options)}
}
if lane.ArtifactKind == "" {
extractor, err := registries.Extractors.BuildLegacyRawWithRequest(lane.Extract.Module, request(lane.Extract))
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
executor.legacy = &preparedLegacyLane{extractor: extractor}
} else {
entry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
if codecErr != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
}
executor.typed = &preparedTypedLane{extractor: module, extract: entry.extract, codec: codec}
extractEntry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(extractEntry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
if codecErr != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
}
executor.typed = &preparedTypedLane{extractor: module, extract: extractEntry.extract, codec: codec}
var err error
executor.extractValidators, err = prepareValidatorChain(pipeline, registries, deps, StageExtract, lane.ID, lane.Extract.Module)
if err != nil {
return preparedLaneExecutor{}, err
}
if lane.ArtifactKind == "" {
module, err := registries.Mergers.BuildLegacyRawWithRequest(lane.Merge.Module, request(lane.Merge))
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.legacy.merger = module
} else {
entry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Merge), lane.Merge.Module, "merger")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.typed.merger = module
executor.typed.merge = entry.merge
mergeEntry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err = buildErasedModule(mergeEntry.builder, request(lane.Merge), lane.Merge.Module, "merger")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.typed.merger = module
executor.typed.merge = mergeEntry.merge
executor.mergeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageMerge, lane.ID, lane.Merge.Module)
if err != nil {
return preparedLaneExecutor{}, err
}
if lane.ArtifactKind == "" {
module, err := registries.Normalizers.BuildLegacyRawWithRequest(lane.Normalize.Module, request(lane.Normalize))
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.legacy.normalizer = module
} else {
entry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.typed.normalizer = module
executor.typed.normalize = entry.normalize
normalizeEntry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err = buildErasedModule(normalizeEntry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.typed.normalizer = module
executor.typed.normalize = normalizeEntry.normalize
executor.normalizeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageNormalize, lane.ID, lane.Normalize.Module)
if err != nil {
return preparedLaneExecutor{}, err
@@ -249,8 +216,7 @@ func buildPreparedValidator(registry *ValidatorRegistry, resolved ResolvedValida
prepared.serialized, err = entry.builder(cloneBuildRequest(request))
implementation = prepared.serialized
default:
prepared.legacy, err = registry.BuildLegacyRawWithRequest(key, request)
implementation = prepared.legacy
return preparedValidator{}, fmt.Errorf("validator construction target %q is not supported", resolved.Target)
}
if err != nil {
return preparedValidator{}, err

10
internal/framework/pipeline/profile.go
View File

@@ -384,7 +384,7 @@ func configuredValidatorsError(pipelineID string, laneID string) error {
func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) {
if extractSpec.ArtifactKind == "" {
return nil, nil
return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", pipelineID, laneID, lane.Extract.Module)
}
if catalog.Extractors == nil {
return nil, fmt.Errorf("pipeline %q lane %q extractor registry must not be nil", pipelineID, laneID)
@@ -393,9 +393,6 @@ func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLa
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q declares artifact kind %q without a typed registration", pipelineID, laneID, lane.Extract.Module, extractSpec.ArtifactKind)
}
if catalog.Extractors.usesRawAdapter(lane.Extract.Module) {
return nil, nil
}
if catalog.ArtifactCodecs == nil {
return nil, fmt.Errorf("pipeline %q lane %q artifact codec registry must not be nil for kind %q", pipelineID, laneID, extractSpec.ArtifactKind)
}
@@ -470,7 +467,7 @@ func validatorSpecForTarget(registry *ValidatorRegistry, stage ModuleStage, key
if spec, ok := registry.Spec(key); ok {
return spec, "", nil
}
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q on legacy raw path", key)
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q without an artifact kind", key)
}
if entry, ok := registry.typedEntry(key, kind); ok {
if entry.valueType != expectedType {
@@ -646,9 +643,6 @@ func validatePipelineReferenceDefaults(
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
var artifactType reflect.Type
artifactKind := extractSpec.ArtifactKind
if catalog.Extractors != nil && catalog.Extractors.usesRawAdapter(extract.Module) {
artifactKind = ""
}
if artifactKind != "" && catalog.Extractors != nil {
if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok {
artifactType = entry.valueType

81
internal/framework/pipeline/profile_test.go
View File

@@ -766,20 +766,22 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
if err := RegisterExtractor[codecNotes](catalog.Extractors, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
ArtifactKind: "test/notes",
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.LegacyRawExtractor, error) {
}, func() (contracts.Extractor[codecNotes], error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
@@ -1177,6 +1179,7 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper()
catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
registerProfileSpecs(t, catalog, defaultProfileSpecs()...)
return catalog
}
@@ -1192,6 +1195,9 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
specs := defaultProfileSpecs()
for _, override := range overrides {
if override.ArtifactKind == "" && (override.Stage == StageExtract || override.Stage == StageMerge || override.Stage == StageNormalize) {
override.ArtifactKind = "test/notes"
}
replaced := false
for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key {
@@ -1206,6 +1212,7 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
}
catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
registerProfileSpecs(t, catalog, specs...)
return catalog
}
@@ -1228,10 +1235,10 @@ func defaultProfileSpecs() []ModuleSpec {
return []ModuleSpec{
ModuleSpec{Key: "text", Stage: StageInput, Provides: []string{"source"}},
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "note-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "appendorder", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "note-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "appendorder", Stage: StageMerge, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "noop", Stage: StageNormalize, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "grounded", Stage: StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}},
ModuleSpec{Key: "json", Stage: StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
}
@@ -1241,30 +1248,40 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
t.Helper()
for _, spec := range specs {
if spec.ArtifactKind == "" && (spec.Stage == StageExtract || spec.Stage == StageMerge || spec.Stage == StageNormalize) {
spec.ArtifactKind = "test/notes"
}
switch spec.Stage {
case StageInput:
if err := catalog.Inputs.RegisterWithSpec(spec, profileInputConstructor(spec.Key)); err != nil {
t.Fatalf("register input spec %#v: %v", spec, err)
}
case StageChunk:
if err := catalog.Chunkers.RegisterWithSpec(spec, profileChunkerConstructor(spec.Key)); err != nil {
validateOptions := func(options map[string]any) error { return RejectUnknownOptions(options, "a", "b", "size") }
if err := catalog.Chunkers.RegisterBuilderWithSpec(spec, validateOptions, func(BuildRequest) (contracts.Chunker, error) { return &typedTestChunker{key: spec.Key}, nil }); err != nil {
t.Fatalf("register chunk spec %#v: %v", spec, err)
}
case StageExtract:
if err := catalog.Extractors.RegisterLegacyRawWithSpec(spec, profileExtractorConstructor(spec.Key)); err != nil {
if err := RegisterExtractor(catalog.Extractors, spec, func() (contracts.Extractor[codecNotes], error) {
return typedTestExtractor[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register extractor spec %#v: %v", spec, err)
}
case StageMerge:
if err := catalog.Mergers.RegisterLegacyRawWithSpec(spec, profileMergerConstructor(spec.Key)); err != nil {
if err := RegisterMerger(catalog.Mergers, spec, func() (contracts.Merger[codecNotes], error) { return typedTestMerger[codecNotes]{key: spec.Key}, nil }); err != nil {
t.Fatalf("register merger spec %#v: %v", spec, err)
}
case StageNormalize:
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(spec, profileNormalizerConstructor(spec.Key)); err != nil {
if err := RegisterNormalizer(catalog.Normalizers, spec, func() (contracts.Normalizer[codecNotes], error) {
return typedTestNormalizer[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register normalizer spec %#v: %v", spec, err)
}
case StageValidate:
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := catalog.Validators.RegisterLegacyRawWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
if err := RegisterTypedValidator(catalog.Validators, "test/notes", validatorSpec, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
case StageOutput:
@@ -1279,7 +1296,9 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper()
if err := catalog.Validators.RegisterLegacyRawWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
if err := RegisterTypedValidator(catalog.Validators, "test/notes", spec, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
}
@@ -1302,36 +1321,6 @@ func (adapter profileInputAdapter) Parse(ctx context.Context, req contracts.Pars
return &source.SourceDocument{}, nil
}
func profileChunkerConstructor(key string) ChunkerConstructor {
return func() (contracts.Chunker, error) {
return registryChunker{key: key}, nil
}
}
func profileExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func profileMergerConstructor(key string) LegacyRawMergerConstructor {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
}
func profileNormalizerConstructor(key string) LegacyRawNormalizerConstructor {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
}
func profileValidatorConstructor(key string) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return registryValidator{name: key}, nil
}
}
func profileOutputConstructor(key string) OutputEncoderConstructor {
return func() (contracts.OutputEncoder, error) {
return registryOutputEncoder{key: key}, nil

14
internal/framework/pipeline/reference_test_helpers_test.go Normal file
View File

@@ -0,0 +1,14 @@
package pipeline
import "gitea.maximumdirect.net/eric/notarius/internal/core/source"
func validSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1", Kind: "document", Format: "text/plain", Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
},
}
}

265
internal/framework/pipeline/registry_integration_test.go
View File

@@ -1,265 +0,0 @@
package pipeline
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerUsesRegistries(t *testing.T) {
var built []string
var executed []string
registries := integrationRegistries(t, &built, &executed)
output, err := newPreparedRunner(t, registries).Run(context.Background(), RunInput{
pipeline: integrationPipeline(),
SourceID: "source-1",
RawInput: []byte("source text"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
wantBuilt := []string{"input", "chunk", "extract-first", "merge", "normalize", "extract-second", "merge", "normalize", "output"}
if !reflect.DeepEqual(built, wantBuilt) {
t.Fatalf("built = %#v, want %#v", built, wantBuilt)
}
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
}
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
}
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
}
}
func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
t.Helper()
registries := Registries{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
*built = append(*built, "input")
return integrationInput{}, nil
}); err != nil {
t.Fatalf("register input: %v", err)
}
if err := registries.Chunkers.Register("chunk", func() (contracts.Chunker, error) {
*built = append(*built, "chunk")
return integrationChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
if err := registries.Mergers.RegisterLegacyRaw("merge", func() (contracts.LegacyRawMerger, error) {
*built = append(*built, "merge")
return integrationMerger{}, nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := registries.Normalizers.RegisterLegacyRaw("normalize", func() (contracts.LegacyRawNormalizer, error) {
*built = append(*built, "normalize")
return integrationNormalizer{}, nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := registries.Outputs.Register("output", func() (contracts.OutputEncoder, error) {
*built = append(*built, "output")
return integrationOutput{}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
return registries
}
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
t.Helper()
if err := registry.RegisterLegacyRaw(key, func() (contracts.LegacyRawExtractor, error) {
*built = append(*built, key)
return integrationExtractor{key: key, executed: executed}, nil
}); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
type integrationInput struct{}
func (input integrationInput) Key() string {
return "input"
}
func (input integrationInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return integrationSourceDocument(), nil
}
type integrationChunker struct{}
func (chunker integrationChunker) Key() string {
return "chunk"
}
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: "chunk-0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: req.Source.Units,
},
},
}, nil
}
type integrationExtractor struct {
key string
executed *[]string
}
func (extractor integrationExtractor) Key() string {
return extractor.key
}
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
}
type integrationNormalizer struct{}
type integrationMerger struct{}
func (merger integrationMerger) Key() string {
return "merge"
}
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
output := contracts.MergeOutput{
LaneID: req.LaneID,
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
if len(req.ExtractOutputs) > 0 {
output.SourceID = req.ExtractOutputs[0].SourceID
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
func (normalizer integrationNormalizer) Key() string {
return "normalize"
}
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type integrationOutput struct{}
func (output integrationOutput) Key() string {
return "output"
}
func (output integrationOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{}`)},
},
}, nil
}
func integrationPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "first",
Extract: Binding("extract-first"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
{
ID: "second",
Extract: Binding("extract-second"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
},
Output: Binding("output"),
}
}
func integrationSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
},
}
}
func normalizeOutputKeys(outputs []contracts.SerializedOutput) []string {
keys := make([]string, 0, len(outputs))
for _, output := range outputs {
keys = append(keys, output.NormalizerKey)
}
return keys
}

775
internal/framework/pipeline/runner.go
View File

@@ -245,7 +245,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}, llmScope))
return false, nil, err
}
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
@@ -323,7 +323,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
} else {
output.Manifest.ValidationStatus = "approved"
}
populateRawOutputManifest(&output)
populateOutputManifest(&output)
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
encoder := input.Prepared.output
@@ -377,534 +377,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
if prepared.typed != nil {
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
return r.runLegacyLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
func (r *Runner) runLegacyLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
lane := prepared.resolved
if prepared.legacy == nil {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
extractor := prepared.legacy.extractor
merger := prepared.legacy.merger
normalizer := prepared.legacy.normalizer
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
extractWarnings := []contracts.Warning{}
extractRejectedStart := len(output.Rejected)
chunksDigest, err := joinedChunkDigest(chunks)
if err != nil {
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractDependencies := digestFingerprints("chunks", chunksDigest)
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
extractStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"decision": extractDecision,
"source": debugSourceDocumentEnvelope(doc),
"chunks": debugSourceChunkEnvelopes(chunks),
"options": redactSensitiveMap(lane.Extract.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if extractDecision.Reused {
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
output.Rejected = append(output.Rejected, cloneRejectedOutputs(extractCheckpoint.Rejected)...)
output.Warnings = append(output.Warnings, extractWarnings...)
} else {
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
for index := range chunks {
chunk := chunks[index]
var acceptedOutput contracts.ExtractOutput
var acceptedWarnings []contracts.Warning
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := extractor.Extract(attemptCtx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
SourceInput: chunkInputMaterial(sourceInput, chunk),
SessionID: sessionID,
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
}
extractOutput := result.Output
extractOutput.LaneID = lane.ID
extractOutput.ExtractorKey = extractor.Key()
extractOutput.SourceID = doc.ID
extractOutput.ChunkID = chunk.ID
extractOutput.ChunkIndex = chunk.Index
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageExtract,
laneID: lane.ID,
moduleKey: extractor.Key(),
source: doc,
sourceID: doc.ID,
chunkID: chunk.ID,
chunkIndex: chunk.Index,
chunk: &chunk,
sourceInput: chunkInputMaterial(sourceInput, chunk),
sessionID: sessionID,
references: lane.ExtractReferences.ReferenceSet,
llmClient: input.llmClient,
schema: extractOutput.Schema,
payload: extractOutput.Payload,
metadata: input.Metadata,
prepared: prepared.extractValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": append(cloneWarnings(result.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedOutput = cloneExtractOutput(extractOutput)
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": acceptedWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
return err
}
if !accepted {
output.Rejected = append(output.Rejected, *rejection)
continue
}
output.Warnings = append(output.Warnings, acceptedWarnings...)
extractWarnings = append(extractWarnings, acceptedWarnings...)
extractOutputs = append(extractOutputs, acceptedOutput)
}
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"outputs": debugExtractOutputEnvelopes(extractOutputs),
"rejected": debugRejectedOutputEnvelopes(output.Rejected[extractRejectedStart:]),
"warnings": extractWarnings,
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if len(extractOutputs) == 0 {
return nil
}
var acceptedMerge contracts.MergeOutput
var mergeWarnings []contracts.Warning
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
mergeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"decision": mergeDecision,
"source": debugSourceDocumentEnvelope(doc),
"extract_outputs": debugExtractOutputEnvelopes(extractOutputs),
"options": redactSensitiveMap(lane.Merge.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
if mergeDecision.Reused {
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("merge", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
mergeResult, err := merger.Merge(attemptCtx, contracts.MergeRequest{
Source: doc,
LaneID: lane.ID,
ExtractOutputs: cloneExtractOutputs(extractOutputs),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
}
mergeOutput := mergeResult.Output
mergeOutput.LaneID = lane.ID
mergeOutput.MergerKey = merger.Key()
mergeOutput.SourceID = doc.ID
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageMerge,
laneID: lane.ID,
moduleKey: merger.Key(),
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.MergeReferences.ReferenceSet,
llmClient: input.llmClient,
schema: mergeOutput.Schema,
payload: mergeOutput.Payload,
extractOutputs: extractOutputs,
metadata: input.Metadata,
prepared: prepared.mergeValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": append(cloneWarnings(mergeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedMerge = cloneMergeOutput(mergeOutput)
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": mergeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
return err
}
if !mergeAccepted {
output.Rejected = append(output.Rejected, *mergeRejection)
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*mergeRejection),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"accepted": true,
"output": debugMergeOutputEnvelope(acceptedMerge),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
var acceptedNormalize contracts.NormalizeOutput
var normalizeWarnings []contracts.Warning
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
normalizeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"decision": normalizeDecision,
"source": debugSourceDocumentEnvelope(doc),
"merge_output": debugMergeOutputEnvelope(acceptedMerge),
"options": redactSensitiveMap(lane.Normalize.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
if normalizeDecision.Reused {
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("normalize", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
normalizeResult, err := normalizer.Normalize(attemptCtx, contracts.NormalizeRequest{
Source: doc,
LaneID: lane.ID,
MergeOutput: cloneMergeOutput(acceptedMerge),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
normalizeOutput := normalizeResult.Output
normalizeOutput.LaneID = lane.ID
normalizeOutput.NormalizerKey = normalizer.Key()
normalizeOutput.SourceID = doc.ID
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageNormalize,
laneID: lane.ID,
moduleKey: normalizer.Key(),
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.NormalizeReferences.ReferenceSet,
llmClient: input.llmClient,
schema: normalizeOutput.Schema,
payload: normalizeOutput.Payload,
mergeOutput: acceptedMerge,
metadata: input.Metadata,
prepared: prepared.normalizeValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": append(cloneWarnings(normalizeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": normalizeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
return err
}
if !normalizeAccepted {
output.Rejected = append(output.Rejected, *normalizeRejection)
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*normalizeRejection),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"accepted": true,
"output": debugNormalizeOutputEnvelope(acceptedNormalize),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
output.NormalizeOutputs = append(output.NormalizeOutputs, serializedOutputFromLegacy(acceptedNormalize))
return nil
}
type rawValidationTarget struct {
stage ModuleStage
laneID string
moduleKey string
source *source.SourceDocument
sourceID string
sourceInput contracts.LLMInputMaterial
sessionID string
references contracts.ReferenceSet
llmClient contracts.StructuredLLMClient
chunkID string
chunkIndex int
chunk *source.Chunk
chunks []source.Chunk
schema contracts.ResponseSchema
payload contracts.RawPayload
extractOutputs []contracts.ExtractOutput
mergeOutput contracts.MergeOutput
metadata map[string]any
prepared preparedValidatorChain
attempt int
debug DebugRecorder
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
attempts := 1
if retries > 0 {
attempts += retries
}
var lastRejection *contracts.RejectedOutput
attempts := retries + 1
var last *contracts.RejectedOutput
for attempt := 1; attempt <= attempts; attempt++ {
if err := ctx.Err(); err != nil {
return false, nil, err
}
accepted, rejection, err := run(attempt)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
@@ -920,55 +402,22 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
}
if rejection != nil {
rejection.AttemptCount = attempt
lastRejection = rejection
last = rejection
}
if ctxErr := ctx.Err(); ctxErr != nil {
return false, nil, ctxErr
}
if attempt == attempts {
if lastRejection == nil {
lastRejection = &contracts.RejectedOutput{
ReasonCode: "raw_output_rejected",
Message: "raw output rejected",
AttemptCount: attempt,
}
if last == nil {
last = &contracts.RejectedOutput{ReasonCode: "output_rejected", Message: "output rejected", AttemptCount: attempt}
}
return false, lastRejection, nil
return false, last, nil
}
}
return false, lastRejection, nil
return false, last, nil
}
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
return r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
moduleKey: moduleKey,
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: references,
llmClient: llmClient,
chunks: chunks,
metadata: metadata,
prepared: prepared,
attempt: attempt,
debug: debug,
})
}
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
allLegacy := true
for _, item := range prepared.validators {
if item.resolved.Target != ValidatorTargetLegacyRaw && item.resolved.Target != "" {
allLegacy = false
break
}
}
if allLegacy {
return r.validateChunksRaw(ctx, doc, moduleKey, chunks, sourceInput, sessionID, references, llmClient, metadata, prepared, attempt, debug)
}
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
content, err := json.Marshal(chunks)
if err != nil {
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
@@ -989,8 +438,9 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
}
debugRequest := contracts.ValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, Payload: contracts.RawPayload{Content: content, MediaType: "application/json"}}
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: debugValidationRequestEnvelope(debugRequest), Result: debugValidationResultEnvelope(result)}
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
debugContent.ContentDigest = debugContentDigest(content)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil {
debugCall.Error = err.Error()
}
@@ -1003,11 +453,11 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if !result.Approved {
reason := result.ReasonCode
if reason == "" {
reason = "raw_output_rejected"
reason = "output_rejected"
}
message := result.Message
if message == "" {
message = "raw output rejected"
message = "output rejected"
}
return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
}
@@ -1016,97 +466,6 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
return warnings, nil, nil
}
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
if len(target.prepared.validators) == 0 {
return nil, nil, nil
}
var warnings []contracts.Warning
for index, preparedValidator := range target.prepared.validators {
validator := preparedValidator.legacy
validatorBinding := preparedValidator.resolved
if validator == nil {
return nil, nil, fmt.Errorf("validator %q is not available on the legacy raw path", validatorBinding.Binding.Module)
}
request := target.validationRequest(validatorBinding.Binding)
started := time.Now().UTC()
attemptPath := path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(validator.Name()), target.attempt))
validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := validator.Validate(validatorCtx, request)
debugPayload := debugValidationCall{
ValidatorName: validator.Name(),
Request: debugValidationRequestEnvelope(request),
Result: debugValidationResultEnvelope(result),
}
if err != nil {
debugPayload.Error = err.Error()
}
if debugErr := writeDebugTimed(target.debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
Attempt: target.attempt,
StartedAt: started,
Payload: debugPayload,
Error: debugPayload.Error,
}, llmScope)); debugErr != nil {
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
}
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
}
if !result.Approved {
reasonCode := strings.TrimSpace(result.ReasonCode)
if reasonCode == "" {
reasonCode = "raw_output_rejected"
}
message := strings.TrimSpace(result.Message)
if message == "" {
message = "raw output rejected"
}
return nil, &contracts.RejectedOutput{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
ChunkID: target.chunkID,
ChunkIndex: target.chunkIndex,
ValidatorName: validator.Name(),
ReasonCode: reasonCode,
Message: message,
AttemptCount: target.attempt,
DiagnosticArtifactPath: result.DiagnosticArtifactPath,
}, nil
}
warnings = append(warnings, result.Warnings...)
}
return warnings, nil, nil
}
func (target rawValidationTarget) validationRequest(binding ModuleBinding) contracts.ValidationRequest {
return contracts.ValidationRequest{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
Source: target.source,
SourceID: target.sourceID,
SourceInput: target.sourceInput.Clone(),
SessionID: target.sessionID,
References: CloneReferenceSet(target.references),
LLMClient: target.llmClient,
LLMProfile: binding.LLMProfile,
Options: cloneOptions(binding.Options),
Metadata: cloneMetadata(target.metadata),
Schema: cloneResponseSchema(target.schema),
Payload: cloneRawPayload(target.payload),
ChunkID: target.chunkID,
ChunkIndex: target.chunkIndex,
Chunk: cloneSourceChunkPtr(target.chunk),
Chunks: cloneSourceChunks(target.chunks),
ExtractOutputs: cloneExtractOutputs(target.extractOutputs),
MergeOutput: cloneMergeOutput(target.mergeOutput),
}
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains {
if chain.Stage != stage {
@@ -1250,7 +609,7 @@ func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.Valida
func failOutput(output RunOutput) RunOutput {
if output.Manifest.PipelineID != "" {
populateRawOutputManifest(&output)
populateOutputManifest(&output)
output.Manifest.ValidationStatus = "failed"
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
}
@@ -1274,7 +633,7 @@ func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage str
})
}
func populateRawOutputManifest(output *RunOutput) {
func populateOutputManifest(output *RunOutput) {
if output == nil {
return
}
@@ -1325,59 +684,18 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
return manifests
}
func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
if output == nil {
return
}
for i := range output.Manifest.ArtifactLanes {
if output.Manifest.ArtifactLanes[i].ID != laneID {
continue
}
metadata := make(map[string]any)
for _, module := range modules {
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
continue
}
key := manifestMetadataKey(module)
if key == "" {
continue
}
metadata[key] = moduleMetadata
}
if len(metadata) > 0 {
output.Manifest.ArtifactLanes[i].Metadata = metadata
}
return
}
}
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil {
return
}
moduleMetadata, ok := moduleManifestMetadata(module)
metadata, ok := moduleManifestMetadata(module)
if !ok {
return
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
}
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.LegacyRawExtractor:
return "extractor"
case contracts.LegacyRawMerger:
return "merger"
case contracts.LegacyRawNormalizer:
return "normalizer"
default:
return ""
}
output.Manifest.ModuleMetadata[moduleKey] = metadata
}
func moduleManifestMetadata(module any) (map[string]any, bool) {
@@ -1557,20 +875,6 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
return append([]contracts.Warning(nil), warnings...)
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: cloneWarnings(payload.Warnings),
}
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
if chunk == nil {
return nil
@@ -1608,35 +912,6 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
return out
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneExtractOutput(output))
}
return out
}
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
if len(outputs) == 0 {
return nil
@@ -1648,16 +923,6 @@ func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.Se
return out
}
func serializedOutputFromLegacy(output contracts.NormalizeOutput) contracts.SerializedOutput {
return contracts.SerializedOutput{
LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
Artifact: contracts.SerializedArtifact{
Schema: contracts.ArtifactSchema{ID: output.Schema.ID, Name: output.Schema.Name, Version: output.Schema.Version, JSONSchema: append([]byte(nil), output.Schema.JSONSchema...)},
MediaType: output.Payload.MediaType, Content: append([]byte(nil), output.Payload.Content...), Metadata: cloneMetadata(output.Payload.Metadata),
},
}
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil

2732
internal/framework/pipeline/runner_test.go
View File

File diff suppressed because it is too large Load Diff

110
internal/framework/pipeline/runner_typed.go
View File

@@ -13,54 +13,30 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func loadArtifactExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader)
if !ok {
return ArtifactExtractCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactExtract(laneID, moduleKey, deps)
func loadExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return loader.Extract(laneID, moduleKey, deps)
}
func loadArtifactMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader)
if !ok {
return ArtifactMergeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactMerge(laneID, moduleKey, deps)
func loadMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return loader.Merge(laneID, moduleKey, deps)
}
func loadArtifactNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader)
if !ok {
return ArtifactNormalizeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactNormalize(laneID, moduleKey, deps)
func loadNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return loader.Normalize(laneID, moduleKey, deps)
}
func recordArtifactExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder)
if !ok {
return nil
}
return typed.ArtifactExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
func recordExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
}
func recordArtifactMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder)
if !ok {
return nil
}
return typed.ArtifactMergeSucceeded(laneID, moduleKey, deps, output, warnings)
func recordMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
}
func recordArtifactNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder)
if !ok {
return nil
}
return typed.ArtifactNormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
func recordNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
}
func cloneArtifactCheckpointOutput(output ArtifactCheckpointOutput) ArtifactCheckpointOutput {
func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
return output
}
func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpointOutput, value any) ArtifactCheckpointOutput {
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) CheckpointArtifact {
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
if codec.metadata != nil {
output.Artifact.Metadata = cloneMetadata(codec.metadata(value))
@@ -69,7 +45,7 @@ func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpoi
}
return output
}
func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointFingerprint {
func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(outputs))
for i, output := range outputs {
sum := sha256.Sum256(output.Artifact.Content)
@@ -77,7 +53,7 @@ func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointF
}
return normalizeCheckpointFingerprints(values)
}
func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]any {
func debugCheckpointArtifact(output CheckpointArtifact) map[string]any {
artifact := output.Artifact
schema := contracts.CloneArtifactSchema(artifact.Schema)
digest := output.SchemaDigest
@@ -89,13 +65,13 @@ func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]a
content.ContentDigest = debugContentDigest(artifact.Content)
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
}
func debugArtifactCheckpointOutputs(outputs []ArtifactCheckpointOutput) []map[string]any {
func debugCheckpointArtifacts(outputs []CheckpointArtifact) []map[string]any {
if len(outputs) == 0 {
return nil
}
out := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugArtifactCheckpointOutput(output))
out = append(out, debugCheckpointArtifact(output))
}
return out
}
@@ -117,7 +93,7 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil
}
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpointOutput) (any, error) {
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
if artifact.Artifact.Kind != codec.spec.Kind {
return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
@@ -134,12 +110,12 @@ func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpo
return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
}
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (ArtifactCheckpointOutput, error) {
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, false)
if err != nil {
return ArtifactCheckpointOutput{}, err
return CheckpointArtifact{}, err
}
return ArtifactCheckpointOutput{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
}
func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
@@ -150,7 +126,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer)
values := make([]erasedExtractArtifact, 0, len(chunks))
serializedExtracts := make([]ArtifactCheckpointOutput, 0, len(chunks))
serializedExtracts := make([]CheckpointArtifact, 0, len(chunks))
extractWarnings := []contracts.Warning{}
rejectedStart := len(output.Rejected)
chunksDigest, err := joinedChunkDigest(chunks)
@@ -158,7 +134,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractDeps := digestFingerprints("chunks", chunksDigest)
cp, decision := loadArtifactExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
cp, decision := loadExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
if decision.Reused {
for _, stored := range cp.Outputs {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil {
@@ -183,7 +159,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
}
values = append(values, artifact)
serializedExtracts = append(serializedExtracts, cloneArtifactCheckpointOutput(stored))
serializedExtracts = append(serializedExtracts, cloneCheckpointArtifact(stored))
}
extractWarnings = cloneWarnings(cp.Warnings)
output.Warnings = append(output.Warnings, extractWarnings...)
@@ -195,7 +171,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
for i := range chunks {
chunk := chunks[i]
var accepted erasedExtractArtifact
var serializedAccepted ArtifactCheckpointOutput
var serializedAccepted CheckpointArtifact
var acceptedWarnings []contracts.Warning
ok, rejection, runErr := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
started := time.Now().UTC()
@@ -218,7 +194,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef
accepted, serializedAccepted = artifact, stored
acceptedWarnings = append(cloneWarnings(result.Warnings), warnings...)
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugArtifactCheckpointOutput(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugCheckpointArtifact(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
return false, nil, debugErr
}
return true, nil, nil
@@ -236,13 +212,13 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
extractWarnings = append(extractWarnings, acceptedWarnings...)
output.Warnings = append(output.Warnings, acceptedWarnings...)
}
if err := recordArtifactExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
if err := recordExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
sort.SliceStable(values, func(i, j int) bool { return values[i].ChunkIndex < values[j].ChunkIndex })
sort.SliceStable(serializedExtracts, func(i, j int) bool { return serializedExtracts[i].ChunkIndex < serializedExtracts[j].ChunkIndex })
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugArtifactCheckpointOutputs(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugCheckpointArtifacts(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
return err
}
if len(values) == 0 {
@@ -254,18 +230,18 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
}
mergeDeps := artifactCheckpointDigests(serializedExtracts)
mergeCP, mergeDecision := loadArtifactMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
mergeCP, mergeDecision := loadMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
if mergeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()}
}
}
recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision)
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugArtifactCheckpointOutputs(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugCheckpointArtifacts(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err
}
var merged erasedMergeArtifact
var serializedMerge ArtifactCheckpointOutput
var serializedMerge CheckpointArtifact
var mergeWarnings []contracts.Warning
if mergeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output)
@@ -273,7 +249,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
}
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(mergeCP.Output), value)
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
mergeWarnings = cloneWarnings(mergeCP.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
@@ -310,33 +286,33 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if err := recordArtifactMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
if err := recordMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
return err
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
return err
}
normalizeDeps := artifactCheckpointDigests([]ArtifactCheckpointOutput{serializedMerge})
normalizeCP, normalizeDecision := loadArtifactNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
normalizeDeps := artifactCheckpointDigests([]CheckpointArtifact{serializedMerge})
normalizeCP, normalizeDecision := loadNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
if normalizeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()}
}
}
recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision)
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugArtifactCheckpointOutput(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugCheckpointArtifact(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err
}
var serializedNormalize ArtifactCheckpointOutput
var serializedNormalize CheckpointArtifact
var normalizeWarnings []contracts.Warning
if normalizeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output)
if decodeErr != nil {
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
}
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
@@ -371,11 +347,11 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := recordArtifactNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
if err := recordNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
return err
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
return err
}
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(serializedNormalize.Artifact)})
@@ -430,7 +406,7 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
}
artifact, _ := serializeArtifact(codec, target.value, true)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugArtifactCheckpointOutput(ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil {
debugCall.Error = err.Error()
}

16
internal/framework/pipeline/runner_typed_checkpoint_test.go
View File

@@ -20,22 +20,22 @@ func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *tes
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
base := ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
base := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
tests := []struct {
name string
mutate func(*ArtifactCheckpointOutput)
mutate func(*CheckpointArtifact)
want string
}{
{name: "missing kind", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Kind = "" }, want: "artifact kind"},
{name: "schema version", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
{name: "schema digest", mutate: func(v *ArtifactCheckpointOutput) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
{name: "media type", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
{name: "decode failure", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
{name: "missing kind", mutate: func(v *CheckpointArtifact) { v.Artifact.Kind = "" }, want: "artifact kind"},
{name: "schema version", mutate: func(v *CheckpointArtifact) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
{name: "schema digest", mutate: func(v *CheckpointArtifact) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
{name: "media type", mutate: func(v *CheckpointArtifact) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
{name: "decode failure", mutate: func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
stored := cloneArtifactCheckpointOutput(base)
stored := cloneCheckpointArtifact(base)
test.mutate(&stored)
if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("decode error = %v, want %q", err, test.want)

98
internal/framework/pipeline/typed_resolution_test.go
View File

@@ -55,6 +55,40 @@ func (typedTestChunkValidator) Validate(context.Context, contracts.ChunkValidati
type typedTestSerializedValidator struct{ key string }
type typedTestInput struct {
key string
doc *source.SourceDocument
}
func (v *typedTestInput) Key() string { return v.key }
func (v *typedTestInput) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
return v.doc, nil
}
type typedTestChunker struct {
key string
chunks []source.Chunk
}
func (v *typedTestChunker) Key() string { return v.key }
func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (v *typedTestChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{Chunks: v.chunks}, nil
}
type typedTestOutput struct{ key string }
func (v *typedTestOutput) Key() string { return v.key }
func (v *typedTestOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func typedTestDocument() *source.SourceDocument {
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "text", Ref: ref}}}
doc.Digest, _ = source.DigestDocument(doc)
return doc
}
func (v typedTestSerializedValidator) Name() string { return v.key }
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
@@ -177,20 +211,6 @@ func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {
}
}
func TestLegacyRawRegistrationCannotSatisfyTypedLane(t *testing.T) {
options := completeTypedCatalogOptions()
options.registerScoreMerger = false
catalog := typedResolutionCatalog(t, options)
if err := catalog.Mergers.RegisterLegacyRaw("typed/merge", func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil {
t.Fatalf("RegisterLegacyRaw() error = %v, want nil", err)
}
_, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err == nil || !strings.Contains(err.Error(), `no typed variant for artifact kind "test/score"`) {
t.Fatalf("ResolvePipeline() error = %v, want typed variant error", err)
}
}
func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
registry := NewMergerRegistry()
spec := ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}
@@ -205,6 +225,48 @@ func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
}
}
func TestConstructorRegistrationsRejectUnconfiguredOptions(t *testing.T) {
extractors := NewExtractorRegistry()
if err := RegisterExtractor(extractors, ModuleSpec{Key: "typed/extract", Stage: StageExtract, ArtifactKind: "test/notes"}, func() (contracts.Extractor[codecNotes], error) {
return typedTestExtractor[codecNotes]{key: "typed/extract"}, nil
}); err != nil {
t.Fatalf("RegisterExtractor() error = %v", err)
}
if err := extractors.validateOptions("typed/extract", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("extractor option validation error = %v, want unknown option", err)
}
mergers := NewMergerRegistry()
if err := RegisterMerger(mergers, ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}, func() (contracts.Merger[codecNotes], error) {
return typedTestMerger[codecNotes]{key: "typed/merge"}, nil
}); err != nil {
t.Fatalf("RegisterMerger() error = %v", err)
}
if err := mergers.validateOptions("typed/merge", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("merger option validation error = %v, want unknown option", err)
}
normalizers := NewNormalizerRegistry()
if err := RegisterNormalizer(normalizers, ModuleSpec{Key: "typed/normalize", Stage: StageNormalize, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
return typedTestNormalizer[codecNotes]{key: "typed/normalize"}, nil
}); err != nil {
t.Fatalf("RegisterNormalizer() error = %v", err)
}
if err := normalizers.validateOptions("typed/normalize", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("normalizer option validation error = %v, want unknown option", err)
}
validators := NewValidatorRegistry()
if err := RegisterTypedValidator(validators, "test/notes", ValidatorSpec{Key: "typed/check", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: "typed/check"}, nil
}); err != nil {
t.Fatalf("RegisterTypedValidator() error = %v", err)
}
if err := validators.validateOptions(ResolvedValidator{Binding: ModuleBinding{Module: "typed/check", Options: map[string]any{"unexpected": true}}, Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("validator option validation error = %v, want unknown option", err)
}
}
func TestResolvedPipelineDigestIncludesArtifactSchemaIdentity(t *testing.T) {
baseOptions := completeTypedCatalogOptions()
base, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, baseOptions))
@@ -302,18 +364,18 @@ func typedResolutionCatalog(t *testing.T, options typedCatalogOptions) ModuleCat
func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
t.Helper()
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{Key: "typed/input", Stage: StageInput}, func() (contracts.InputAdapter, error) {
return &runnerInputAdapter{key: "typed/input", doc: validSourceDocument()}, nil
return &typedTestInput{key: "typed/input", doc: typedTestDocument()}, nil
}); err != nil {
t.Fatalf("register input: %v", err)
}
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
doc := validSourceDocument()
return &runnerChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
doc := typedTestDocument()
return &typedTestChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{Key: "typed/output", Stage: StageOutput}, func() (contracts.OutputEncoder, error) {
return &runnerOutputEncoder{key: "typed/output"}, nil
return &typedTestOutput{key: "typed/output"}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}

113
internal/framework/pipeline/validator_registry.go
View File

@@ -10,9 +10,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type LegacyRawValidatorConstructor func() (contracts.LegacyRawValidator, error)
type LegacyRawValidatorBuilder func(BuildRequest) (contracts.LegacyRawValidator, error)
type ValidatorSpec struct {
Key string `json:"key"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
@@ -27,16 +24,12 @@ type SerializedValidatorSpec struct {
type ValidatorTarget string
const (
ValidatorTargetLegacyRaw ValidatorTarget = "legacy_raw"
ValidatorTargetChunk ValidatorTarget = "chunk"
ValidatorTargetSerialized ValidatorTarget = "serialized"
ValidatorTargetTyped ValidatorTarget = "typed"
)
type ValidatorRegistry struct {
legacyBuilders map[string]LegacyRawValidatorBuilder
legacyValidators map[string]OptionValidator
legacySpecs map[string]ValidatorSpec
typedEntries map[artifactVariantKey]typedValidatorEntry
chunkEntries map[string]chunkValidatorEntry
serializedEntries map[string]serializedValidatorEntry
@@ -65,65 +58,17 @@ type serializedValidatorEntry struct {
func NewValidatorRegistry() *ValidatorRegistry {
return &ValidatorRegistry{
legacyBuilders: make(map[string]LegacyRawValidatorBuilder),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ValidatorSpec),
typedEntries: make(map[artifactVariantKey]typedValidatorEntry),
chunkEntries: make(map[string]chunkValidatorEntry),
serializedEntries: make(map[string]serializedValidatorEntry),
}
}
func (r *ValidatorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawValidatorConstructor) error {
return r.RegisterLegacyRawWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
}
func (r *ValidatorRegistry) RegisterLegacyRawWithSpec(spec ValidatorSpec, constructor LegacyRawValidatorConstructor) error {
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawValidator, error) {
return constructor()
})
}
func (r *ValidatorRegistry) RegisterLegacyRawBuilderWithSpec(spec ValidatorSpec, validateOptions OptionValidator, builder LegacyRawValidatorBuilder) error {
if r == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
}
if validateOptions == nil {
return fmt.Errorf("validator option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("validator builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw validator %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawValidatorBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ValidatorSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = normalizedSpec
return nil
}
func RegisterTypedValidator[T any](registry *ValidatorRegistry, kind contracts.ArtifactKind, spec ValidatorSpec, constructor func() (contracts.TypedValidator[T], error)) error {
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterTypedValidatorBuilder(registry, kind, spec, allowLegacyOptions, func(BuildRequest) (contracts.TypedValidator[T], error) {
return RegisterTypedValidatorBuilder(registry, kind, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.TypedValidator[T], error) {
return constructor()
})
}
@@ -180,7 +125,7 @@ func RegisterChunkValidator(registry *ValidatorRegistry, spec ValidatorSpec, con
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterChunkValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.ChunkValidator, error) {
return RegisterChunkValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.ChunkValidator, error) {
return constructor()
})
}
@@ -213,7 +158,7 @@ func RegisterSerializedValidator(registry *ValidatorRegistry, spec SerializedVal
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterSerializedValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.SerializedValidator, error) {
return RegisterSerializedValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.SerializedValidator, error) {
return constructor()
})
}
@@ -246,39 +191,6 @@ func RegisterSerializedValidatorBuilder(registry *ValidatorRegistry, spec Serial
return nil
}
func (r *ValidatorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawValidator, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *ValidatorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawValidator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("validator key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw validator %q is not registered", normalizedKey)
}
validator, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build validator %q: %w", normalizedKey, err)
}
if validator == nil {
return nil, fmt.Errorf("validator %q constructor returned nil", normalizedKey)
}
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
spec := r.legacySpecs[normalizedKey]
if validator.ExecutionClass() != spec.ExecutionClass {
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
}
return validator, nil
}
func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
if r == nil {
return fmt.Errorf("validator registry must not be nil")
@@ -301,8 +213,6 @@ func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
if ok {
validator = entry.validateOptions
}
default:
validator = r.legacyValidators[key]
}
if validator == nil {
return fmt.Errorf("validator %q construction entry is not registered", key)
@@ -314,10 +224,6 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
if r == nil {
return ValidatorSpec{}, false
}
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if ok {
return spec, true
}
normalized := strings.TrimSpace(key)
if entry, found := r.chunkEntries[normalized]; found {
return entry.spec, true
@@ -329,7 +235,7 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
entry, found := r.typedEntry(normalized, kinds[0])
return entry.spec, found
}
return spec, ok
return ValidatorSpec{}, false
}
func (r *ValidatorRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedValidatorEntry, bool) {
@@ -372,13 +278,15 @@ func (r *ValidatorRegistry) registeredTypedKinds(key string) []contracts.Artifac
}
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
if r == nil || len(r.legacySpecs) == 0 {
if r == nil {
return nil
}
keys := sortedRegistryKeys(r.legacySpecs)
keys := r.RegisteredKeys()
specs := make([]ValidatorSpec, 0, len(keys))
for _, key := range keys {
specs = append(specs, r.legacySpecs[key])
if spec, ok := r.Spec(key); ok {
specs = append(specs, spec)
}
}
return specs
}
@@ -388,9 +296,6 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
return nil
}
keys := make(map[string]struct{})
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries {
keys[key.module] = struct{}{}
}

117
internal/framework/pipeline/validator_registry_test.go
View File

@@ -1,117 +0,0 @@
package pipeline
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidatorRegistryBehavior(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterLegacyRaw(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw("generic-validator")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if validator.Name() != "generic-validator" {
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
}
spec, ok := registry.Spec(" generic-validator ")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("Spec() = %#v, want %#v", spec, want)
}
}
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
registry := NewValidatorRegistry()
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
if err := registry.RegisterLegacyRawWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("llm-validator")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
registry := NewValidatorRegistry()
for _, key := range []string{"zeta", "alpha"} {
if err := registry.RegisterLegacyRaw(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register(%q) error = %v", key, err)
}
}
specs := registry.RegisteredSpecs()
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
}
}
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
)
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
}
}
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("validator")
if err == nil {
t.Fatal("Build() error = nil, want execution class mismatch")
}
if !strings.Contains(err.Error(), "execution class") {
t.Fatalf("Build() error = %q, want execution class context", err.Error())
}
}
type testValidator struct {
name string
executionClass contracts.ExecutionClass
}
func validatorConstructor(name string, executionClass contracts.ExecutionClass) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return testValidator{name: name, executionClass: executionClass}, nil
}
}
func (validator testValidator) Name() string {
return validator.name
}
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
return validator.executionClass
}
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

467
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -1,467 +0,0 @@
package pipeline
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestWalkingSkeletonFixture(t *testing.T) {
inputBytes := readTestFixture(t, "testdata/walking_skeleton_input.json")
expectedBytes := readTestFixture(t, "testdata/walking_skeleton_output.json")
llmClient := &walkingSkeletonLLMClient{}
resolved, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, walkingSkeletonCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
output, err := newPreparedRunner(t, walkingSkeletonRegistries(t)).Run(context.Background(), RunInput{
pipeline: resolved,
SourceID: "fixture-source",
Path: "walking_skeleton_input.json",
RawInput: inputBytes,
llmClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
if llmClient.calls != 3 {
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
}
}
func TestWalkingSkeletonResolutionRejectsMissingCapability(t *testing.T) {
catalog := walkingSkeletonCatalog(t)
catalog.Extractors = NewExtractorRegistry()
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"missing"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("ResolvePipeline() error = %q, want missing capability", err.Error())
}
}
func TestWalkingSkeletonResolutionRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{Only: []string{"missing"}}, walkingSkeletonCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
if !strings.Contains(err.Error(), "missing") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("ResolvePipeline() error = %q, want unknown lane error", err.Error())
}
}
func walkingSkeletonProfile() PipelineProfile {
return PipelineProfile{
ID: "walking-skeleton",
Input: Binding("fake/input"),
Chunk: Binding("fake/chunk"),
Output: Binding("json"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("fake/extract")},
},
}
}
func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
t.Helper()
catalog := ModuleCatalog{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
Key: "fake/input",
Stage: StageInput,
Provides: []string{"plain_text"},
}, func() (contracts.InputAdapter, error) {
return walkingSkeletonInput{}, nil
}); err != nil {
t.Fatalf("register fake input: %v", err)
}
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{
Key: "fake/chunk",
Stage: StageChunk,
Requires: []string{"plain_text"},
Provides: []string{"chunks"},
}, func() (contracts.Chunker, error) {
return walkingSkeletonChunker{}, nil
}); err != nil {
t.Fatalf("register fake chunker: %v", err)
}
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"chunks"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("register fake extractor: %v", err)
}
if err := catalog.Mergers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultMergeModule,
Stage: StageMerge,
Requires: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawMerger, error) {
return walkingSkeletonMerger{}, nil
}); err != nil {
t.Fatalf("register append-order merger: %v", err)
}
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultNormalizeModule,
Stage: StageNormalize,
}, func() (contracts.LegacyRawNormalizer, error) {
return walkingSkeletonNormalizer{}, nil
}); err != nil {
t.Fatalf("register no-op normalizer: %v", err)
}
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{
Key: "json",
Stage: StageOutput,
}, func() (contracts.OutputEncoder, error) {
return walkingSkeletonOutput{}, nil
}); err != nil {
t.Fatalf("register fake output: %v", err)
}
return catalog
}
func walkingSkeletonRegistries(t *testing.T) Registries {
t.Helper()
catalog := walkingSkeletonCatalog(t)
return Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Outputs: catalog.Outputs,
}
}
type walkingSkeletonInput struct{}
func (input walkingSkeletonInput) Key() string {
return "fake/input"
}
func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
var fixture struct {
ID string `json:"id"`
Units []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"units"`
}
if err := json.Unmarshal(req.Raw, &fixture); err != nil {
return nil, err
}
units := make([]source.SourceUnit, 0, len(fixture.Units))
for _, unit := range fixture.Units {
units = append(units, source.SourceUnit{
ID: unit.ID,
Kind: "unit",
Text: unit.Text,
Ref: source.SourceRef{
SourceID: fixture.ID,
StartUnitID: unit.ID,
EndUnitID: unit.ID,
},
})
}
return &source.SourceDocument{
ID: fixture.ID,
Kind: "fixture",
Format: "application/json",
Digest: rawDigest(req.Raw),
Units: units,
}, nil
}
type walkingSkeletonChunker struct{}
func (chunker walkingSkeletonChunker) Key() string {
return "fake/chunk"
}
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if len(req.Source.Units) < 3 {
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
}
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[1].ID},
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
},
{
ID: req.Source.ID + ":chunk:1",
SourceID: req.Source.ID,
Index: 1,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[2].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
Content: []byte(`{"units":[3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
},
},
}, nil
}
type walkingSkeletonExtractor struct{}
func (extractor walkingSkeletonExtractor) Key() string {
return "fake/extract"
}
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/extract",
PromptID: "fake.event",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.ExtractionResult{}, err
}
payload, err := json.Marshal(map[string]any{
"chunk_id": req.Chunk.ID,
"llm_call": response.Call,
"text": chunkText(req.Chunk.Units),
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonLLMClient struct {
calls int
}
func (client *walkingSkeletonLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.calls++
if response, ok := out.(*struct {
Call int `json:"call"`
}); ok {
response.Call = client.calls
}
content, err := json.Marshal(map[string]any{"call": client.calls})
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{
Content: content,
}, nil
}
type walkingSkeletonMerger struct{}
func (merger walkingSkeletonMerger) Key() string {
return DefaultMergeModule
}
func (merger walkingSkeletonMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
outputs := make([]json.RawMessage, 0, len(req.ExtractOutputs))
for _, output := range req.ExtractOutputs {
outputs = append(outputs, json.RawMessage(output.Payload.Content))
}
content, err := json.Marshal(map[string]any{"outputs": outputs})
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonNormalizer struct{}
func (normalizer walkingSkeletonNormalizer) Key() string {
return DefaultNormalizeModule
}
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
PromptID: "fake.normalize",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type walkingSkeletonOutput struct{}
func (output walkingSkeletonOutput) Key() string {
return "json"
}
func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
type rawOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id"`
Schema contracts.ResponseSchema `json:"schema"`
MediaType string `json:"media_type"`
Content json.RawMessage `json:"content"`
}
rawOutputs := make([]rawOutput, 0, len(req.NormalizeOutputs))
for _, output := range req.NormalizeOutputs {
rawOutputs = append(rawOutputs, rawOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: contracts.ResponseSchema{ID: output.Artifact.Schema.ID, Name: output.Artifact.Schema.Name, Version: output.Artifact.Schema.Version},
MediaType: output.Artifact.MediaType,
Content: json.RawMessage(output.Artifact.Content),
})
}
encoded, err := json.Marshal(struct {
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []rawOutput `json:"normalize_outputs"`
}{
Manifest: artifacts.RunManifest{
PipelineID: req.Manifest.PipelineID,
PipelineDigest: req.Manifest.PipelineDigest,
ArtifactLanes: req.Manifest.ArtifactLanes,
ValidationStatus: req.Manifest.ValidationStatus,
},
NormalizeOutputs: rawOutputs,
})
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: encoded},
},
}, nil
}
func readTestFixture(t *testing.T, path string) []byte {
t.Helper()
bytes, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %q: %v", path, err)
}
return bytes
}
func assertStructuralJSONEqual(t *testing.T, gotBytes, wantBytes []byte) {
t.Helper()
var got any
if err := json.Unmarshal(gotBytes, &got); err != nil {
t.Fatalf("unmarshal actual JSON: %v\n%s", err, gotBytes)
}
var want any
if err := json.Unmarshal(wantBytes, &want); err != nil {
t.Fatalf("unmarshal expected JSON: %v\n%s", err, wantBytes)
}
if !reflect.DeepEqual(got, want) {
gotFormatted, _ := json.MarshalIndent(got, "", " ")
wantFormatted, _ := json.MarshalIndent(want, "", " ")
t.Fatalf("actual JSON:\n%s\nwant:\n%s", gotFormatted, wantFormatted)
}
}
func chunkText(units []source.SourceUnit) string {
parts := make([]string, 0, len(units))
for _, unit := range units {
parts = append(parts, unit.Text)
}
return strings.Join(parts, " ")
}
func rawDigest(raw []byte) string {
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}