Remove legacy raw pipeline contracts

This commit is contained in:
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions

View File

@@ -88,147 +88,77 @@ func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if decision := l.readJSON(laneManifestPath("extract", laneID), &manifest); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
var payload extractOutputsEnvelope
if decision := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
outputs, err := extractOutputsFromEnvelope(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests(extractPayloads(outputs))) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint output digests do not match payload")
}
return pipeline.ExtractCheckpoint{
Outputs: outputs,
Rejected: cloneRejectedOutputs(payload.Rejected),
Warnings: cloneWarnings(payload.Warnings),
}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactExtract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactExtractCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
return pipeline.ExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
}
return pipeline.ArtifactExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
var payload mergeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
output, err := mergeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint output digest does not match payload")
}
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactMerge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactMergeCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
return pipeline.MergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactMergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
var payload normalizeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
output, err := normalizeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint output digest does not match payload")
}
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactNormalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactNormalizeCheckpoint, pipeline.CheckpointDecision) {
func (l *WorkspaceLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
return pipeline.NormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactNormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.ArtifactCheckpointOutput, error) {
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.CheckpointArtifact, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]pipeline.ArtifactCheckpointOutput, 0, len(values))
out := make([]pipeline.CheckpointArtifact, 0, len(values))
for _, v := range values {
content, err := contentFromEnvelope(v.Content)
if err != nil {
@@ -237,7 +167,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
}
out = append(out, pipeline.ArtifactCheckpointOutput{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
return out, nil
}
@@ -326,70 +256,6 @@ func sourceChunksFromEnvelope(values []chunkEnvelope) ([]source.Chunk, error) {
return out, nil
}
func extractOutputsFromEnvelope(values []extractOutputEnvelope) ([]contracts.ExtractOutput, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]contracts.ExtractOutput, 0, len(values))
for _, value := range values {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return nil, err
}
out = append(out, contracts.ExtractOutput{
LaneID: value.LaneID,
ExtractorKey: value.ExtractorKey,
SourceID: value.SourceID,
ChunkID: value.ChunkID,
ChunkIndex: value.ChunkIndex,
Schema: value.Schema,
Payload: payload,
})
}
return out, nil
}
func mergeOutputFromEnvelope(value mergeOutputPayload) (contracts.MergeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.MergeOutput{}, err
}
return contracts.MergeOutput{
LaneID: value.LaneID,
MergerKey: value.MergerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func normalizeOutputFromEnvelope(value normalizeOutputPayload) (contracts.NormalizeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.NormalizeOutput{}, err
}
return contracts.NormalizeOutput{
LaneID: value.LaneID,
NormalizerKey: value.NormalizerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func rawPayloadFromEnvelope(value binaryEnvelope) (contracts.RawPayload, error) {
content, err := contentFromEnvelope(value)
if err != nil {
return contracts.RawPayload{}, err
}
return contracts.RawPayload{
Content: content,
MediaType: value.MediaType,
Metadata: cloneMetadata(value.Metadata),
Warnings: cloneWarnings(value.Warnings),
}, nil
}
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil {

View File

@@ -119,28 +119,7 @@ func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, depe
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := extractOutputsEnvelope{
Outputs: extractOutputEnvelopes(outputs),
Rejected: cloneRejectedOutputs(rejected),
Warnings: cloneWarnings(warnings),
}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
StageManifest: manifest,
ChunkCount: len(outputs) + len(rejected),
OutputCount: len(outputs),
})
}
func (r *WorkspaceRecorder) ArtifactExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
@@ -166,27 +145,12 @@ func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, depend
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error {
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
StageManifest: manifest,
InputCount: len(dependencies),
})
}
func (r *WorkspaceRecorder) ArtifactMergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -213,24 +177,12 @@ func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, de
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error {
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) ArtifactNormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -308,48 +260,6 @@ type chunkEnvelope struct {
Metadata map[string]any `json:"metadata,omitempty"`
}
type extractOutputsEnvelope struct {
Outputs []extractOutputEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type extractOutputEnvelope struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type mergeOutputEnvelope struct {
Output mergeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type mergeOutputPayload struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type normalizeOutputEnvelope struct {
Output normalizeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type normalizeOutputPayload struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type binaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
@@ -380,12 +290,12 @@ type artifactSingleEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
func artifactCheckpointEnvelopeFromOutput(output pipeline.ArtifactCheckpointOutput) artifactCheckpointEnvelope {
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
}
func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []artifactCheckpointEnvelope {
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
if len(outputs) == 0 {
return nil
}
@@ -395,7 +305,7 @@ func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []
}
return out
}
func artifactOutputDigests(outputs []pipeline.ArtifactCheckpointOutput) []pipeline.CheckpointFingerprint {
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs {
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
@@ -422,54 +332,6 @@ func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
return out
}
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
if len(outputs) == 0 {
return nil
}
out := make([]extractOutputEnvelope, 0, len(outputs))
for _, output := range outputs {
out = append(out, extractOutputEnvelope{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
})
}
return out
}
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
return mergeOutputPayload{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
return normalizeOutputPayload{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = nil
return schema
}
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
return binaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
@@ -528,28 +390,6 @@ func cloneMetadata(metadata map[string]any) map[string]any {
return out
}
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
for i, payload := range payloads {
values = append(values, pipeline.CheckpointFingerprint{
Name: fmt.Sprintf("payload[%d]", i),
Value: contentDigest(payload.Content),
})
}
return normalizeFingerprints(values)
}
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
if len(outputs) == 0 {
return nil
}
payloads := make([]contracts.RawPayload, 0, len(outputs))
for _, output := range outputs {
payloads = append(payloads, output.Payload)
}
return payloads
}
func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) {
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
for _, chunk := range chunks {

View File

@@ -79,117 +79,19 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
}
}
func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []source.Chunk{
{
ID: "chunk-1",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"),
MediaType: "text/plain",
Units: doc.Units,
},
}
extractOutput := contracts.ExtractOutput{
LaneID: "spells",
ExtractorKey: "dnd/spells",
SourceID: doc.ID,
ChunkID: "chunk-1",
Payload: contracts.RawPayload{
Content: []byte(`{"spell":"cure wounds"}`),
MediaType: "application/json",
},
}
mergeOutput := contracts.MergeOutput{
LaneID: "spells",
MergerKey: "appendorder",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
normalizeOutput := contracts.NormalizeOutput{
LaneID: "spells",
NormalizerKey: "noop",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"normalized":true}`),
MediaType: "application/json",
},
}
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
t.Fatalf("SourceSucceeded: %v", err)
}
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []contracts.ExtractOutput{extractOutput}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
mergeDeps := rawOutputDigests([]contracts.RawPayload{extractOutput.Payload})
if err := recorder.MergeSucceeded("spells", "appendorder", mergeDeps, mergeOutput, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
normalizeDeps := rawOutputDigests([]contracts.RawPayload{mergeOutput.Payload})
if err := recorder.NormalizeSucceeded("spells", "noop", normalizeDeps, normalizeOutput, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
sourceCheckpoint, decision := loader.Source("seriatim")
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
}
if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want {
t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want)
}
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)
}
if got, want := chunkCheckpoint.Chunks[0].Ref, chunks[0].Ref; got != want {
t.Fatalf("checkpoint chunk ref = %#v, want %#v", got, want)
}
extractCheckpoint, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extractCheckpoint.Outputs) != 1 || string(extractCheckpoint.Outputs[0].Payload.Content) != `{"spell":"cure wounds"}` {
t.Fatalf("extract decision = %#v checkpoint=%#v, want reused", decision, extractCheckpoint)
}
mergeCheckpoint, decision := loader.Merge("spells", "appendorder", mergeDeps)
if !decision.Reused || string(mergeCheckpoint.Output.Payload.Content) != `{"merged":true}` {
t.Fatalf("merge decision = %#v checkpoint=%#v, want reused", decision, mergeCheckpoint)
}
normalizeCheckpoint, decision := loader.Normalize("spells", "noop", normalizeDeps)
if !decision.Reused || string(normalizeCheckpoint.Output.Payload.Content) != `{"normalized":true}` {
t.Fatalf("normalize decision = %#v checkpoint=%#v, want reused", decision, normalizeCheckpoint)
}
}
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}}
stored := pipeline.ArtifactCheckpointOutput{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
stored := pipeline.CheckpointArtifact{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ArtifactExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.ArtifactCheckpointOutput{stored}, nil, nil); err != nil {
t.Fatalf("ArtifactExtractSucceeded: %v", err)
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.CheckpointArtifact{stored}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
extracted, decision := loader.ArtifactExtract("spells", "dnd/spells", extractDeps)
extracted, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extracted.Outputs) != 1 {
t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted)
}
@@ -198,20 +100,20 @@ func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T
t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got)
}
mergeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactMergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactMergeSucceeded: %v", err)
mergeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.MergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
merged, decision := loader.ArtifactMerge("spells", "merge", mergeDeps)
merged, decision := loader.Merge("spells", "merge", mergeDeps)
if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) {
t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged)
}
normalizeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactNormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactNormalizeSucceeded: %v", err)
normalizeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.NormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
normalized, decision := loader.ArtifactNormalize("spells", "normalize", normalizeDeps)
normalized, decision := loader.Normalize("spells", "normalize", normalizeDeps)
if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest {
t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized)
}
@@ -361,23 +263,20 @@ func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
output := contracts.NormalizeOutput{
LaneID: "spells",
NormalizerKey: "noop",
SourceID: "source-1",
Payload: contracts.RawPayload{
Content: []byte(`{"ok":true}`),
MediaType: "application/json",
},
schema := contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
output := pipeline.CheckpointArtifact{
LaneID: "events", ModuleKey: "noop", SourceID: "source-1",
Artifact: contracts.SerializedArtifact{Kind: "test/artifact", Schema: schema, MediaType: "application/json", Content: []byte(`{"ok":true}`)},
SchemaDigest: contracts.DigestArtifactSchema(schema),
}
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil {
if err := recorder.NormalizeSucceeded("events", "noop", nil, output, warnings); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
var manifest coreworkspace.NormalizeLaneManifest
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest)
readJSON(t, filepath.Join(root, "normalize", "events", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
}