eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serialize typed artifacts at durable boundaries

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 07:33:49 +00:00
parent 66de1a5520
commit 814fcdc6ba
23 changed files with 624 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
internal/framework/checkpoint/loader.go
View File

@@ -114,6 +114,28 @@ func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies
}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactExtract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
}
return pipeline.ArtifactExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
@@ -136,6 +158,28 @@ func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactMerge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactMergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactMergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
@@ -158,6 +202,46 @@ func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencie
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactNormalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactNormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
}
return pipeline.ArtifactNormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.ArtifactCheckpointOutput, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]pipeline.ArtifactCheckpointOutput, 0, len(values))
for _, v := range values {
content, err := contentFromEnvelope(v.Content)
if err != nil {
return nil, err
}
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
}
out = append(out, pipeline.ArtifactCheckpointOutput{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
return out, nil
}
func (l *WorkspaceLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}

80
internal/framework/checkpoint/recorder.go
View File

@@ -140,6 +140,19 @@ func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, de
})
}
func (r *WorkspaceRecorder) ArtifactExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests(outputs))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
}
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
@@ -168,6 +181,17 @@ func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, depe
})
}
func (r *WorkspaceRecorder) ArtifactMergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
@@ -201,6 +225,17 @@ func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string,
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) ArtifactNormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
@@ -323,6 +358,51 @@ type binaryEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type artifactCheckpointEnvelope struct {
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content binaryEnvelope `json:"content"`
}
type artifactExtractEnvelope struct {
Outputs []artifactCheckpointEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type artifactSingleEnvelope struct {
Output artifactCheckpointEnvelope `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
func artifactCheckpointEnvelopeFromOutput(output pipeline.ArtifactCheckpointOutput) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
}
func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []artifactCheckpointEnvelope {
if len(outputs) == 0 {
return nil
}
out := make([]artifactCheckpointEnvelope, 0, len(outputs))
for _, v := range outputs {
out = append(out, artifactCheckpointEnvelopeFromOutput(v))
}
return out
}
func artifactOutputDigests(outputs []pipeline.ArtifactCheckpointOutput) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs {
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
}
return normalizeFingerprints(values)
}
func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
if len(chunks) == 0 {
return nil

40
internal/framework/checkpoint/recorder_test.go
View File

@@ -177,6 +177,46 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
}
}
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}}
stored := pipeline.ArtifactCheckpointOutput{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ArtifactExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.ArtifactCheckpointOutput{stored}, nil, nil); err != nil {
t.Fatalf("ArtifactExtractSucceeded: %v", err)
}
extracted, decision := loader.ArtifactExtract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extracted.Outputs) != 1 {
t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted)
}
got := extracted.Outputs[0]
if got.Artifact.Kind != artifact.Kind || got.Artifact.Schema.ID != schema.ID || got.Artifact.Schema.Version != schema.Version || got.SchemaDigest != stored.SchemaDigest || string(got.Artifact.Content) != string(artifact.Content) || got.ChunkRef != stored.ChunkRef {
t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got)
}
mergeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactMergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactMergeSucceeded: %v", err)
}
merged, decision := loader.ArtifactMerge("spells", "merge", mergeDeps)
if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) {
t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged)
}
normalizeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored})
if err := recorder.ArtifactNormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactNormalizeSucceeded: %v", err)
}
normalized, decision := loader.ArtifactNormalize("spells", "normalize", normalizeDeps)
if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest {
t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized)
}
}
func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *testing.T) {
t.Run("missing", func(t *testing.T) {
loader := &WorkspaceLoader{root: t.TempDir()}