Add stable checkpoint decision diagnostics

This commit is contained in:
2026-07-22 02:13:50 +00:00
parent 5bdd56cfb1
commit 9de399432e
10 changed files with 391 additions and 155 deletions

View File

@@ -43,6 +43,17 @@ not permit stale downstream reuse. The CLI's selective recomputation policy
records `forced_recompute` for the selected step and its dependents while
requiring compatible predecessor checkpoints.
The loader assigns a typed category and reason code at each validation site;
diagnostic prose is not classified after the fact. The runner then applies
forced-execution policy, validates reusable artifact bytes through the prepared
codec, and records the final decision before enforcing a required-predecessor
failure. That failure names only the step, lane, and stable reason code. Decision
detail passes through one UTF-8-safe bounded sanitizer and contains only
allowlisted diagnostic context, never payloads, references, credentials,
environment values, or physical paths. Typed categories and codes remain intact
through pipeline events and become strings only in manifest and debug-summary
JSON.
`internal/core/fileio` provides confined atomic file writes used by state
collaborators. The chunk-plan store retains its stronger entry validation.

View File

@@ -173,6 +173,33 @@ then inspect checkpoint decision categories and reason codes. A
`dependency_invalidated` decision means the stored producer, codec identity,
schema, or canonical content no longer matches; a missing or rejected producer
requires rerunning it rather than copying an artifact into the checkpoint root.
The decision that caused a required-predecessor failure is retained before the
run returns, and the CLI error identifies its step, lane, and reason code.
Checkpoint reason codes are stable diagnostic identifiers:
| Reason code | Operator meaning |
| --- | --- |
| `loading_disabled` | This invocation did not enable checkpoint loading. |
| `checkpoint_missing` | The requested checkpoint file does not exist. |
| `checkpoint_path_invalid` | The requested checkpoint location failed confinement validation. |
| `checkpoint_read_failed` | An existing checkpoint could not be read. |
| `checkpoint_decode_failed` | Checkpoint JSON could not be decoded. |
| `workspace_schema_incompatible` | The stored workspace schema is not supported by this build. |
| `identity_mismatch` | The stored invocation identity differs from the current invocation. |
| `stage_mismatch`, `step_mismatch`, `lane_mismatch`, `module_mismatch` | Stored scope does not match the requested pipeline scope. |
| `status_not_reusable` | The stored operation did not finish in a reusable status. |
| `dependency_mismatch` | Stored dependencies differ; the category is `dependency_invalidated`. |
| `artifact_payload_invalid` | Stored artifact payload structure or encoding is invalid. |
| `artifact_digest_mismatch` | Stored artifact bytes do not match their recorded digest. |
| `artifact_codec_incompatible` | Stored artifact identity is incomplete or incompatible with the codec contract. |
| `artifact_not_canonical` | The codec can decode the artifact, but its bytes are not canonical. |
| `checkpoint_reused` | The stored checkpoint passed validation and was reused. |
| `recompute_step` | Selective recomputation forced execution of this lane. |
Decision detail is bounded explanatory text, not a data-recovery channel. It
never contains checkpoint paths, artifact or reference content, source content,
credentials, or environment values.
## Debug Bundles

View File

@@ -8,6 +8,7 @@ import (
"runtime"
"strings"
"testing"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -137,7 +138,7 @@ func TestFilesystemCheckpointRejectsMissingAndCorruptState(t *testing.T) {
if err := os.Remove(stage.manifest(fixture)); err != nil {
t.Fatal(err)
}
assertStageNotReused(t, stage, fixture, "missing")
assertStageNotReused(t, stage, fixture, pipeline.CheckpointReasonMissing)
})
t.Run(stage.name+" malformed manifest", func(t *testing.T) {
@@ -145,7 +146,7 @@ func TestFilesystemCheckpointRejectsMissingAndCorruptState(t *testing.T) {
if err := os.WriteFile(stage.manifest(fixture), []byte("{"), 0o600); err != nil {
t.Fatal(err)
}
assertStageNotReused(t, stage, fixture, "decode")
assertStageNotReused(t, stage, fixture, pipeline.CheckpointReasonDecodeFailed)
})
}
}
@@ -154,18 +155,18 @@ func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
for _, tt := range []struct {
name string
edit func(map[string]any)
want string
want pipeline.CheckpointReasonCode
}{
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, "workspace schema"},
{"v2 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV2 }, "workspace schema"},
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, "workspace schema"},
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, "identity"},
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, "stage"},
{"lane", func(m map[string]any) { m["lane_id"] = "lane-other" }, "lane"},
{"module", func(m map[string]any) { m["module_key"] = "module-other" }, "module"},
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"v2 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV2 }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, pipeline.CheckpointReasonIdentityMismatch},
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, pipeline.CheckpointReasonStageMismatch},
{"lane", func(m map[string]any) { m["lane_id"] = "lane-other" }, pipeline.CheckpointReasonLaneMismatch},
{"module", func(m map[string]any) { m["module_key"] = "module-other" }, pipeline.CheckpointReasonModuleMismatch},
{"dependency", func(m map[string]any) {
m["dependency_fingerprints"] = []map[string]string{{"name": "input", "value": "other"}}
}, "dependency"},
}, pipeline.CheckpointReasonDependencyMismatch},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
@@ -180,7 +181,7 @@ func TestFilesystemCheckpointRejectsNonTerminalStatuses(t *testing.T) {
t.Run(string(status), func(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) { m["status"] = string(status) })
assertStageNotReused(t, checkpointStages()[1], fixture, "status")
assertStageNotReused(t, checkpointStages()[1], fixture, pipeline.CheckpointReasonStatusNotReusable)
})
}
}
@@ -189,20 +190,20 @@ func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T)
for _, tt := range []struct {
name string
edit func(map[string]any)
want string
want pipeline.CheckpointReasonCode
}{
{"artifact kind", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = "" }, "artifact codec identity"},
{"schema id", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["id"] = "" }, "artifact codec identity"},
{"artifact kind", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = "" }, pipeline.CheckpointReasonArtifactCodecIncompatible},
{"schema id", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["id"] = "" }, pipeline.CheckpointReasonArtifactCodecIncompatible},
{"schema version", func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["version"] = ""
}, "artifact codec identity"},
{"schema digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema_digest"] = "" }, "artifact codec identity"},
}, pipeline.CheckpointReasonArtifactCodecIncompatible},
{"schema digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema_digest"] = "" }, pipeline.CheckpointReasonArtifactCodecIncompatible},
{"base64", func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_base64"] = "%"
}, "base64"},
}, pipeline.CheckpointReasonArtifactPayloadInvalid},
{"content digest", func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other"
}, "content digest"},
}, pipeline.CheckpointReasonArtifactDigestMismatch},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
@@ -218,7 +219,7 @@ func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
m["document"].(map[string]any)["units"].([]any)[0].(map[string]any)["text"] = ""
})
assertStageNotReused(t, checkpointStages()[0], fixture, "source checkpoint document")
assertStageNotReused(t, checkpointStages()[0], fixture, pipeline.CheckpointReasonArtifactPayloadInvalid)
})
t.Run("source output digest", func(t *testing.T) {
@@ -226,7 +227,7 @@ func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
m["document"].(map[string]any)["digest"] = "sha256:other"
})
assertStageNotReused(t, checkpointStages()[0], fixture, "output digest")
assertStageNotReused(t, checkpointStages()[0], fixture, pipeline.CheckpointReasonArtifactDigestMismatch)
})
for _, stage := range checkpointStages()[1:] {
@@ -235,7 +236,7 @@ func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T
editManifest(t, stage.manifest(fixture), func(m map[string]any) {
m["output_digests"].([]any)[0].(map[string]any)["value"] = "sha256:other"
})
assertStageNotReused(t, stage, fixture, "output digest")
assertStageNotReused(t, stage, fixture, pipeline.CheckpointReasonArtifactDigestMismatch)
})
}
}
@@ -259,6 +260,68 @@ func TestFilesystemCheckpointDependencyDecisionIsBoundedAndCategorized(t *testin
}
}
func TestFilesystemCheckpointDecisionFamiliesAreStableAndSafe(t *testing.T) {
const secretSentinel = "do-not-expose-checkpoint-secret"
tests := []struct {
name string
prepare func(*testing.T, filesystemCheckpointFixture) pipeline.CheckpointDecision
category pipeline.CheckpointDecisionCategory
code pipeline.CheckpointReasonCode
}{
{"loading disabled", func(t *testing.T, _ filesystemCheckpointFixture) pipeline.CheckpointDecision {
_, decision := pipeline.NoopCheckpointLoader().Source("source")
return decision
}, pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLoadingDisabled},
{"checkpoint unavailable", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
if err := os.Remove(checkpointStages()[1].manifest(fixture)); err != nil {
t.Fatal(err)
}
return checkpointStages()[1].load(fixture)
}, pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing},
{"workspace identity", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) {
m["workspace_schema_version"] = secretSentinel
})
return checkpointStages()[1].load(fixture)
}, pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible},
{"manifest scope", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) { m["module_key"] = secretSentinel })
return checkpointStages()[1].load(fixture)
}, pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonModuleMismatch},
{"dependency invalidated", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
_, decision := fixture.loader.Extract("lane-a", "extract-module", []pipeline.CheckpointFingerprint{{Name: "input", Value: secretSentinel}})
return decision
}, pipeline.CheckpointDecisionDependencyInvalidated, pipeline.CheckpointReasonDependencyMismatch},
{"artifact payload", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "outputs.json"), func(m map[string]any) {
m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = ""
m["outputs"].([]any)[0].(map[string]any)["source_id"] = secretSentinel
})
return checkpointStages()[1].load(fixture)
}, pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactCodecIncompatible},
{"checkpoint reused", func(t *testing.T, fixture filesystemCheckpointFixture) pipeline.CheckpointDecision {
return checkpointStages()[1].load(fixture)
}, pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
decision := test.prepare(t, fixture)
if decision.Category != test.category || decision.ReasonCode != test.code {
t.Fatalf("decision = %#v, want category %q and code %q", decision, test.category, test.code)
}
if len([]byte(decision.Detail)) > 512 || !utf8.ValidString(decision.Detail) {
t.Fatalf("decision detail is not bounded valid UTF-8: %#v", decision)
}
for _, forbidden := range []string{secretSentinel, fixture.root} {
if strings.Contains(decision.Detail, forbidden) || strings.Contains(decision.Reason, forbidden) {
t.Fatalf("decision leaked %q: %#v", forbidden, decision)
}
}
})
}
}
type checkpointStage struct {
name string
manifest func(filesystemCheckpointFixture) string
@@ -367,11 +430,11 @@ func checkpointArtifact(module, content string) pipeline.CheckpointArtifact {
}
}
func assertStageNotReused(t *testing.T, stage checkpointStage, fixture filesystemCheckpointFixture, want string) {
func assertStageNotReused(t *testing.T, stage checkpointStage, fixture filesystemCheckpointFixture, want pipeline.CheckpointReasonCode) {
t.Helper()
decision := stage.load(fixture)
if decision.Reused || !strings.Contains(strings.ToLower(decision.Reason), strings.ToLower(want)) {
t.Fatalf("%s decision=%#v, want non-reused reason containing %q", stage.name, decision, want)
if decision.Reused || decision.ReasonCode != want {
t.Fatalf("%s decision=%#v, want non-reused reason code %q", stage.name, decision, want)
}
}

View File

@@ -52,13 +52,13 @@ func (l *FilesystemLoader) Source(moduleKey string) (pipeline.SourceCheckpoint,
}
doc := cloneSourceDocument(payload.Document)
if err := source.ValidateDocument(&doc); err != nil {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint document is invalid: %v", err)
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactPayloadInvalid, "source checkpoint document is invalid")
}
if strings.TrimSpace(manifest.SourceID) != "" && manifest.SourceID != doc.ID {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint source id does not match payload")
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactPayloadInvalid, "source checkpoint identity does not match its payload")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), digestFingerprints("source_document", doc.Digest)) {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint output digest does not match payload")
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "source checkpoint output digest does not match its payload")
}
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
}
@@ -81,10 +81,10 @@ func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, depe
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
return pipeline.ExtractCheckpoint{}, artifactDecision(err, "extract checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
return pipeline.ExtractCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "extract checkpoint output digest does not match its payload")
}
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -107,10 +107,10 @@ func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, depend
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
return pipeline.MergeCheckpoint{}, artifactDecision(err, "merge checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
return pipeline.MergeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "merge checkpoint output digest does not match its payload")
}
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -133,10 +133,10 @@ func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, de
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
return pipeline.NormalizeCheckpoint{}, artifactDecision(err, "normalize checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "normalize checkpoint output digest does not match its payload")
}
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -152,7 +152,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
return nil, err
}
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactCodecIncompatible, err: fmt.Errorf("artifact codec identity is incomplete")}
}
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
@@ -161,21 +161,21 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
target, err := fileio.SafePath(l.root, name)
if err != nil {
return invalidDecision("checkpoint path is invalid: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonPathInvalid, "checkpoint path is invalid")
}
data, err := os.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "checkpoint_missing", Reason: "checkpoint artifact is missing"}
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing, "checkpoint artifact is missing")
}
return invalidDecision("read checkpoint artifact: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonReadFailed, "checkpoint artifact could not be read")
}
if err := json.Unmarshal(data, out); err != nil {
return invalidDecision("decode checkpoint artifact: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonDecodeFailed, "checkpoint artifact could not be decoded")
}
return reusedDecision()
}
@@ -186,28 +186,28 @@ func (l *FilesystemLoader) validateManifest(manifest StageManifest, stage StageN
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, stepID string, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV1 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV2 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if strings.TrimSpace(l.identityDigest) != "" && manifest.Metadata["checkpoint_identity_digest"] != l.identityDigest {
return invalidDecision("checkpoint identity digest does not match current invocation")
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonIdentityMismatch, "checkpoint identity does not match the current invocation")
}
if manifest.Stage != stage {
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStageMismatch, "checkpoint stage does not match the requested stage")
}
if strings.TrimSpace(stepID) != "" && manifest.StepID != stepID {
return invalidDecision("checkpoint step does not match requested step")
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStepMismatch, "checkpoint step does not match the requested step")
}
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLaneMismatch, "checkpoint lane does not match the requested lane")
}
if strings.TrimSpace(moduleKey) != "" && manifest.ModuleKey != moduleKey {
return invalidDecision("checkpoint module %q does not match %q", manifest.ModuleKey, moduleKey)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonModuleMismatch, "checkpoint module does not match the requested module")
}
statusOK := false
for _, status := range statuses {
@@ -217,10 +217,10 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
}
}
if !statusOK {
return invalidDecision("checkpoint status %q cannot be reused", manifest.Status)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStatusNotReusable, "checkpoint status cannot be reused")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.DependencyFingerprints), dependencies) {
return invalidDecision("checkpoint dependency fingerprints do not match")
return decision(pipeline.CheckpointDecisionDependencyInvalidated, pipeline.CheckpointReasonDependencyMismatch, "checkpoint dependencies do not match")
}
return reusedDecision()
}
@@ -228,10 +228,10 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil {
return nil, fmt.Errorf("decode content_base64: %w", err)
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactPayloadInvalid, err: fmt.Errorf("decode content_base64: %w", err)}
}
if digest := strings.TrimSpace(value.ContentDigest); digest != "" && digest != contentDigest(content) {
return nil, fmt.Errorf("content digest mismatch")
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactDigestMismatch, err: fmt.Errorf("content digest mismatch")}
}
return content, nil
}
@@ -262,65 +262,24 @@ func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.Checkpoi
}
func reusedDecision() pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reused: true, Category: "reused", ReasonCode: "checkpoint_valid", Reason: "checkpoint is valid"}
return decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused, "checkpoint is reusable")
}
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
reason := fmt.Sprintf(format, args...)
code := "checkpoint_invalid"
category := "executed"
switch {
case strings.Contains(reason, "dependency fingerprints"):
code, category = "dependency_mismatch", "dependency_invalidated"
case strings.Contains(reason, "missing"):
code = "checkpoint_missing"
case strings.Contains(reason, "schema"):
code = "workspace_schema_incompatible"
case strings.Contains(reason, "codec"):
code = "artifact_codec_incompatible"
case strings.Contains(reason, "content"):
code = "artifact_content_invalid"
case strings.Contains(reason, "identity"):
code = "identity_mismatch"
}
safe := safeReasonText(reason, code)
return pipeline.CheckpointDecision{Category: category, ReasonCode: code, Detail: safe, Reason: safe}
func decision(category pipeline.CheckpointDecisionCategory, code pipeline.CheckpointReasonCode, detail string) pipeline.CheckpointDecision {
return pipeline.NewCheckpointDecision(category, code, detail)
}
func safeReasonText(reason, code string) string {
lower := strings.ToLower(reason)
switch {
case strings.Contains(lower, "workspace schema"):
return "checkpoint workspace schema is incompatible"
case strings.Contains(lower, "source checkpoint document"):
return "source checkpoint document is invalid"
case strings.Contains(lower, "artifact codec identity"):
return "artifact codec identity is incomplete"
case strings.Contains(lower, "base64"):
return "checkpoint content base64 is invalid"
case strings.Contains(lower, "content digest"):
return "checkpoint content digest is invalid"
case strings.Contains(lower, "output digest"):
return "checkpoint output digest does not match payload"
case strings.Contains(lower, "identity"):
return "checkpoint identity does not match"
case strings.Contains(lower, "dependency"):
return "checkpoint dependency fingerprints do not match"
case strings.Contains(lower, "stage"):
return "checkpoint stage does not match"
case strings.Contains(lower, "step"):
return "checkpoint step does not match"
case strings.Contains(lower, "lane"):
return "checkpoint lane does not match"
case strings.Contains(lower, "module"):
return "checkpoint module does not match"
case strings.Contains(lower, "status"):
return "checkpoint status cannot be reused"
case strings.Contains(lower, "payload"):
return "checkpoint artifact payload is invalid"
case strings.Contains(lower, "decode"):
return "checkpoint artifact decode failed"
default:
return "checkpoint is not reusable (" + code + ")"
}
type artifactPayloadError struct {
code pipeline.CheckpointReasonCode
err error
}
func (e *artifactPayloadError) Error() string { return e.err.Error() }
func artifactDecision(err error, detail string) pipeline.CheckpointDecision {
code := pipeline.CheckpointReasonArtifactPayloadInvalid
if payloadErr, ok := err.(*artifactPayloadError); ok {
code = payloadErr.code
}
return decision(pipeline.CheckpointDecisionExecuted, code, detail)
}

View File

@@ -57,26 +57,96 @@ type StepCheckpointRecorder interface {
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecisionCategory string
const (
CheckpointDecisionExecuted CheckpointDecisionCategory = "executed"
CheckpointDecisionReused CheckpointDecisionCategory = "reused"
CheckpointDecisionForcedRecompute CheckpointDecisionCategory = "forced_recompute"
CheckpointDecisionDependencyInvalidated CheckpointDecisionCategory = "dependency_invalidated"
)
type CheckpointReasonCode string
const (
CheckpointReasonLoadingDisabled CheckpointReasonCode = "loading_disabled"
CheckpointReasonMissing CheckpointReasonCode = "checkpoint_missing"
CheckpointReasonPathInvalid CheckpointReasonCode = "checkpoint_path_invalid"
CheckpointReasonReadFailed CheckpointReasonCode = "checkpoint_read_failed"
CheckpointReasonDecodeFailed CheckpointReasonCode = "checkpoint_decode_failed"
CheckpointReasonWorkspaceSchemaIncompatible CheckpointReasonCode = "workspace_schema_incompatible"
CheckpointReasonIdentityMismatch CheckpointReasonCode = "identity_mismatch"
CheckpointReasonStageMismatch CheckpointReasonCode = "stage_mismatch"
CheckpointReasonStepMismatch CheckpointReasonCode = "step_mismatch"
CheckpointReasonLaneMismatch CheckpointReasonCode = "lane_mismatch"
CheckpointReasonModuleMismatch CheckpointReasonCode = "module_mismatch"
CheckpointReasonStatusNotReusable CheckpointReasonCode = "status_not_reusable"
CheckpointReasonDependencyMismatch CheckpointReasonCode = "dependency_mismatch"
CheckpointReasonArtifactPayloadInvalid CheckpointReasonCode = "artifact_payload_invalid"
CheckpointReasonArtifactDigestMismatch CheckpointReasonCode = "artifact_digest_mismatch"
CheckpointReasonArtifactCodecIncompatible CheckpointReasonCode = "artifact_codec_incompatible"
CheckpointReasonArtifactNotCanonical CheckpointReasonCode = "artifact_not_canonical"
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
)
type CheckpointDecision struct {
Reused bool `json:"reused"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reused bool `json:"reused"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
// Reason is retained as a compatibility/debug field for existing callers.
// New checkpoint stores should put bounded, non-sensitive text in Detail.
Reason string `json:"reason,omitempty"`
}
const checkpointDecisionDetailLimit = 512
func NewCheckpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode, detail string) CheckpointDecision {
return checkpointDecision(category, reasonCode, detail)
}
func checkpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode, detail string) CheckpointDecision {
detail = sanitizeCheckpointDecisionDetail(detail)
return CheckpointDecision{
Reused: category == CheckpointDecisionReused,
Category: category,
ReasonCode: reasonCode,
Detail: detail,
Reason: detail,
}
}
func sanitizeCheckpointDecisionDetail(detail string) string {
detail = strings.TrimSpace(strings.ToValidUTF8(detail, "?"))
lower := strings.ToLower(detail)
if strings.ContainsAny(detail, `/\\`) || strings.Contains(lower, "secret") || strings.Contains(lower, "token") || strings.Contains(lower, "password") || strings.Contains(lower, "credential") || strings.Contains(lower, "environment") {
return "checkpoint decision detail redacted"
}
var b strings.Builder
for _, r := range detail {
if r < 0x20 || r == 0x7f {
r = ' '
}
if b.Len()+len(string(r)) > checkpointDecisionDetailLimit {
break
}
b.WriteRune(r)
}
return strings.Join(strings.Fields(b.String()), " ")
}
type CheckpointEvent struct {
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action string `json:"action"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action CheckpointDecisionCategory `json:"action"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
}
type CheckpointExecutionPolicy struct {
@@ -100,14 +170,14 @@ func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string)
func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision {
if policy.forced(stepID, laneID) {
return CheckpointDecision{Category: "forced_recompute", ReasonCode: "recompute_step", Detail: "selected step requires execution"}
return checkpointDecision(CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep, "selected step requires execution")
}
return decision
}
func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error {
if policy.requiresReusable(stepID, laneID) && !decision.Reused {
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q", strings.TrimSpace(stepID), strings.TrimSpace(laneID))
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q (%s)", strings.TrimSpace(stepID), strings.TrimSpace(laneID), decision.ReasonCode)
}
return nil
}
@@ -116,23 +186,20 @@ func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID
// validation at the single point where a stage's observable decision is made.
func resolveCheckpointDecision(output *RunOutput, loader CheckpointLoader, policy CheckpointExecutionPolicy, stage ModuleStage, stepID, laneID, moduleKey string, decision CheckpointDecision, codec artifactCodecEntry, artifacts []CheckpointArtifact) (CheckpointDecision, error) {
decision = forceCheckpointDecision(policy, stepID, laneID, decision)
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
if decision.Reused {
for _, artifact := range artifacts {
if _, _, err := decodeCanonicalCheckpointArtifact(codec, artifact); err != nil {
decision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored " + string(stage) + " artifact failed canonical codec validation", Reason: string(stage) + " artifact checkpoint is not canonical"}
decision = checkpointDecision(CheckpointDecisionExecuted, checkpointArtifactReasonCode(err), "stored "+string(stage)+" artifact failed canonical codec validation")
break
}
}
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
if output != nil {
recordCheckpointEvent(output, loader, string(stage), stepID, laneID, moduleKey, decision)
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
return decision, nil
}
@@ -228,16 +295,16 @@ func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFinger
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return SourceCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return ExtractCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return MergeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {

View File

@@ -623,14 +623,14 @@ func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage str
})
}
func checkpointDecisionCategory(decision CheckpointDecision) string {
func checkpointDecisionCategory(decision CheckpointDecision) CheckpointDecisionCategory {
if decision.Category != "" {
return decision.Category
}
if decision.Reused {
return "reused"
return CheckpointDecisionReused
}
return "executed"
return CheckpointDecisionExecuted
}
func populateOutputManifest(output *RunOutput) {
@@ -642,7 +642,7 @@ func populateOutputManifest(output *RunOutput) {
if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents {
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: event.Category, ReasonCode: event.ReasonCode, Detail: event.Detail})
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: string(event.Category), ReasonCode: string(event.ReasonCode), Detail: event.Detail})
}
output.Manifest.CheckpointDecisions = decisions
}

View File

@@ -100,7 +100,7 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
if err := setTypedLaneManifestMetadata(output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return nil, err
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared)
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared, output)
if err != nil {
return nil, err
}
@@ -175,6 +175,13 @@ func (r *Runner) runLaneEngine(parent context.Context, input RunInput, checkpoin
}()
}
completedOutputs, runErrors := collectLaneResults(ctx, cancel, input, checkpoints, chunks, states, results, completions, continuations)
close(continuations)
continuationWorkers.Wait()
return completedOutputs, runErrors
}
func collectLaneResults(ctx context.Context, cancel context.CancelFunc, input RunInput, checkpoints CheckpointRecorder, chunks []source.Chunk, states []*laneExtractState, results <-chan extractJobResult, completions <-chan laneCompletion, continuations chan<- *laneExtractState) ([]RunOutput, []orderedRunError) {
completedOutputs := make([]RunOutput, len(states))
var runErrors []orderedRunError
var pendingContinuations []*laneExtractState
@@ -184,7 +191,6 @@ func (r *Runner) runLaneEngine(parent context.Context, input RunInput, checkpoin
pendingContinuations = append(pendingContinuations, state)
}
}
resultChannel := results
for resultChannel != nil || len(pendingContinuations) > 0 || completed < launched {
var continuationChannel chan<- *laneExtractState
@@ -232,8 +238,6 @@ func (r *Runner) runLaneEngine(parent context.Context, input RunInput, checkpoin
}
}
}
close(continuations)
continuationWorkers.Wait()
return completedOutputs, runErrors
}
@@ -246,7 +250,7 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error
return nil
}
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor) (*laneExtractState, error) {
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor, output *RunOutput) (*laneExtractState, error) {
lane, typed := prepared.resolved, prepared.typed
digest, err := joinedChunkDigest(chunks)
if err != nil {
@@ -256,7 +260,7 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
decision, err = resolveCheckpointDecision(nil, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
decision, err = resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
if err != nil {
return nil, err
}
@@ -385,7 +389,6 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints C
}
local.Warnings = append(local.Warnings, cloneWarnings(results.warnings)...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
recordCheckpointEvent(&local, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, results.decision)
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}

View File

@@ -252,7 +252,7 @@ func TestRunnerPromotesOnlyAcceptedExtractRetryWarnings(t *testing.T) {
}
}
func assertExtractDecision(t *testing.T, events []CheckpointEvent, action string, reason string) {
func assertExtractDecision(t *testing.T, events []CheckpointEvent, action CheckpointDecisionCategory, reason string) {
t.Helper()
for _, event := range events {
if event.Stage == string(StageExtract) {

View File

@@ -100,21 +100,45 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: metadata}, nil
}
type checkpointArtifactValidationError struct {
code CheckpointReasonCode
err error
}
func (e *checkpointArtifactValidationError) Error() string { return e.err.Error() }
func (e *checkpointArtifactValidationError) Unwrap() error { return e.err }
func checkpointArtifactValidationFailure(code CheckpointReasonCode, format string, args ...any) error {
return &checkpointArtifactValidationError{code: code, err: fmt.Errorf(format, args...)}
}
func checkpointArtifactReasonCode(err error) CheckpointReasonCode {
var validationErr *checkpointArtifactValidationError
if errors.As(err, &validationErr) {
return validationErr.code
}
return CheckpointReasonArtifactPayloadInvalid
}
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
if artifact.Artifact.Kind != codec.spec.Kind {
return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
}
if artifact.Artifact.Schema.ID != codec.spec.Schema.ID || artifact.Artifact.Schema.Version != codec.spec.Schema.Version {
return nil, fmt.Errorf("artifact schema %q version %q does not match codec schema %q version %q", artifact.Artifact.Schema.ID, artifact.Artifact.Schema.Version, codec.spec.Schema.ID, codec.spec.Schema.Version)
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema %q version %q does not match codec schema %q version %q", artifact.Artifact.Schema.ID, artifact.Artifact.Schema.Version, codec.spec.Schema.ID, codec.spec.Schema.Version)
}
if artifact.SchemaDigest != expectedDigest {
return nil, fmt.Errorf("artifact schema digest %q does not match codec schema digest %q", artifact.SchemaDigest, expectedDigest)
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema digest %q does not match codec schema digest %q", artifact.SchemaDigest, expectedDigest)
}
if artifact.Artifact.MediaType != codec.spec.MediaType {
return nil, fmt.Errorf("artifact media type %q does not match codec media type %q", artifact.Artifact.MediaType, codec.spec.MediaType)
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact media type %q does not match codec media type %q", artifact.Artifact.MediaType, codec.spec.MediaType)
}
return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
value, err := codec.decode(append([]byte(nil), artifact.Artifact.Content...))
if err != nil {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "decode artifact payload: %w", err)
}
return value, nil
}
func decodeCanonicalCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, CheckpointArtifact, error) {
@@ -124,14 +148,14 @@ func decodeCanonicalCheckpointArtifact(codec artifactCodecEntry, artifact Checkp
}
canonical, err := serializeArtifact(codec, value, false)
if err != nil {
return nil, CheckpointArtifact{}, err
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "encode canonical artifact: %w", err)
}
if canonical.Kind != artifact.Artifact.Kind || canonical.MediaType != artifact.Artifact.MediaType || checkpointContentDigest(canonical.Content) != checkpointContentDigest(artifact.Artifact.Content) {
return nil, CheckpointArtifact{}, fmt.Errorf("stored artifact is not canonical")
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactNotCanonical, "stored artifact is not canonical")
}
hydrated, err := hydrateCheckpointArtifact(codec, cloneCheckpointArtifact(artifact), value)
if err != nil {
return nil, CheckpointArtifact{}, err
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "hydrate checkpoint artifact: %w", err)
}
return value, hydrated, nil
}

View File

@@ -1,12 +1,94 @@
package pipeline
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type requiredCheckpointLoader struct {
CheckpointLoader
checkpoint ExtractCheckpoint
decision CheckpointDecision
}
func (l requiredCheckpointLoader) Enabled() bool { return true }
func (l requiredCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return l.checkpoint, l.decision
}
func TestRequiredCheckpointFailureRetainsDecision(t *testing.T) {
const unsafeDetail = "unsafe-loader-detail-/private/checkpoint/path"
tests := []struct {
name string
decision CheckpointDecision
corrupt bool
wantCode CheckpointReasonCode
wantAction CheckpointDecisionCategory
}{
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing, unsafeDetail), false, CheckpointReasonMissing, CheckpointDecisionExecuted},
{"corrupt", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonReused, "checkpoint reusable"), true, CheckpointReasonArtifactNotCanonical, CheckpointDecisionExecuted},
{"incompatible", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactCodecIncompatible, unsafeDetail), false, CheckpointReasonArtifactCodecIncompatible, CheckpointDecisionExecuted},
{"dependency invalidated", NewCheckpointDecision(CheckpointDecisionDependencyInvalidated, CheckpointReasonDependencyMismatch, unsafeDetail), false, CheckpointReasonDependencyMismatch, CheckpointDecisionDependencyInvalidated},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
step := prepared.Steps[0]
lane := step.lanes[0]
checkpoint := ExtractCheckpoint{}
if test.corrupt {
stored, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Extract.Module, "source", codecNotes{Items: []string{"stored"}})
if err != nil {
t.Fatal(err)
}
stored.Artifact.Content = []byte(`{"items": ["stored"]}`)
checkpoint.Outputs = []CheckpointArtifact{stored}
}
loader := requiredCheckpointLoader{CheckpointLoader: NoopCheckpointLoader(), checkpoint: checkpoint, decision: test.decision}
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{CheckpointLaneKey(step.ID, lane.resolved.ID): {}}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err == nil || !strings.Contains(err.Error(), step.ID) || !strings.Contains(err.Error(), lane.resolved.ID) || !strings.Contains(err.Error(), string(test.wantCode)) {
t.Fatalf("Run() error = %v, want step, lane, and reason code %q", err, test.wantCode)
}
if strings.Contains(err.Error(), unsafeDetail) {
t.Fatalf("Run() error leaked loader detail: %v", err)
}
var found bool
for _, event := range output.CheckpointEvents {
if event.Stage == string(StageExtract) && event.StepID == step.ID && event.LaneID == lane.resolved.ID {
found = true
if event.Action != test.wantAction || event.ReasonCode != test.wantCode {
t.Fatalf("checkpoint event = %#v, want action %q and reason %q", event, test.wantAction, test.wantCode)
}
if strings.Contains(event.Detail, unsafeDetail) {
t.Fatalf("checkpoint event leaked loader detail: %#v", event)
}
}
}
if !found {
t.Fatalf("required checkpoint decision missing from %#v", output.CheckpointEvents)
}
var manifestFound bool
for _, decision := range output.Manifest.CheckpointDecisions {
if decision.Stage == string(StageExtract) && decision.StepID == step.ID && decision.LaneID == lane.resolved.ID {
manifestFound = decision.Category == string(test.wantAction) && decision.ReasonCode == string(test.wantCode)
}
}
if !manifestFound {
t.Fatalf("manifest checkpoint decision missing category %q and code %q: %#v", test.wantAction, test.wantCode, output.Manifest.CheckpointDecisions)
}
encoded, marshalErr := json.Marshal(output.CheckpointEvents)
if marshalErr != nil || !strings.Contains(string(encoded), `"category":"`+string(test.wantAction)+`"`) || !strings.Contains(string(encoded), `"reason_code":"`+string(test.wantCode)+`"`) {
t.Fatalf("checkpoint event JSON = %s, error = %v", encoded, marshalErr)
}
})
}
}
func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {