Harden checkpoint reuse and combat validation
This commit is contained in:
@@ -33,13 +33,13 @@ func (l *acceptedCheckpointLoader) AcceptedNormalize(stepID, laneID, _ string) (
|
||||
}
|
||||
func (l *acceptedCheckpointLoader) Extract(laneID string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
l.extractDeps[laneID] = append([]CheckpointFingerprint(nil), dependencies...)
|
||||
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing, "checkpoint missing")
|
||||
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
func (l *acceptedCheckpointLoader) Merge(_ string, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing, "checkpoint missing")
|
||||
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
func (l *acceptedCheckpointLoader) Normalize(_ string, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing, "checkpoint missing")
|
||||
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func TestRunnerHydratesRequiredNormalizedArtifact(t *testing.T) {
|
||||
@@ -85,7 +85,7 @@ func TestRunnerHydratesRequiredNormalizedArtifact(t *testing.T) {
|
||||
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
|
||||
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
|
||||
loader.accepted[producerKey] = NormalizeCheckpoint{Output: stored, Warnings: []contracts.Warning{{Scope: "normalize", ReasonCode: "stored-warning", Message: "stored normalize warning"}}}
|
||||
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable")
|
||||
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
|
||||
policy := CheckpointExecutionPolicy{
|
||||
RequireReusableLanes: map[string]struct{}{producerKey: {}},
|
||||
ForcedLanes: map[string]struct{}{consumerKey: {}},
|
||||
@@ -154,12 +154,12 @@ func TestRunnerRejectsInvalidRequiredNormalizedArtifactBeforeConsumer(t *testing
|
||||
mutate func(*CheckpointArtifact)
|
||||
wantCode CheckpointReasonCode
|
||||
}{
|
||||
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing, "checkpoint missing"), nil, CheckpointReasonMissing},
|
||||
{"rejected status", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonStatusNotReusable, "status rejected"), nil, CheckpointReasonStatusNotReusable},
|
||||
{"corrupt payload", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable"), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, CheckpointReasonArtifactPayloadInvalid},
|
||||
{"non canonical", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable"), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items": ["stored"]}`) }, CheckpointReasonArtifactNotCanonical},
|
||||
{"wrong codec identity", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable"), func(v *CheckpointArtifact) { v.Artifact.Kind = "test/score" }, CheckpointReasonArtifactCodecIncompatible},
|
||||
{"wrong content digest", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactDigestMismatch, "content digest mismatch"), nil, CheckpointReasonArtifactDigestMismatch},
|
||||
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing), nil, CheckpointReasonMissing},
|
||||
{"rejected status", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonStatusNotReusable), nil, CheckpointReasonStatusNotReusable},
|
||||
{"corrupt payload", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, CheckpointReasonArtifactPayloadInvalid},
|
||||
{"non canonical", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items": ["stored"]}`) }, CheckpointReasonArtifactNotCanonical},
|
||||
{"wrong codec identity", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Kind = "test/score" }, CheckpointReasonArtifactCodecIncompatible},
|
||||
{"wrong content digest", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactDigestMismatch), nil, CheckpointReasonArtifactDigestMismatch},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -205,6 +205,100 @@ func TestRunnerRejectsInvalidRequiredNormalizedArtifactBeforeConsumer(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetainsEarlierHydratedProducerWhenLaterRequiredProducerFails(t *testing.T) {
|
||||
prepared := preparedPipelineWithSharedProducerStep(t)
|
||||
first := &prepared.Steps[0].lanes[0]
|
||||
second := &prepared.Steps[0].lanes[1]
|
||||
consumer := &prepared.Steps[1].lanes[0]
|
||||
doc := prepared.input.(*typedTestInput).doc
|
||||
|
||||
stored, err := checkpointArtifact(first.typed.codec, first.resolved.ID, first.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"retained producer"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumerCalls := 0
|
||||
consumer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
consumerCalls++
|
||||
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
|
||||
}
|
||||
|
||||
loader := newAcceptedCheckpointLoader()
|
||||
firstKey := CheckpointLaneKey(first.resolved.StepID, first.resolved.ID)
|
||||
secondKey := CheckpointLaneKey(second.resolved.StepID, second.resolved.ID)
|
||||
loader.accepted[firstKey] = NormalizeCheckpoint{Output: stored, Warnings: []contracts.Warning{{Scope: "normalize", ReasonCode: "retained-warning", Message: "retained warning"}}}
|
||||
loader.acceptedDecision[firstKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
|
||||
loader.acceptedDecision[secondKey] = NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{firstKey: {}, secondKey: {}}}
|
||||
|
||||
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
|
||||
if runErr == nil || !strings.Contains(runErr.Error(), string(CheckpointReasonMissing)) {
|
||||
t.Fatalf("Run() error = %v, want missing required producer", runErr)
|
||||
}
|
||||
if consumerCalls != 0 {
|
||||
t.Fatalf("consumer calls = %d, want zero", consumerCalls)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || output.NormalizeOutputs[0].LaneID != first.resolved.ID || string(output.NormalizeOutputs[0].Artifact.Content) != string(stored.Artifact.Content) {
|
||||
t.Fatalf("retained normalize outputs = %#v, want first producer", output.NormalizeOutputs)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "retained-warning" {
|
||||
t.Fatalf("retained warnings = %#v", output.Warnings)
|
||||
}
|
||||
type decisionExpectation struct {
|
||||
step, lane string
|
||||
category CheckpointDecisionCategory
|
||||
reason CheckpointReasonCode
|
||||
}
|
||||
want := []decisionExpectation{
|
||||
{first.resolved.StepID, first.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused},
|
||||
{second.resolved.StepID, second.resolved.ID, CheckpointDecisionExecuted, CheckpointReasonMissing},
|
||||
}
|
||||
var got []decisionExpectation
|
||||
for _, event := range output.CheckpointEvents {
|
||||
if event.Stage == string(StageNormalize) {
|
||||
got = append(got, decisionExpectation{event.StepID, event.LaneID, event.Category, event.ReasonCode})
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
|
||||
}
|
||||
var manifestGot []decisionExpectation
|
||||
for _, decision := range output.Manifest.CheckpointDecisions {
|
||||
if decision.Stage == string(StageNormalize) {
|
||||
manifestGot = append(manifestGot, decisionExpectation{
|
||||
decision.StepID, decision.LaneID,
|
||||
CheckpointDecisionCategory(decision.Category), CheckpointReasonCode(decision.ReasonCode),
|
||||
})
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(manifestGot, want) {
|
||||
t.Fatalf("manifest normalize decisions = %#v, want %#v", manifestGot, want)
|
||||
}
|
||||
}
|
||||
|
||||
func preparedPipelineWithSharedProducerStep(t *testing.T) *PreparedPipeline {
|
||||
t.Helper()
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
base := typedResolutionProfile()
|
||||
profile := base
|
||||
profile.Artifacts = nil
|
||||
profile.Steps = []PipelineStepProfile{
|
||||
{ID: "producers", Artifacts: map[string]ArtifactLaneProfile{"first": base.Artifacts["notes"], "second": base.Artifacts["notes"]}},
|
||||
{ID: "consumer", Artifacts: map[string]ArtifactLaneProfile{"score": base.Artifacts["score"]}},
|
||||
}
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v", err)
|
||||
}
|
||||
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
doc := typedTestDocumentWithUnits(1)
|
||||
prepared.input.(*typedTestInput).doc = doc
|
||||
prepared.chunker = &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func TestForcedRequiredLaneExecutesInsteadOfHydrating(t *testing.T) {
|
||||
prepared := preparedOrderedPipeline(t, 1,
|
||||
orderedLaneSpec{id: "unrelated", profile: "score"},
|
||||
@@ -247,9 +341,9 @@ func TestForcedRequiredLaneExecutesInsteadOfHydrating(t *testing.T) {
|
||||
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
|
||||
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
|
||||
loader.accepted[unrelatedKey] = NormalizeCheckpoint{Output: unrelatedArtifact}
|
||||
loader.acceptedDecision[unrelatedKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable")
|
||||
loader.acceptedDecision[unrelatedKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
|
||||
loader.accepted[producerKey] = NormalizeCheckpoint{Output: producerArtifact}
|
||||
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused, "accepted artifact reusable")
|
||||
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
|
||||
policy := CheckpointExecutionPolicy{
|
||||
ForcedLanes: map[string]struct{}{producerKey: {}, consumerKey: {}},
|
||||
RequireReusableLanes: map[string]struct{}{unrelatedKey: {}, producerKey: {}},
|
||||
|
||||
Reference in New Issue
Block a user