eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Implement catalog idempotent publish skips

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-19 16:57:38 +00:00
parent 484fda2514
commit 2e0d903626
7 changed files with 365 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
internal/publish/plan_test.go
View File

@@ -7,11 +7,13 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
var (
@@ -108,6 +110,189 @@ func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T)
}
}
func TestBuildAdditiveSkipsMatchingCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionSkipSame {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
}
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildReplacementSkipsMatchingCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionSkipSame {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
}
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildReplacementDoesNotSkipWhenCurrentOwnerOutputWouldBeDeleted(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
}
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
}
}
func TestBuildDoesNotSkipWhenCatalogMetadataDiffers(t *testing.T) {
tests := []struct {
name string
configure func(*Request)
mutate func(*state.CatalogState)
}{
{
name: "pipeline id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].PipelineID = "other"
},
},
{
name: "destination id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].DestinationID = "web"
},
},
{
name: "source id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.ID = "reports.other"
},
},
{
name: "source digest",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
},
{
name: "source created",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.Created = catalog.Outputs[0].Source.Created.Add(time.Second)
},
},
{
name: "output kind",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Kind = state.OutputKindGenerated
catalog.Outputs[0].SourcePath = "report.md"
catalog.Outputs[0].Transform = transform.MarkdownToHTML
},
},
{
name: "output digest",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].SHA256 = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
},
{
name: "output size",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Size++
},
},
{
name: "url",
configure: func(req *Request) {
req.Links = &config.Links{BaseURL: "https://reports.example.com/archive", Primary: config.LinkPrimaryAuto}
},
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].URL = "https://reports.example.com/archive/old-report.md"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
if tt.configure != nil {
tt.configure(&req)
}
existing := matchingCatalogForRequest(t, req)
tt.mutate(&existing)
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action == ActionSkipSame {
t.Fatalf("plan action = %s, want write action after metadata change", plan.Action)
}
})
}
}
func TestBuildDoesNotSkipWhenGeneratedCatalogMetadataDiffers(t *testing.T) {
tests := []struct {
name string
mutate func(*state.CatalogState)
}{
{
name: "source path",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].SourcePath = "summary.md"
},
},
{
name: "transform",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Transform = "markdown_to_html_index"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogGeneratedPlanRequest(t, config.WorkflowAdditive)
existing := matchingCatalogForRequest(t, req)
tt.mutate(&existing)
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action == ActionSkipSame {
t.Fatalf("plan action = %s, want write action after generated metadata change", plan.Action)
}
})
}
}
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
@@ -342,6 +527,41 @@ func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Bac
}
}
func catalogGeneratedPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
t.Helper()
sourceBackend, destinationBackend, req := catalogPlanRequest(t, workflow)
data := []byte("<p>Generated</p>\n")
req.Publish = config.PublishPolicy{HTML: true}
req.Transform = config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
Enabled: true,
Mode: config.TransformModeSidecar,
}}
req.Transformers = testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: data,
SHA256: bundle.FileDigest(data),
Size: int64(len(data)),
}}}}
return sourceBackend, destinationBackend, req
}
func matchingCatalogForRequest(t *testing.T, req Request) state.CatalogState {
t.Helper()
outputs, err := PlanOutputs(context.Background(), req)
if err != nil {
t.Fatalf("PlanOutputs() error = %v", err)
}
outputs, _, err = PlanLinks(req, outputs)
if err != nil {
t.Fatalf("PlanLinks() error = %v", err)
}
catalog := baseCatalog(req)
catalog.Outputs = catalogOutputsForPlan(req, outputs, nil, state.CurrentOwnerScope(req.PipelineID, req.DestinationID), planCreatedAt)
return catalog
}
func baseCatalog(req Request) state.CatalogState {
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,