eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add reconciliation state foundation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 18:24:12 +00:00
parent c04432e40b
commit b7db3993fb
13 changed files with 557 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
internal/state/distributor_test.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
func TestParseValidState(t *testing.T) {
@@ -24,9 +25,24 @@ func TestParseValidState(t *testing.T) {
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
}
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
}
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := len(state.Outputs), 1; got != want {
t.Fatalf("output count = %d, want %d", got, want)
}
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
}
}
func TestParseValidStateWithLinks(t *testing.T) {
@@ -62,6 +78,10 @@ func TestParseRejectsMissingFields(t *testing.T) {
"pipeline_id": `"pipeline_id"`,
"destination_id": `"destination_id"`,
"published_at": `"published_at"`,
"created_at": `"created_at"`,
"updated_at": `"updated_at"`,
"state": `"state"`,
"reconciliation": `"reconciliation"`,
"source": `"source"`,
"outputs": `"outputs"`,
}
@@ -75,9 +95,31 @@ func TestParseRejectsMissingFields(t *testing.T) {
}
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1)
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "schema_version must be 1")
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
}
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
state, err := Parse([]byte(legacyStateJSON(t)))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got, want := state.SchemaVersion, SchemaVersion; got != want {
t.Fatalf("schema version = %d, want normalized %d", got, want)
}
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("created_at = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("output updated_at = %q, want %q", got, want)
}
}
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
@@ -161,6 +203,12 @@ func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
"negative size": func(s *DistributorState) {
s.Outputs[0].Size = -1
},
"missing output created_at": func(s *DistributorState) {
s.Outputs[0].CreatedAt = time.Time{}
},
"missing output updated_at": func(s *DistributorState) {
s.Outputs[0].UpdatedAt = time.Time{}
},
"invalid output url": func(s *DistributorState) {
s.Outputs[0].URL = "file:///tmp/report.md"
},
@@ -202,20 +250,33 @@ func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
assertStateErrorContains(t, err, "published_at must be RFC3339")
}
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "created_at must be RFC3339")
}
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
source := validManifest(t)
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60)),
Source: SourceState{Manifest: source},
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: StatePolicy{Mode: StateModeSingleOwner},
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: source},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
}},
}
data, err := json.Marshal(state)
@@ -229,13 +290,18 @@ func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
source := validManifest(t)
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
Source: SourceState{Manifest: source},
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: StatePolicy{Mode: StateModeSingleOwner},
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: source},
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
@@ -243,6 +309,8 @@ func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
URL: "https://reports.example.com/archive/report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
}},
}
data, err := json.Marshal(state)
@@ -257,6 +325,42 @@ func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
}
}
func TestOutputHelpers(t *testing.T) {
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
updatedAt := createdAt.Add(time.Hour)
retained := []OutputFile{{
Path: "old.md",
Kind: OutputKindSource,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "report.md",
Kind: OutputKindSource,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}}
projected := ProjectOutputs([]OutputProjection{{
Path: "report.md",
Kind: OutputKindSource,
}, {
Path: "new.md",
Kind: OutputKindSource,
}}, retained, updatedAt)
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
}
merged, err := MergeOutputFiles(retained, projected)
if err != nil {
t.Fatalf("MergeOutputFiles() error = %v", err)
}
if got, want := len(merged), 3; got != want {
t.Fatalf("merged count = %d, want %d", got, want)
}
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("managed paths = %#v, want %#v", got, want)
}
}
func validStateJSON(t *testing.T) string {
t.Helper()
return validStateWithManifestJSON(t, manifestJSON(t))
@@ -265,11 +369,15 @@ func validStateJSON(t *testing.T) string {
func validStateWithManifestJSON(t *testing.T, manifest string) string {
t.Helper()
return `{
"schema_version": 1,
"schema_version": 2,
"distributor_version": "dev",
"pipeline_id": "reports",
"destination_id": "archive",
"published_at": "2026-05-30T11:12:00Z",
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z",
"state": {"mode": "single_owner"},
"reconciliation": {"mode": "replace"},
"source": {
"manifest": ` + manifest + `
},
@@ -279,12 +387,31 @@ func validStateWithManifestJSON(t *testing.T, manifest string) string {
"kind": "source",
"source_path": "report.md",
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
"size": 16
"size": 16,
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z"
}
]
}`
}
func legacyStateJSON(t *testing.T) string {
t.Helper()
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
`"schema_version": 2`, `"schema_version": 1`, 1),
` "created_at": "2026-05-30T11:12:00Z",
`, "", 1),
` "updated_at": "2026-05-30T11:12:00Z",
`, "", 1),
` "state": {"mode": "single_owner"},
`, "", 1),
` "reconciliation": {"mode": "replace"},
`, "", 1),
`,
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
}
func manifestJSON(t *testing.T) string {
t.Helper()
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")