eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add reconciliation state foundation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 18:24:12 +00:00
parent c04432e40b
commit b7db3993fb
13 changed files with 557 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
internal/state/distributor.go
View File

@@ -8,9 +8,14 @@ import (
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
const SchemaVersion = 1
const (
SchemaVersion = 2
legacySchemaVersion = 1
StateModeSingleOwner = "single_owner"
)
type DistributorState struct {
SchemaVersion int
@@ -18,11 +23,23 @@ type DistributorState struct {
PipelineID string
DestinationID string
PublishedAt time.Time
CreatedAt time.Time
UpdatedAt time.Time
State StatePolicy
Reconciliation ReconciliationPolicy
Source SourceState
Links *LinkState
Outputs []OutputFile
}
type StatePolicy struct {
Mode string
}
type ReconciliationPolicy struct {
Mode string
}
type SourceState struct {
Manifest bundle.Manifest
}
@@ -39,17 +56,31 @@ type OutputFile struct {
URL string
SHA256 string
Size int64
CreatedAt time.Time
UpdatedAt time.Time
}
type rawDistributorState struct {
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
PublishedAt *string `json:"published_at"`
Source *rawSourceState `json:"source"`
Links *rawLinkState `json:"links"`
Outputs []rawOutputFile `json:"outputs"`
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
PublishedAt *string `json:"published_at"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
State *rawStatePolicy `json:"state"`
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
Source *rawSourceState `json:"source"`
Links *rawLinkState `json:"links"`
Outputs []rawOutputFile `json:"outputs"`
}
type rawStatePolicy struct {
Mode string `json:"mode"`
}
type rawReconciliationPolicy struct {
Mode string `json:"mode"`
}
type rawSourceState struct {
@@ -68,6 +99,8 @@ type rawOutputFile struct {
URL string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
}
func Parse(data []byte) (DistributorState, error) {
@@ -96,9 +129,10 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
return DistributorState{}, fmt.Errorf("state schema_version is required")
}
state.SchemaVersion = *raw.SchemaVersion
if state.SchemaVersion != SchemaVersion {
return DistributorState{}, fmt.Errorf("state schema_version must be %d", SchemaVersion)
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
}
legacy := state.SchemaVersion == legacySchemaVersion
state.DistributorVersion = raw.DistributorVersion
if raw.PipelineID == nil || *raw.PipelineID == "" {
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
@@ -116,6 +150,32 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
}
state.PublishedAt = publishedAt.UTC()
if legacy {
state.SchemaVersion = SchemaVersion
state.CreatedAt = state.PublishedAt
state.UpdatedAt = state.PublishedAt
state.State.Mode = StateModeSingleOwner
state.Reconciliation.Mode = config.ReconciliationModeReplace
} else {
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
if err != nil {
return DistributorState{}, err
}
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
if err != nil {
return DistributorState{}, err
}
state.CreatedAt = createdAt
state.UpdatedAt = updatedAt
if raw.State == nil || raw.State.Mode == "" {
return DistributorState{}, fmt.Errorf("state state.mode is required")
}
state.State.Mode = raw.State.Mode
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
}
state.Reconciliation.Mode = raw.Reconciliation.Mode
}
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
return DistributorState{}, fmt.Errorf("state source.manifest is required")
}
@@ -130,7 +190,7 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
if raw.Outputs == nil {
return DistributorState{}, fmt.Errorf("state outputs is required")
}
outputs, err := parseOutputs(raw.Outputs)
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
if err != nil {
return DistributorState{}, err
}
@@ -138,11 +198,22 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
return state, nil
}
func parseOutputs(rawOutputs []rawOutputFile) ([]OutputFile, error) {
func parseRequiredTime(context string, raw *string) (time.Time, error) {
if raw == nil || *raw == "" {
return time.Time{}, fmt.Errorf("%s is required", context)
}
parsed, err := time.Parse(time.RFC3339, *raw)
if err != nil {
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
}
return parsed.UTC(), nil
}
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
outputs := make([]OutputFile, 0, len(rawOutputs))
seen := make(map[string]struct{}, len(rawOutputs))
for index, raw := range rawOutputs {
output, err := parseOutput(index, raw)
output, err := parseOutput(index, raw, legacy, publishedAt)
if err != nil {
return nil, err
}
@@ -155,7 +226,7 @@ func parseOutputs(rawOutputs []rawOutputFile) ([]OutputFile, error) {
return outputs, nil
}
func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
if raw.Path == nil || *raw.Path == "" {
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
}
@@ -171,6 +242,19 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
if raw.Size == nil {
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
}
createdAt := publishedAt
updatedAt := publishedAt
if !legacy {
var err error
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
if err != nil {
return OutputFile{}, err
}
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
if err != nil {
return OutputFile{}, err
}
}
return OutputFile{
Path: *raw.Path,
Kind: *raw.Kind,
@@ -179,6 +263,8 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
URL: raw.URL,
SHA256: *raw.SHA256,
Size: *raw.Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}, nil
}
@@ -186,19 +272,39 @@ func (s DistributorState) PublishedAtString() string {
return s.PublishedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) CreatedAtString() string {
return s.CreatedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) UpdatedAtString() string {
return s.UpdatedAt.UTC().Format(time.RFC3339)
}
func (o OutputFile) CreatedAtString() string {
return o.CreatedAt.UTC().Format(time.RFC3339)
}
func (o OutputFile) UpdatedAtString() string {
return o.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) MarshalJSON() ([]byte, error) {
type sourceJSON struct {
Manifest bundle.Manifest `json:"manifest"`
}
type stateJSON struct {
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
PublishedAt string `json:"published_at"`
Source sourceJSON `json:"source"`
Links *LinkState `json:"links,omitempty"`
Outputs []OutputFile `json:"outputs"`
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
PublishedAt string `json:"published_at"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
State StatePolicy `json:"state"`
Reconciliation ReconciliationPolicy `json:"reconciliation"`
Source sourceJSON `json:"source"`
Links *LinkState `json:"links,omitempty"`
Outputs []OutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
SchemaVersion: s.SchemaVersion,
@@ -206,12 +312,30 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
PipelineID: s.PipelineID,
DestinationID: s.DestinationID,
PublishedAt: s.PublishedAtString(),
CreatedAt: s.CreatedAtString(),
UpdatedAt: s.UpdatedAtString(),
State: s.State,
Reconciliation: s.Reconciliation,
Source: sourceJSON{Manifest: s.Source.Manifest},
Links: s.Links,
Outputs: s.Outputs,
})
}
func (p StatePolicy) MarshalJSON() ([]byte, error) {
type policyJSON struct {
Mode string `json:"mode"`
}
return json.Marshal(policyJSON{Mode: p.Mode})
}
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
type policyJSON struct {
Mode string `json:"mode"`
}
return json.Marshal(policyJSON{Mode: p.Mode})
}
func (l LinkState) MarshalJSON() ([]byte, error) {
type linkJSON struct {
PrimaryURL string `json:"primary_url,omitempty"`
@@ -228,6 +352,8 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
return json.Marshal(outputJSON{
Path: o.Path,
@@ -237,5 +363,7 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
})
}