Remove legacy publish and state paths
This commit is contained in:
@@ -265,9 +265,6 @@ func unsupportedStateDocumentError(document state.StateDocument) error {
|
||||
if document.SupersededLegacy != nil {
|
||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
if document.SingleOwner != nil || document.SharedRoot != nil {
|
||||
return fmt.Errorf("legacy destination state is not supported by this command")
|
||||
}
|
||||
return fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
|
||||
@@ -12,14 +12,13 @@ func shouldNotify(action publish.Action) bool {
|
||||
func notifyEvent(plan publish.Plan) notify.Event {
|
||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||
for _, output := range plan.Outputs {
|
||||
stateOutput := output.StateOutputFile()
|
||||
outputs = append(outputs, notify.Output{
|
||||
Path: stateOutput.Path,
|
||||
Kind: stateOutput.Kind,
|
||||
SourcePath: stateOutput.SourcePath,
|
||||
Transform: stateOutput.Transform,
|
||||
SHA256: stateOutput.SHA256,
|
||||
Size: stateOutput.Size,
|
||||
Path: output.DestinationPath,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return notify.Event{
|
||||
|
||||
@@ -220,15 +220,14 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
|
||||
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
||||
results := make([]RunOutputRecord, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
stateOutput := output.StateOutputFile()
|
||||
results = append(results, RunOutputRecord{
|
||||
Path: stateOutput.Path,
|
||||
Kind: stateOutput.Kind,
|
||||
SourcePath: stateOutput.SourcePath,
|
||||
Transform: stateOutput.Transform,
|
||||
URL: stateOutput.URL,
|
||||
SHA256: stateOutput.SHA256,
|
||||
Size: stateOutput.Size,
|
||||
Path: output.DestinationPath,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return results
|
||||
|
||||
@@ -31,7 +31,7 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
||||
s.replaceCatalog++
|
||||
case publish.ActionForceReplace:
|
||||
s.forceReplace++
|
||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||
case publish.ActionSkipSame:
|
||||
s.skipSame++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,14 +210,14 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
||||
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
||||
}
|
||||
if destinationState.Source.Manifest.ID != manifest.ID {
|
||||
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
|
||||
if destinationState.SourceID != manifest.ID {
|
||||
t.Fatalf("state source id = %q, want %q", destinationState.SourceID, manifest.ID)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
|
||||
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
|
||||
if destinationState.PrimaryURL != "" || destinationState.Outputs[0].URL != "" {
|
||||
t.Fatalf("state primary URL = %q output URL=%q, want absent", destinationState.PrimaryURL, destinationState.Outputs[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,8 +362,8 @@ func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
|
||||
if destinationState.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
t.Fatalf("state primary URL = %q, want source primary URL", destinationState.PrimaryURL)
|
||||
}
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
@@ -387,8 +387,8 @@ func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
|
||||
if destinationState.PrimaryURL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("state primary URL = %q, want fixed index primary URL", destinationState.PrimaryURL)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
@@ -427,8 +427,8 @@ func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
|
||||
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||
if destinationState.SourceID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,8 +457,8 @@ func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.a" {
|
||||
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
|
||||
if destinationState.SourceID != "reports.a" {
|
||||
t.Fatalf("state source id = %q, want reports.a", destinationState.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1807,11 +1807,6 @@ func writeConfigFile(t *testing.T, body string) string {
|
||||
return path
|
||||
}
|
||||
|
||||
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
|
||||
}
|
||||
|
||||
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
@@ -1829,8 +1824,10 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
type testDestinationState struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
Source state.SourceState
|
||||
Links *state.LinkState
|
||||
SourceID string
|
||||
SourceDigest string
|
||||
SourceCreated time.Time
|
||||
PrimaryURL string
|
||||
Outputs []testStateOutput
|
||||
}
|
||||
|
||||
@@ -1859,11 +1856,11 @@ func readStateFile(t *testing.T, path string) testDestinationState {
|
||||
if index == 0 {
|
||||
view.PipelineID = output.PipelineID
|
||||
view.DestinationID = output.DestinationID
|
||||
view.Source.Manifest.ID = output.Source.ID
|
||||
view.Source.Manifest.Digest = output.Source.Digest
|
||||
view.Source.Manifest.Created = output.Source.Created
|
||||
view.SourceID = output.Source.ID
|
||||
view.SourceDigest = output.Source.Digest
|
||||
view.SourceCreated = output.Source.Created
|
||||
if output.URL != "" {
|
||||
view.Links = &state.LinkState{PrimaryURL: output.URL}
|
||||
view.PrimaryURL = output.URL
|
||||
}
|
||||
}
|
||||
view.Outputs = append(view.Outputs, testStateOutput{
|
||||
|
||||
@@ -38,29 +38,25 @@ type Pipeline struct {
|
||||
}
|
||||
|
||||
type Destination struct {
|
||||
ID string `yaml:"id"`
|
||||
Backend string `yaml:"backend"`
|
||||
Host string `yaml:"host"`
|
||||
User string `yaml:"user"`
|
||||
Port int `yaml:"port"`
|
||||
Path string `yaml:"path"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Region string `yaml:"region"`
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
Publish *PublishPolicy `yaml:"publish"`
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
Workflow string `yaml:"workflow"`
|
||||
State StatePolicy `yaml:"-"`
|
||||
Reconciliation ReconciliationPolicy `yaml:"-"`
|
||||
Takeover TakeoverPolicy `yaml:"-"`
|
||||
Retention RetentionPolicy `yaml:"retention"`
|
||||
Transfer TransferPolicy `yaml:"-"`
|
||||
ID string `yaml:"id"`
|
||||
Backend string `yaml:"backend"`
|
||||
Host string `yaml:"host"`
|
||||
User string `yaml:"user"`
|
||||
Port int `yaml:"port"`
|
||||
Path string `yaml:"path"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Region string `yaml:"region"`
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
Publish *PublishPolicy `yaml:"publish"`
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
Workflow string `yaml:"workflow"`
|
||||
Retention RetentionPolicy `yaml:"retention"`
|
||||
}
|
||||
|
||||
type Backend struct {
|
||||
@@ -124,18 +120,6 @@ type Links struct {
|
||||
Primary string `yaml:"primary"`
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type TakeoverPolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type RetentionPolicy struct {
|
||||
Prune PrunePolicy `yaml:"prune"`
|
||||
}
|
||||
@@ -145,10 +129,3 @@ type PrunePolicy struct {
|
||||
OlderThan *Duration `yaml:"older_than"`
|
||||
KeepLatest *int `yaml:"keep_latest"`
|
||||
}
|
||||
|
||||
type TransferPolicy struct {
|
||||
OnDestinationSame string `yaml:"on_destination_same"`
|
||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
||||
OnDestinationNewer string `yaml:"on_destination_newer"`
|
||||
OnConflict string `yaml:"on_conflict"`
|
||||
}
|
||||
|
||||
@@ -20,12 +20,6 @@ const (
|
||||
ValidationActionFail = "fail"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferActionSkip = "skip"
|
||||
TransferActionReplace = "replace"
|
||||
TransferActionFail = "fail"
|
||||
)
|
||||
|
||||
const (
|
||||
TransformModeSidecar = transform.MarkdownModeSidecar
|
||||
TransformModeIndex = transform.MarkdownModeIndex
|
||||
@@ -47,23 +41,6 @@ const (
|
||||
WorkflowReplacement = "replacement"
|
||||
)
|
||||
|
||||
const (
|
||||
ReconciliationModeReplace = "replace"
|
||||
ReconciliationModeMerge = "merge"
|
||||
)
|
||||
|
||||
const (
|
||||
TakeoverModeSamePipeline = "same_pipeline"
|
||||
TakeoverModeSameSource = "same_source"
|
||||
TakeoverModeAnyManaged = "any_managed"
|
||||
TakeoverModeNever = "never"
|
||||
)
|
||||
|
||||
const (
|
||||
StateModeSingleOwner = "single_owner"
|
||||
StateModeSharedRoot = "shared_root"
|
||||
)
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
const (
|
||||
@@ -104,27 +81,6 @@ func ApplyDefaults(cfg *Config) {
|
||||
if destination.Workflow == "" {
|
||||
destination.Workflow = WorkflowAdditive
|
||||
}
|
||||
if destination.State.Mode == "" {
|
||||
destination.State.Mode = StateModeSingleOwner
|
||||
}
|
||||
if destination.Reconciliation.Mode == "" {
|
||||
destination.Reconciliation.Mode = ReconciliationModeReplace
|
||||
}
|
||||
if destination.Takeover.Mode == "" {
|
||||
destination.Takeover.Mode = TakeoverModeSamePipeline
|
||||
}
|
||||
if destination.Transfer.OnDestinationSame == "" {
|
||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||
}
|
||||
if destination.Transfer.OnDestinationOlder == "" {
|
||||
destination.Transfer.OnDestinationOlder = TransferActionReplace
|
||||
}
|
||||
if destination.Transfer.OnDestinationNewer == "" {
|
||||
destination.Transfer.OnDestinationNewer = TransferActionSkip
|
||||
}
|
||||
if destination.Transfer.OnConflict == "" {
|
||||
destination.Transfer.OnConflict = TransferActionFail
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,140 +5,20 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
switch plan.Action {
|
||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||
case ActionSkipSame:
|
||||
return nil
|
||||
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
|
||||
return executeCatalog(ctx, req, plan)
|
||||
case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover:
|
||||
if usesSharedRootState(req, plan) {
|
||||
return executeSharedRoot(ctx, req, plan)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
|
||||
if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceNewer || plan.Action == ActionReplaceTakeover {
|
||||
if plan.ExistingState == nil {
|
||||
return fmt.Errorf("replace requires existing destination state")
|
||||
}
|
||||
if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceTakeover {
|
||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if usesMergeRetention(plan) {
|
||||
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if usesMergeRetention(plan) {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
}
|
||||
managed := outputManagedByExistingState(output, plan.ExistingState)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
createdAt := now
|
||||
if plan.ExistingState != nil {
|
||||
createdAt = plan.ExistingState.CreatedAt
|
||||
}
|
||||
stateOutputs, err := stateOutputsForPlan(plan, now)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
destinationState := state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
PublishedAt: now,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
Outputs: stateOutputs,
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
}
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
||||
@@ -293,264 +173,3 @@ func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionReplaceTakeover || plan.Action == ActionReplaceConflict || (isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeReplace) {
|
||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
}
|
||||
managed := outputManagedBySharedRootPlan(output, plan)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
||||
for _, output := range plan.Outputs {
|
||||
if outputManagedByExistingState(output, plan.ExistingState) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func usesSharedRootState(req Request, plan Plan) bool {
|
||||
return plan.StateMode == config.StateModeSharedRoot
|
||||
}
|
||||
|
||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
||||
if existing == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
||||
return ok
|
||||
}
|
||||
|
||||
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
||||
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
||||
if existing.Path == output.DestinationPath {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, existing := range plan.TakenOverOwnerOutputs {
|
||||
if existing.Path == output.DestinationPath {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
||||
existingOutputs := []state.OutputFile(nil)
|
||||
if plan.ExistingState != nil {
|
||||
existingOutputs = plan.ExistingState.Outputs
|
||||
}
|
||||
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
||||
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
||||
return planned, nil
|
||||
}
|
||||
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
||||
}
|
||||
|
||||
func usesMergeRetention(plan Plan) bool {
|
||||
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && isReconciliationReplacementAction(plan.Action)
|
||||
}
|
||||
|
||||
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
||||
now = now.UTC()
|
||||
scope := plan.OwnerScope
|
||||
if scope.PipelineID == "" && scope.DestinationID == "" {
|
||||
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
}
|
||||
base := sharedRootBaseState(req, plan, now)
|
||||
base = removeTakenOverSharedRootOutputs(base, plan.TakenOverOwnerOutputs)
|
||||
owner := state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
}
|
||||
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
|
||||
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
||||
if plan.Action == ActionForceReplace {
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
base := *plan.ExistingSharedRoot
|
||||
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
||||
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
||||
base.DistributorVersion = req.DistributorVersion
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
base := newSharedRootState(req, now)
|
||||
base.CreatedAt = plan.ExistingState.CreatedAt
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
|
||||
func removeTakenOverSharedRootOutputs(sharedRoot state.SharedRootState, takenOver []state.SharedRootOutputFile) state.SharedRootState {
|
||||
if len(takenOver) == 0 {
|
||||
return sharedRoot
|
||||
}
|
||||
paths := sharedRootOutputPathSet(takenOver)
|
||||
next := sharedRoot
|
||||
next.Outputs = make([]state.SharedRootOutputFile, 0, len(sharedRoot.Outputs))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
if _, remove := paths[output.Path]; remove {
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{},
|
||||
Outputs: []state.SharedRootOutputFile{},
|
||||
}
|
||||
}
|
||||
|
||||
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
||||
for _, output := range plan.ExistingSharedRoot.Outputs {
|
||||
if output.Owner == plan.OwnerScope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
||||
for _, output := range plan.ExistingState.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: plan.OwnerScope,
|
||||
SourceID: plan.ExistingState.Source.Manifest.ID,
|
||||
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
||||
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
||||
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing
|
||||
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
req.DestinationBundlePath = "bundle"
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
@@ -110,7 +110,7 @@ func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||
|
||||
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||
|
||||
@@ -98,50 +98,10 @@ func rejectOutputCollisions(outputs []Output) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o Output) StateOutputFile() state.OutputFile {
|
||||
return state.OutputFile{
|
||||
Path: o.DestinationPath,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) StateOutputProjection() state.OutputProjection {
|
||||
return state.OutputProjection{
|
||||
Path: o.DestinationPath,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) ManagedPath() string {
|
||||
return o.DestinationPath
|
||||
}
|
||||
|
||||
func StateOutputFiles(outputs []Output) []state.OutputFile {
|
||||
files := make([]state.OutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
files = append(files, output.StateOutputFile())
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func StateOutputProjections(outputs []Output) []state.OutputProjection {
|
||||
projections := make([]state.OutputProjection, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
projections = append(projections, output.StateOutputProjection())
|
||||
}
|
||||
return projections
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(outputs []Output) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
|
||||
@@ -12,46 +12,12 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
func TestOutputStateProjection(t *testing.T) {
|
||||
sourceOutput := Output{
|
||||
SourcePath: "report.md",
|
||||
DestinationPath: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
URL: "https://reports.example.com/report.md",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 123,
|
||||
}
|
||||
sourceState := sourceOutput.StateOutputFile()
|
||||
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
|
||||
t.Fatalf("source state output = %#v", sourceState)
|
||||
}
|
||||
|
||||
generatedOutput := Output{
|
||||
SourcePath: "report.md",
|
||||
DestinationPath: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
Transform: transform.MarkdownToHTML,
|
||||
URL: "https://reports.example.com/report.html",
|
||||
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
Size: 456,
|
||||
}
|
||||
generatedState := generatedOutput.StateOutputFile()
|
||||
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
|
||||
t.Fatalf("generated state output = %#v", generatedState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputSliceProjections(t *testing.T) {
|
||||
func TestManagedOutputPaths(t *testing.T) {
|
||||
outputs := []Output{
|
||||
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
||||
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
||||
}
|
||||
|
||||
stateOutputs := StateOutputFiles(outputs)
|
||||
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
|
||||
t.Fatalf("state outputs = %#v", stateOutputs)
|
||||
}
|
||||
|
||||
paths := ManagedOutputPaths(outputs)
|
||||
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
||||
t.Fatalf("managed paths = %#v", paths)
|
||||
|
||||
@@ -15,18 +15,13 @@ import (
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionPublishNew Action = "publish_new"
|
||||
ActionReplaceOlder Action = "replace_older"
|
||||
ActionReplaceConflict Action = "replace_conflict"
|
||||
ActionReplaceNewer Action = "replace_newer"
|
||||
ActionSkipSame Action = "skip_same"
|
||||
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
||||
ActionFailConflict Action = "fail_conflict"
|
||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||
ActionForceReplace Action = "force_replace"
|
||||
ActionReplaceTakeover Action = "replace_takeover"
|
||||
ActionUpsertAdditive Action = "upsert_additive"
|
||||
ActionReplaceCatalog Action = "replace_catalog"
|
||||
ActionPublishNew Action = "publish_new"
|
||||
ActionSkipSame Action = "skip_same"
|
||||
ActionFailConflict Action = "fail_conflict"
|
||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||
ActionForceReplace Action = "force_replace"
|
||||
ActionUpsertAdditive Action = "upsert_additive"
|
||||
ActionReplaceCatalog Action = "replace_catalog"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
@@ -82,18 +77,6 @@ type Plan struct {
|
||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||
ClearDestinationRoot bool
|
||||
|
||||
// Retained while the executor is migrated to catalog state.
|
||||
StateMode string
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
TakeoverMode string
|
||||
ExistingState *state.DistributorState
|
||||
ExistingSharedRoot *state.SharedRootState
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
TakenOverOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
type catalogPlanDetails struct {
|
||||
@@ -396,22 +379,3 @@ func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.ReconciliationModeReplace
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func isReconciliationReplacementAction(action Action) bool {
|
||||
return action == ActionReplaceOlder || action == ActionReplaceNewer
|
||||
}
|
||||
|
||||
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
|
||||
paths := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths[output.Path] = struct{}{}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
@@ -385,7 +385,7 @@ func TestBuildForceReplacesNoStateNonEmptyDestination(t *testing.T) {
|
||||
|
||||
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
||||
|
||||
@@ -393,7 +393,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != state.SchemaVersion {
|
||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != 2 {
|
||||
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
||||
}
|
||||
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
||||
@@ -403,7 +403,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||
|
||||
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
@@ -606,6 +606,10 @@ func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, cat
|
||||
writeJSONState(t, backend, relative, catalog)
|
||||
}
|
||||
|
||||
func legacyStateDocument(schemaVersion int) map[string]int {
|
||||
return map[string]int{"schema_version": schemaVersion}
|
||||
}
|
||||
|
||||
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
|
||||
@@ -19,8 +19,6 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||
}
|
||||
return state.DestinationStatus{
|
||||
State: document.SingleOwner,
|
||||
SharedRoot: document.SharedRoot,
|
||||
Catalog: document.Catalog,
|
||||
SupersededLegacy: document.SupersededLegacy,
|
||||
HasContents: true,
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
func TestParseCatalogState(t *testing.T) {
|
||||
@@ -45,7 +47,7 @@ func TestCatalogMarshalIsDeterministic(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||
if string(data) != want {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
@@ -56,14 +58,14 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
||||
}
|
||||
if catalog.Catalog == nil || catalog.SingleOwner != nil || catalog.SharedRoot != nil || catalog.SupersededLegacy != nil {
|
||||
if catalog.Catalog == nil || catalog.SupersededLegacy != nil {
|
||||
t.Fatalf("catalog document = %#v", catalog)
|
||||
}
|
||||
|
||||
tests := map[string]string{
|
||||
"schema 1": legacyStateJSON(t),
|
||||
"schema 2": validStateJSON(t),
|
||||
"schema 3": validSharedRootStateJSON(t),
|
||||
"schema 1": schemaOnlyStateJSON(legacySchemaVersion),
|
||||
"schema 2": schemaOnlyStateJSON(legacyOwnerSchema),
|
||||
"schema 3": schemaOnlyStateJSON(legacyMultiOwnerSchema),
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
@@ -71,7 +73,7 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument() error = %v", err)
|
||||
}
|
||||
if document.SupersededLegacy == nil || document.Catalog != nil || document.SingleOwner != nil || document.SharedRoot != nil {
|
||||
if document.SupersededLegacy == nil || document.Catalog != nil {
|
||||
t.Fatalf("document = %#v, want superseded legacy only", document)
|
||||
}
|
||||
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
||||
@@ -317,6 +319,41 @@ func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
|
||||
return data
|
||||
}
|
||||
|
||||
func schemaOnlyStateJSON(schemaVersion int) string {
|
||||
data, err := json.Marshal(map[string]int{"schema_version": schemaVersion})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func assertStateErrorContains(t *testing.T, err error, substring string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want substring %q", substring)
|
||||
}
|
||||
if !strings.Contains(err.Error(), substring) {
|
||||
t.Fatalf("error = %v, want substring %q", err, substring)
|
||||
}
|
||||
}
|
||||
|
||||
func validManifest(t *testing.T) bundle.Manifest {
|
||||
t.Helper()
|
||||
files := []bundle.ManifestFile{{
|
||||
Path: "report.md",
|
||||
SHA256: bundle.FileDigest([]byte("# Report\nSunny.\n")),
|
||||
Size: int64(len("# Report\nSunny.\n")),
|
||||
}}
|
||||
manifest := bundle.Manifest{
|
||||
SchemaVersion: bundle.SchemaVersion,
|
||||
ID: "weather.daily.brentwood.2026-05-30",
|
||||
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||
Files: files,
|
||||
}
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
}
|
||||
|
||||
func validCatalogState(t *testing.T) CatalogState {
|
||||
t.Helper()
|
||||
manifest := validManifest(t)
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
type Outcome string
|
||||
|
||||
const (
|
||||
OutcomeDestinationAbsent Outcome = "destination_absent"
|
||||
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
|
||||
OutcomeInvalidState Outcome = "invalid_destination_state"
|
||||
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
|
||||
OutcomeSameSource Outcome = "same_source_manifest"
|
||||
OutcomeDestinationOlder Outcome = "destination_older"
|
||||
OutcomeDestinationNewer Outcome = "destination_newer"
|
||||
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
|
||||
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
|
||||
)
|
||||
|
||||
type DestinationStatus struct {
|
||||
State *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
|
||||
type Comparison struct {
|
||||
Outcome Outcome
|
||||
Reason string
|
||||
Detail ComparisonDetail
|
||||
}
|
||||
|
||||
type ComparisonDetailKind string
|
||||
|
||||
const (
|
||||
ComparisonDetailNone ComparisonDetailKind = ""
|
||||
ComparisonDetailInvalidState ComparisonDetailKind = "invalid_state"
|
||||
ComparisonDetailUnmanagedContent ComparisonDetailKind = "unmanaged_content"
|
||||
ComparisonDetailPipelineIDMismatch ComparisonDetailKind = "pipeline_id_mismatch"
|
||||
ComparisonDetailDestinationIDMismatch ComparisonDetailKind = "destination_id_mismatch"
|
||||
ComparisonDetailDifferentSourceID ComparisonDetailKind = "different_source_id"
|
||||
ComparisonDetailSameCreatedDigestConflict ComparisonDetailKind = "same_created_digest_conflict"
|
||||
ComparisonDetailDestinationNewer ComparisonDetailKind = "destination_newer"
|
||||
ComparisonDetailSharedRootOwnerAbsent ComparisonDetailKind = "shared_root_owner_absent"
|
||||
ComparisonDetailSharedRootOutputOwner ComparisonDetailKind = "shared_root_output_owner_conflict"
|
||||
)
|
||||
|
||||
type ComparisonDetail struct {
|
||||
Kind ComparisonDetailKind
|
||||
CurrentPipelineID string
|
||||
CurrentDestinationID string
|
||||
DestinationPipelineID string
|
||||
DestinationDestinationID string
|
||||
CurrentSourceID string
|
||||
DestinationSourceID string
|
||||
CurrentSourceDigest string
|
||||
DestinationSourceDigest string
|
||||
Path string
|
||||
CurrentOwner OwnerScope
|
||||
ConflictingOwner OwnerScope
|
||||
}
|
||||
|
||||
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
if status.SharedRoot != nil {
|
||||
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
owner, ok := status.SharedRoot.Owner(scope)
|
||||
if !ok {
|
||||
return Comparison{
|
||||
Outcome: OutcomeDestinationAbsent,
|
||||
Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID),
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailSharedRootOwnerAbsent,
|
||||
CurrentOwner: scope,
|
||||
},
|
||||
}
|
||||
}
|
||||
return compareManifests(source, owner.Source.Manifest)
|
||||
}
|
||||
if status.State != nil {
|
||||
destinationState := *status.State
|
||||
if err := Validate(destinationState); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
if destinationState.PipelineID != scope.PipelineID {
|
||||
return pipelineIDMismatchComparison(destinationState.PipelineID, scope.PipelineID)
|
||||
}
|
||||
if destinationState.DestinationID != scope.DestinationID {
|
||||
return destinationIDMismatchComparison(destinationState.DestinationID, scope.DestinationID)
|
||||
}
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
if status.HasContents {
|
||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||
}
|
||||
|
||||
func compareManifests(source, destination bundle.Manifest) Comparison {
|
||||
if manifestsEqual(source, destination) {
|
||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
||||
}
|
||||
if destination.ID != source.ID {
|
||||
return Comparison{
|
||||
Outcome: OutcomeDifferentSourceConflict,
|
||||
Reason: "destination source id differs from source",
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailDifferentSourceID,
|
||||
CurrentSourceID: source.ID,
|
||||
DestinationSourceID: destination.ID,
|
||||
},
|
||||
}
|
||||
}
|
||||
if destination.Created.Before(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
||||
}
|
||||
if destination.Created.After(source.Created) {
|
||||
return Comparison{
|
||||
Outcome: OutcomeDestinationNewer,
|
||||
Reason: "destination source is newer than source",
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailDestinationNewer,
|
||||
CurrentSourceID: source.ID,
|
||||
DestinationSourceID: destination.ID,
|
||||
},
|
||||
}
|
||||
}
|
||||
if destination.Digest != source.Digest {
|
||||
return Comparison{
|
||||
Outcome: OutcomeSameCreatedConflict,
|
||||
Reason: "destination source has same id and created time but different digest",
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailSameCreatedDigestConflict,
|
||||
CurrentSourceID: source.ID,
|
||||
DestinationSourceID: destination.ID,
|
||||
CurrentSourceDigest: source.Digest,
|
||||
DestinationSourceDigest: destination.Digest,
|
||||
},
|
||||
}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome", Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
|
||||
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
if status.State == nil {
|
||||
if status.HasContents {
|
||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||
}
|
||||
|
||||
destinationState := *status.State
|
||||
if err := Validate(destinationState); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
||||
}
|
||||
if destinationState.PipelineID != pipelineID {
|
||||
return pipelineIDMismatchComparison(destinationState.PipelineID, pipelineID)
|
||||
}
|
||||
if destinationState.DestinationID != destinationID {
|
||||
return destinationIDMismatchComparison(destinationState.DestinationID, destinationID)
|
||||
}
|
||||
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
|
||||
func pipelineIDMismatchComparison(destinationPipelineID, currentPipelineID string) Comparison {
|
||||
return Comparison{
|
||||
Outcome: OutcomeIdentityMismatch,
|
||||
Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationPipelineID, currentPipelineID),
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailPipelineIDMismatch,
|
||||
CurrentPipelineID: currentPipelineID,
|
||||
DestinationPipelineID: destinationPipelineID,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func destinationIDMismatchComparison(destinationDestinationID, currentDestinationID string) Comparison {
|
||||
return Comparison{
|
||||
Outcome: OutcomeIdentityMismatch,
|
||||
Reason: fmt.Sprintf("destination id %q does not match %q", destinationDestinationID, currentDestinationID),
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailDestinationIDMismatch,
|
||||
CurrentDestinationID: currentDestinationID,
|
||||
DestinationDestinationID: destinationDestinationID,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func manifestsEqual(a, b bundle.Manifest) bool {
|
||||
if a.SchemaVersion != b.SchemaVersion ||
|
||||
a.ID != b.ID ||
|
||||
a.Digest != b.Digest ||
|
||||
!a.Created.Equal(b.Created) ||
|
||||
len(a.Files) != len(b.Files) {
|
||||
return false
|
||||
}
|
||||
for index := range a.Files {
|
||||
if a.Files[index] != b.Files[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestCompareOutcomes(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
tests := []struct {
|
||||
name string
|
||||
status DestinationStatus
|
||||
want Outcome
|
||||
}{
|
||||
{
|
||||
name: "destination absent",
|
||||
status: DestinationStatus{},
|
||||
want: OutcomeDestinationAbsent,
|
||||
},
|
||||
{
|
||||
name: "destination unmanaged",
|
||||
status: DestinationStatus{HasContents: true},
|
||||
want: OutcomeDestinationUnmanaged,
|
||||
},
|
||||
{
|
||||
name: "invalid destination state",
|
||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
||||
want: OutcomeInvalidState,
|
||||
},
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
||||
want: OutcomeIdentityMismatch,
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
||||
want: OutcomeIdentityMismatch,
|
||||
},
|
||||
{
|
||||
name: "same source manifest",
|
||||
status: DestinationStatus{State: withState(t, source, nil)},
|
||||
want: OutcomeSameSource,
|
||||
},
|
||||
{
|
||||
name: "destination older",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
|
||||
})},
|
||||
want: OutcomeDestinationOlder,
|
||||
},
|
||||
{
|
||||
name: "destination newer",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
||||
})},
|
||||
want: OutcomeDestinationNewer,
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
||||
})},
|
||||
want: OutcomeSameCreatedConflict,
|
||||
},
|
||||
{
|
||||
name: "different source id",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.ID = "other.source"
|
||||
})},
|
||||
want: OutcomeDifferentSourceConflict,
|
||||
},
|
||||
{
|
||||
name: "invalid state object",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = "other"
|
||||
})},
|
||||
want: OutcomeInvalidState,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Compare(source, "reports", "archive", tt.status)
|
||||
if got.Outcome != tt.want {
|
||||
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
|
||||
}
|
||||
if got.Reason == "" {
|
||||
t.Fatal("Compare() reason is empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareReportsStructuredDetails(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
tests := []struct {
|
||||
name string
|
||||
status DestinationStatus
|
||||
wantKind ComparisonDetailKind
|
||||
assertions func(t *testing.T, detail ComparisonDetail)
|
||||
}{
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
||||
wantKind: ComparisonDetailPipelineIDMismatch,
|
||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
||||
t.Helper()
|
||||
if detail.DestinationPipelineID != "other" || detail.CurrentPipelineID != "reports" {
|
||||
t.Fatalf("detail = %#v, want pipeline ids", detail)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
||||
wantKind: ComparisonDetailDestinationIDMismatch,
|
||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
||||
t.Helper()
|
||||
if detail.DestinationDestinationID != "other" || detail.CurrentDestinationID != "archive" {
|
||||
t.Fatalf("detail = %#v, want destination ids", detail)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different source id",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.Source.Manifest.ID = "other.source" })},
|
||||
wantKind: ComparisonDetailDifferentSourceID,
|
||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
||||
t.Helper()
|
||||
if detail.DestinationSourceID != "other.source" || detail.CurrentSourceID != source.ID {
|
||||
t.Fatalf("detail = %#v, want source ids", detail)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
||||
})},
|
||||
wantKind: ComparisonDetailSameCreatedDigestConflict,
|
||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
||||
t.Helper()
|
||||
if detail.CurrentSourceDigest == "" || detail.DestinationSourceDigest == "" || detail.CurrentSourceDigest == detail.DestinationSourceDigest {
|
||||
t.Fatalf("detail = %#v, want different source digests", detail)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "destination newer",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
||||
})},
|
||||
wantKind: ComparisonDetailDestinationNewer,
|
||||
},
|
||||
{
|
||||
name: "invalid state",
|
||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
||||
wantKind: ComparisonDetailInvalidState,
|
||||
},
|
||||
{
|
||||
name: "unmanaged content",
|
||||
status: DestinationStatus{HasContents: true},
|
||||
wantKind: ComparisonDetailUnmanagedContent,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Compare(source, "reports", "archive", tt.status)
|
||||
if got.Detail.Kind != tt.wantKind {
|
||||
t.Fatalf("Compare() detail kind = %q, want %q; comparison=%#v", got.Detail.Kind, tt.wantKind, got)
|
||||
}
|
||||
if tt.assertions != nil {
|
||||
tt.assertions(t, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
|
||||
t.Helper()
|
||||
stateManifest := source
|
||||
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: stateManifest},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&state)
|
||||
}
|
||||
return &state
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemaVersion = 2
|
||||
SharedRootSchemaVersion = 3
|
||||
CatalogSchemaVersion = 4
|
||||
legacySchemaVersion = 1
|
||||
StateModeSingleOwner = config.StateModeSingleOwner
|
||||
StateModeSharedRoot = config.StateModeSharedRoot
|
||||
StateModeCatalog = "catalog"
|
||||
)
|
||||
|
||||
type DistributorState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
PublishedAt time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
Outputs []OutputFile
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type SourceState struct {
|
||||
Manifest bundle.Manifest
|
||||
}
|
||||
|
||||
type LinkState struct {
|
||||
PrimaryURL string
|
||||
}
|
||||
|
||||
type OutputFile struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type rawDistributorState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
PublishedAt *string `json:"published_at"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
Outputs []rawOutputFile `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawStatePolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawReconciliationPolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawSourceState struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
}
|
||||
|
||||
type rawLinkState struct {
|
||||
PrimaryURL string `json:"primary_url"`
|
||||
}
|
||||
|
||||
type rawOutputFile struct {
|
||||
Path *string `json:"path"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform string `json:"transform"`
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func Parse(data []byte) (DistributorState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw rawDistributorState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseRaw(raw)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
if err := Validate(state); err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
var state DistributorState
|
||||
if raw.SchemaVersion == nil {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state.SchemaVersion = *raw.SchemaVersion
|
||||
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
|
||||
}
|
||||
legacy := state.SchemaVersion == legacySchemaVersion
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
||||
}
|
||||
state.PipelineID = *raw.PipelineID
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return DistributorState{}, fmt.Errorf("state destination_id is required")
|
||||
}
|
||||
state.DestinationID = *raw.DestinationID
|
||||
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
|
||||
return DistributorState{}, fmt.Errorf("state published_at is required")
|
||||
}
|
||||
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
||||
}
|
||||
state.PublishedAt = publishedAt.UTC()
|
||||
if legacy {
|
||||
state.SchemaVersion = SchemaVersion
|
||||
state.CreatedAt = state.PublishedAt
|
||||
state.UpdatedAt = state.PublishedAt
|
||||
state.State.Mode = StateModeSingleOwner
|
||||
state.Reconciliation.Mode = config.ReconciliationModeReplace
|
||||
} else {
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
|
||||
}
|
||||
state.Reconciliation.Mode = raw.Reconciliation.Mode
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
||||
}
|
||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||
if err != nil {
|
||||
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
||||
}
|
||||
state.Source.Manifest = manifest
|
||||
if raw.Links != nil {
|
||||
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||
}
|
||||
if raw.Outputs == nil {
|
||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseRequiredTime(context string, raw *string) (time.Time, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", context)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
|
||||
}
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
|
||||
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(rawOutputs))
|
||||
seen := make(map[string]struct{}, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseOutput(index, raw, legacy, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
createdAt := publishedAt
|
||||
updatedAt := publishedAt
|
||||
if !legacy {
|
||||
var err error
|
||||
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
}
|
||||
return OutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
SourcePath: *raw.SourcePath,
|
||||
Transform: raw.Transform,
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s DistributorState) PublishedAtString() string {
|
||||
return s.PublishedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
}
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
Outputs []OutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
PipelineID: s.PipelineID,
|
||||
DestinationID: s.DestinationID,
|
||||
PublishedAt: s.PublishedAtString(),
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Reconciliation: s.Reconciliation,
|
||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
||||
Links: s.Links,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (l LinkState) MarshalJSON() ([]byte, error) {
|
||||
type linkJSON struct {
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
}
|
||||
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
|
||||
}
|
||||
|
||||
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
|
||||
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
@@ -1,473 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseValidState(t *testing.T) {
|
||||
state, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if state.SchemaVersion != SchemaVersion {
|
||||
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
|
||||
}
|
||||
if state.PipelineID != "reports" || state.DestinationID != "archive" {
|
||||
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
|
||||
}
|
||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 1; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseValidStateWithLinks(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
|
||||
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
|
||||
|
||||
state, err := Parse([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
|
||||
t.Fatalf("links = %#v, want primary URL", state.Links)
|
||||
}
|
||||
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
|
||||
t.Fatalf("output URL = %q", state.Outputs[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingOutputs(t *testing.T) {
|
||||
state, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
state.Outputs = append(state.Outputs, OutputFile{
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 10,
|
||||
CreatedAt: state.CreatedAt,
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
})
|
||||
|
||||
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
|
||||
if changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = true, want false")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
||||
state, err := Parse([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsMissingFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"schema_version": `"schema_version"`,
|
||||
"pipeline_id": `"pipeline_id"`,
|
||||
"destination_id": `"destination_id"`,
|
||||
"published_at": `"published_at"`,
|
||||
"created_at": `"created_at"`,
|
||||
"updated_at": `"updated_at"`,
|
||||
"state": `"state"`,
|
||||
"reconciliation": `"reconciliation"`,
|
||||
"source": `"source"`,
|
||||
"outputs": `"outputs"`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "required")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
|
||||
}
|
||||
|
||||
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
|
||||
state, err := Parse([]byte(legacyStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want normalized %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output updated_at = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "source.manifest")
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||
tests := map[string]func(bundle.Manifest) bundle.Manifest{
|
||||
"schema version": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.SchemaVersion = 2
|
||||
return manifest
|
||||
},
|
||||
"empty id": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.ID = ""
|
||||
return manifest
|
||||
},
|
||||
"bad digest": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
||||
return manifest
|
||||
},
|
||||
"zero created": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Created = time.Time{}
|
||||
return manifest
|
||||
},
|
||||
"empty files": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files = nil
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"unsafe path": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[0].Path = "../report.md"
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"duplicate path": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[1].Path = manifest.Files[0].Path
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"negative size": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[0].Size = -1
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"digest mismatch": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||
return manifest
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
source := mutate(validManifest(t))
|
||||
state := *withState(t, validManifest(t), func(*DistributorState) {})
|
||||
state.Source.Manifest = source
|
||||
err := Validate(state)
|
||||
assertStateErrorContains(t, err, "source.manifest")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
tests := map[string]func(*DistributorState){
|
||||
"unsafe path": func(s *DistributorState) {
|
||||
s.Outputs[0].Path = "../report.md"
|
||||
},
|
||||
"invalid kind": func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = "other"
|
||||
},
|
||||
"invalid source": func(s *DistributorState) {
|
||||
s.Outputs[0].SourcePath = "../report.md"
|
||||
},
|
||||
"generated missing": func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = OutputKindGenerated
|
||||
},
|
||||
"invalid digest": func(s *DistributorState) {
|
||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
||||
},
|
||||
"negative size": func(s *DistributorState) {
|
||||
s.Outputs[0].Size = -1
|
||||
},
|
||||
"missing output created_at": func(s *DistributorState) {
|
||||
s.Outputs[0].CreatedAt = time.Time{}
|
||||
},
|
||||
"missing output updated_at": func(s *DistributorState) {
|
||||
s.Outputs[0].UpdatedAt = time.Time{}
|
||||
},
|
||||
"invalid output url": func(s *DistributorState) {
|
||||
s.Outputs[0].URL = "file:///tmp/report.md"
|
||||
},
|
||||
"invalid primary url": func(s *DistributorState) {
|
||||
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
state := *withState(t, source, mutate)
|
||||
err := Validate(state)
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReportsURLFieldContext(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
state := *withState(t, source, func(s *DistributorState) {
|
||||
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
|
||||
})
|
||||
err := Validate(state)
|
||||
assertStateErrorContains(t, err, "state links.primary_url")
|
||||
assertStateErrorContains(t, err, "must not include a fragment")
|
||||
|
||||
state = *withState(t, source, func(s *DistributorState) {
|
||||
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
|
||||
})
|
||||
err = Validate(state)
|
||||
assertStateErrorContains(t, err, "state outputs[0].url")
|
||||
assertStateErrorContains(t, err, "must not include a query string")
|
||||
}
|
||||
|
||||
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "created_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
|
||||
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
URL: "https://reports.example.com/archive/report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
|
||||
t.Fatalf("json = %s, want links primary URL", data)
|
||||
}
|
||||
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
|
||||
t.Fatalf("json = %s, want output URL", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputHelpers(t *testing.T) {
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
updatedAt := createdAt.Add(time.Hour)
|
||||
retained := []OutputFile{{
|
||||
Path: "old.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}}
|
||||
projected := ProjectOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
}, {
|
||||
Path: "new.md",
|
||||
Kind: OutputKindSource,
|
||||
}}, retained, updatedAt)
|
||||
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
|
||||
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
|
||||
}
|
||||
merged, err := MergeOutputFiles(retained, projected)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOutputFiles() error = %v", err)
|
||||
}
|
||||
if got, want := len(merged), 3; got != want {
|
||||
t.Fatalf("merged count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("managed paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func validStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
||||
}
|
||||
|
||||
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
||||
t.Helper()
|
||||
return `{
|
||||
"schema_version": 2,
|
||||
"distributor_version": "dev",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"published_at": "2026-05-30T11:12:00Z",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z",
|
||||
"state": {"mode": "single_owner"},
|
||||
"reconciliation": {"mode": "replace"},
|
||||
"source": {
|
||||
"manifest": ` + manifest + `
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"path": "report.md",
|
||||
"kind": "source",
|
||||
"source_path": "report.md",
|
||||
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||
"size": 16,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"
|
||||
}
|
||||
]
|
||||
}`
|
||||
}
|
||||
|
||||
func legacyStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
|
||||
`"schema_version": 2`, `"schema_version": 1`, 1),
|
||||
` "created_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "updated_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "state": {"mode": "single_owner"},
|
||||
`, "", 1),
|
||||
` "reconciliation": {"mode": "replace"},
|
||||
`, "", 1),
|
||||
`,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest fixture: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func validManifest(t *testing.T) bundle.Manifest {
|
||||
t.Helper()
|
||||
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseManifest() error = %v", err)
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func assertStateErrorContains(t *testing.T, err error, want string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want substring %q", want)
|
||||
}
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
107
internal/state/document.go
Normal file
107
internal/state/document.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
legacyOwnerSchema = 2
|
||||
legacyMultiOwnerSchema = 3
|
||||
CatalogSchemaVersion = 4
|
||||
legacySchemaVersion = 1
|
||||
StateModeCatalog = "catalog"
|
||||
OutputKindSource = "source"
|
||||
OutputKindGenerated = "generated"
|
||||
)
|
||||
|
||||
type StateDocument struct {
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
}
|
||||
|
||||
type DestinationStatus struct {
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
|
||||
type SupersededLegacyState struct {
|
||||
SchemaVersion int
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type OwnerScope struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
}
|
||||
|
||||
type rawStatePolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
func ParseDocument(data []byte) (StateDocument, error) {
|
||||
schemaVersion, err := parseSchemaVersion(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
switch schemaVersion {
|
||||
case legacySchemaVersion, legacyOwnerSchema, legacyMultiOwnerSchema:
|
||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||
case CatalogSchemaVersion:
|
||||
catalog, err := ParseCatalog(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{Catalog: &catalog}, nil
|
||||
default:
|
||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
return *raw.SchemaVersion, nil
|
||||
}
|
||||
|
||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||
}
|
||||
|
||||
func parseRequiredTime(field string, raw *string) (time.Time, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", field, err)
|
||||
}
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
@@ -1,31 +1,5 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
type OutputProjection struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return OutputFile{}, false
|
||||
}
|
||||
|
||||
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
@@ -35,158 +9,6 @@ func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogO
|
||||
return CatalogOutputFile{}, false
|
||||
}
|
||||
|
||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
||||
for _, output := range retained {
|
||||
if _, exists := indexByPath[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
seenPlanned := make(map[string]struct{}, len(planned))
|
||||
for _, output := range planned {
|
||||
if _, exists := seenPlanned[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seenPlanned[output.Path] = struct{}{}
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(s DistributorState) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]OutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
files = append(files, OutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||
}
|
||||
|
||||
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
|
||||
for _, owner := range s.Owners {
|
||||
if owner.Scope == scope {
|
||||
return owner, true
|
||||
}
|
||||
}
|
||||
return OwnerRecord{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
|
||||
owner, ok := s.Owner(scope)
|
||||
if !ok {
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
return owner.Source.Manifest, true
|
||||
}
|
||||
|
||||
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func (s SharedRootState) AllManagedOutputPaths() []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
||||
selected := make([]CatalogOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
@@ -243,159 +65,6 @@ func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (Catalog
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
||||
for _, output := range s.Outputs {
|
||||
if output.Path == path {
|
||||
return output.Owner, true
|
||||
}
|
||||
}
|
||||
return OwnerScope{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
|
||||
for _, path := range paths {
|
||||
owner, exists := s.OutputOwner(path)
|
||||
if exists && owner != scope {
|
||||
return PathOwnershipConflict{
|
||||
Path: path,
|
||||
Owner: owner,
|
||||
CurrentOwner: scope,
|
||||
Detail: ComparisonDetail{
|
||||
Kind: ComparisonDetailSharedRootOutputOwner,
|
||||
Path: path,
|
||||
CurrentOwner: scope,
|
||||
ConflictingOwner: owner,
|
||||
},
|
||||
}, true
|
||||
}
|
||||
}
|
||||
return PathOwnershipConflict{}, false
|
||||
}
|
||||
|
||||
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
files = append(files, SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, planned...)
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
for _, output := range planned {
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
next.Outputs = outputs
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
|
||||
seen := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
|
||||
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, output := range outputs {
|
||||
if output.Owner != scope {
|
||||
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
|
||||
next := append([]OwnerRecord(nil), owners...)
|
||||
for index, existing := range next {
|
||||
if existing.Scope == owner.Scope {
|
||||
next[index] = owner
|
||||
return next
|
||||
}
|
||||
}
|
||||
return append(next, owner)
|
||||
}
|
||||
|
||||
func pathSet(paths []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
|
||||
@@ -22,33 +22,6 @@ type PrunePlan struct {
|
||||
Preserved []PruneCandidate
|
||||
}
|
||||
|
||||
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
continue
|
||||
}
|
||||
owner := output.Owner
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
Owner: &owner,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
|
||||
@@ -74,19 +74,6 @@ func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
|
||||
sharedRoot := validSharedRootState(t)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
candidates := SharedRootPruneCandidates(sharedRoot, scope)
|
||||
|
||||
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
|
||||
t.Fatalf("candidates = %q, want %q", got, want)
|
||||
}
|
||||
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
|
||||
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
||||
paths := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
|
||||
@@ -1,531 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type StateDocument struct {
|
||||
SingleOwner *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
}
|
||||
|
||||
type SupersededLegacyState struct {
|
||||
SchemaVersion int
|
||||
}
|
||||
|
||||
type SharedRootState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Owners []OwnerRecord
|
||||
Outputs []SharedRootOutputFile
|
||||
}
|
||||
|
||||
type OwnerScope struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
}
|
||||
|
||||
type OwnerRecord struct {
|
||||
Scope OwnerScope
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
}
|
||||
|
||||
type SharedRootOutputFile struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
Owner OwnerScope
|
||||
SourceID string
|
||||
SourceDigest string
|
||||
SourceCreated time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type PathOwnershipConflict struct {
|
||||
Path string
|
||||
Owner OwnerScope
|
||||
CurrentOwner OwnerScope
|
||||
Detail ComparisonDetail
|
||||
}
|
||||
|
||||
type rawSharedRootState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Owners []rawOwnerRecord `json:"owners"`
|
||||
Outputs []rawSharedRootOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawOwnerRecord struct {
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
}
|
||||
|
||||
type rawSharedRootOutput struct {
|
||||
Path *string `json:"path"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform string `json:"transform"`
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
SourceID *string `json:"source_id"`
|
||||
SourceDigest *string `json:"source_digest"`
|
||||
SourceCreated *string `json:"source_created"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func ParseDocument(data []byte) (StateDocument, error) {
|
||||
schemaVersion, err := parseSchemaVersion(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
switch schemaVersion {
|
||||
case legacySchemaVersion, SchemaVersion, SharedRootSchemaVersion:
|
||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||
case CatalogSchemaVersion:
|
||||
catalog, err := ParseCatalog(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{Catalog: &catalog}, nil
|
||||
default:
|
||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
return *raw.SchemaVersion, nil
|
||||
}
|
||||
|
||||
func ParseSharedRoot(data []byte) (SharedRootState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw rawSharedRootState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseSharedRootRaw(raw)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
if err := ValidateSharedRoot(state); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
|
||||
if raw.SchemaVersion == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
|
||||
if state.SchemaVersion != SharedRootSchemaVersion {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return SharedRootState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Owners == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners, err := parseOwnerRecords(raw.Owners)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Owners = owners
|
||||
if raw.Outputs == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseSharedRootOutputs(raw.Outputs)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
|
||||
owners := make([]OwnerRecord, 0, len(rawOwners))
|
||||
for index, raw := range rawOwners {
|
||||
owner, err := parseOwnerRecord(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owners = append(owners, owner)
|
||||
}
|
||||
return owners, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
|
||||
}
|
||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||
if err != nil {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
owner := OwnerRecord{
|
||||
Scope: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
|
||||
Source: SourceState{Manifest: manifest},
|
||||
}
|
||||
if raw.Links != nil {
|
||||
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||
}
|
||||
return owner, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
|
||||
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseSharedRootOutput(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.SourceID == nil || *raw.SourceID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
|
||||
}
|
||||
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
return SharedRootOutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
SourcePath: *raw.SourcePath,
|
||||
Transform: raw.Transform,
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
Owner: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
SourceID: *raw.SourceID,
|
||||
SourceDigest: *raw.SourceDigest,
|
||||
SourceCreated: sourceCreated,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s SharedRootState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) SourceCreatedString() string {
|
||||
return o.SourceCreated.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) MarshalJSON() ([]byte, error) {
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Owners []OwnerRecord `json:"owners"`
|
||||
Outputs []SharedRootOutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Owners: s.Owners,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
}
|
||||
type ownerJSON struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
}
|
||||
return json.Marshal(ownerJSON{
|
||||
PipelineID: o.Scope.PipelineID,
|
||||
DestinationID: o.Scope.DestinationID,
|
||||
Reconciliation: o.Reconciliation,
|
||||
Source: sourceJSON{Manifest: o.Source.Manifest},
|
||||
Links: o.Links,
|
||||
})
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceDigest string `json:"source_digest"`
|
||||
SourceCreated string `json:"source_created"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
PipelineID: o.Owner.PipelineID,
|
||||
DestinationID: o.Owner.DestinationID,
|
||||
SourceID: o.SourceID,
|
||||
SourceDigest: o.SourceDigest,
|
||||
SourceCreated: o.SourceCreatedString(),
|
||||
CreatedAt: o.CreatedAtString(),
|
||||
UpdatedAt: o.UpdatedAtString(),
|
||||
})
|
||||
}
|
||||
|
||||
func ValidateSharedRoot(s SharedRootState) error {
|
||||
if s.SchemaVersion != SharedRootSchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSharedRoot {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
|
||||
}
|
||||
if s.Owners == nil {
|
||||
return fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
|
||||
for index, owner := range s.Owners {
|
||||
if err := validateOwnerRecord(index, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := owners[owner.Scope]; exists {
|
||||
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
|
||||
}
|
||||
owners[owner.Scope] = owner
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateSharedRootOutput(index, output, owners); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seenPaths[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seenPaths[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOwnerRecord(index int, owner OwnerRecord) error {
|
||||
if owner.Scope.PipelineID == "" {
|
||||
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if owner.Scope.DestinationID == "" {
|
||||
return fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
if owner.Links != nil && owner.Links.PrimaryURL != "" {
|
||||
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
|
||||
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource, OutputKindGenerated:
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.Owner.PipelineID == "" {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if output.Owner.DestinationID == "" {
|
||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if _, exists := owners[output.Owner]; !exists {
|
||||
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
|
||||
}
|
||||
if output.SourceID == "" {
|
||||
return fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
|
||||
}
|
||||
if output.SourceCreated.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].source_created is required", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,314 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseSharedRootState(t *testing.T) {
|
||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseSharedRoot() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 2; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
manifest, ok := state.SourceManifest(scope)
|
||||
if !ok {
|
||||
t.Fatal("SourceManifest() ok = false, want true")
|
||||
}
|
||||
if got, want := manifest.ID, validManifest(t).ID; got != want {
|
||||
t.Fatalf("source manifest id = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
|
||||
tests := map[string]func(string) string{
|
||||
"schema": func(body string) string {
|
||||
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
|
||||
},
|
||||
"state mode": func(body string) string {
|
||||
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
|
||||
},
|
||||
"duplicate owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
|
||||
},
|
||||
"unknown output owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
|
||||
},
|
||||
"duplicate output": func(body string) string {
|
||||
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
|
||||
},
|
||||
"invalid source digest": func(body string) string {
|
||||
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
|
||||
},
|
||||
"invalid owner link": func(body string) string {
|
||||
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
|
||||
if err == nil {
|
||||
t.Fatal("ParseSharedRoot() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state.UpdatedAt = state.CreatedAt
|
||||
state.Outputs[0].SourceCreated = state.CreatedAt
|
||||
state.Outputs[0].CreatedAt = state.CreatedAt
|
||||
state.Outputs[0].UpdatedAt = state.CreatedAt
|
||||
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`"created_at":"2026-05-30T11:12:00Z"`,
|
||||
`"updated_at":"2026-05-30T11:12:00Z"`,
|
||||
`"source_created":"2026-05-30T11:12:00Z"`,
|
||||
} {
|
||||
if !strings.Contains(string(data), want) {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOutputHelpers(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
html := CurrentOwnerScope("reports", "html")
|
||||
|
||||
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("archive paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("all paths = %#v, want %#v", got, want)
|
||||
}
|
||||
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
|
||||
if !ok || conflict.Owner != html {
|
||||
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
|
||||
}
|
||||
if conflict.Detail.Kind != ComparisonDetailSharedRootOutputOwner {
|
||||
t.Fatalf("conflict detail kind = %q, want %q", conflict.Detail.Kind, ComparisonDetailSharedRootOutputOwner)
|
||||
}
|
||||
if conflict.Detail.Path != "report.html" || conflict.Detail.CurrentOwner != archive || conflict.Detail.ConflictingOwner != html {
|
||||
t.Fatalf("conflict detail = %#v, want path and owners", conflict.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if _, ok := next.OutputOwner("report.html"); !ok {
|
||||
t.Fatal("report.html owner missing, want unrelated owner preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
|
||||
}
|
||||
if got := next.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("paths = %#v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
|
||||
planned := ProjectSharedRootOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: validManifest(t).Files[0].SHA256,
|
||||
Size: validManifest(t).Files[0].Size,
|
||||
}, {
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: validManifest(t).Files[1].SHA256,
|
||||
Size: validManifest(t).Files[1].Size,
|
||||
}}, state.Outputs, archive, validManifest(t), now)
|
||||
|
||||
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("merged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if merged.Outputs[0].CreatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
|
||||
}
|
||||
if !merged.Outputs[0].UpdatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
|
||||
}
|
||||
|
||||
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("replaced paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
planned := []SharedRootOutputFile{{
|
||||
Path: "report.html",
|
||||
Kind: OutputKindSource,
|
||||
Owner: archive,
|
||||
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
}}
|
||||
|
||||
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
source := validManifest(t)
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
|
||||
}
|
||||
|
||||
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if missing.Outcome != OutcomeDestinationAbsent {
|
||||
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
|
||||
}
|
||||
if missing.Detail.Kind != ComparisonDetailSharedRootOwnerAbsent {
|
||||
t.Fatalf("missing owner detail kind = %q, want %q", missing.Detail.Kind, ComparisonDetailSharedRootOwnerAbsent)
|
||||
}
|
||||
if missing.Detail.CurrentOwner != CurrentOwnerScope("missing", "archive") {
|
||||
t.Fatalf("missing owner detail = %#v, want missing/archive", missing.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
|
||||
singleOwner, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
source := singleOwner.Source.Manifest
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older", comparison)
|
||||
}
|
||||
}
|
||||
|
||||
func validSharedRootStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared root state: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func validSharedRootState(t *testing.T) SharedRootState {
|
||||
t.Helper()
|
||||
source := validManifest(t)
|
||||
htmlSource := source
|
||||
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return SharedRootState{
|
||||
SchemaVersion: SharedRootSchemaVersion,
|
||||
DistributorVersion: "dev",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: StatePolicy{Mode: StateModeSharedRoot},
|
||||
Owners: []OwnerRecord{{
|
||||
Scope: CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
}, {
|
||||
Scope: CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
URL: "https://reports.example.com/html/report.html",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 128,
|
||||
Owner: CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
OutputKindSource = "source"
|
||||
OutputKindGenerated = "generated"
|
||||
)
|
||||
|
||||
func Validate(s DistributorState) error {
|
||||
if s.SchemaVersion != SchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
||||
}
|
||||
if s.PipelineID == "" {
|
||||
return fmt.Errorf("state pipeline_id is required")
|
||||
}
|
||||
if s.DestinationID == "" {
|
||||
return fmt.Errorf("state destination_id is required")
|
||||
}
|
||||
if s.PublishedAt.IsZero() {
|
||||
return fmt.Errorf("state published_at is required")
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSingleOwner {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
|
||||
}
|
||||
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state source.manifest: %w", err)
|
||||
}
|
||||
if s.Links != nil && s.Links.PrimaryURL != "" {
|
||||
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
|
||||
return fmt.Errorf("state links.primary_url: %w", err)
|
||||
}
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateOutput(index, output); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateEmbeddedManifest(manifest bundle.Manifest) error {
|
||||
return bundle.ValidateManifest(manifest)
|
||||
}
|
||||
|
||||
func validateOutput(index int, output OutputFile) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource, OutputKindGenerated:
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
)
|
||||
@@ -33,13 +31,6 @@ type BundleOptions struct {
|
||||
ExtraFiles []SourceFile
|
||||
}
|
||||
|
||||
type DestinationStateOptions struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
DistributorVersion string
|
||||
PublishedAt time.Time
|
||||
}
|
||||
|
||||
func DefaultSourceFiles() []SourceFile {
|
||||
return []SourceFile{
|
||||
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
||||
@@ -133,28 +124,6 @@ func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
|
||||
}
|
||||
}
|
||||
|
||||
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
t.Helper()
|
||||
destinationState := DestinationState(manifest, opts)
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal destination state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
WriteFakeFile(t, backend, path, "old")
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
@@ -318,47 +287,6 @@ pipelines:
|
||||
`)
|
||||
}
|
||||
|
||||
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
t.Helper()
|
||||
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir destination bundle: %v", err)
|
||||
}
|
||||
destinationState := DestinationState(manifest, opts)
|
||||
writeJSONFile(t, filepath.Join(bundleRoot, storage.StateFileName), destinationState)
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
publishedAt := defaultPublishedAt(opts.PublishedAt)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: opts.DistributorVersion,
|
||||
PipelineID: defaultString(opts.PipelineID, "reports"),
|
||||
DestinationID: defaultString(opts.DestinationID, "archive"),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
Outputs: sourceOutputs(manifest, publishedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func ReadDestinationState(t testing.TB, path string) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read destination state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse destination state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func AssertFile(t testing.TB, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
@@ -392,22 +320,6 @@ func sourceFiles(opts BundleOptions) []SourceFile {
|
||||
return files
|
||||
}
|
||||
|
||||
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
|
||||
outputs := make([]state.OutputFile, 0, len(manifest.Files))
|
||||
for _, file := range manifest.Files {
|
||||
outputs = append(outputs, state.OutputFile{
|
||||
Path: file.Path,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
func writeConfigFile(t testing.TB, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
@@ -455,10 +367,3 @@ func defaultCreated(value time.Time) time.Time {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultPublishedAt(value time.Time) time.Time {
|
||||
if value.IsZero() {
|
||||
return time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user