Remove legacy publish and state paths
This commit is contained in:
@@ -4,15 +4,15 @@ Audience: developers and LLM coding agents changing `internal/state`.
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/state` parses, validates, serializes, and transforms `.distributor.json` destination catalog state.
|
`internal/state` parses, validates, serializes, and updates `.distributor.json` destination catalog state records.
|
||||||
|
|
||||||
## Inputs And Outputs
|
## Inputs And Outputs
|
||||||
|
|
||||||
Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair projections, and prune candidate plans.
|
Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair results, and prune candidate plans.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and helper projections.
|
The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and catalog output helpers.
|
||||||
|
|
||||||
The external destination state contract is documented in `docs/integrations/destination-state.md`.
|
The external destination state contract is documented in `docs/integrations/destination-state.md`.
|
||||||
|
|
||||||
@@ -36,7 +36,7 @@ The package identifies schema versions older than the current catalog schema as
|
|||||||
|
|
||||||
The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates.
|
The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates.
|
||||||
|
|
||||||
Catalog helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution.
|
Publish execution owns catalog output projection and timestamp preservation for rewritten outputs. State helpers only parse, validate, filter, and remove catalog records supplied by callers.
|
||||||
|
|
||||||
## Skip And Resume Behavior
|
## Skip And Resume Behavior
|
||||||
|
|
||||||
|
|||||||
@@ -199,7 +199,7 @@ Use this current layout unless the project has a documented reason to differ:
|
|||||||
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
||||||
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||||
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
||||||
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
- `internal/state`: `.distributor.json` catalog parsing, validation, and output metadata.
|
||||||
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
||||||
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
@@ -207,7 +207,7 @@ Use this current layout unless the project has a documented reason to differ:
|
|||||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||||
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
||||||
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
||||||
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
|
- `internal/publish`: destination planning, catalog workflow safety checks, and publish execution.
|
||||||
- `internal/notify`: notification interface and MVP no-op notifier.
|
- `internal/notify`: notification interface and MVP no-op notifier.
|
||||||
- `internal/logging`: logging setup and shared logging helpers.
|
- `internal/logging`: logging setup and shared logging helpers.
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
|||||||
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
||||||
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
||||||
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
||||||
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
- `internal/state`: destination `.distributor.json` catalog parsing, validation, and output metadata.
|
||||||
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||||
|
|||||||
@@ -265,9 +265,6 @@ func unsupportedStateDocumentError(document state.StateDocument) error {
|
|||||||
if document.SupersededLegacy != nil {
|
if document.SupersededLegacy != nil {
|
||||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||||
}
|
}
|
||||||
if document.SingleOwner != nil || document.SharedRoot != nil {
|
|
||||||
return fmt.Errorf("legacy destination state is not supported by this command")
|
|
||||||
}
|
|
||||||
return fmt.Errorf("destination state document is empty")
|
return fmt.Errorf("destination state document is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,13 @@ func shouldNotify(action publish.Action) bool {
|
|||||||
func notifyEvent(plan publish.Plan) notify.Event {
|
func notifyEvent(plan publish.Plan) notify.Event {
|
||||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||||
for _, output := range plan.Outputs {
|
for _, output := range plan.Outputs {
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
outputs = append(outputs, notify.Output{
|
outputs = append(outputs, notify.Output{
|
||||||
Path: stateOutput.Path,
|
Path: output.DestinationPath,
|
||||||
Kind: stateOutput.Kind,
|
Kind: output.Kind,
|
||||||
SourcePath: stateOutput.SourcePath,
|
SourcePath: output.SourcePath,
|
||||||
Transform: stateOutput.Transform,
|
Transform: output.Transform,
|
||||||
SHA256: stateOutput.SHA256,
|
SHA256: output.SHA256,
|
||||||
Size: stateOutput.Size,
|
Size: output.Size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return notify.Event{
|
return notify.Event{
|
||||||
|
|||||||
@@ -220,15 +220,14 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
|
|||||||
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
||||||
results := make([]RunOutputRecord, 0, len(outputs))
|
results := make([]RunOutputRecord, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
results = append(results, RunOutputRecord{
|
results = append(results, RunOutputRecord{
|
||||||
Path: stateOutput.Path,
|
Path: output.DestinationPath,
|
||||||
Kind: stateOutput.Kind,
|
Kind: output.Kind,
|
||||||
SourcePath: stateOutput.SourcePath,
|
SourcePath: output.SourcePath,
|
||||||
Transform: stateOutput.Transform,
|
Transform: output.Transform,
|
||||||
URL: stateOutput.URL,
|
URL: output.URL,
|
||||||
SHA256: stateOutput.SHA256,
|
SHA256: output.SHA256,
|
||||||
Size: stateOutput.Size,
|
Size: output.Size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return results
|
return results
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
|||||||
s.replaceCatalog++
|
s.replaceCatalog++
|
||||||
case publish.ActionForceReplace:
|
case publish.ActionForceReplace:
|
||||||
s.forceReplace++
|
s.forceReplace++
|
||||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
case publish.ActionSkipSame:
|
||||||
s.skipSame++
|
s.skipSame++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -210,14 +210,14 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
|||||||
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
||||||
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
||||||
}
|
}
|
||||||
if destinationState.Source.Manifest.ID != manifest.ID {
|
if destinationState.SourceID != manifest.ID {
|
||||||
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
|
t.Fatalf("state source id = %q, want %q", destinationState.SourceID, manifest.ID)
|
||||||
}
|
}
|
||||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
|
if destinationState.PrimaryURL != "" || destinationState.Outputs[0].URL != "" {
|
||||||
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
|
t.Fatalf("state primary URL = %q output URL=%q, want absent", destinationState.PrimaryURL, destinationState.Outputs[0].URL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,8 +362,8 @@ func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
||||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
if destinationState.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
|
t.Fatalf("state primary URL = %q, want source primary URL", destinationState.PrimaryURL)
|
||||||
}
|
}
|
||||||
outputs := outputsByPath(destinationState.Outputs)
|
outputs := outputsByPath(destinationState.Outputs)
|
||||||
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
@@ -387,8 +387,8 @@ func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
|
if destinationState.PrimaryURL != "https://reports.example.com/latest/" {
|
||||||
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
|
t.Fatalf("state primary URL = %q, want fixed index primary URL", destinationState.PrimaryURL)
|
||||||
}
|
}
|
||||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
@@ -427,8 +427,8 @@ func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
|
|||||||
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
if destinationState.SourceID != "reports.new" {
|
||||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
t.Fatalf("state source id = %q, want reports.new", destinationState.SourceID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,8 +457,8 @@ func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Source.Manifest.ID != "reports.a" {
|
if destinationState.SourceID != "reports.a" {
|
||||||
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
|
t.Fatalf("state source id = %q, want reports.a", destinationState.SourceID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1807,11 +1807,6 @@ func writeConfigFile(t *testing.T, body string) string {
|
|||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
|
|
||||||
t.Helper()
|
|
||||||
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||||
@@ -1829,8 +1824,10 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
|||||||
type testDestinationState struct {
|
type testDestinationState struct {
|
||||||
PipelineID string
|
PipelineID string
|
||||||
DestinationID string
|
DestinationID string
|
||||||
Source state.SourceState
|
SourceID string
|
||||||
Links *state.LinkState
|
SourceDigest string
|
||||||
|
SourceCreated time.Time
|
||||||
|
PrimaryURL string
|
||||||
Outputs []testStateOutput
|
Outputs []testStateOutput
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1859,11 +1856,11 @@ func readStateFile(t *testing.T, path string) testDestinationState {
|
|||||||
if index == 0 {
|
if index == 0 {
|
||||||
view.PipelineID = output.PipelineID
|
view.PipelineID = output.PipelineID
|
||||||
view.DestinationID = output.DestinationID
|
view.DestinationID = output.DestinationID
|
||||||
view.Source.Manifest.ID = output.Source.ID
|
view.SourceID = output.Source.ID
|
||||||
view.Source.Manifest.Digest = output.Source.Digest
|
view.SourceDigest = output.Source.Digest
|
||||||
view.Source.Manifest.Created = output.Source.Created
|
view.SourceCreated = output.Source.Created
|
||||||
if output.URL != "" {
|
if output.URL != "" {
|
||||||
view.Links = &state.LinkState{PrimaryURL: output.URL}
|
view.PrimaryURL = output.URL
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
view.Outputs = append(view.Outputs, testStateOutput{
|
view.Outputs = append(view.Outputs, testStateOutput{
|
||||||
|
|||||||
@@ -38,29 +38,25 @@ type Pipeline struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Destination struct {
|
type Destination struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
Backend string `yaml:"backend"`
|
Backend string `yaml:"backend"`
|
||||||
Host string `yaml:"host"`
|
Host string `yaml:"host"`
|
||||||
User string `yaml:"user"`
|
User string `yaml:"user"`
|
||||||
Port int `yaml:"port"`
|
Port int `yaml:"port"`
|
||||||
Path string `yaml:"path"`
|
Path string `yaml:"path"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Bucket string `yaml:"bucket"`
|
Bucket string `yaml:"bucket"`
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
Region string `yaml:"region"`
|
Region string `yaml:"region"`
|
||||||
ForcePath *bool `yaml:"force_path_style"`
|
ForcePath *bool `yaml:"force_path_style"`
|
||||||
Creds Credentials `yaml:"credentials"`
|
Creds Credentials `yaml:"credentials"`
|
||||||
SSH SSH `yaml:",inline"`
|
SSH SSH `yaml:",inline"`
|
||||||
Publish *PublishPolicy `yaml:"publish"`
|
Publish *PublishPolicy `yaml:"publish"`
|
||||||
Transform Transform `yaml:"transform"`
|
Transform Transform `yaml:"transform"`
|
||||||
PathMap PathMapping `yaml:"path_mapping"`
|
PathMap PathMapping `yaml:"path_mapping"`
|
||||||
Links *Links `yaml:"links"`
|
Links *Links `yaml:"links"`
|
||||||
Workflow string `yaml:"workflow"`
|
Workflow string `yaml:"workflow"`
|
||||||
State StatePolicy `yaml:"-"`
|
Retention RetentionPolicy `yaml:"retention"`
|
||||||
Reconciliation ReconciliationPolicy `yaml:"-"`
|
|
||||||
Takeover TakeoverPolicy `yaml:"-"`
|
|
||||||
Retention RetentionPolicy `yaml:"retention"`
|
|
||||||
Transfer TransferPolicy `yaml:"-"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Backend struct {
|
type Backend struct {
|
||||||
@@ -124,18 +120,6 @@ type Links struct {
|
|||||||
Primary string `yaml:"primary"`
|
Primary string `yaml:"primary"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReconciliationPolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TakeoverPolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type StatePolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RetentionPolicy struct {
|
type RetentionPolicy struct {
|
||||||
Prune PrunePolicy `yaml:"prune"`
|
Prune PrunePolicy `yaml:"prune"`
|
||||||
}
|
}
|
||||||
@@ -145,10 +129,3 @@ type PrunePolicy struct {
|
|||||||
OlderThan *Duration `yaml:"older_than"`
|
OlderThan *Duration `yaml:"older_than"`
|
||||||
KeepLatest *int `yaml:"keep_latest"`
|
KeepLatest *int `yaml:"keep_latest"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TransferPolicy struct {
|
|
||||||
OnDestinationSame string `yaml:"on_destination_same"`
|
|
||||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
|
||||||
OnDestinationNewer string `yaml:"on_destination_newer"`
|
|
||||||
OnConflict string `yaml:"on_conflict"`
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -20,12 +20,6 @@ const (
|
|||||||
ValidationActionFail = "fail"
|
ValidationActionFail = "fail"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
TransferActionSkip = "skip"
|
|
||||||
TransferActionReplace = "replace"
|
|
||||||
TransferActionFail = "fail"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
TransformModeSidecar = transform.MarkdownModeSidecar
|
TransformModeSidecar = transform.MarkdownModeSidecar
|
||||||
TransformModeIndex = transform.MarkdownModeIndex
|
TransformModeIndex = transform.MarkdownModeIndex
|
||||||
@@ -47,23 +41,6 @@ const (
|
|||||||
WorkflowReplacement = "replacement"
|
WorkflowReplacement = "replacement"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
ReconciliationModeReplace = "replace"
|
|
||||||
ReconciliationModeMerge = "merge"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
TakeoverModeSamePipeline = "same_pipeline"
|
|
||||||
TakeoverModeSameSource = "same_source"
|
|
||||||
TakeoverModeAnyManaged = "any_managed"
|
|
||||||
TakeoverModeNever = "never"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
StateModeSingleOwner = "single_owner"
|
|
||||||
StateModeSharedRoot = "shared_root"
|
|
||||||
)
|
|
||||||
|
|
||||||
const DefaultS3Region = "us-east-1"
|
const DefaultS3Region = "us-east-1"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -104,27 +81,6 @@ func ApplyDefaults(cfg *Config) {
|
|||||||
if destination.Workflow == "" {
|
if destination.Workflow == "" {
|
||||||
destination.Workflow = WorkflowAdditive
|
destination.Workflow = WorkflowAdditive
|
||||||
}
|
}
|
||||||
if destination.State.Mode == "" {
|
|
||||||
destination.State.Mode = StateModeSingleOwner
|
|
||||||
}
|
|
||||||
if destination.Reconciliation.Mode == "" {
|
|
||||||
destination.Reconciliation.Mode = ReconciliationModeReplace
|
|
||||||
}
|
|
||||||
if destination.Takeover.Mode == "" {
|
|
||||||
destination.Takeover.Mode = TakeoverModeSamePipeline
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationSame == "" {
|
|
||||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationOlder == "" {
|
|
||||||
destination.Transfer.OnDestinationOlder = TransferActionReplace
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationNewer == "" {
|
|
||||||
destination.Transfer.OnDestinationNewer = TransferActionSkip
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnConflict == "" {
|
|
||||||
destination.Transfer.OnConflict = TransferActionFail
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,140 +5,20 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Execute(ctx context.Context, req Request, plan Plan) error {
|
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
||||||
switch plan.Action {
|
switch plan.Action {
|
||||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
case ActionSkipSame:
|
||||||
return nil
|
return nil
|
||||||
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
|
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
|
||||||
return executeCatalog(ctx, req, plan)
|
return executeCatalog(ctx, req, plan)
|
||||||
case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover:
|
|
||||||
if usesSharedRootState(req, plan) {
|
|
||||||
return executeSharedRoot(ctx, req, plan)
|
|
||||||
}
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceNewer || plan.Action == ActionReplaceTakeover {
|
|
||||||
if plan.ExistingState == nil {
|
|
||||||
return fmt.Errorf("replace requires existing destination state")
|
|
||||||
}
|
|
||||||
if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceTakeover {
|
|
||||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if plan.Action == ActionForceReplace {
|
|
||||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if usesMergeRetention(plan) {
|
|
||||||
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
cleanup := func() {
|
|
||||||
outputs := writtenOutputs
|
|
||||||
if usesMergeRetention(plan) {
|
|
||||||
outputs = newOutputs
|
|
||||||
}
|
|
||||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
||||||
}
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data := output.Data
|
|
||||||
if output.Kind == state.OutputKindSource {
|
|
||||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
managed := outputManagedByExistingState(output, plan.ExistingState)
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
writtenOutputs = append(writtenOutputs, output)
|
|
||||||
if !managed {
|
|
||||||
newOutputs = append(newOutputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
createdAt := now
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
createdAt = plan.ExistingState.CreatedAt
|
|
||||||
}
|
|
||||||
stateOutputs, err := stateOutputsForPlan(plan, now)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
destinationState := state.DistributorState{
|
|
||||||
SchemaVersion: state.SchemaVersion,
|
|
||||||
DistributorVersion: req.DistributorVersion,
|
|
||||||
PipelineID: req.PipelineID,
|
|
||||||
DestinationID: req.DestinationID,
|
|
||||||
PublishedAt: now,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
||||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
||||||
Outputs: stateOutputs,
|
|
||||||
}
|
|
||||||
if plan.PrimaryURL != "" {
|
|
||||||
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
||||||
}
|
|
||||||
if err := state.Validate(destinationState); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data = append(data, '\n')
|
|
||||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
||||||
@@ -293,264 +173,3 @@ func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
|
|||||||
}
|
}
|
||||||
return paths
|
return paths
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
|
||||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
||||||
if plan.Action == ActionForceReplace {
|
|
||||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if plan.Action == ActionReplaceTakeover || plan.Action == ActionReplaceConflict || (isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeReplace) {
|
|
||||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
cleanup := func() {
|
|
||||||
outputs := writtenOutputs
|
|
||||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
||||||
outputs = newOutputs
|
|
||||||
}
|
|
||||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
||||||
}
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data := output.Data
|
|
||||||
if output.Kind == state.OutputKindSource {
|
|
||||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
managed := outputManagedBySharedRootPlan(output, plan)
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
writtenOutputs = append(writtenOutputs, output)
|
|
||||||
if !managed {
|
|
||||||
newOutputs = append(newOutputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data = append(data, '\n')
|
|
||||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
if outputManagedByExistingState(output, plan.ExistingState) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
|
||||||
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
|
||||||
} else if !storage.IsNotFound(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func usesSharedRootState(req Request, plan Plan) bool {
|
|
||||||
return plan.StateMode == config.StateModeSharedRoot
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
|
||||||
if existing == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
|
||||||
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
|
||||||
if existing.Path == output.DestinationPath {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, existing := range plan.TakenOverOwnerOutputs {
|
|
||||||
if existing.Path == output.DestinationPath {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
|
||||||
existingOutputs := []state.OutputFile(nil)
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
existingOutputs = plan.ExistingState.Outputs
|
|
||||||
}
|
|
||||||
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
|
||||||
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
|
||||||
return planned, nil
|
|
||||||
}
|
|
||||||
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
|
||||||
}
|
|
||||||
|
|
||||||
func usesMergeRetention(plan Plan) bool {
|
|
||||||
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && isReconciliationReplacementAction(plan.Action)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
|
||||||
now = now.UTC()
|
|
||||||
scope := plan.OwnerScope
|
|
||||||
if scope.PipelineID == "" && scope.DestinationID == "" {
|
|
||||||
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
|
||||||
}
|
|
||||||
base := sharedRootBaseState(req, plan, now)
|
|
||||||
base = removeTakenOverSharedRootOutputs(base, plan.TakenOverOwnerOutputs)
|
|
||||||
owner := state.OwnerRecord{
|
|
||||||
Scope: scope,
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
||||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
||||||
}
|
|
||||||
if plan.PrimaryURL != "" {
|
|
||||||
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
||||||
}
|
|
||||||
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
|
||||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
||||||
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
|
||||||
}
|
|
||||||
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
|
||||||
if plan.Action == ActionForceReplace {
|
|
||||||
return newSharedRootState(req, now)
|
|
||||||
}
|
|
||||||
if plan.ExistingSharedRoot != nil {
|
|
||||||
base := *plan.ExistingSharedRoot
|
|
||||||
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
|
||||||
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
|
||||||
base.DistributorVersion = req.DistributorVersion
|
|
||||||
base.UpdatedAt = now
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
base := newSharedRootState(req, now)
|
|
||||||
base.CreatedAt = plan.ExistingState.CreatedAt
|
|
||||||
base.UpdatedAt = now
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return newSharedRootState(req, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
func removeTakenOverSharedRootOutputs(sharedRoot state.SharedRootState, takenOver []state.SharedRootOutputFile) state.SharedRootState {
|
|
||||||
if len(takenOver) == 0 {
|
|
||||||
return sharedRoot
|
|
||||||
}
|
|
||||||
paths := sharedRootOutputPathSet(takenOver)
|
|
||||||
next := sharedRoot
|
|
||||||
next.Outputs = make([]state.SharedRootOutputFile, 0, len(sharedRoot.Outputs))
|
|
||||||
for _, output := range sharedRoot.Outputs {
|
|
||||||
if _, remove := paths[output.Path]; remove {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
|
|
||||||
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
|
||||||
return state.SharedRootState{
|
|
||||||
SchemaVersion: state.SharedRootSchemaVersion,
|
|
||||||
DistributorVersion: req.DistributorVersion,
|
|
||||||
CreatedAt: now,
|
|
||||||
UpdatedAt: now,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
|
||||||
Owners: []state.OwnerRecord{},
|
|
||||||
Outputs: []state.SharedRootOutputFile{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
|
||||||
if plan.ExistingSharedRoot != nil {
|
|
||||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
|
||||||
for _, output := range plan.ExistingSharedRoot.Outputs {
|
|
||||||
if output.Owner == plan.OwnerScope {
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
|
||||||
for _, output := range plan.ExistingState.Outputs {
|
|
||||||
outputs = append(outputs, state.SharedRootOutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
Owner: plan.OwnerScope,
|
|
||||||
SourceID: plan.ExistingState.Source.Manifest.ID,
|
|
||||||
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
|
||||||
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
|
||||||
CreatedAt: output.CreatedAt,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
|
||||||
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing
|
|||||||
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
req.DestinationBundlePath = "bundle"
|
req.DestinationBundlePath = "bundle"
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||||
@@ -110,7 +110,7 @@ func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||||
|
|||||||
@@ -98,50 +98,10 @@ func rejectOutputCollisions(outputs []Output) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o Output) StateOutputFile() state.OutputFile {
|
|
||||||
return state.OutputFile{
|
|
||||||
Path: o.DestinationPath,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o Output) StateOutputProjection() state.OutputProjection {
|
|
||||||
return state.OutputProjection{
|
|
||||||
Path: o.DestinationPath,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o Output) ManagedPath() string {
|
func (o Output) ManagedPath() string {
|
||||||
return o.DestinationPath
|
return o.DestinationPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func StateOutputFiles(outputs []Output) []state.OutputFile {
|
|
||||||
files := make([]state.OutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
files = append(files, output.StateOutputFile())
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func StateOutputProjections(outputs []Output) []state.OutputProjection {
|
|
||||||
projections := make([]state.OutputProjection, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
projections = append(projections, output.StateOutputProjection())
|
|
||||||
}
|
|
||||||
return projections
|
|
||||||
}
|
|
||||||
|
|
||||||
func ManagedOutputPaths(outputs []Output) []string {
|
func ManagedOutputPaths(outputs []Output) []string {
|
||||||
paths := make([]string, 0, len(outputs))
|
paths := make([]string, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
|
|||||||
@@ -12,46 +12,12 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOutputStateProjection(t *testing.T) {
|
func TestManagedOutputPaths(t *testing.T) {
|
||||||
sourceOutput := Output{
|
|
||||||
SourcePath: "report.md",
|
|
||||||
DestinationPath: "report.md",
|
|
||||||
Kind: state.OutputKindSource,
|
|
||||||
URL: "https://reports.example.com/report.md",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 123,
|
|
||||||
}
|
|
||||||
sourceState := sourceOutput.StateOutputFile()
|
|
||||||
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
|
|
||||||
t.Fatalf("source state output = %#v", sourceState)
|
|
||||||
}
|
|
||||||
|
|
||||||
generatedOutput := Output{
|
|
||||||
SourcePath: "report.md",
|
|
||||||
DestinationPath: "report.html",
|
|
||||||
Kind: state.OutputKindGenerated,
|
|
||||||
Transform: transform.MarkdownToHTML,
|
|
||||||
URL: "https://reports.example.com/report.html",
|
|
||||||
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
||||||
Size: 456,
|
|
||||||
}
|
|
||||||
generatedState := generatedOutput.StateOutputFile()
|
|
||||||
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
|
|
||||||
t.Fatalf("generated state output = %#v", generatedState)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutputSliceProjections(t *testing.T) {
|
|
||||||
outputs := []Output{
|
outputs := []Output{
|
||||||
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
||||||
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
||||||
}
|
}
|
||||||
|
|
||||||
stateOutputs := StateOutputFiles(outputs)
|
|
||||||
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
|
|
||||||
t.Fatalf("state outputs = %#v", stateOutputs)
|
|
||||||
}
|
|
||||||
|
|
||||||
paths := ManagedOutputPaths(outputs)
|
paths := ManagedOutputPaths(outputs)
|
||||||
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
||||||
t.Fatalf("managed paths = %#v", paths)
|
t.Fatalf("managed paths = %#v", paths)
|
||||||
|
|||||||
@@ -15,18 +15,13 @@ import (
|
|||||||
type Action string
|
type Action string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ActionPublishNew Action = "publish_new"
|
ActionPublishNew Action = "publish_new"
|
||||||
ActionReplaceOlder Action = "replace_older"
|
ActionSkipSame Action = "skip_same"
|
||||||
ActionReplaceConflict Action = "replace_conflict"
|
ActionFailConflict Action = "fail_conflict"
|
||||||
ActionReplaceNewer Action = "replace_newer"
|
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||||
ActionSkipSame Action = "skip_same"
|
ActionForceReplace Action = "force_replace"
|
||||||
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
ActionUpsertAdditive Action = "upsert_additive"
|
||||||
ActionFailConflict Action = "fail_conflict"
|
ActionReplaceCatalog Action = "replace_catalog"
|
||||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
|
||||||
ActionForceReplace Action = "force_replace"
|
|
||||||
ActionReplaceTakeover Action = "replace_takeover"
|
|
||||||
ActionUpsertAdditive Action = "upsert_additive"
|
|
||||||
ActionReplaceCatalog Action = "replace_catalog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
@@ -82,18 +77,6 @@ type Plan struct {
|
|||||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||||
ClearDestinationRoot bool
|
ClearDestinationRoot bool
|
||||||
|
|
||||||
// Retained while the executor is migrated to catalog state.
|
|
||||||
StateMode string
|
|
||||||
Reconciliation config.ReconciliationPolicy
|
|
||||||
TakeoverMode string
|
|
||||||
ExistingState *state.DistributorState
|
|
||||||
ExistingSharedRoot *state.SharedRootState
|
|
||||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
TakenOverOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
|
||||||
OwnerOutputsToWrite []Output
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type catalogPlanDetails struct {
|
type catalogPlanDetails struct {
|
||||||
@@ -396,22 +379,3 @@ func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{}
|
|||||||
}
|
}
|
||||||
return paths
|
return paths
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
|
||||||
if policy.Mode == "" {
|
|
||||||
policy.Mode = config.ReconciliationModeReplace
|
|
||||||
}
|
|
||||||
return policy
|
|
||||||
}
|
|
||||||
|
|
||||||
func isReconciliationReplacementAction(action Action) bool {
|
|
||||||
return action == ActionReplaceOlder || action == ActionReplaceNewer
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
|
|
||||||
paths := make(map[string]struct{}, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -385,7 +385,7 @@ func TestBuildForceReplacesNoStateNonEmptyDestination(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
||||||
|
|
||||||
@@ -393,7 +393,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Build() error = %v", err)
|
t.Fatalf("Build() error = %v", err)
|
||||||
}
|
}
|
||||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != state.SchemaVersion {
|
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != 2 {
|
||||||
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
||||||
}
|
}
|
||||||
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
||||||
@@ -403,7 +403,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
|
|
||||||
plan, err := Build(context.Background(), req)
|
plan, err := Build(context.Background(), req)
|
||||||
@@ -606,6 +606,10 @@ func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, cat
|
|||||||
writeJSONState(t, backend, relative, catalog)
|
writeJSONState(t, backend, relative, catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func legacyStateDocument(schemaVersion int) map[string]int {
|
||||||
|
return map[string]int{"schema_version": schemaVersion}
|
||||||
|
}
|
||||||
|
|
||||||
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := json.MarshalIndent(value, "", " ")
|
data, err := json.MarshalIndent(value, "", " ")
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
|||||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||||
}
|
}
|
||||||
return state.DestinationStatus{
|
return state.DestinationStatus{
|
||||||
State: document.SingleOwner,
|
|
||||||
SharedRoot: document.SharedRoot,
|
|
||||||
Catalog: document.Catalog,
|
Catalog: document.Catalog,
|
||||||
SupersededLegacy: document.SupersededLegacy,
|
SupersededLegacy: document.SupersededLegacy,
|
||||||
HasContents: true,
|
HasContents: true,
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseCatalogState(t *testing.T) {
|
func TestParseCatalogState(t *testing.T) {
|
||||||
@@ -45,7 +47,7 @@ func TestCatalogMarshalIsDeterministic(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
}
|
}
|
||||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||||
if string(data) != want {
|
if string(data) != want {
|
||||||
t.Fatalf("json = %s, want %s", data, want)
|
t.Fatalf("json = %s, want %s", data, want)
|
||||||
}
|
}
|
||||||
@@ -56,14 +58,14 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
||||||
}
|
}
|
||||||
if catalog.Catalog == nil || catalog.SingleOwner != nil || catalog.SharedRoot != nil || catalog.SupersededLegacy != nil {
|
if catalog.Catalog == nil || catalog.SupersededLegacy != nil {
|
||||||
t.Fatalf("catalog document = %#v", catalog)
|
t.Fatalf("catalog document = %#v", catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
tests := map[string]string{
|
tests := map[string]string{
|
||||||
"schema 1": legacyStateJSON(t),
|
"schema 1": schemaOnlyStateJSON(legacySchemaVersion),
|
||||||
"schema 2": validStateJSON(t),
|
"schema 2": schemaOnlyStateJSON(legacyOwnerSchema),
|
||||||
"schema 3": validSharedRootStateJSON(t),
|
"schema 3": schemaOnlyStateJSON(legacyMultiOwnerSchema),
|
||||||
}
|
}
|
||||||
for name, body := range tests {
|
for name, body := range tests {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
@@ -71,7 +73,7 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseDocument() error = %v", err)
|
t.Fatalf("ParseDocument() error = %v", err)
|
||||||
}
|
}
|
||||||
if document.SupersededLegacy == nil || document.Catalog != nil || document.SingleOwner != nil || document.SharedRoot != nil {
|
if document.SupersededLegacy == nil || document.Catalog != nil {
|
||||||
t.Fatalf("document = %#v, want superseded legacy only", document)
|
t.Fatalf("document = %#v, want superseded legacy only", document)
|
||||||
}
|
}
|
||||||
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
||||||
@@ -317,6 +319,41 @@ func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func schemaOnlyStateJSON(schemaVersion int) string {
|
||||||
|
data, err := json.Marshal(map[string]int{"schema_version": schemaVersion})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStateErrorContains(t *testing.T, err error, substring string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want substring %q", substring)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), substring) {
|
||||||
|
t.Fatalf("error = %v, want substring %q", err, substring)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validManifest(t *testing.T) bundle.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
files := []bundle.ManifestFile{{
|
||||||
|
Path: "report.md",
|
||||||
|
SHA256: bundle.FileDigest([]byte("# Report\nSunny.\n")),
|
||||||
|
Size: int64(len("# Report\nSunny.\n")),
|
||||||
|
}}
|
||||||
|
manifest := bundle.Manifest{
|
||||||
|
SchemaVersion: bundle.SchemaVersion,
|
||||||
|
ID: "weather.daily.brentwood.2026-05-30",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
func validCatalogState(t *testing.T) CatalogState {
|
func validCatalogState(t *testing.T) CatalogState {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
manifest := validManifest(t)
|
manifest := validManifest(t)
|
||||||
|
|||||||
@@ -1,216 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Outcome string
|
|
||||||
|
|
||||||
const (
|
|
||||||
OutcomeDestinationAbsent Outcome = "destination_absent"
|
|
||||||
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
|
|
||||||
OutcomeInvalidState Outcome = "invalid_destination_state"
|
|
||||||
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
|
|
||||||
OutcomeSameSource Outcome = "same_source_manifest"
|
|
||||||
OutcomeDestinationOlder Outcome = "destination_older"
|
|
||||||
OutcomeDestinationNewer Outcome = "destination_newer"
|
|
||||||
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
|
|
||||||
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DestinationStatus struct {
|
|
||||||
State *DistributorState
|
|
||||||
SharedRoot *SharedRootState
|
|
||||||
Catalog *CatalogState
|
|
||||||
SupersededLegacy *SupersededLegacyState
|
|
||||||
StateErr error
|
|
||||||
HasContents bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type Comparison struct {
|
|
||||||
Outcome Outcome
|
|
||||||
Reason string
|
|
||||||
Detail ComparisonDetail
|
|
||||||
}
|
|
||||||
|
|
||||||
type ComparisonDetailKind string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ComparisonDetailNone ComparisonDetailKind = ""
|
|
||||||
ComparisonDetailInvalidState ComparisonDetailKind = "invalid_state"
|
|
||||||
ComparisonDetailUnmanagedContent ComparisonDetailKind = "unmanaged_content"
|
|
||||||
ComparisonDetailPipelineIDMismatch ComparisonDetailKind = "pipeline_id_mismatch"
|
|
||||||
ComparisonDetailDestinationIDMismatch ComparisonDetailKind = "destination_id_mismatch"
|
|
||||||
ComparisonDetailDifferentSourceID ComparisonDetailKind = "different_source_id"
|
|
||||||
ComparisonDetailSameCreatedDigestConflict ComparisonDetailKind = "same_created_digest_conflict"
|
|
||||||
ComparisonDetailDestinationNewer ComparisonDetailKind = "destination_newer"
|
|
||||||
ComparisonDetailSharedRootOwnerAbsent ComparisonDetailKind = "shared_root_owner_absent"
|
|
||||||
ComparisonDetailSharedRootOutputOwner ComparisonDetailKind = "shared_root_output_owner_conflict"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ComparisonDetail struct {
|
|
||||||
Kind ComparisonDetailKind
|
|
||||||
CurrentPipelineID string
|
|
||||||
CurrentDestinationID string
|
|
||||||
DestinationPipelineID string
|
|
||||||
DestinationDestinationID string
|
|
||||||
CurrentSourceID string
|
|
||||||
DestinationSourceID string
|
|
||||||
CurrentSourceDigest string
|
|
||||||
DestinationSourceDigest string
|
|
||||||
Path string
|
|
||||||
CurrentOwner OwnerScope
|
|
||||||
ConflictingOwner OwnerScope
|
|
||||||
}
|
|
||||||
|
|
||||||
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
|
|
||||||
if status.StateErr != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if status.SharedRoot != nil {
|
|
||||||
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
owner, ok := status.SharedRoot.Owner(scope)
|
|
||||||
if !ok {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDestinationAbsent,
|
|
||||||
Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSharedRootOwnerAbsent,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return compareManifests(source, owner.Source.Manifest)
|
|
||||||
}
|
|
||||||
if status.State != nil {
|
|
||||||
destinationState := *status.State
|
|
||||||
if err := Validate(destinationState); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if destinationState.PipelineID != scope.PipelineID {
|
|
||||||
return pipelineIDMismatchComparison(destinationState.PipelineID, scope.PipelineID)
|
|
||||||
}
|
|
||||||
if destinationState.DestinationID != scope.DestinationID {
|
|
||||||
return destinationIDMismatchComparison(destinationState.DestinationID, scope.DestinationID)
|
|
||||||
}
|
|
||||||
return compareManifests(source, destinationState.Source.Manifest)
|
|
||||||
}
|
|
||||||
if status.HasContents {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
|
||||||
}
|
|
||||||
|
|
||||||
func compareManifests(source, destination bundle.Manifest) Comparison {
|
|
||||||
if manifestsEqual(source, destination) {
|
|
||||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
|
||||||
}
|
|
||||||
if destination.ID != source.ID {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDifferentSourceConflict,
|
|
||||||
Reason: "destination source id differs from source",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDifferentSourceID,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if destination.Created.Before(source.Created) {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
|
||||||
}
|
|
||||||
if destination.Created.After(source.Created) {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDestinationNewer,
|
|
||||||
Reason: "destination source is newer than source",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDestinationNewer,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if destination.Digest != source.Digest {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeSameCreatedConflict,
|
|
||||||
Reason: "destination source has same id and created time but different digest",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSameCreatedDigestConflict,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
CurrentSourceDigest: source.Digest,
|
|
||||||
DestinationSourceDigest: destination.Digest,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome", Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
|
||||||
if status.StateErr != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if status.State == nil {
|
|
||||||
if status.HasContents {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
|
||||||
}
|
|
||||||
|
|
||||||
destinationState := *status.State
|
|
||||||
if err := Validate(destinationState); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if destinationState.PipelineID != pipelineID {
|
|
||||||
return pipelineIDMismatchComparison(destinationState.PipelineID, pipelineID)
|
|
||||||
}
|
|
||||||
if destinationState.DestinationID != destinationID {
|
|
||||||
return destinationIDMismatchComparison(destinationState.DestinationID, destinationID)
|
|
||||||
}
|
|
||||||
|
|
||||||
return compareManifests(source, destinationState.Source.Manifest)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pipelineIDMismatchComparison(destinationPipelineID, currentPipelineID string) Comparison {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeIdentityMismatch,
|
|
||||||
Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationPipelineID, currentPipelineID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailPipelineIDMismatch,
|
|
||||||
CurrentPipelineID: currentPipelineID,
|
|
||||||
DestinationPipelineID: destinationPipelineID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func destinationIDMismatchComparison(destinationDestinationID, currentDestinationID string) Comparison {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeIdentityMismatch,
|
|
||||||
Reason: fmt.Sprintf("destination id %q does not match %q", destinationDestinationID, currentDestinationID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDestinationIDMismatch,
|
|
||||||
CurrentDestinationID: currentDestinationID,
|
|
||||||
DestinationDestinationID: destinationDestinationID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func manifestsEqual(a, b bundle.Manifest) bool {
|
|
||||||
if a.SchemaVersion != b.SchemaVersion ||
|
|
||||||
a.ID != b.ID ||
|
|
||||||
a.Digest != b.Digest ||
|
|
||||||
!a.Created.Equal(b.Created) ||
|
|
||||||
len(a.Files) != len(b.Files) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for index := range a.Files {
|
|
||||||
if a.Files[index] != b.Files[index] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
@@ -1,216 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompareOutcomes(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status DestinationStatus
|
|
||||||
want Outcome
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "destination absent",
|
|
||||||
status: DestinationStatus{},
|
|
||||||
want: OutcomeDestinationAbsent,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination unmanaged",
|
|
||||||
status: DestinationStatus{HasContents: true},
|
|
||||||
want: OutcomeDestinationUnmanaged,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid destination state",
|
|
||||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
|
||||||
want: OutcomeInvalidState,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pipeline mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
|
||||||
want: OutcomeIdentityMismatch,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
|
||||||
want: OutcomeIdentityMismatch,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same source manifest",
|
|
||||||
status: DestinationStatus{State: withState(t, source, nil)},
|
|
||||||
want: OutcomeSameSource,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination older",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
|
|
||||||
})},
|
|
||||||
want: OutcomeDestinationOlder,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination newer",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
|
||||||
})},
|
|
||||||
want: OutcomeDestinationNewer,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same created digest conflict",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
|
||||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
|
||||||
})},
|
|
||||||
want: OutcomeSameCreatedConflict,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "different source id",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.ID = "other.source"
|
|
||||||
})},
|
|
||||||
want: OutcomeDifferentSourceConflict,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid state object",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = "other"
|
|
||||||
})},
|
|
||||||
want: OutcomeInvalidState,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := Compare(source, "reports", "archive", tt.status)
|
|
||||||
if got.Outcome != tt.want {
|
|
||||||
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
|
|
||||||
}
|
|
||||||
if got.Reason == "" {
|
|
||||||
t.Fatal("Compare() reason is empty")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareReportsStructuredDetails(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status DestinationStatus
|
|
||||||
wantKind ComparisonDetailKind
|
|
||||||
assertions func(t *testing.T, detail ComparisonDetail)
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "pipeline mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
|
||||||
wantKind: ComparisonDetailPipelineIDMismatch,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationPipelineID != "other" || detail.CurrentPipelineID != "reports" {
|
|
||||||
t.Fatalf("detail = %#v, want pipeline ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
|
||||||
wantKind: ComparisonDetailDestinationIDMismatch,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationDestinationID != "other" || detail.CurrentDestinationID != "archive" {
|
|
||||||
t.Fatalf("detail = %#v, want destination ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "different source id",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.Source.Manifest.ID = "other.source" })},
|
|
||||||
wantKind: ComparisonDetailDifferentSourceID,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationSourceID != "other.source" || detail.CurrentSourceID != source.ID {
|
|
||||||
t.Fatalf("detail = %#v, want source ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same created digest conflict",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
|
||||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
|
||||||
})},
|
|
||||||
wantKind: ComparisonDetailSameCreatedDigestConflict,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.CurrentSourceDigest == "" || detail.DestinationSourceDigest == "" || detail.CurrentSourceDigest == detail.DestinationSourceDigest {
|
|
||||||
t.Fatalf("detail = %#v, want different source digests", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination newer",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
|
||||||
})},
|
|
||||||
wantKind: ComparisonDetailDestinationNewer,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid state",
|
|
||||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
|
||||||
wantKind: ComparisonDetailInvalidState,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unmanaged content",
|
|
||||||
status: DestinationStatus{HasContents: true},
|
|
||||||
wantKind: ComparisonDetailUnmanagedContent,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := Compare(source, "reports", "archive", tt.status)
|
|
||||||
if got.Detail.Kind != tt.wantKind {
|
|
||||||
t.Fatalf("Compare() detail kind = %q, want %q; comparison=%#v", got.Detail.Kind, tt.wantKind, got)
|
|
||||||
}
|
|
||||||
if tt.assertions != nil {
|
|
||||||
tt.assertions(t, got.Detail)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
stateManifest := source
|
|
||||||
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: stateManifest},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if mutate != nil {
|
|
||||||
mutate(&state)
|
|
||||||
}
|
|
||||||
return &state
|
|
||||||
}
|
|
||||||
@@ -1,373 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
SchemaVersion = 2
|
|
||||||
SharedRootSchemaVersion = 3
|
|
||||||
CatalogSchemaVersion = 4
|
|
||||||
legacySchemaVersion = 1
|
|
||||||
StateModeSingleOwner = config.StateModeSingleOwner
|
|
||||||
StateModeSharedRoot = config.StateModeSharedRoot
|
|
||||||
StateModeCatalog = "catalog"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DistributorState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
DistributorVersion string
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
PublishedAt time.Time
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
State StatePolicy
|
|
||||||
Reconciliation ReconciliationPolicy
|
|
||||||
Source SourceState
|
|
||||||
Links *LinkState
|
|
||||||
Outputs []OutputFile
|
|
||||||
}
|
|
||||||
|
|
||||||
type StatePolicy struct {
|
|
||||||
Mode string
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReconciliationPolicy struct {
|
|
||||||
Mode string
|
|
||||||
}
|
|
||||||
|
|
||||||
type SourceState struct {
|
|
||||||
Manifest bundle.Manifest
|
|
||||||
}
|
|
||||||
|
|
||||||
type LinkState struct {
|
|
||||||
PrimaryURL string
|
|
||||||
}
|
|
||||||
|
|
||||||
type OutputFile struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawDistributorState struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version"`
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
PublishedAt *string `json:"published_at"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
State *rawStatePolicy `json:"state"`
|
|
||||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source *rawSourceState `json:"source"`
|
|
||||||
Links *rawLinkState `json:"links"`
|
|
||||||
Outputs []rawOutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawStatePolicy struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawReconciliationPolicy struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSourceState struct {
|
|
||||||
Manifest json.RawMessage `json:"manifest"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawLinkState struct {
|
|
||||||
PrimaryURL string `json:"primary_url"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawOutputFile struct {
|
|
||||||
Path *string `json:"path"`
|
|
||||||
Kind *string `json:"kind"`
|
|
||||||
SourcePath *string `json:"source_path"`
|
|
||||||
Transform string `json:"transform"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
SHA256 *string `json:"sha256"`
|
|
||||||
Size *int64 `json:"size"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func Parse(data []byte) (DistributorState, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw rawDistributorState
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
state, err := parseRaw(raw)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
if err := Validate(state); err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
|
||||||
var state DistributorState
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
state.SchemaVersion = *raw.SchemaVersion
|
|
||||||
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
|
|
||||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
|
|
||||||
}
|
|
||||||
legacy := state.SchemaVersion == legacySchemaVersion
|
|
||||||
state.DistributorVersion = raw.DistributorVersion
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
|
||||||
}
|
|
||||||
state.PipelineID = *raw.PipelineID
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state destination_id is required")
|
|
||||||
}
|
|
||||||
state.DestinationID = *raw.DestinationID
|
|
||||||
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state published_at is required")
|
|
||||||
}
|
|
||||||
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
|
||||||
}
|
|
||||||
state.PublishedAt = publishedAt.UTC()
|
|
||||||
if legacy {
|
|
||||||
state.SchemaVersion = SchemaVersion
|
|
||||||
state.CreatedAt = state.PublishedAt
|
|
||||||
state.UpdatedAt = state.PublishedAt
|
|
||||||
state.State.Mode = StateModeSingleOwner
|
|
||||||
state.Reconciliation.Mode = config.ReconciliationModeReplace
|
|
||||||
} else {
|
|
||||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
state.CreatedAt = createdAt
|
|
||||||
state.UpdatedAt = updatedAt
|
|
||||||
if raw.State == nil || raw.State.Mode == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state state.mode is required")
|
|
||||||
}
|
|
||||||
state.State.Mode = raw.State.Mode
|
|
||||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
|
|
||||||
}
|
|
||||||
state.Reconciliation.Mode = raw.Reconciliation.Mode
|
|
||||||
}
|
|
||||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
|
||||||
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
|
||||||
}
|
|
||||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
|
||||||
}
|
|
||||||
state.Source.Manifest = manifest
|
|
||||||
if raw.Links != nil {
|
|
||||||
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
|
||||||
}
|
|
||||||
if raw.Outputs == nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
state.Outputs = outputs
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRequiredTime(context string, raw *string) (time.Time, error) {
|
|
||||||
if raw == nil || *raw == "" {
|
|
||||||
return time.Time{}, fmt.Errorf("%s is required", context)
|
|
||||||
}
|
|
||||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
|
||||||
if err != nil {
|
|
||||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
|
|
||||||
}
|
|
||||||
return parsed.UTC(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
|
|
||||||
outputs := make([]OutputFile, 0, len(rawOutputs))
|
|
||||||
seen := make(map[string]struct{}, len(rawOutputs))
|
|
||||||
for index, raw := range rawOutputs {
|
|
||||||
output, err := parseOutput(index, raw, legacy, publishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
|
|
||||||
if raw.Path == nil || *raw.Path == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
|
||||||
}
|
|
||||||
if raw.Kind == nil || *raw.Kind == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
|
||||||
}
|
|
||||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
|
||||||
}
|
|
||||||
if raw.Size == nil {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
|
||||||
}
|
|
||||||
createdAt := publishedAt
|
|
||||||
updatedAt := publishedAt
|
|
||||||
if !legacy {
|
|
||||||
var err error
|
|
||||||
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return OutputFile{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return OutputFile{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OutputFile{
|
|
||||||
Path: *raw.Path,
|
|
||||||
Kind: *raw.Kind,
|
|
||||||
SourcePath: *raw.SourcePath,
|
|
||||||
Transform: raw.Transform,
|
|
||||||
URL: raw.URL,
|
|
||||||
SHA256: *raw.SHA256,
|
|
||||||
Size: *raw.Size,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) PublishedAtString() string {
|
|
||||||
return s.PublishedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) CreatedAtString() string {
|
|
||||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) UpdatedAtString() string {
|
|
||||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) CreatedAtString() string {
|
|
||||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) UpdatedAtString() string {
|
|
||||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
|
||||||
type sourceJSON struct {
|
|
||||||
Manifest bundle.Manifest `json:"manifest"`
|
|
||||||
}
|
|
||||||
type stateJSON struct {
|
|
||||||
SchemaVersion int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
PublishedAt string `json:"published_at"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
State StatePolicy `json:"state"`
|
|
||||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source sourceJSON `json:"source"`
|
|
||||||
Links *LinkState `json:"links,omitempty"`
|
|
||||||
Outputs []OutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
return json.Marshal(stateJSON{
|
|
||||||
SchemaVersion: s.SchemaVersion,
|
|
||||||
DistributorVersion: s.DistributorVersion,
|
|
||||||
PipelineID: s.PipelineID,
|
|
||||||
DestinationID: s.DestinationID,
|
|
||||||
PublishedAt: s.PublishedAtString(),
|
|
||||||
CreatedAt: s.CreatedAtString(),
|
|
||||||
UpdatedAt: s.UpdatedAtString(),
|
|
||||||
State: s.State,
|
|
||||||
Reconciliation: s.Reconciliation,
|
|
||||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
|
||||||
Links: s.Links,
|
|
||||||
Outputs: s.Outputs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
|
||||||
type policyJSON struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
|
|
||||||
type policyJSON struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l LinkState) MarshalJSON() ([]byte, error) {
|
|
||||||
type linkJSON struct {
|
|
||||||
PrimaryURL string `json:"primary_url,omitempty"`
|
|
||||||
}
|
|
||||||
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
|
||||||
type outputJSON struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
SourcePath string `json:"source_path"`
|
|
||||||
Transform string `json:"transform,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
return json.Marshal(outputJSON{
|
|
||||||
Path: o.Path,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
|
|
||||||
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,473 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseValidState(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if state.SchemaVersion != SchemaVersion {
|
|
||||||
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
|
|
||||||
}
|
|
||||||
if state.PipelineID != "reports" || state.DestinationID != "archive" {
|
|
||||||
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
|
|
||||||
}
|
|
||||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
|
||||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Outputs), 1; got != want {
|
|
||||||
t.Fatalf("output count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseValidStateWithLinks(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
|
|
||||||
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
|
|
||||||
|
|
||||||
state, err := Parse([]byte(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
|
|
||||||
t.Fatalf("links = %#v, want primary URL", state.Links)
|
|
||||||
}
|
|
||||||
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
|
|
||||||
t.Fatalf("output URL = %q", state.Outputs[0].URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingOutputs(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
state.Outputs = append(state.Outputs, OutputFile{
|
|
||||||
Path: "summary.txt",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "summary.txt",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 10,
|
|
||||||
CreatedAt: state.CreatedAt,
|
|
||||||
UpdatedAt: state.UpdatedAt,
|
|
||||||
})
|
|
||||||
|
|
||||||
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
|
|
||||||
if changed {
|
|
||||||
t.Fatal("RemoveMissingOutputs() changed = true, want false")
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
|
||||||
state, err := Parse([]byte(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMissingFields(t *testing.T) {
|
|
||||||
tests := map[string]string{
|
|
||||||
"schema_version": `"schema_version"`,
|
|
||||||
"pipeline_id": `"pipeline_id"`,
|
|
||||||
"destination_id": `"destination_id"`,
|
|
||||||
"published_at": `"published_at"`,
|
|
||||||
"created_at": `"created_at"`,
|
|
||||||
"updated_at": `"updated_at"`,
|
|
||||||
"state": `"state"`,
|
|
||||||
"reconciliation": `"reconciliation"`,
|
|
||||||
"source": `"source"`,
|
|
||||||
"outputs": `"outputs"`,
|
|
||||||
}
|
|
||||||
for name, field := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "required")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(legacyStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.SchemaVersion, SchemaVersion; got != want {
|
|
||||||
t.Fatalf("schema version = %d, want normalized %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("created_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
|
||||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("output updated_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
|
||||||
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "source.manifest")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateRejectsInvalidEmbeddedManifest(t *testing.T) {
|
|
||||||
tests := map[string]func(bundle.Manifest) bundle.Manifest{
|
|
||||||
"schema version": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.SchemaVersion = 2
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"empty id": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.ID = ""
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"bad digest": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"zero created": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Created = time.Time{}
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"empty files": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files = nil
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"unsafe path": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[0].Path = "../report.md"
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"duplicate path": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[1].Path = manifest.Files[0].Path
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"negative size": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[0].Size = -1
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"digest mismatch": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
source := mutate(validManifest(t))
|
|
||||||
state := *withState(t, validManifest(t), func(*DistributorState) {})
|
|
||||||
state.Source.Manifest = source
|
|
||||||
err := Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "source.manifest")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := map[string]func(*DistributorState){
|
|
||||||
"unsafe path": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Path = "../report.md"
|
|
||||||
},
|
|
||||||
"invalid kind": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = "other"
|
|
||||||
},
|
|
||||||
"invalid source": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].SourcePath = "../report.md"
|
|
||||||
},
|
|
||||||
"generated missing": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = OutputKindGenerated
|
|
||||||
},
|
|
||||||
"invalid digest": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
|
||||||
},
|
|
||||||
"negative size": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Size = -1
|
|
||||||
},
|
|
||||||
"missing output created_at": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].CreatedAt = time.Time{}
|
|
||||||
},
|
|
||||||
"missing output updated_at": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].UpdatedAt = time.Time{}
|
|
||||||
},
|
|
||||||
"invalid output url": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].URL = "file:///tmp/report.md"
|
|
||||||
},
|
|
||||||
"invalid primary url": func(s *DistributorState) {
|
|
||||||
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
state := *withState(t, source, mutate)
|
|
||||||
err := Validate(state)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Validate() error = nil, want error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateReportsURLFieldContext(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
state := *withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
|
|
||||||
})
|
|
||||||
err := Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "state links.primary_url")
|
|
||||||
assertStateErrorContains(t, err, "must not include a fragment")
|
|
||||||
|
|
||||||
state = *withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
|
|
||||||
})
|
|
||||||
err = Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "state outputs[0].url")
|
|
||||||
assertStateErrorContains(t, err, "must not include a query string")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "created_at must be RFC3339")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
|
|
||||||
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
URL: "https://reports.example.com/archive/report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
|
|
||||||
t.Fatalf("json = %s, want links primary URL", data)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
|
|
||||||
t.Fatalf("json = %s, want output URL", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutputHelpers(t *testing.T) {
|
|
||||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
updatedAt := createdAt.Add(time.Hour)
|
|
||||||
retained := []OutputFile{{
|
|
||||||
Path: "old.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}, {
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}}
|
|
||||||
projected := ProjectOutputs([]OutputProjection{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
}, {
|
|
||||||
Path: "new.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
}}, retained, updatedAt)
|
|
||||||
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
|
|
||||||
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
|
|
||||||
}
|
|
||||||
merged, err := MergeOutputFiles(retained, projected)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("MergeOutputFiles() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(merged), 3; got != want {
|
|
||||||
t.Fatalf("merged count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("managed paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
|
||||||
}
|
|
||||||
|
|
||||||
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
|
||||||
t.Helper()
|
|
||||||
return `{
|
|
||||||
"schema_version": 2,
|
|
||||||
"distributor_version": "dev",
|
|
||||||
"pipeline_id": "reports",
|
|
||||||
"destination_id": "archive",
|
|
||||||
"published_at": "2026-05-30T11:12:00Z",
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z",
|
|
||||||
"state": {"mode": "single_owner"},
|
|
||||||
"reconciliation": {"mode": "replace"},
|
|
||||||
"source": {
|
|
||||||
"manifest": ` + manifest + `
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"path": "report.md",
|
|
||||||
"kind": "source",
|
|
||||||
"source_path": "report.md",
|
|
||||||
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
|
||||||
"size": 16,
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}`
|
|
||||||
}
|
|
||||||
|
|
||||||
func legacyStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
|
|
||||||
`"schema_version": 2`, `"schema_version": 1`, 1),
|
|
||||||
` "created_at": "2026-05-30T11:12:00Z",
|
|
||||||
`, "", 1),
|
|
||||||
` "updated_at": "2026-05-30T11:12:00Z",
|
|
||||||
`, "", 1),
|
|
||||||
` "state": {"mode": "single_owner"},
|
|
||||||
`, "", 1),
|
|
||||||
` "reconciliation": {"mode": "replace"},
|
|
||||||
`, "", 1),
|
|
||||||
`,
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func manifestJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read manifest fixture: %v", err)
|
|
||||||
}
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validManifest(t *testing.T) bundle.Manifest {
|
|
||||||
t.Helper()
|
|
||||||
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseManifest() error = %v", err)
|
|
||||||
}
|
|
||||||
return manifest
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertStateErrorContains(t *testing.T, err error, want string) {
|
|
||||||
t.Helper()
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("error = nil, want substring %q", want)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
107
internal/state/document.go
Normal file
107
internal/state/document.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
legacyOwnerSchema = 2
|
||||||
|
legacyMultiOwnerSchema = 3
|
||||||
|
CatalogSchemaVersion = 4
|
||||||
|
legacySchemaVersion = 1
|
||||||
|
StateModeCatalog = "catalog"
|
||||||
|
OutputKindSource = "source"
|
||||||
|
OutputKindGenerated = "generated"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StateDocument struct {
|
||||||
|
Catalog *CatalogState
|
||||||
|
SupersededLegacy *SupersededLegacyState
|
||||||
|
}
|
||||||
|
|
||||||
|
type DestinationStatus struct {
|
||||||
|
Catalog *CatalogState
|
||||||
|
SupersededLegacy *SupersededLegacyState
|
||||||
|
StateErr error
|
||||||
|
HasContents bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type SupersededLegacyState struct {
|
||||||
|
SchemaVersion int
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatePolicy struct {
|
||||||
|
Mode string
|
||||||
|
}
|
||||||
|
|
||||||
|
type OwnerScope struct {
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawStatePolicy struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseDocument(data []byte) (StateDocument, error) {
|
||||||
|
schemaVersion, err := parseSchemaVersion(data)
|
||||||
|
if err != nil {
|
||||||
|
return StateDocument{}, err
|
||||||
|
}
|
||||||
|
switch schemaVersion {
|
||||||
|
case legacySchemaVersion, legacyOwnerSchema, legacyMultiOwnerSchema:
|
||||||
|
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||||
|
case CatalogSchemaVersion:
|
||||||
|
catalog, err := ParseCatalog(data)
|
||||||
|
if err != nil {
|
||||||
|
return StateDocument{}, err
|
||||||
|
}
|
||||||
|
return StateDocument{Catalog: &catalog}, nil
|
||||||
|
default:
|
||||||
|
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSchemaVersion(data []byte) (int, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
var raw struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
}
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||||
|
}
|
||||||
|
if raw.SchemaVersion == nil {
|
||||||
|
return 0, fmt.Errorf("state schema_version is required")
|
||||||
|
}
|
||||||
|
return *raw.SchemaVersion, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||||
|
type policyJSON struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
}
|
||||||
|
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||||
|
}
|
||||||
|
|
||||||
|
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||||
|
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRequiredTime(field string, raw *string) (time.Time, error) {
|
||||||
|
if raw == nil || *raw == "" {
|
||||||
|
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||||
|
}
|
||||||
|
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", field, err)
|
||||||
|
}
|
||||||
|
return parsed.UTC(), nil
|
||||||
|
}
|
||||||
@@ -1,31 +1,5 @@
|
|||||||
package state
|
package state
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OutputProjection struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OutputFile{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
if output.Path == path {
|
if output.Path == path {
|
||||||
@@ -35,158 +9,6 @@ func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogO
|
|||||||
return CatalogOutputFile{}, false
|
return CatalogOutputFile{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
|
||||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
|
||||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
|
||||||
for _, output := range retained {
|
|
||||||
if _, exists := indexByPath[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
seenPlanned := make(map[string]struct{}, len(planned))
|
|
||||||
for _, output := range planned {
|
|
||||||
if _, exists := seenPlanned[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
seenPlanned[output.Path] = struct{}{}
|
|
||||||
if index, exists := indexByPath[output.Path]; exists {
|
|
||||||
outputs[index] = output
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ManagedOutputPaths(s DistributorState) []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
|
|
||||||
now = now.UTC()
|
|
||||||
files := make([]OutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
createdAt := now
|
|
||||||
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
|
|
||||||
createdAt = existingOutput.CreatedAt
|
|
||||||
}
|
|
||||||
files = append(files, OutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
|
||||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
|
|
||||||
for _, owner := range s.Owners {
|
|
||||||
if owner.Scope == scope {
|
|
||||||
return owner, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OwnerRecord{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
|
|
||||||
owner, ok := s.Owner(scope)
|
|
||||||
if !ok {
|
|
||||||
return bundle.Manifest{}, false
|
|
||||||
}
|
|
||||||
return owner.Source.Manifest, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner == scope {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) AllManagedOutputPaths() []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner == scope {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
||||||
selected := make([]CatalogOutputFile, 0, len(outputs))
|
selected := make([]CatalogOutputFile, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
@@ -243,159 +65,6 @@ func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (Catalog
|
|||||||
return next, changed
|
return next, changed
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output.Owner, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OwnerScope{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
|
|
||||||
for _, path := range paths {
|
|
||||||
owner, exists := s.OutputOwner(path)
|
|
||||||
if exists && owner != scope {
|
|
||||||
return PathOwnershipConflict{
|
|
||||||
Path: path,
|
|
||||||
Owner: owner,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSharedRootOutputOwner,
|
|
||||||
Path: path,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
ConflictingOwner: owner,
|
|
||||||
},
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return PathOwnershipConflict{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
|
|
||||||
now = now.UTC()
|
|
||||||
files := make([]SharedRootOutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
createdAt := now
|
|
||||||
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
|
|
||||||
createdAt = existingOutput.CreatedAt
|
|
||||||
}
|
|
||||||
files = append(files, SharedRootOutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
Owner: scope,
|
|
||||||
SourceID: source.ID,
|
|
||||||
SourceDigest: source.Digest,
|
|
||||||
SourceCreated: source.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
|
||||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
next := s
|
|
||||||
next.Owners = upsertOwner(s.Owners, owner)
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, planned...)
|
|
||||||
return next, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
|
||||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
next := s
|
|
||||||
next.Owners = upsertOwner(s.Owners, owner)
|
|
||||||
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
|
||||||
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
for _, output := range planned {
|
|
||||||
if index, exists := indexByPath[output.Path]; exists {
|
|
||||||
outputs[index] = output
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
next.Outputs = outputs
|
|
||||||
return next, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return SharedRootOutputFile{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
|
|
||||||
seen := make(map[string]struct{}, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
|
|
||||||
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
|
|
||||||
next := append([]OwnerRecord(nil), owners...)
|
|
||||||
for index, existing := range next {
|
|
||||||
if existing.Scope == owner.Scope {
|
|
||||||
next[index] = owner
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return append(next, owner)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pathSet(paths []string) map[string]struct{} {
|
func pathSet(paths []string) map[string]struct{} {
|
||||||
set := make(map[string]struct{}, len(paths))
|
set := make(map[string]struct{}, len(paths))
|
||||||
for _, path := range paths {
|
for _, path := range paths {
|
||||||
|
|||||||
@@ -22,33 +22,6 @@ type PrunePlan struct {
|
|||||||
Preserved []PruneCandidate
|
Preserved []PruneCandidate
|
||||||
}
|
}
|
||||||
|
|
||||||
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
|
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
candidates = append(candidates, PruneCandidate{
|
|
||||||
Path: output.Path,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return candidates
|
|
||||||
}
|
|
||||||
|
|
||||||
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
|
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
owner := output.Owner
|
|
||||||
candidates = append(candidates, PruneCandidate{
|
|
||||||
Path: output.Path,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
Owner: &owner,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return candidates
|
|
||||||
}
|
|
||||||
|
|
||||||
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||||
for _, output := range s.Outputs {
|
for _, output := range s.Outputs {
|
||||||
|
|||||||
@@ -74,19 +74,6 @@ func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
|
|
||||||
sharedRoot := validSharedRootState(t)
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
candidates := SharedRootPruneCandidates(sharedRoot, scope)
|
|
||||||
|
|
||||||
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
|
|
||||||
t.Fatalf("candidates = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
|
|
||||||
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
||||||
paths := make([]string, 0, len(candidates))
|
paths := make([]string, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
|
|||||||
@@ -1,531 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StateDocument struct {
|
|
||||||
SingleOwner *DistributorState
|
|
||||||
SharedRoot *SharedRootState
|
|
||||||
Catalog *CatalogState
|
|
||||||
SupersededLegacy *SupersededLegacyState
|
|
||||||
}
|
|
||||||
|
|
||||||
type SupersededLegacyState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
}
|
|
||||||
|
|
||||||
type SharedRootState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
DistributorVersion string
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
State StatePolicy
|
|
||||||
Owners []OwnerRecord
|
|
||||||
Outputs []SharedRootOutputFile
|
|
||||||
}
|
|
||||||
|
|
||||||
type OwnerScope struct {
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
}
|
|
||||||
|
|
||||||
type OwnerRecord struct {
|
|
||||||
Scope OwnerScope
|
|
||||||
Reconciliation ReconciliationPolicy
|
|
||||||
Source SourceState
|
|
||||||
Links *LinkState
|
|
||||||
}
|
|
||||||
|
|
||||||
type SharedRootOutputFile struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
Owner OwnerScope
|
|
||||||
SourceID string
|
|
||||||
SourceDigest string
|
|
||||||
SourceCreated time.Time
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type PathOwnershipConflict struct {
|
|
||||||
Path string
|
|
||||||
Owner OwnerScope
|
|
||||||
CurrentOwner OwnerScope
|
|
||||||
Detail ComparisonDetail
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSharedRootState struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
State *rawStatePolicy `json:"state"`
|
|
||||||
Owners []rawOwnerRecord `json:"owners"`
|
|
||||||
Outputs []rawSharedRootOutput `json:"outputs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawOwnerRecord struct {
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source *rawSourceState `json:"source"`
|
|
||||||
Links *rawLinkState `json:"links"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSharedRootOutput struct {
|
|
||||||
Path *string `json:"path"`
|
|
||||||
Kind *string `json:"kind"`
|
|
||||||
SourcePath *string `json:"source_path"`
|
|
||||||
Transform string `json:"transform"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
SHA256 *string `json:"sha256"`
|
|
||||||
Size *int64 `json:"size"`
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
SourceID *string `json:"source_id"`
|
|
||||||
SourceDigest *string `json:"source_digest"`
|
|
||||||
SourceCreated *string `json:"source_created"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseDocument(data []byte) (StateDocument, error) {
|
|
||||||
schemaVersion, err := parseSchemaVersion(data)
|
|
||||||
if err != nil {
|
|
||||||
return StateDocument{}, err
|
|
||||||
}
|
|
||||||
switch schemaVersion {
|
|
||||||
case legacySchemaVersion, SchemaVersion, SharedRootSchemaVersion:
|
|
||||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
|
||||||
case CatalogSchemaVersion:
|
|
||||||
catalog, err := ParseCatalog(data)
|
|
||||||
if err != nil {
|
|
||||||
return StateDocument{}, err
|
|
||||||
}
|
|
||||||
return StateDocument{Catalog: &catalog}, nil
|
|
||||||
default:
|
|
||||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSchemaVersion(data []byte) (int, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
}
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return 0, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
return *raw.SchemaVersion, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSharedRoot(data []byte) (SharedRootState, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw rawSharedRootState
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
state, err := parseSharedRootRaw(raw)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
if err := ValidateSharedRoot(state); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
|
|
||||||
if state.SchemaVersion != SharedRootSchemaVersion {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
|
||||||
}
|
|
||||||
state.DistributorVersion = raw.DistributorVersion
|
|
||||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.CreatedAt = createdAt
|
|
||||||
state.UpdatedAt = updatedAt
|
|
||||||
if raw.State == nil || raw.State.Mode == "" {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state state.mode is required")
|
|
||||||
}
|
|
||||||
state.State.Mode = raw.State.Mode
|
|
||||||
if raw.Owners == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state owners is required")
|
|
||||||
}
|
|
||||||
owners, err := parseOwnerRecords(raw.Owners)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.Owners = owners
|
|
||||||
if raw.Outputs == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
outputs, err := parseSharedRootOutputs(raw.Outputs)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.Outputs = outputs
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
|
|
||||||
owners := make([]OwnerRecord, 0, len(rawOwners))
|
|
||||||
for index, raw := range rawOwners {
|
|
||||||
owner, err := parseOwnerRecord(index, raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
owners = append(owners, owner)
|
|
||||||
}
|
|
||||||
return owners, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
|
|
||||||
}
|
|
||||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
|
|
||||||
}
|
|
||||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
|
||||||
if err != nil {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
|
||||||
}
|
|
||||||
owner := OwnerRecord{
|
|
||||||
Scope: OwnerScope{
|
|
||||||
PipelineID: *raw.PipelineID,
|
|
||||||
DestinationID: *raw.DestinationID,
|
|
||||||
},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
|
|
||||||
Source: SourceState{Manifest: manifest},
|
|
||||||
}
|
|
||||||
if raw.Links != nil {
|
|
||||||
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
|
||||||
}
|
|
||||||
return owner, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
|
|
||||||
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
|
|
||||||
for index, raw := range rawOutputs {
|
|
||||||
output, err := parseSharedRootOutput(index, raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
|
|
||||||
if raw.Path == nil || *raw.Path == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
|
||||||
}
|
|
||||||
if raw.Kind == nil || *raw.Kind == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
|
||||||
}
|
|
||||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
|
||||||
}
|
|
||||||
if raw.Size == nil {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
|
||||||
}
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourceID == nil || *raw.SourceID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
|
|
||||||
}
|
|
||||||
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
return SharedRootOutputFile{
|
|
||||||
Path: *raw.Path,
|
|
||||||
Kind: *raw.Kind,
|
|
||||||
SourcePath: *raw.SourcePath,
|
|
||||||
Transform: raw.Transform,
|
|
||||||
URL: raw.URL,
|
|
||||||
SHA256: *raw.SHA256,
|
|
||||||
Size: *raw.Size,
|
|
||||||
Owner: OwnerScope{
|
|
||||||
PipelineID: *raw.PipelineID,
|
|
||||||
DestinationID: *raw.DestinationID,
|
|
||||||
},
|
|
||||||
SourceID: *raw.SourceID,
|
|
||||||
SourceDigest: *raw.SourceDigest,
|
|
||||||
SourceCreated: sourceCreated,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) CreatedAtString() string {
|
|
||||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) UpdatedAtString() string {
|
|
||||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) SourceCreatedString() string {
|
|
||||||
return o.SourceCreated.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) CreatedAtString() string {
|
|
||||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) UpdatedAtString() string {
|
|
||||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) MarshalJSON() ([]byte, error) {
|
|
||||||
type stateJSON struct {
|
|
||||||
SchemaVersion int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
State StatePolicy `json:"state"`
|
|
||||||
Owners []OwnerRecord `json:"owners"`
|
|
||||||
Outputs []SharedRootOutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
return json.Marshal(stateJSON{
|
|
||||||
SchemaVersion: s.SchemaVersion,
|
|
||||||
DistributorVersion: s.DistributorVersion,
|
|
||||||
CreatedAt: s.CreatedAtString(),
|
|
||||||
UpdatedAt: s.UpdatedAtString(),
|
|
||||||
State: s.State,
|
|
||||||
Owners: s.Owners,
|
|
||||||
Outputs: s.Outputs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
|
|
||||||
type sourceJSON struct {
|
|
||||||
Manifest bundle.Manifest `json:"manifest"`
|
|
||||||
}
|
|
||||||
type ownerJSON struct {
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source sourceJSON `json:"source"`
|
|
||||||
Links *LinkState `json:"links,omitempty"`
|
|
||||||
}
|
|
||||||
return json.Marshal(ownerJSON{
|
|
||||||
PipelineID: o.Scope.PipelineID,
|
|
||||||
DestinationID: o.Scope.DestinationID,
|
|
||||||
Reconciliation: o.Reconciliation,
|
|
||||||
Source: sourceJSON{Manifest: o.Source.Manifest},
|
|
||||||
Links: o.Links,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
|
|
||||||
type outputJSON struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
SourcePath string `json:"source_path"`
|
|
||||||
Transform string `json:"transform,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
SourceID string `json:"source_id"`
|
|
||||||
SourceDigest string `json:"source_digest"`
|
|
||||||
SourceCreated string `json:"source_created"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
return json.Marshal(outputJSON{
|
|
||||||
Path: o.Path,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
PipelineID: o.Owner.PipelineID,
|
|
||||||
DestinationID: o.Owner.DestinationID,
|
|
||||||
SourceID: o.SourceID,
|
|
||||||
SourceDigest: o.SourceDigest,
|
|
||||||
SourceCreated: o.SourceCreatedString(),
|
|
||||||
CreatedAt: o.CreatedAtString(),
|
|
||||||
UpdatedAt: o.UpdatedAtString(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func ValidateSharedRoot(s SharedRootState) error {
|
|
||||||
if s.SchemaVersion != SharedRootSchemaVersion {
|
|
||||||
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
|
||||||
}
|
|
||||||
if s.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state created_at is required")
|
|
||||||
}
|
|
||||||
if s.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state updated_at is required")
|
|
||||||
}
|
|
||||||
if s.State.Mode != StateModeSharedRoot {
|
|
||||||
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
|
|
||||||
}
|
|
||||||
if s.Owners == nil {
|
|
||||||
return fmt.Errorf("state owners is required")
|
|
||||||
}
|
|
||||||
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
|
|
||||||
for index, owner := range s.Owners {
|
|
||||||
if err := validateOwnerRecord(index, owner); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := owners[owner.Scope]; exists {
|
|
||||||
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
|
|
||||||
}
|
|
||||||
owners[owner.Scope] = owner
|
|
||||||
}
|
|
||||||
if s.Outputs == nil {
|
|
||||||
return fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
|
||||||
for index, output := range s.Outputs {
|
|
||||||
if err := validateSharedRootOutput(index, output, owners); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := seenPaths[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seenPaths[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOwnerRecord(index int, owner OwnerRecord) error {
|
|
||||||
if owner.Scope.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if owner.Scope.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state owners[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
|
|
||||||
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
|
||||||
}
|
|
||||||
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
|
|
||||||
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
|
||||||
}
|
|
||||||
if owner.Links != nil && owner.Links.PrimaryURL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
|
|
||||||
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
|
|
||||||
if err := storage.ValidatePath(output.Path); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
|
||||||
}
|
|
||||||
switch output.Kind {
|
|
||||||
case OutputKindSource, OutputKindGenerated:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
|
||||||
}
|
|
||||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
|
||||||
}
|
|
||||||
if output.URL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Size < 0 {
|
|
||||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
|
||||||
}
|
|
||||||
if output.Owner.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if output.Owner.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if _, exists := owners[output.Owner]; !exists {
|
|
||||||
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if output.SourceID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_id is required", index)
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.SourceCreated.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_created is required", index)
|
|
||||||
}
|
|
||||||
if output.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
|
||||||
}
|
|
||||||
if output.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,314 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseSharedRootState(t *testing.T) {
|
|
||||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseSharedRoot() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
|
|
||||||
t.Fatalf("schema version = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("created_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Owners), 2; got != want {
|
|
||||||
t.Fatalf("owner count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Outputs), 2; got != want {
|
|
||||||
t.Fatalf("output count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
manifest, ok := state.SourceManifest(scope)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("SourceManifest() ok = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := manifest.ID, validManifest(t).ID; got != want {
|
|
||||||
t.Fatalf("source manifest id = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
|
|
||||||
tests := map[string]func(string) string{
|
|
||||||
"schema": func(body string) string {
|
|
||||||
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
|
|
||||||
},
|
|
||||||
"state mode": func(body string) string {
|
|
||||||
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
|
|
||||||
},
|
|
||||||
"duplicate owner": func(body string) string {
|
|
||||||
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
|
|
||||||
},
|
|
||||||
"unknown output owner": func(body string) string {
|
|
||||||
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
|
|
||||||
},
|
|
||||||
"duplicate output": func(body string) string {
|
|
||||||
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
|
|
||||||
},
|
|
||||||
"invalid source digest": func(body string) string {
|
|
||||||
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
|
|
||||||
},
|
|
||||||
"invalid owner link": func(body string) string {
|
|
||||||
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("ParseSharedRoot() error = nil, want error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
|
||||||
state.UpdatedAt = state.CreatedAt
|
|
||||||
state.Outputs[0].SourceCreated = state.CreatedAt
|
|
||||||
state.Outputs[0].CreatedAt = state.CreatedAt
|
|
||||||
state.Outputs[0].UpdatedAt = state.CreatedAt
|
|
||||||
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
`"created_at":"2026-05-30T11:12:00Z"`,
|
|
||||||
`"updated_at":"2026-05-30T11:12:00Z"`,
|
|
||||||
`"source_created":"2026-05-30T11:12:00Z"`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(string(data), want) {
|
|
||||||
t.Fatalf("json = %s, want %s", data, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootOutputHelpers(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
html := CurrentOwnerScope("reports", "html")
|
|
||||||
|
|
||||||
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("archive paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("all paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
|
|
||||||
if !ok || conflict.Owner != html {
|
|
||||||
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
|
|
||||||
}
|
|
||||||
if conflict.Detail.Kind != ComparisonDetailSharedRootOutputOwner {
|
|
||||||
t.Fatalf("conflict detail kind = %q, want %q", conflict.Detail.Kind, ComparisonDetailSharedRootOutputOwner)
|
|
||||||
}
|
|
||||||
if conflict.Detail.Path != "report.html" || conflict.Detail.CurrentOwner != archive || conflict.Detail.ConflictingOwner != html {
|
|
||||||
t.Fatalf("conflict detail = %#v, want path and owners", conflict.Detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if _, ok := next.OutputOwner("report.html"); !ok {
|
|
||||||
t.Fatal("report.html owner missing, want unrelated owner preserved")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got := next.AllManagedOutputPaths(); len(got) != 0 {
|
|
||||||
t.Fatalf("paths = %#v, want none", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
owner, ok := state.Owner(archive)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("Owner() ok = false, want true")
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
|
|
||||||
planned := ProjectSharedRootOutputs([]OutputProjection{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: validManifest(t).Files[0].SHA256,
|
|
||||||
Size: validManifest(t).Files[0].Size,
|
|
||||||
}, {
|
|
||||||
Path: "summary.txt",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "summary.txt",
|
|
||||||
SHA256: validManifest(t).Files[1].SHA256,
|
|
||||||
Size: validManifest(t).Files[1].Size,
|
|
||||||
}}, state.Outputs, archive, validManifest(t), now)
|
|
||||||
|
|
||||||
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("MergeOwnerOutputs() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("merged paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if merged.Outputs[0].CreatedAt.Equal(now) {
|
|
||||||
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
|
|
||||||
}
|
|
||||||
if !merged.Outputs[0].UpdatedAt.Equal(now) {
|
|
||||||
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("replaced paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
owner, ok := state.Owner(archive)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("Owner() ok = false, want true")
|
|
||||||
}
|
|
||||||
planned := []SharedRootOutputFile{{
|
|
||||||
Path: "report.html",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
Owner: archive,
|
|
||||||
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
|
||||||
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
|
||||||
}}
|
|
||||||
|
|
||||||
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
|
|
||||||
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
|
|
||||||
}
|
|
||||||
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
|
|
||||||
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
source := validManifest(t)
|
|
||||||
source.Created = source.Created.Add(time.Hour)
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
|
|
||||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
|
|
||||||
if comparison.Outcome != OutcomeDestinationOlder {
|
|
||||||
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
|
|
||||||
}
|
|
||||||
|
|
||||||
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
|
|
||||||
if missing.Outcome != OutcomeDestinationAbsent {
|
|
||||||
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
|
|
||||||
}
|
|
||||||
if missing.Detail.Kind != ComparisonDetailSharedRootOwnerAbsent {
|
|
||||||
t.Fatalf("missing owner detail kind = %q, want %q", missing.Detail.Kind, ComparisonDetailSharedRootOwnerAbsent)
|
|
||||||
}
|
|
||||||
if missing.Detail.CurrentOwner != CurrentOwnerScope("missing", "archive") {
|
|
||||||
t.Fatalf("missing owner detail = %#v, want missing/archive", missing.Detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
|
|
||||||
singleOwner, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
source := singleOwner.Source.Manifest
|
|
||||||
source.Created = source.Created.Add(time.Hour)
|
|
||||||
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
|
|
||||||
|
|
||||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
|
|
||||||
if comparison.Outcome != OutcomeDestinationOlder {
|
|
||||||
t.Fatalf("comparison = %#v, want destination older", comparison)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSharedRootStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal shared root state: %v", err)
|
|
||||||
}
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSharedRootState(t *testing.T) SharedRootState {
|
|
||||||
t.Helper()
|
|
||||||
source := validManifest(t)
|
|
||||||
htmlSource := source
|
|
||||||
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
|
||||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
return SharedRootState{
|
|
||||||
SchemaVersion: SharedRootSchemaVersion,
|
|
||||||
DistributorVersion: "dev",
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSharedRoot},
|
|
||||||
Owners: []OwnerRecord{{
|
|
||||||
Scope: CurrentOwnerScope("reports", "archive"),
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
|
||||||
}, {
|
|
||||||
Scope: CurrentOwnerScope("reports", "html"),
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
|
||||||
Source: SourceState{Manifest: htmlSource},
|
|
||||||
}},
|
|
||||||
Outputs: []SharedRootOutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
Owner: CurrentOwnerScope("reports", "archive"),
|
|
||||||
SourceID: source.ID,
|
|
||||||
SourceDigest: source.Digest,
|
|
||||||
SourceCreated: source.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}, {
|
|
||||||
Path: "report.html",
|
|
||||||
Kind: OutputKindGenerated,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
Transform: "markdown_to_html",
|
|
||||||
URL: "https://reports.example.com/html/report.html",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 128,
|
|
||||||
Owner: CurrentOwnerScope("reports", "html"),
|
|
||||||
SourceID: htmlSource.ID,
|
|
||||||
SourceDigest: htmlSource.Digest,
|
|
||||||
SourceCreated: htmlSource.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
OutputKindSource = "source"
|
|
||||||
OutputKindGenerated = "generated"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Validate(s DistributorState) error {
|
|
||||||
if s.SchemaVersion != SchemaVersion {
|
|
||||||
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
|
||||||
}
|
|
||||||
if s.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state pipeline_id is required")
|
|
||||||
}
|
|
||||||
if s.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state destination_id is required")
|
|
||||||
}
|
|
||||||
if s.PublishedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state published_at is required")
|
|
||||||
}
|
|
||||||
if s.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state created_at is required")
|
|
||||||
}
|
|
||||||
if s.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state updated_at is required")
|
|
||||||
}
|
|
||||||
if s.State.Mode != StateModeSingleOwner {
|
|
||||||
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
|
|
||||||
}
|
|
||||||
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
|
|
||||||
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
|
||||||
}
|
|
||||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
|
||||||
return fmt.Errorf("state source.manifest: %w", err)
|
|
||||||
}
|
|
||||||
if s.Links != nil && s.Links.PrimaryURL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
|
|
||||||
return fmt.Errorf("state links.primary_url: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.Outputs == nil {
|
|
||||||
return fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
seen := make(map[string]struct{}, len(s.Outputs))
|
|
||||||
for index, output := range s.Outputs {
|
|
||||||
if err := validateOutput(index, output); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateEmbeddedManifest(manifest bundle.Manifest) error {
|
|
||||||
return bundle.ValidateManifest(manifest)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOutput(index int, output OutputFile) error {
|
|
||||||
if err := storage.ValidatePath(output.Path); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
|
||||||
}
|
|
||||||
switch output.Kind {
|
|
||||||
case OutputKindSource, OutputKindGenerated:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
|
||||||
}
|
|
||||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
|
||||||
}
|
|
||||||
if output.URL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Size < 0 {
|
|
||||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
|
||||||
}
|
|
||||||
if output.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
|
||||||
}
|
|
||||||
if output.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -11,8 +11,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
)
|
)
|
||||||
@@ -33,13 +31,6 @@ type BundleOptions struct {
|
|||||||
ExtraFiles []SourceFile
|
ExtraFiles []SourceFile
|
||||||
}
|
}
|
||||||
|
|
||||||
type DestinationStateOptions struct {
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
DistributorVersion string
|
|
||||||
PublishedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func DefaultSourceFiles() []SourceFile {
|
func DefaultSourceFiles() []SourceFile {
|
||||||
return []SourceFile{
|
return []SourceFile{
|
||||||
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
||||||
@@ -133,28 +124,6 @@ func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
destinationState := DestinationState(manifest, opts)
|
|
||||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal destination state: %v", err)
|
|
||||||
}
|
|
||||||
statePath, err := storage.StatePath(relative)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("state path: %v", err)
|
|
||||||
}
|
|
||||||
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
|
||||||
for _, output := range destinationState.Outputs {
|
|
||||||
path, err := storage.Join(relative, output.Path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("join output path: %v", err)
|
|
||||||
}
|
|
||||||
WriteFakeFile(t, backend, path, "old")
|
|
||||||
}
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return writeConfigFile(t, `
|
return writeConfigFile(t, `
|
||||||
@@ -318,47 +287,6 @@ pipelines:
|
|||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
|
||||||
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
|
||||||
t.Fatalf("mkdir destination bundle: %v", err)
|
|
||||||
}
|
|
||||||
destinationState := DestinationState(manifest, opts)
|
|
||||||
writeJSONFile(t, filepath.Join(bundleRoot, storage.StateFileName), destinationState)
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
publishedAt := defaultPublishedAt(opts.PublishedAt)
|
|
||||||
return state.DistributorState{
|
|
||||||
SchemaVersion: state.SchemaVersion,
|
|
||||||
DistributorVersion: opts.DistributorVersion,
|
|
||||||
PipelineID: defaultString(opts.PipelineID, "reports"),
|
|
||||||
DestinationID: defaultString(opts.DestinationID, "archive"),
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: state.SourceState{Manifest: manifest},
|
|
||||||
Outputs: sourceOutputs(manifest, publishedAt),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReadDestinationState(t testing.TB, path string) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read destination state: %v", err)
|
|
||||||
}
|
|
||||||
destinationState, err := state.Parse(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse destination state: %v", err)
|
|
||||||
}
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func AssertFile(t testing.TB, path, want string) {
|
func AssertFile(t testing.TB, path, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
@@ -392,22 +320,6 @@ func sourceFiles(opts BundleOptions) []SourceFile {
|
|||||||
return files
|
return files
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
|
|
||||||
outputs := make([]state.OutputFile, 0, len(manifest.Files))
|
|
||||||
for _, file := range manifest.Files {
|
|
||||||
outputs = append(outputs, state.OutputFile{
|
|
||||||
Path: file.Path,
|
|
||||||
Kind: state.OutputKindSource,
|
|
||||||
SourcePath: file.Path,
|
|
||||||
SHA256: file.SHA256,
|
|
||||||
Size: file.Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigFile(t testing.TB, body string) string {
|
func writeConfigFile(t testing.TB, body string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
@@ -455,10 +367,3 @@ func defaultCreated(value time.Time) time.Time {
|
|||||||
}
|
}
|
||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaultPublishedAt(value time.Time) time.Time {
|
|
||||||
if value.IsZero() {
|
|
||||||
return time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user