eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add destination link metadata

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-01 21:42:00 +00:00
parent a8564035d3
commit 980ae15249
24 changed files with 737 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
internal/state/distributor.go
View File

@@ -19,6 +19,7 @@ type DistributorState struct {
DestinationID string
PublishedAt time.Time
Source SourceState
Links *LinkState
Outputs []OutputFile
}
@@ -26,11 +27,16 @@ type SourceState struct {
Manifest bundle.Manifest
}
type LinkState struct {
PrimaryURL string
}
type OutputFile struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
}
@@ -42,6 +48,7 @@ type rawDistributorState struct {
DestinationID *string `json:"destination_id"`
PublishedAt *string `json:"published_at"`
Source *rawSourceState `json:"source"`
Links *rawLinkState `json:"links"`
Outputs []rawOutputFile `json:"outputs"`
}
@@ -49,11 +56,16 @@ type rawSourceState struct {
Manifest json.RawMessage `json:"manifest"`
}
type rawLinkState struct {
PrimaryURL string `json:"primary_url"`
}
type rawOutputFile struct {
Path *string `json:"path"`
Kind *string `json:"kind"`
SourcePath *string `json:"source_path"`
Transform string `json:"transform"`
URL string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
}
@@ -112,6 +124,9 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
}
state.Source.Manifest = manifest
if raw.Links != nil {
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
}
if raw.Outputs == nil {
return DistributorState{}, fmt.Errorf("state outputs is required")
}
@@ -161,6 +176,7 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
Kind: *raw.Kind,
SourcePath: *raw.SourcePath,
Transform: raw.Transform,
URL: raw.URL,
SHA256: *raw.SHA256,
Size: *raw.Size,
}, nil
@@ -181,6 +197,7 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
DestinationID string `json:"destination_id"`
PublishedAt string `json:"published_at"`
Source sourceJSON `json:"source"`
Links *LinkState `json:"links,omitempty"`
Outputs []OutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
@@ -190,16 +207,25 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
DestinationID: s.DestinationID,
PublishedAt: s.PublishedAtString(),
Source: sourceJSON{Manifest: s.Source.Manifest},
Links: s.Links,
Outputs: s.Outputs,
})
}
func (l LinkState) MarshalJSON() ([]byte, error) {
type linkJSON struct {
PrimaryURL string `json:"primary_url,omitempty"`
}
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
}
func (o OutputFile) MarshalJSON() ([]byte, error) {
type outputJSON struct {
Path string `json:"path"`
Kind string `json:"kind"`
SourcePath string `json:"source_path"`
Transform string `json:"transform,omitempty"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
}
@@ -208,6 +234,7 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
})

52
internal/state/distributor_test.go
View File

@@ -29,6 +29,22 @@ func TestParseValidState(t *testing.T) {
}
}
func TestParseValidStateWithLinks(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
state, err := Parse([]byte(body))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
t.Fatalf("links = %#v, want primary URL", state.Links)
}
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
t.Fatalf("output URL = %q", state.Outputs[0].URL)
}
}
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
state, err := Parse([]byte(body))
@@ -145,6 +161,12 @@ func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
"negative size": func(s *DistributorState) {
s.Outputs[0].Size = -1
},
"invalid output url": func(s *DistributorState) {
s.Outputs[0].URL = "file:///tmp/report.md"
},
"invalid primary url": func(s *DistributorState) {
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
@@ -188,6 +210,36 @@ func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
}
}
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
source := validManifest(t)
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
Source: SourceState{Manifest: source},
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
URL: "https://reports.example.com/archive/report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
}},
}
data, err := json.Marshal(state)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
t.Fatalf("json = %s, want links primary URL", data)
}
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
t.Fatalf("json = %s, want output URL", data)
}
}
func validStateJSON(t *testing.T) string {
t.Helper()
return validStateWithManifestJSON(t, manifestJSON(t))

31
internal/state/validate.go
View File

@@ -2,6 +2,7 @@ package state
import (
"fmt"
"net/url"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
@@ -28,6 +29,11 @@ func Validate(s DistributorState) error {
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
return fmt.Errorf("state source.manifest: %w", err)
}
if s.Links != nil && s.Links.PrimaryURL != "" {
if err := validateStateURL(s.Links.PrimaryURL); err != nil {
return fmt.Errorf("state links.primary_url: %w", err)
}
}
if s.Outputs == nil {
return fmt.Errorf("state outputs is required")
}
@@ -63,6 +69,11 @@ func validateOutput(index int, output OutputFile) error {
if output.Kind == OutputKindGenerated && output.Transform == "" {
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
if output.URL != "" {
if err := validateStateURL(output.URL); err != nil {
return fmt.Errorf("state outputs[%d].url: %w", index, err)
}
}
if err := bundle.ValidateDigest(output.SHA256); err != nil {
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
}
@@ -71,3 +82,23 @@ func validateOutput(index int, output OutputFile) error {
}
return nil
}
func validateStateURL(value string) error {
parsed, err := url.Parse(value)
if err != nil {
return err
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("must use http or https")
}
if parsed.Host == "" {
return fmt.Errorf("must include a host")
}
if parsed.RawQuery != "" {
return fmt.Errorf("must not include a query string")
}
if parsed.Fragment != "" {
return fmt.Errorf("must not include a fragment")
}
return nil
}