Inspect hosted assets against build manifests

This commit is contained in:
2026-10-05 19:56:32 +00:00
parent 717d6d0d04
commit 692a503c26
9 changed files with 695 additions and 13 deletions

266
internal/gitea/assets.go Normal file
View File

@@ -0,0 +1,266 @@
package gitea
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"sort"
"strconv"
"strings"
)
const ManifestFilename = "SHA256SUMS"
type AssetState string
const (
AssetMatching AssetState = "matching"
AssetMissing AssetState = "missing"
AssetConflicting AssetState = "conflicting"
AssetUnverifiable AssetState = "unverifiable"
)
// ExpectedAsset describes one declared file. Content is populated for the
// generated checksum manifest; Path identifies local binary content for later
// upload operations.
type ExpectedAsset struct {
Filename string
SHA256 string
Size int64
Path string
Content []byte
}
type RemoteAsset struct {
ID int64
Filename string
Size int64
DownloadURL string
}
type AssetInspection struct {
Expected ExpectedAsset
State AssetState
Existing []RemoteAsset
Err error
}
type Inspection struct {
Assets []AssetInspection
Unrelated []RemoteAsset
}
type assetResponse struct {
ID int64 `json:"id"`
Name string `json:"name"`
Size int64 `json:"size"`
BrowserDownloadURL string `json:"browser_download_url"`
}
// BuildManifest creates the deterministic checksum file for the declared
// binaries. An empty binary set produces no manifest.
func BuildManifest(binaries []ExpectedAsset) (ExpectedAsset, error) {
if len(binaries) == 0 {
return ExpectedAsset{}, nil
}
ordered := append([]ExpectedAsset(nil), binaries...)
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Filename < ordered[j].Filename })
var body strings.Builder
seen := make(map[string]struct{}, len(ordered))
for _, binary := range ordered {
if !validAssetName(binary.Filename) || binary.Filename == ManifestFilename || !validSHA256(binary.SHA256) {
return ExpectedAsset{}, failure(FailureInvalid, "build checksum manifest", 0, fmt.Errorf("invalid binary identity for %q", binary.Filename))
}
if _, exists := seen[binary.Filename]; exists {
return ExpectedAsset{}, failure(FailureInvalid, "build checksum manifest", 0, fmt.Errorf("duplicate binary filename %q", binary.Filename))
}
seen[binary.Filename] = struct{}{}
fmt.Fprintf(&body, "%s %s\n", binary.SHA256, binary.Filename)
}
content := []byte(body.String())
return ExpectedAsset{Filename: ManifestFilename, SHA256: digest(content), Size: int64(len(content)), Content: content}, nil
}
// InspectAssets lists the release's attachments and compares every declared
// asset by streamed SHA-256 content. It does not create or modify remote data.
func (c Client) InspectAssets(ctx context.Context, destination Destination, releaseID int64, expected []ExpectedAsset) (Inspection, error) {
if releaseID <= 0 {
return Inspection{}, failure(FailureInvalid, "inspect release assets", 0, errors.New("release ID must be positive"))
}
if strings.TrimSpace(destination.Token) == "" {
return Inspection{}, failure(FailureInvalid, "inspect release assets", 0, errors.New("Gitea token is required"))
}
base, err := parseBaseURL(destination.BaseURL)
if err != nil {
return Inspection{}, failure(FailureInvalid, "validate destination", 0, err)
}
if strings.TrimSpace(destination.Owner) == "" || strings.ContainsAny(destination.Owner, "/\\") || strings.TrimSpace(destination.Repository) == "" || strings.ContainsAny(destination.Repository, "/\\") {
return Inspection{}, failure(FailureInvalid, "inspect release assets", 0, errors.New("Gitea owner and repository must be path segments"))
}
if err := validateExpectedAssets(expected); err != nil {
return Inspection{}, err
}
apiRoot := strings.TrimRight(base.String(), "/") + "/api/v1/repos/" + url.PathEscape(destination.Owner) + "/" + url.PathEscape(destination.Repository)
assetsURL := apiRoot + "/releases/" + strconv.FormatInt(releaseID, 10) + "/assets"
var listed []assetResponse
if err := c.getJSON(ctx, c.httpClient(), destination.Token, "list release assets", assetsURL, &listed); err != nil {
return Inspection{}, err
}
if listed == nil {
return Inspection{}, failure(FailureMalformed, "decode release assets", http.StatusOK, errors.New("attachment response must be a JSON array"))
}
remoteByName := make(map[string][]RemoteAsset, len(listed))
for _, item := range listed {
if item.ID <= 0 || item.Name == "" || item.Size < 0 {
return Inspection{}, failure(FailureMalformed, "decode release assets", http.StatusOK, errors.New("attachment entry is missing its ID, name, or size"))
}
remote := RemoteAsset{ID: item.ID, Filename: item.Name, Size: item.Size, DownloadURL: item.BrowserDownloadURL}
remoteByName[item.Name] = append(remoteByName[item.Name], remote)
}
inspection := Inspection{Assets: make([]AssetInspection, 0, len(expected))}
declared := make(map[string]struct{}, len(expected))
for _, item := range expected {
declared[item.Filename] = struct{}{}
matches := remoteByName[item.Filename]
observation := AssetInspection{Expected: item, Existing: matches}
switch len(matches) {
case 0:
observation.State = AssetMissing
case 1:
remoteHash, _, err := c.downloadHash(ctx, c.httpClient(), base, destination.Token, matches[0].DownloadURL)
if err != nil {
observation.State = AssetUnverifiable
observation.Err = err
} else if remoteHash == item.SHA256 {
observation.State = AssetMatching
} else {
observation.State = AssetConflicting
}
default:
observation.State = AssetConflicting
}
inspection.Assets = append(inspection.Assets, observation)
}
for _, item := range listed {
if _, expected := declared[item.Name]; !expected {
inspection.Unrelated = append(inspection.Unrelated, RemoteAsset{ID: item.ID, Filename: item.Name, Size: item.Size, DownloadURL: item.BrowserDownloadURL})
}
}
return inspection, nil
}
func (c Client) downloadHash(ctx context.Context, client *http.Client, base *url.URL, token, rawURL string) (string, int64, error) {
if strings.TrimSpace(rawURL) == "" {
return "", 0, failure(FailureUnverifiable, "download release asset", 0, errors.New("attachment has no download URL"))
}
assetURL, err := base.Parse(rawURL)
if err != nil || !sameOrigin(base, assetURL) || assetURL.User != nil {
return "", 0, failure(FailureUnverifiable, "download release asset", 0, errors.New("asset URL is outside the configured service origin"))
}
assetURL.Fragment = ""
visited := make(map[string]struct{})
for {
key := assetURL.String()
if _, exists := visited[key]; exists {
return "", 0, failure(FailureUnverifiable, "download release asset", 0, errors.New("asset redirect loop"))
}
visited[key] = struct{}{}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, key, nil)
if err != nil {
return "", 0, failure(FailureUnverifiable, "download release asset", 0, err)
}
request.Header.Set("Authorization", "token "+token)
request.Header.Set("Accept", "application/octet-stream")
response, err := client.Do(request)
if err != nil {
return "", 0, failure(FailureUnverifiable, "download release asset", 0, err)
}
if isRedirect(response.StatusCode) {
location, locationErr := response.Location()
_ = response.Body.Close()
if locationErr != nil || !sameOrigin(base, location) || location.User != nil {
return "", 0, failure(FailureUnverifiable, "redirect release asset", response.StatusCode, errors.New("asset redirect is outside the configured service origin"))
}
location.Fragment = ""
assetURL = location
continue
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return "", 0, responseFailure("download release asset", response.StatusCode)
}
hasher := sha256.New()
size, err := io.Copy(hasher, response.Body)
if err != nil {
return "", size, failure(FailureUnverifiable, "read release asset", response.StatusCode, err)
}
return hex.EncodeToString(hasher.Sum(nil)), size, nil
}
}
func validateExpectedAssets(expected []ExpectedAsset) error {
seen := make(map[string]struct{}, len(expected))
for _, item := range expected {
if !validAssetName(item.Filename) || !validSHA256(item.SHA256) || item.Size < 0 {
return failure(FailureInvalid, "inspect release assets", 0, fmt.Errorf("invalid expected asset %q", item.Filename))
}
if _, duplicate := seen[item.Filename]; duplicate {
return failure(FailureInvalid, "inspect release assets", 0, fmt.Errorf("duplicate expected asset %q", item.Filename))
}
seen[item.Filename] = struct{}{}
if len(item.Content) > 0 && (int64(len(item.Content)) != item.Size || digest(item.Content) != item.SHA256) {
return failure(FailureInvalid, "inspect release assets", 0, fmt.Errorf("expected content does not match identity for %q", item.Filename))
}
}
return nil
}
func validAssetName(name string) bool {
return name != "" && name != "." && name != ".." && path.Base(name) == name && !strings.ContainsAny(name, "\\\r\n\x00")
}
func validSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(value)
return err == nil && strings.ToLower(value) == value
}
func digest(content []byte) string {
sum := sha256.Sum256(content)
return hex.EncodeToString(sum[:])
}
func sameOrigin(base, target *url.URL) bool {
if target == nil || target.User != nil || target.Host == "" {
return false
}
return strings.EqualFold(base.Scheme, target.Scheme) && strings.EqualFold(base.Hostname(), target.Hostname()) && effectivePort(base) == effectivePort(target)
}
func effectivePort(parsed *url.URL) string {
if port := parsed.Port(); port != "" {
return port
}
if strings.EqualFold(parsed.Scheme, "https") {
return "443"
}
return "80"
}
func isRedirect(status int) bool {
switch status {
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
return true
default:
return false
}
}

View File

@@ -0,0 +1,303 @@
package gitea
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
)
const assetListPath = "/api/v1/repos/owner/project/releases/10/assets"
func TestBuildManifestSortsNamesAndOmitsEmptySourceOnlyManifest(t *testing.T) {
manifest, err := BuildManifest([]ExpectedAsset{
assetFor("zeta.bin", []byte("zeta")),
assetFor("alpha.bin", []byte("alpha")),
})
if err != nil {
t.Fatal(err)
}
want := fmt.Sprintf("%s alpha.bin\n%s zeta.bin\n", digest([]byte("alpha")), digest([]byte("zeta")))
if string(manifest.Content) != want || manifest.Filename != ManifestFilename || manifest.SHA256 != digest([]byte(want)) || manifest.Size != int64(len(want)) {
t.Fatalf("manifest = %#v, content %q", manifest, manifest.Content)
}
empty, err := BuildManifest(nil)
if err != nil || empty.Filename != "" || len(empty.Content) != 0 {
t.Fatalf("empty manifest = %#v, error = %v", empty, err)
}
}
func TestInspectAssetsAcceptsMatchingContentAndPreservesUnrelatedFiles(t *testing.T) {
content := []byte("selected binary")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
switch r.URL.Path {
case assetListPath:
writeAssetList(t, w, []assetResponse{
{ID: 4, Name: "app-v1.2.3-linux-amd64", Size: int64(len(content)), BrowserDownloadURL: "/files/app"},
{ID: 8, Name: "notes.txt", Size: 9, BrowserDownloadURL: "/files/notes"},
})
case "/files/app":
_, _ = w.Write(content)
case "/files/notes":
_, _ = w.Write([]byte("unrelated"))
default:
t.Fatalf("unexpected path %s", r.URL.Path)
}
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{assetFor("app-v1.2.3-linux-amd64", content)})
if err != nil {
t.Fatal(err)
}
if len(inspection.Assets) != 1 || inspection.Assets[0].State != AssetMatching {
t.Fatalf("inspection = %#v", inspection)
}
if len(inspection.Unrelated) != 1 || inspection.Unrelated[0].Filename != "notes.txt" {
t.Fatalf("unrelated = %#v", inspection.Unrelated)
}
}
func TestInspectAssetsDetectsSameNameAndSizeWithDifferentBytes(t *testing.T) {
expected := []ExpectedAsset{assetFor("app.bin", []byte("alpha")), assetFor("other.bin", []byte("beta"))}
var appDownloads, otherDownloads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{
{ID: 2, Name: expected[0].Filename, Size: 5, BrowserDownloadURL: "/files/app"},
{ID: 3, Name: expected[1].Filename, Size: 4, BrowserDownloadURL: "/files/other"},
})
return
}
if r.URL.Path == "/files/app" {
appDownloads.Add(1)
_, _ = w.Write([]byte("bravo"))
return
}
otherDownloads.Add(1)
_, _ = w.Write([]byte("beta"))
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, expected)
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetConflicting || inspection.Assets[1].State != AssetMatching || appDownloads.Load() != 1 || otherDownloads.Load() != 1 {
t.Fatalf("inspection = %#v, downloads = %d and %d", inspection, appDownloads.Load(), otherDownloads.Load())
}
}
func TestInspectAssetsAcceptsFullManifestWhenBinariesAreMissing(t *testing.T) {
binaries := []ExpectedAsset{assetFor("alpha.bin", []byte("alpha")), assetFor("beta.bin", []byte("beta"))}
manifest, err := BuildManifest(binaries)
if err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 3, Name: manifest.Filename, Size: manifest.Size, BrowserDownloadURL: "/files/manifest"}})
return
}
if r.URL.Path != "/files/manifest" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
_, _ = w.Write(manifest.Content)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, append(binaries, manifest))
if err != nil {
t.Fatal(err)
}
if len(inspection.Assets) != 3 || inspection.Assets[0].State != AssetMissing || inspection.Assets[1].State != AssetMissing || inspection.Assets[2].State != AssetMatching {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsRejectsSubsetManifest(t *testing.T) {
binaries := []ExpectedAsset{assetFor("alpha.bin", []byte("alpha")), assetFor("beta.bin", []byte("beta"))}
full, err := BuildManifest(binaries)
if err != nil {
t.Fatal(err)
}
subset, err := BuildManifest(binaries[:1])
if err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 3, Name: ManifestFilename, Size: subset.Size, BrowserDownloadURL: "/files/manifest"}})
return
}
_, _ = w.Write(subset.Content)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{full})
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetConflicting {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsRejectsDifferentManifest(t *testing.T) {
expected, err := BuildManifest([]ExpectedAsset{assetFor("alpha.bin", []byte("alpha"))})
if err != nil {
t.Fatal(err)
}
different := []byte("not the declared checksum list\n")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 3, Name: ManifestFilename, Size: int64(len(different)), BrowserDownloadURL: "/files/manifest"}})
return
}
_, _ = w.Write(different)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{expected})
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetConflicting {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsMarksUnreadableContentUnverifiable(t *testing.T) {
expected := assetFor("app.bin", []byte("alpha"))
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 2, Name: expected.Filename, Size: expected.Size, BrowserDownloadURL: "/files/app"}})
return
}
http.Error(w, "forbidden", http.StatusForbidden)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{expected})
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetUnverifiable || inspection.Assets[0].Err == nil {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsRejectsMalformedAttachmentList(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, assetListPath)
_, _ = w.Write([]byte("null"))
}))
defer server.Close()
_, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{assetFor("app.bin", []byte("alpha"))})
if failureKind(t, err) != FailureMalformed {
t.Fatalf("error = %v", err)
}
}
func TestInspectAssetsSourceOnlyDoesNotDeclareManifest(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, assetListPath)
writeAssetList(t, w, []assetResponse{{ID: 7, Name: "historical.txt", Size: 3}})
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, nil)
if err != nil {
t.Fatal(err)
}
if len(inspection.Assets) != 0 || len(inspection.Unrelated) != 1 || inspection.Unrelated[0].Filename != "historical.txt" {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsFollowsSameOriginRedirectWithAuthorization(t *testing.T) {
content := []byte("alpha")
expected := assetFor("app.bin", content)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 1, Name: expected.Filename, Size: expected.Size, BrowserDownloadURL: "/files/redirect"}})
return
}
if r.URL.Path == "/files/redirect" {
http.Redirect(w, r, "/files/content", http.StatusFound)
return
}
if r.URL.Path == "/files/content" {
_, _ = w.Write(content)
return
}
t.Fatalf("unexpected path %s", r.URL.Path)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{expected})
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetMatching {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestInspectAssetsNeverSendsTokenToCrossOriginRedirect(t *testing.T) {
var externalCalled atomic.Bool
external := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { externalCalled.Store(true) }))
defer external.Close()
expected := assetFor("app.bin", []byte("alpha"))
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assertAssetGET(t, r, "")
if r.URL.Path == assetListPath {
writeAssetList(t, w, []assetResponse{{ID: 1, Name: expected.Filename, Size: expected.Size, BrowserDownloadURL: "/files/redirect"}})
return
}
http.Redirect(w, r, external.URL+"/private", http.StatusFound)
}))
defer server.Close()
inspection, err := testClient(server).InspectAssets(context.Background(), testDestination(server.URL), 10, []ExpectedAsset{expected})
if err != nil {
t.Fatal(err)
}
if inspection.Assets[0].State != AssetUnverifiable || externalCalled.Load() {
t.Fatalf("inspection = %#v, cross-origin called = %t", inspection, externalCalled.Load())
}
}
func assetFor(filename string, content []byte) ExpectedAsset {
sum := sha256.Sum256(content)
return ExpectedAsset{Filename: filename, SHA256: hex.EncodeToString(sum[:]), Size: int64(len(content)), Content: append([]byte(nil), content...)}
}
func writeAssetList(t *testing.T, w http.ResponseWriter, assets []assetResponse) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(assets); err != nil {
t.Error(err)
}
}
func assertAssetGET(t *testing.T, r *http.Request, expectedPath string) {
t.Helper()
if r.Method != http.MethodGet {
t.Errorf("method = %q, want GET", r.Method)
}
if expectedPath != "" && r.URL.Path != expectedPath {
t.Errorf("path = %q, want %q", r.URL.Path, expectedPath)
}
if got := r.Header.Get("Authorization"); got != "token "+testToken {
t.Errorf("Authorization = %q", got)
}
}
func testDestination(base string) Destination {
return Destination{BaseURL: base, Owner: "owner", Repository: "project", Token: testToken}
}

View File

@@ -23,6 +23,7 @@ const (
FailureUnauthorized FailureKind = "unauthorized"
FailureConflict FailureKind = "conflict"
FailureMalformed FailureKind = "malformed_response"
FailureUnverifiable FailureKind = "unverifiable"
FailureTransport FailureKind = "transport"
FailureRejected FailureKind = "rejected"
FailureUnknown FailureKind = "unknown"