eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Bind audio cache reuse to remote identity

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 21:09:21 +00:00
parent 8375ad83f3
commit d9fa1d9328
9 changed files with 414 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
internal/audio/cache_identity.go Normal file
View File

@@ -0,0 +1,177 @@
package audio
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
const audioCacheIdentityVersion = 1
type audioCacheIdentity struct {
Version int `json:"version"`
Bucket string `json:"bucket"`
ObjectKey string `json:"object_key"`
Generation string `json:"generation"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
func cacheIdentityPath(cachePath string) string {
return cachePath + ".identity.json"
}
func cacheIdentityForRequest(req S3MaterializeRequest, checksum string) audioCacheIdentity {
return audioCacheIdentity{
Version: audioCacheIdentityVersion,
Bucket: strings.TrimSpace(req.Bucket),
ObjectKey: strings.TrimSpace(req.Object.Key),
Generation: strings.TrimSpace(req.Object.ETag),
Size: req.Object.Size,
SHA256: strings.ToLower(strings.TrimSpace(checksum)),
}
}
func (identity audioCacheIdentity) matchesRequest(req S3MaterializeRequest) bool {
return identity.Version == audioCacheIdentityVersion &&
identity.Bucket == strings.TrimSpace(req.Bucket) &&
identity.ObjectKey == strings.TrimSpace(req.Object.Key) &&
identity.Generation != "" && identity.Generation == strings.TrimSpace(req.Object.ETag) &&
identity.Size > 0 && identity.Size == req.Object.Size &&
validSHA256(identity.SHA256)
}
func cacheIdentityEligible(req S3MaterializeRequest) bool {
return strings.TrimSpace(req.Object.Key) != "" && strings.TrimSpace(req.Bucket) != "" &&
strings.TrimSpace(req.Object.ETag) != "" && req.Object.Size > 0
}
func loadAudioCacheIdentity(path string) (audioCacheIdentity, bool, error) {
data, err := fileops.ReadRegularFile(path, 64*1024)
if os.IsNotExist(err) {
return audioCacheIdentity{}, false, nil
}
if err != nil {
return audioCacheIdentity{}, false, nil
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var identity audioCacheIdentity
if err := decoder.Decode(&identity); err != nil {
return audioCacheIdentity{}, false, nil
}
if err := decoder.Decode(&struct{}{}); err != io.EOF || !validSHA256(identity.SHA256) {
return audioCacheIdentity{}, false, nil
}
return identity, true, nil
}
func writeAudioCacheIdentity(path string, identity audioCacheIdentity) error {
data, err := json.Marshal(identity)
if err != nil {
return fmt.Errorf("encode audio cache identity: %w", err)
}
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write audio cache identity: %w", err)
}
return nil
}
func materializeVerifiedCachedAudio(ctx context.Context, req S3MaterializeRequest, cachePath string) (string, bool, error) {
if err := ctx.Err(); err != nil {
return "", false, err
}
identity, ok, err := loadAudioCacheIdentity(cacheIdentityPath(cachePath))
if err != nil {
return "", false, err
}
if !ok || !identity.matchesRequest(req) {
return "", false, nil
}
relativePath, err := cacheRelativePath(req.CacheRoot, cachePath)
if err != nil {
return "", false, err
}
source, err := fileops.OpenConfinedRegularFile(req.CacheRoot, relativePath)
if err != nil {
return "", false, nil
}
defer func() { _ = source.Close() }()
info, err := source.Stat()
if err != nil {
return "", false, fmt.Errorf("inspect cached audio: %w", err)
}
if !info.Mode().IsRegular() || info.Size() != identity.Size {
return "", false, nil
}
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(req.DestPath)); err != nil {
return "", false, fmt.Errorf("create audio destination directory: %w", err)
}
temporary, err := fileops.DownloadToSiblingTemp(req.DestPath, func(destination io.Writer) error {
_, err := io.Copy(destination, source)
return err
})
if err != nil {
return "", false, fmt.Errorf("copy verified cached audio: %w", err)
}
defer func() { _ = temporary.Cleanup() }()
checksum, size, err := temporaryChecksum(temporary)
if err != nil {
return "", false, err
}
if size != identity.Size || checksum != identity.SHA256 {
return "", false, nil
}
if err := temporary.Install(filepath.Base(req.DestPath), fileops.WorkspaceFileMode); err != nil {
return "", false, fmt.Errorf("install verified cached audio: %w", err)
}
return checksum, true, nil
}
func cacheRelativePath(cacheRoot, cachePath string) (string, error) {
relativePath, err := filepath.Rel(filepath.Clean(cacheRoot), filepath.Clean(cachePath))
if err != nil {
return "", fmt.Errorf("resolve audio cache path: %w", err)
}
normalized, err := pathsafe.NormalizeRelativeDestination(filepath.ToSlash(relativePath))
if err != nil {
return "", fmt.Errorf("audio cache path escapes cache root: %w", err)
}
return normalized, nil
}
func temporaryChecksum(temporary *fileops.DownloadedTempFile) (string, int64, error) {
file, err := temporary.Open()
if err != nil {
return "", 0, fmt.Errorf("open cached audio copy: %w", err)
}
digest := sha256.New()
size, copyErr := io.Copy(digest, file)
closeErr := file.Close()
if copyErr != nil {
return "", 0, fmt.Errorf("checksum cached audio copy: %w", copyErr)
}
if closeErr != nil {
return "", 0, fmt.Errorf("close cached audio copy: %w", closeErr)
}
return hex.EncodeToString(digest.Sum(nil)), size, nil
}
func validSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}

37
internal/audio/s3_audio.go
View File

@@ -51,19 +51,15 @@ func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3Materi
}
result := S3MaterializeResult{}
if req.CacheEnabled {
if req.CacheEnabled && cacheIdentityEligible(req) {
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("resolve audio cache path: %w", err)
}
result.CachePath = cachePath
if ok, err := validCachedAudio(cachePath, req.Object.Size); err != nil {
if checksum, ok, err := materializeVerifiedCachedAudio(ctx, req, cachePath); err != nil {
return S3MaterializeResult{}, err
} else if ok {
checksum, err := fileops.CopyFileAtomicWithChecksum(cachePath, req.DestPath, fileops.WorkspaceFileMode)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("materialize cached audio %q: %w", cachePath, err)
}
result.Checksum = checksum
result.CacheHit = true
return result, nil
@@ -90,34 +86,21 @@ func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3Materi
result.Downloaded = true
if result.CachePath != "" {
if _, err := fileops.CopyFileAtomicWithChecksum(spoolPath, result.CachePath, fileops.WorkspaceFileMode); err != nil {
cacheChecksum, err := fileops.CopyFileAtomicWithChecksum(spoolPath, result.CachePath, fileops.WorkspaceFileMode)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("populate audio cache %q: %w", result.CachePath, err)
}
if cacheChecksum != result.Checksum {
return S3MaterializeResult{}, fmt.Errorf("audio cache checksum differs from downloaded audio")
}
if err := writeAudioCacheIdentity(cacheIdentityPath(result.CachePath), cacheIdentityForRequest(req, cacheChecksum)); err != nil {
return S3MaterializeResult{}, err
}
}
return result, nil
}
func validCachedAudio(path string, expectedSize int64) (bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("stat cached audio %q: %w", path, err)
}
if info.IsDir() {
return false, fmt.Errorf("cached audio path is a directory: %q", path)
}
if info.Size() <= 0 {
return false, nil
}
if expectedSize > 0 && info.Size() != expectedSize {
return false, nil
}
return true, nil
}
func validateLocalAudio(path string, expectedSize int64) error {
info, err := os.Stat(path)
if err != nil {

206
internal/audio/s3_audio_test.go
View File

@@ -40,12 +40,11 @@ func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
t.Fatalf("cache path: %v", err)
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("populate cache: %v", err)
}
writeAudioTestFile(t, cachePath, "audio")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
@@ -54,8 +53,8 @@ func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
if !result.CacheHit || result.Downloaded {
t.Fatalf("result = %#v, want cache hit", result)
}
if len(fake.Downloads) != 0 {
t.Fatalf("downloads = %d, want 0", len(fake.Downloads))
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want only the initial population download", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio")
if result.SpoolPath != "" {
@@ -63,6 +62,192 @@ func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
}
}
func TestMaterializeS3AudioRefreshesSameSizeChangedGeneration(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio-one"), ETag: "generation-one"})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio-one")))
req.Object.ETag = "generation-one"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("initial MaterializeS3Audio() error = %v", err)
}
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio-two"), ETag: "generation-two"})
req.Object.ETag = "generation-two"
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("refreshed MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit || len(fake.Downloads) != 2 {
t.Fatalf("result=%#v downloads=%d, want refreshed download", result, len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio-two")
}
func TestMaterializeS3AudioRefreshesCorruptedSameSizeCache(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "generation"})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
req.Object.ETag = "generation"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("initial MaterializeS3Audio() error = %v", err)
}
writeAudioTestFile(t, cachePathForTest(t, req), "stale-audio")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("refreshed MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit || len(fake.Downloads) != 2 {
t.Fatalf("result=%#v downloads=%d, want refreshed download", result, len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "fresh-audio")
}
func TestMaterializeS3AudioMissingIdentityDoesNotReuseCache(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "generation"})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
req.Object.ETag = "generation"
writeAudioTestFile(t, cachePathForTest(t, req), "stale-audio")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit || len(fake.Downloads) != 1 {
t.Fatalf("result=%#v downloads=%d, want cache refresh", result, len(fake.Downloads))
}
}
func TestMaterializeS3AudioTreatsMultipartETagAsGeneration(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "not-a-checksum-2"})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
req.Object.ETag = "not-a-checksum-2"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("populate cache: %v", err)
}
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("reuse cache: %v", err)
}
if !result.CacheHit || result.Downloaded || len(fake.Downloads) != 1 {
t.Fatalf("result=%#v downloads=%d, want verified cache hit", result, len(fake.Downloads))
}
}
func TestMaterializeS3AudioUnknownRemoteSizeDoesNotReuseCache(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "generation"})
req := testMaterializeRequest(t, root, fake, key, 0)
req.Object.ETag = "generation"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("first MaterializeS3Audio() error = %v", err)
}
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("second MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit || len(fake.Downloads) != 2 {
t.Fatalf("result=%#v downloads=%d, want refresh without identity evidence", result, len(fake.Downloads))
}
}
func TestMaterializeS3AudioSymlinkedCacheRefreshesWithoutFollowingLink(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "generation"})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
req.Object.ETag = "generation"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("populate cache: %v", err)
}
cachePath := cachePathForTest(t, req)
outsidePath := filepath.Join(root, "outside-audio")
writeAudioTestFile(t, outsidePath, "untrusted-audio")
if err := os.Remove(cachePath); err != nil {
t.Fatalf("remove cache file: %v", err)
}
if err := os.Symlink(outsidePath, cachePath); err != nil {
t.Fatalf("create cache symlink: %v", err)
}
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("refresh symlinked cache: %v", err)
}
if !result.Downloaded || result.CacheHit || len(fake.Downloads) != 2 {
t.Fatalf("result=%#v downloads=%d, want safe refresh", result, len(fake.Downloads))
}
assertFileEquals(t, outsidePath, "untrusted-audio")
assertFileEquals(t, req.DestPath, "fresh-audio")
}
func TestMaterializeS3AudioNonRegularCacheFailsClearly(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio"), ETag: "generation"})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
req.Object.ETag = "generation"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("populate cache: %v", err)
}
cachePath := cachePathForTest(t, req)
if err := os.Remove(cachePath); err != nil {
t.Fatalf("remove cache file: %v", err)
}
if err := os.Mkdir(cachePath, 0o755); err != nil {
t.Fatalf("create cache directory: %v", err)
}
_, err := MaterializeS3Audio(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "populate audio cache") {
t.Fatalf("error = %v, want clear cache refresh error", err)
}
}
func TestMaterializeS3AudioInterruptedRefreshCanRetry(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio-one"), ETag: "generation-one"})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio-one")))
req.Object.ETag = "generation-one"
if _, err := MaterializeS3Audio(context.Background(), req); err != nil {
t.Fatalf("populate cache: %v", err)
}
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio-two"), ETag: "generation-two"})
req.Object.ETag = "generation-two"
fake.DownloadErr = os.ErrPermission
if _, err := MaterializeS3Audio(context.Background(), req); err == nil {
t.Fatal("MaterializeS3Audio() error = nil, want interrupted refresh failure")
}
fake.DownloadErr = nil
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("retry refresh: %v", err)
}
if !result.Downloaded || result.CacheHit {
t.Fatalf("result = %#v, want refreshed download after retry", result)
}
assertFileEquals(t, req.DestPath, "audio-two")
}
func TestMaterializeS3AudioInvalidCacheRefreshesFromS3(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
@@ -116,6 +301,15 @@ func testMaterializeRequest(t *testing.T, root string, fake *storage.FakeBackend
}
}
func cachePathForTest(t *testing.T, req S3MaterializeRequest) string {
t.Helper()
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, req.Object.Key)
if err != nil {
t.Fatalf("cache path: %v", err)
}
return cachePath
}
func writeAudioTestFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {