diff --git a/README.md b/README.md index 7b9e4be..56d9919 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,35 @@ Current boundaries: - local development audio (`audio_dir` / `audio_files`) still works - `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive -- no real S3 backend or AWS SDK integration yet +- real S3-compatible backend now exists in the storage adapter package +- storage backend tests use fake storage and do not require live S3 - no prepare-stage S3 list/download behavior yet - no archive-stage S3 upload/promotion behavior yet - no `current/manifest.json` or `current/run_id.txt` uploads yet +## Remote Storage Backend + +Narratio includes an object-store backend layer for future prepare/archive work: + +- `List(ctx, prefix)` +- `Download(ctx, key, localPath)` +- `Upload(ctx, localPath, key, opts)` +- `Exists(ctx, key)` + +Implemented backends: + +- fake storage backend for deterministic tests +- S3-compatible backend built from `pipeline.storage.s3` + +Key invariant: + +- callers pass full bucket-relative object keys +- storage backends do not prepend `root_prefix` and do not infer session/campaign paths + +Current boundary: + +- this backend layer is implemented but not yet wired into prepare-stage audio retrieval or archive-stage uploads + ## Canonical Stage Order 1. `prepare` diff --git a/architecture.md b/architecture.md index d071c17..592a162 100644 --- a/architecture.md +++ b/architecture.md @@ -32,11 +32,15 @@ Implemented: - S3 session/run/current key builders - campaign/session/run local work/spool path helpers - manifest run/path identity fields (`campaign`, `run_id`, local and S3 prefixes) +- remote storage backend layer: + - narrow object-store interface (`List`, `Download`, `Upload`, `Exists`) + - fake storage backend for deterministic tests (no network dependency) + - S3-compatible backend using AWS SDK v2 + - config-based object-store construction helper Still placeholder/future: - `archive` stage behavior -- real S3 storage backend (list/download/upload/exists) - prepare-stage S3 audio download behavior - archive-stage S3 upload/promotion behavior - `notify` stage behavior @@ -157,6 +161,15 @@ Cross-config validation scope: - `pipeline.storage.s3.bucket` is required only when an S3-dependent feature is explicitly configured (for current foundations, that includes `session.inputs.audio_s3`, and archive upload intent when using `storage.backend: s3`) - no AWS credentials are stored in Narratio config; credential resolution remains an external runtime concern +Remote object-store backend scope: + +- remote storage APIs are isolated to `internal/adapters/storage` +- AWS SDK types remain contained within the S3 backend implementation package +- S3 key/session path semantics remain outside the backend, with this invariant: + - callers pass full bucket-relative object keys + - backend methods do not prepend `root_prefix` or infer campaign/session/run paths +- the backend layer is available for future prepare/archive usage, but no stage currently invokes `List`/`Download`/`Upload`/`Exists` + `pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work. `pipeline.trim` is optional. Existing pipelines without trim config continue to work. diff --git a/docs/roadmap/narratio-s3-archive-implementation-plan.md b/docs/roadmap/narratio-s3-archive-implementation-plan.md index 8fd6948..0b765d9 100644 --- a/docs/roadmap/narratio-s3-archive-implementation-plan.md +++ b/docs/roadmap/narratio-s3-archive-implementation-plan.md @@ -95,11 +95,14 @@ Implemented in repository: - campaign/session/run local work and spool path helpers - manifest run/path identity fields - examples and tests for the above foundations +- remote storage backend layer: + - object-store abstraction with `List`, `Download`, `Upload`, and `Exists` + - fake storage backend for deterministic, no-network testing + - S3-compatible backend built from `storage.s3` config + - backend construction helper from resolved config Not implemented yet: -- real S3 backend -- AWS SDK integration - prepare-stage S3 list/download behavior - archive-stage S3 upload behavior - promotion uploads @@ -849,9 +852,9 @@ Expected commit: Add archive storage path configuration ``` -### Storage Backend Interface and S3 Backend +### Storage Backend Interface and S3 Backend (Implemented) -Implement: +Implemented: - storage backend interface - object metadata type @@ -859,7 +862,7 @@ Implement: - real S3 backend using AWS SDK or existing project dependency policy - backend construction from config -No stage behavior yet. +No prepare/archive stage behavior yet. Expected commit: diff --git a/docs/runbooks/s3-archive-foundations.md b/docs/runbooks/s3-archive-foundations.md index 3cc7cf6..73babaa 100644 --- a/docs/runbooks/s3-archive-foundations.md +++ b/docs/runbooks/s3-archive-foundations.md @@ -12,11 +12,14 @@ This runbook documents the currently implemented storage/archive foundations and - promotion rule validation for safe relative paths - run ID generation and path/key helper functions - manifest run/path identity fields +- remote storage backend layer: + - object-store interface (`List`, `Download`, `Upload`, `Exists`) + - fake backend for deterministic tests + - S3-compatible backend using AWS SDK v2 + - config-based backend construction helper ## Not Implemented Yet -- real S3 backend integration -- AWS SDK wiring - prepare-stage S3 object listing or download - archive-stage S3 upload or promotion writes - writing `current/manifest.json` or `current/run_id.txt` in S3 @@ -29,4 +32,4 @@ This runbook documents the currently implemented storage/archive foundations and ## Next Implementation Target -Build the storage backend layer that can list/download/upload S3 objects through a fake-tested interface, then wire prepare/archive stages to use that backend. +Use the storage backend layer in prepare-stage session audio discovery/download flow, while preserving local audio input support. diff --git a/docs/storage-backends.md b/docs/storage-backends.md new file mode 100644 index 0000000..c6e7460 --- /dev/null +++ b/docs/storage-backends.md @@ -0,0 +1,58 @@ +# Storage Backends + +This document describes the currently implemented remote object storage backend layer used by Narratio, and its intended role in later prepare/archive work. + +## Implemented + +Remote object store abstraction: + +- `List(ctx, prefix)` +- `Download(ctx, key, localPath)` +- `Upload(ctx, localPath, key, opts)` +- `Exists(ctx, key)` + +Object metadata model includes: + +- key +- size +- ETag (provider metadata only) +- last modified time when available + +Backends: + +- fake storage backend for deterministic tests +- S3-compatible backend implemented with AWS SDK for Go v2 + +Construction: + +- config-based constructor builds S3 backend from `pipeline.storage.s3` values: + - bucket + - region + - endpoint + - force_path_style + +## Key Invariant + +- callers pass full bucket-relative object keys +- storage backends do not prepend `root_prefix` +- storage backends do not infer campaign/session/run paths + +S3 session/run key builders remain separate and continue to live outside backend implementations. + +## Security Boundary + +- do not store AWS credentials in Narratio config +- AWS credentials are resolved through standard AWS SDK credential chains +- AWS SDK-specific types remain isolated to the storage adapter package + +## Testing + +- fake storage tests cover list/download/upload/exists and error paths +- S3 backend tests use injected fake S3 API clients +- tests do not require live S3 services, AWS credentials, or network access + +## Not Implemented Yet + +- prepare-stage S3 object listing or downloads +- archive-stage S3 uploads or promotion writes +- writing `current/manifest.json` or `current/run_id.txt` to S3 diff --git a/go.mod b/go.mod index 4058cfd..e4f74a6 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,24 @@ module gitea.maximumdirect.net/eric/narratio go 1.25.0 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect +) diff --git a/go.sum b/go.sum index a62c313..48a2f5c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,39 @@ +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/adapters/storage/factory.go b/internal/adapters/storage/factory.go new file mode 100644 index 0000000..e7adc14 --- /dev/null +++ b/internal/adapters/storage/factory.go @@ -0,0 +1,29 @@ +package storage + +import ( + "context" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +// NewObjectStoreFromConfig constructs a remote object store from resolved config. +func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) { + if cfg == nil || cfg.Pipeline == nil { + return nil, fmt.Errorf("pipeline config is required") + } + + if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") { + if cfg.Pipeline.Storage.S3 == nil { + return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3") + } + return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3) + } + + if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" { + return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3) + } + + return nil, fmt.Errorf("no remote object store backend is configured") +} diff --git a/internal/adapters/storage/factory_test.go b/internal/adapters/storage/factory_test.go new file mode 100644 index 0000000..7c03cac --- /dev/null +++ b/internal/adapters/storage/factory_test.go @@ -0,0 +1,56 @@ +package storage + +import ( + "context" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) { + original := newS3Client + t.Cleanup(func() { newS3Client = original }) + newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) { + return &fakeS3API{}, nil + } + + store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{ + Pipeline: &config.PipelineConfig{ + Storage: config.StorageConfig{ + Backend: "s3", + S3: &config.StorageS3Config{ + Bucket: "my-archive", + }, + }, + }, + }) + if err != nil { + t.Fatalf("NewObjectStoreFromConfig() error = %v", err) + } + if _, ok := store.(*S3Backend); !ok { + t.Fatalf("store type = %T, want *S3Backend", store) + } +} + +func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) { + _, err := NewObjectStoreFromConfig(context.Background(), &config.Config{ + Pipeline: &config.PipelineConfig{ + Storage: config.StorageConfig{Backend: "s3"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") { + t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err) + } +} + +func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) { + _, err := NewObjectStoreFromConfig(context.Background(), &config.Config{ + Pipeline: &config.PipelineConfig{ + Storage: config.StorageConfig{Backend: "local"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") { + t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err) + } +} diff --git a/internal/adapters/storage/fake.go b/internal/adapters/storage/fake.go index be11724..b25f58b 100644 --- a/internal/adapters/storage/fake.go +++ b/internal/adapters/storage/fake.go @@ -1,6 +1,14 @@ package storage -import "context" +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) // NoopBackend is a deterministic no-op archive/storage adapter. type NoopBackend struct{} @@ -18,6 +26,13 @@ type FakeBackend struct { Requests []ArchiveRequest Err error Result ArchiveResult + + Objects map[string]FakeObject + + ListErr error + DownloadErr error + UploadErr error + ExistsErr error } // Archive records request and returns configured response. @@ -38,3 +53,140 @@ func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveR } return res, nil } + +// FakeObject is a deterministic fake object-store record. +type FakeObject struct { + Key string + Data []byte + Metadata map[string]string + ETag string + LastModified *time.Time +} + +// SeedObject inserts or replaces an object in the fake object store. +func (f *FakeBackend) SeedObject(obj FakeObject) { + if f.Objects == nil { + f.Objects = map[string]FakeObject{} + } + key := normalizeObjectKey(obj.Key) + obj.Key = key + obj.Data = append([]byte(nil), obj.Data...) + obj.Metadata = copyMetadata(obj.Metadata) + f.Objects[key] = obj +} + +// List returns deterministic prefix-filtered objects. +func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if f.ListErr != nil { + return nil, f.ListErr + } + + normalizedPrefix := normalizeObjectKey(prefix) + keys := make([]string, 0, len(f.Objects)) + for key := range f.Objects { + if strings.HasPrefix(key, normalizedPrefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + + out := make([]ObjectInfo, 0, len(keys)) + for _, key := range keys { + obj := f.Objects[key] + out = append(out, ObjectInfo{ + Key: obj.Key, + Size: int64(len(obj.Data)), + ETag: obj.ETag, + LastModified: obj.LastModified, + }) + } + return out, nil +} + +// Download writes one object to a local path. +func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error { + if err := ctx.Err(); err != nil { + return err + } + if f.DownloadErr != nil { + return f.DownloadErr + } + if strings.TrimSpace(localPath) == "" { + return fmt.Errorf("download object: local path is required") + } + + obj, ok := f.Objects[normalizeObjectKey(key)] + if !ok { + return fmt.Errorf("download object %q: %w", key, os.ErrNotExist) + } + + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return fmt.Errorf("download object %q: create parent directory: %w", key, err) + } + if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil { + return fmt.Errorf("download object %q: write local file: %w", key, err) + } + return nil +} + +// Upload reads a local file and stores it under key. +func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) { + if err := ctx.Err(); err != nil { + return ObjectInfo{}, err + } + if f.UploadErr != nil { + return ObjectInfo{}, f.UploadErr + } + if strings.TrimSpace(localPath) == "" { + return ObjectInfo{}, fmt.Errorf("upload object: local path is required") + } + if strings.TrimSpace(key) == "" { + return ObjectInfo{}, fmt.Errorf("upload object: key is required") + } + + data, err := os.ReadFile(localPath) + if err != nil { + return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err) + } + + normalizedKey := normalizeObjectKey(key) + now := time.Now().UTC() + obj := FakeObject{ + Key: normalizedKey, + Data: data, + Metadata: copyMetadata(opts.Metadata), + LastModified: &now, + } + f.SeedObject(obj) + return ObjectInfo{ + Key: normalizedKey, + Size: int64(len(data)), + LastModified: &now, + }, nil +} + +// Exists checks object presence. +func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + if f.ExistsErr != nil { + return false, f.ExistsErr + } + _, ok := f.Objects[normalizeObjectKey(key)] + return ok, nil +} + +func copyMetadata(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + return out +} diff --git a/internal/adapters/storage/fake_test.go b/internal/adapters/storage/fake_test.go index eee58df..22a1bd2 100644 --- a/internal/adapters/storage/fake_test.go +++ b/internal/adapters/storage/fake_test.go @@ -3,6 +3,9 @@ package storage import ( "context" "errors" + "os" + "path/filepath" + "strings" "testing" ) @@ -29,3 +32,84 @@ func TestFakeBackendError(t *testing.T) { t.Fatal("expected error, got nil") } } + +func TestFakeBackendListPrefixFiltering(t *testing.T) { + fake := &FakeBackend{} + fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")}) + fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")}) + fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")}) + + items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/") + if err != nil { + t.Fatalf("List() error = %v", err) + } + if len(items) != 2 { + t.Fatalf("List() len = %d, want 2", len(items)) + } + if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" { + t.Fatalf("List() keys = %#v", items) + } +} + +func TestFakeBackendDownload(t *testing.T) { + fake := &FakeBackend{} + fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")}) + + dst := filepath.Join(t.TempDir(), "nested", "a.flac") + if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil { + t.Fatalf("Download() error = %v", err) + } + data, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if string(data) != "audio-a" { + t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a") + } +} + +func TestFakeBackendUploadAndExists(t *testing.T) { + fake := &FakeBackend{} + + local := filepath.Join(t.TempDir(), "upload.txt") + if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{ + Metadata: map[string]string{"kind": "artifact"}, + }) + if err != nil { + t.Fatalf("Upload() error = %v", err) + } + if info.Key != "runs/id/artifact.txt" { + t.Fatalf("Upload() key = %q, want normalized key", info.Key) + } + + ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt") + if err != nil { + t.Fatalf("Exists() error = %v", err) + } + if !ok { + t.Fatal("Exists() = false, want true") + } +} + +func TestFakeBackendObjectErrors(t *testing.T) { + fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")} + + if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") { + t.Fatalf("List() error = %v, want list fail", err) + } + if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") { + t.Fatalf("Download() error = %v, want download fail", err) + } + local := filepath.Join(t.TempDir(), "x.txt") + _ = os.WriteFile(local, []byte("x"), 0o644) + if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") { + t.Fatalf("Upload() error = %v, want upload fail", err) + } + if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") { + t.Fatalf("Exists() error = %v, want exists fail", err) + } +} diff --git a/internal/adapters/storage/keys.go b/internal/adapters/storage/keys.go new file mode 100644 index 0000000..4325187 --- /dev/null +++ b/internal/adapters/storage/keys.go @@ -0,0 +1,8 @@ +package storage + +import "strings" + +func normalizeObjectKey(key string) string { + normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/") + return strings.TrimLeft(normalized, "/") +} diff --git a/internal/adapters/storage/keys_test.go b/internal/adapters/storage/keys_test.go new file mode 100644 index 0000000..a16f03e --- /dev/null +++ b/internal/adapters/storage/keys_test.go @@ -0,0 +1,21 @@ +package storage + +import "testing" + +func TestNormalizeObjectKey(t *testing.T) { + tests := []struct { + in string + want string + }{ + {in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"}, + {in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"}, + {in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"}, + {in: "", want: ""}, + } + + for _, tt := range tests { + if got := normalizeObjectKey(tt.in); got != tt.want { + t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/internal/adapters/storage/object_store.go b/internal/adapters/storage/object_store.go new file mode 100644 index 0000000..7ecd977 --- /dev/null +++ b/internal/adapters/storage/object_store.go @@ -0,0 +1,32 @@ +package storage + +import ( + "context" + "time" +) + +// ObjectStore is a remote object storage boundary used by future prepare/archive work. +// +// Key invariant: +// callers pass full bucket-relative object keys. Backend implementations do not +// infer Narratio session semantics and do not prepend root prefixes. +type ObjectStore interface { + List(ctx context.Context, prefix string) ([]ObjectInfo, error) + Download(ctx context.Context, key, localPath string) error + Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) + Exists(ctx context.Context, key string) (bool, error) +} + +// ObjectInfo describes one object in remote storage. +type ObjectInfo struct { + Key string + Size int64 + ETag string + LastModified *time.Time +} + +// UploadOptions configures optional object upload metadata. +type UploadOptions struct { + Metadata map[string]string + ContentType string +} diff --git a/internal/adapters/storage/s3_backend.go b/internal/adapters/storage/s3_backend.go new file mode 100644 index 0000000..9aca0be --- /dev/null +++ b/internal/adapters/storage/s3_backend.go @@ -0,0 +1,241 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +type s3API interface { + ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) +} + +// S3Backend is an ObjectStore implementation backed by S3-compatible APIs. +type S3Backend struct { + bucket string + client s3API +} + +type s3ClientOptions struct { + Region string + Endpoint string + ForcePathStyle bool +} + +var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) { + loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1) + if strings.TrimSpace(opts.Region) != "" { + loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region))) + } + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) + if err != nil { + return nil, fmt.Errorf("load aws config: %w", err) + } + + return s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if strings.TrimSpace(opts.Endpoint) != "" { + endpoint := strings.TrimSpace(opts.Endpoint) + o.BaseEndpoint = &endpoint + } + o.UsePathStyle = opts.ForcePathStyle + }), nil +} + +// NewS3BackendFromConfig builds an S3 backend from resolved config. +func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) { + bucket := strings.TrimSpace(cfg.Bucket) + if bucket == "" { + return nil, fmt.Errorf("storage.s3.bucket is required") + } + + client, err := newS3Client(ctx, s3ClientOptions{ + Region: cfg.Region, + Endpoint: cfg.Endpoint, + ForcePathStyle: cfg.ForcePathStyle, + }) + if err != nil { + return nil, fmt.Errorf("build s3 client: %w", err) + } + + return &S3Backend{ + bucket: bucket, + client: client, + }, nil +} + +// List returns objects under prefix. +func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { + normalizedPrefix := normalizeObjectKey(prefix) + out := make([]ObjectInfo, 0) + var token *string + + for { + resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &b.bucket, + Prefix: &normalizedPrefix, + ContinuationToken: token, + }) + if err != nil { + return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err) + } + + for _, item := range resp.Contents { + var lastModified *time.Time + if item.LastModified != nil { + t := *item.LastModified + lastModified = &t + } + out = append(out, ObjectInfo{ + Key: normalizeObjectKey(valueOrEmpty(item.Key)), + Size: valueOrZeroInt64(item.Size), + ETag: strings.Trim(valueOrEmpty(item.ETag), "\""), + LastModified: lastModified, + }) + } + + if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil { + break + } + token = resp.NextContinuationToken + } + return out, nil +} + +// Download retrieves one object to localPath, creating parent directories as needed. +func (b *S3Backend) Download(ctx context.Context, key, localPath string) error { + normalizedKey := normalizeObjectKey(key) + if strings.TrimSpace(localPath) == "" { + return fmt.Errorf("download object: local path is required") + } + + resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &b.bucket, + Key: &normalizedKey, + }) + if err != nil { + return fmt.Errorf("download object %q: %w", normalizedKey, err) + } + defer resp.Body.Close() + + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err) + } + dst, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err) + } + defer dst.Close() + + if _, err := io.Copy(dst, resp.Body); err != nil { + return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err) + } + if err := dst.Sync(); err != nil { + return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err) + } + return nil +} + +// Upload sends a local file to key. +func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) { + normalizedKey := normalizeObjectKey(key) + if strings.TrimSpace(localPath) == "" { + return ObjectInfo{}, fmt.Errorf("upload object: local path is required") + } + if normalizedKey == "" { + return ObjectInfo{}, fmt.Errorf("upload object: key is required") + } + + file, err := os.Open(localPath) + if err != nil { + return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err) + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err) + } + + input := &s3.PutObjectInput{ + Bucket: &b.bucket, + Key: &normalizedKey, + Body: file, + Metadata: copyMetadata(opts.Metadata), + } + if strings.TrimSpace(opts.ContentType) != "" { + ct := strings.TrimSpace(opts.ContentType) + input.ContentType = &ct + } + + resp, err := b.client.PutObject(ctx, input) + if err != nil { + return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err) + } + + return ObjectInfo{ + Key: normalizedKey, + Size: stat.Size(), + ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), + }, nil +} + +// Exists checks whether one object key exists. +func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) { + normalizedKey := normalizeObjectKey(key) + _, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &b.bucket, + Key: &normalizedKey, + }) + if err == nil { + return true, nil + } + + var notFound *types.NotFound + if errors.As(err, ¬Found) { + return false, nil + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NotFound", "NoSuchKey", "404": + return false, nil + } + } + return false, fmt.Errorf("head object %q: %w", normalizedKey, err) +} + +func valueOrEmpty(v *string) string { + if v == nil { + return "" + } + return *v +} + +func valueOrZeroInt64(v *int64) int64 { + if v == nil { + return 0 + } + return *v +} + +func valueOrFalseBool(v *bool) bool { + if v == nil { + return false + } + return *v +} diff --git a/internal/adapters/storage/s3_backend_test.go b/internal/adapters/storage/s3_backend_test.go new file mode 100644 index 0000000..33f8fba --- /dev/null +++ b/internal/adapters/storage/s3_backend_test.go @@ -0,0 +1,224 @@ +package storage + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +type fakeS3API struct { + listOut *s3.ListObjectsV2Output + listErr error + + getBody io.ReadCloser + getErr error + + putOut *s3.PutObjectOutput + putErr error + + headErr error + + lastList *s3.ListObjectsV2Input + lastGet *s3.GetObjectInput + lastPut *s3.PutObjectInput + lastHead *s3.HeadObjectInput +} + +func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + f.lastList = params + if f.listErr != nil { + return nil, f.listErr + } + if f.listOut == nil { + return &s3.ListObjectsV2Output{}, nil + } + return f.listOut, nil +} + +func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + f.lastGet = params + if f.getErr != nil { + return nil, f.getErr + } + body := f.getBody + if body == nil { + body = io.NopCloser(strings.NewReader("")) + } + return &s3.GetObjectOutput{Body: body}, nil +} + +func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + f.lastPut = params + if f.putErr != nil { + return nil, f.putErr + } + if f.putOut == nil { + return &s3.PutObjectOutput{}, nil + } + return f.putOut, nil +} + +func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + f.lastHead = params + if f.headErr != nil { + return nil, f.headErr + } + return &s3.HeadObjectOutput{}, nil +} + +func TestS3BackendListAndKeyNormalization(t *testing.T) { + lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + client := &fakeS3API{ + listOut: &s3.ListObjectsV2Output{ + Contents: []types.Object{ + {Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified}, + }, + }, + } + backend := &S3Backend{bucket: "bucket-1", client: client} + + items, err := backend.List(context.Background(), `dnd\campaigns\`) + if err != nil { + t.Fatalf("List() error = %v", err) + } + if len(items) != 1 { + t.Fatalf("List() len = %d, want 1", len(items)) + } + if items[0].Key != "dnd/campaigns/forsaken/a.flac" { + t.Fatalf("List() key = %q, want normalized slash key", items[0].Key) + } + if items[0].ETag != "abc" { + t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc") + } + if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" { + t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList) + } +} + +func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) { + client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))} + backend := &S3Backend{bucket: "bucket-1", client: client} + + dst := filepath.Join(t.TempDir(), "nested", "clip.flac") + if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil { + t.Fatalf("Download() error = %v", err) + } + + data, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if string(data) != "audio" { + t.Fatalf("downloaded content = %q, want %q", string(data), "audio") + } + if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" { + t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet) + } +} + +func TestS3BackendUploadAndExists(t *testing.T) { + client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}} + backend := &S3Backend{bucket: "bucket-1", client: client} + + local := filepath.Join(t.TempDir(), "artifact.txt") + if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{ + Metadata: map[string]string{"kind": "artifact"}, + }) + if err != nil { + t.Fatalf("Upload() error = %v", err) + } + if info.Key != "runs/id/artifact.txt" { + t.Fatalf("Upload key = %q, want normalized key", info.Key) + } + if info.ETag != "etag123" { + t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123") + } + if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" { + t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut) + } + + ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt") + if err != nil { + t.Fatalf("Exists() error = %v", err) + } + if !ok { + t.Fatal("Exists() = false, want true") + } +} + +func TestS3BackendUploadMissingLocalFile(t *testing.T) { + backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}} + _, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{}) + if err == nil || !strings.Contains(err.Error(), "no such file") { + t.Fatalf("Upload() error = %v, want missing local file error", err) + } +} + +func TestS3BackendExistsNotFound(t *testing.T) { + backend := &S3Backend{ + bucket: "bucket-1", + client: &fakeS3API{ + headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"}, + }, + } + ok, err := backend.Exists(context.Background(), "missing-key") + if err != nil { + t.Fatalf("Exists() error = %v", err) + } + if ok { + t.Fatal("Exists() = true, want false") + } +} + +func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) { + original := newS3Client + t.Cleanup(func() { newS3Client = original }) + + var got s3ClientOptions + newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) { + got = opts + return &fakeS3API{}, nil + } + + backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{ + Bucket: "my-archive", + Region: "us-east-1", + Endpoint: "http://localhost:9000", + ForcePathStyle: true, + }) + if err != nil { + t.Fatalf("NewS3BackendFromConfig() error = %v", err) + } + if backend.bucket != "my-archive" { + t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive") + } + if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle { + t.Fatalf("client options = %#v, want region/endpoint/path-style values", got) + } +} + +func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) { + _, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{}) + if err == nil || !strings.Contains(err.Error(), "bucket is required") { + t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err) + } +} + +func strPtr(v string) *string { return &v } +func int64Ptr(v int64) *int64 { return &v } + +var _ s3API = (*fakeS3API)(nil)