Add secrets directory credential resolver

This commit is contained in:
2026-05-31 17:00:47 +00:00
parent 84f77ec0d0
commit 052aa8a64a
14 changed files with 665 additions and 6 deletions

View File

@@ -72,6 +72,7 @@ Sidecar generation writes `report.html` for `report.md`. It does not mutate the
Top level: Top level:
- `secrets.directory`: optional credential secrets directory.
- `pipelines`: required non-empty list. - `pipelines`: required non-empty list.
Pipeline: Pipeline:
@@ -168,7 +169,18 @@ Defaults are applied after YAML decoding and before validation:
## Secrets ## Secrets
Do not put literal secrets in config files. S3 credentials may name environment variables: Do not put literal secrets in config files. `secrets.directory` lets deployments provide credential values as files:
```yaml
secrets:
directory: /run/secrets/distributor
```
Each regular file in the directory becomes an internal credential environment value named by the filename. Valid filenames must match `[A-Za-z_][A-Za-z0-9_]*`. Directories are ignored, and symlinks to regular files are followed. Exactly one trailing LF or CRLF is trimmed from each file; other whitespace is preserved.
The resolver checks the real process environment first, then the secrets directory. If both define the same variable with different values, `run` prints a warning with the variable name and uses the real environment value. Secret values are not printed. The process environment is not modified, so SDK default credential chains see only real environment variables.
S3 credentials may name environment variables:
- `credentials.access_key_id_env` - `credentials.access_key_id_env`
- `credentials.secret_access_key_env` - `credentials.secret_access_key_env`

View File

@@ -6,7 +6,7 @@
## Inputs and outputs ## Inputs and outputs
Input is a YAML file containing `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation. Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
## Loading flow ## Loading flow
@@ -14,6 +14,8 @@ Input is a YAML file containing `pipelines`. Output is a `Config` value with def
Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run. Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run.
`LoadFile` does not read secret files. `Run` loads the configured secrets directory after config validation and before backend construction.
## Defaults ## Defaults
Defaults are applied in `ApplyDefaults`: Defaults are applied in `ApplyDefaults`:
@@ -39,6 +41,14 @@ Config validation accepts `local`, `ssh`, and `s3` backend shapes so config file
SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`. SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`.
## Secrets and credential resolution
`secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata.
The resolver checks the real process environment first and loaded secret values second. Differing process/secret conflicts are reported by variable name only. The resolver does not mutate `os.Environ`; default SDK credential chains continue to see only real process environment values.
Future credential-consuming backend code should resolve explicit credential environment variable references through `Environment.ResolveCredentials` or the same resolver pattern instead of calling `os.Getenv` directly.
The user-facing configuration reference is `docs/config.md`; this file documents package behavior for maintainers. The user-facing configuration reference is `docs/config.md`; this file documents package behavior for maintainers.
## Failure behavior ## Failure behavior

View File

@@ -96,6 +96,19 @@ The default host key policy is `accept-new`. New host keys are written to `known
Recovery boundaries are the same as local storage: replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, and failed writes are cleaned up where practical. Distributor never performs broad recursive remote deletion. Recovery boundaries are the same as local storage: replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, and failed writes are cleaned up where practical. Distributor never performs broad recursive remote deletion.
## Secrets Directory
Configure `secrets.directory` when credential values should come from mounted files, such as deployment secrets:
```yaml
secrets:
directory: /run/secrets/distributor
```
The directory is loaded during `run` before any source or destination backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the run fails before publication work starts.
Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` prints a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
## Caveats ## Caveats
S3 execution, external notification adapters, and force overwrite behavior are unavailable. S3 execution, external notification adapters, and force overwrite behavior are unavailable.

View File

@@ -166,7 +166,7 @@ For example, one destination may publish source files only as a long-term archiv
## Backend Abstraction ## Backend Abstraction
Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem backend. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented. Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem and SSH/SFTP backends. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented.
Application logic must interact with storage through internal backend interfaces. Backend-specific behavior belongs in adapter packages. Pipeline, bundle, state, publish, and transform packages must not import service-specific or filesystem adapter implementation details. Application logic must interact with storage through internal backend interfaces. Backend-specific behavior belongs in adapter packages. Pipeline, bundle, state, publish, and transform packages must not import service-specific or filesystem adapter implementation details.
@@ -194,6 +194,7 @@ Use this current layout unless the project has a documented reason to differ:
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata. - `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors. - `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
- `internal/adapters/local`: local filesystem backend. - `internal/adapters/local`: local filesystem backend.
- `internal/adapters/ssh`: SSH/SFTP backend.
- `internal/transform`: transform interfaces, registry, planning, and shared transform models. - `internal/transform`: transform interfaces, registry, planning, and shared transform models.
- `internal/transform/markdown`: Markdown-to-HTML implementation. - `internal/transform/markdown`: Markdown-to-HTML implementation.
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution. - `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.

View File

@@ -13,6 +13,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison. - `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers. - `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
- `internal/adapters/local`: local filesystem backend. - `internal/adapters/local`: local filesystem backend.
- `internal/adapters/ssh`: SSH/SFTP backend.
- `internal/storage/fake`: in-memory backend for tests. - `internal/storage/fake`: in-memory backend for tests.
- `internal/publish`: destination inspection, output planning, reconciliation, execution, and managed cleanup. - `internal/publish`: destination inspection, output planning, reconciliation, execution, and managed cleanup.
- `internal/transform`: transform interface and registry. - `internal/transform`: transform interface and registry.
@@ -82,6 +83,8 @@ The project currently depends on:
- `gopkg.in/yaml.v3` for YAML configuration loading. - `gopkg.in/yaml.v3` for YAML configuration loading.
- `github.com/yuin/goldmark` for Markdown rendering. - `github.com/yuin/goldmark` for Markdown rendering.
- `golang.org/x/crypto/ssh`, `golang.org/x/crypto/ssh/agent`, and `golang.org/x/crypto/ssh/knownhosts` for native SSH support.
- `github.com/pkg/sftp` for native SFTP support.
Add external dependencies only when they materially improve correctness, Add external dependencies only when they materially improve correctness,
security, interoperability, or implementation complexity. Avoid dependencies security, interoperability, or implementation complexity. Avoid dependencies
@@ -104,6 +107,12 @@ Config validation may accept fields for backends that are not executable yet,
but user-facing docs and examples must clearly state execution support. At the but user-facing docs and examples must clearly state execution support. At the
time of this policy, local and SSH backends are executable. time of this policy, local and SSH backends are executable.
Credential-consuming code must use the config-owned environment resolver for
explicit credential environment variable references. Do not call `os.Getenv`
directly for backend credentials, because `secrets.directory` values are
intentionally available through the resolver without mutating the process
environment.
## CLI Changes ## CLI Changes
The CLI is hand-written with the Go standard library. Do not introduce a CLI The CLI is hand-written with the Go standard library. Do not introduce a CLI

View File

@@ -48,6 +48,69 @@ go run ./cmd/distributor run --config <config-path> --dry-run
Safe fix: use `local` or `ssh` for executable workflows. See [configuration](config.md). Safe fix: use `local` or `ssh` for executable workflows. See [configuration](config.md).
## `load secrets directory ... no such file or directory`
Likely cause: `secrets.directory` points to a missing directory.
Diagnostic:
```sh
ls -ld <secrets-directory>
```
Safe fix: create or mount the directory before running, or remove `secrets.directory` if no credential files are needed.
## `load secrets directory ... permission denied`
Likely cause: the service user cannot read the configured secrets directory.
Diagnostic:
```sh
ls -ld <secrets-directory>
namei -l <secrets-directory>
```
Safe fix: adjust the directory path or deployment permissions so the service user can read the directory. Distributor does not enforce owner, group, or mode policy beyond OS read access.
## `secret filename ... is invalid`
Likely cause: a regular file in `secrets.directory` does not match `[A-Za-z_][A-Za-z0-9_]*`.
Diagnostic:
```sh
find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
```
Safe fix: rename the file to a valid credential environment variable name, or remove it from the secrets directory.
## `credential environment variable ... is not set`
Likely cause: a backend credential field references an environment variable that is absent from both the real process environment and the configured secrets directory.
Diagnostic:
```sh
printenv <variable-name>
ls -l <secrets-directory>/<variable-name>
```
Safe fix: set the real environment variable or create a readable secrets-directory file with the same name.
## `secret ... ignored because the real environment already has that variable`
Likely cause: the real process environment and secrets directory both define the variable with different values.
Diagnostic:
```sh
printenv <variable-name>
ls -l <secrets-directory>/<variable-name>
```
Safe fix: remove one source of the credential or make the deployment intentionally prefer the real environment value. Distributor does not print either value.
## `host is required for ssh backend` ## `host is required for ssh backend`
Likely cause: SSH config is missing the structured `host` field, or an old URI-based SSH config is still in use. Likely cause: SSH config is missing the structured `host` field, or an old URI-based SSH config is still in use.

View File

@@ -23,10 +23,15 @@ const (
) )
type backendFactory struct { type backendFactory struct {
registry *storage.Registry registry *storage.Registry
environment config.Environment
} }
func newBackendFactory() *backendFactory { func newBackendFactory() *backendFactory {
return newBackendFactoryWithEnvironment(config.ProcessEnvironment())
}
func newBackendFactoryWithEnvironment(environment config.Environment) *backendFactory {
registry := storage.NewRegistry() registry := storage.NewRegistry()
_ = registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { _ = registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -49,7 +54,7 @@ func newBackendFactory() *backendFactory {
HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]), HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
}) })
}) })
return &backendFactory{registry: registry} return &backendFactory{registry: registry, environment: environment}
} }
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) { func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
@@ -70,6 +75,10 @@ func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storag
return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path}) return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path})
} }
func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.ResolvedCredentials, error) {
return f.environment.ResolveCredentials(creds)
}
func sourceOpenConfig(source config.Backend) storage.OpenConfig { func sourceOpenConfig(source config.Backend) storage.OpenConfig {
cfg := storage.OpenConfig{storagePathKey: source.Path} cfg := storage.OpenConfig{storagePathKey: source.Path}
if source.Backend == config.BackendSSH { if source.Backend == config.BackendSSH {

View File

@@ -131,6 +131,26 @@ func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) {
} }
} }
func TestBackendFactoryResolvesCredentialsThroughEnvironment(t *testing.T) {
factory := newBackendFactoryWithEnvironment(config.NewEnvironment(map[string]string{
"ACCESS_KEY_ID": "secret-access",
"SECRET_ACCESS_KEY": "secret-secret",
}, func(string) (string, bool) {
return "", false
}))
creds, err := factory.resolveCredentials(config.Credentials{
AccessKeyIDEnv: "ACCESS_KEY_ID",
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
})
if err != nil {
t.Fatalf("resolveCredentials() error = %v", err)
}
if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
t.Fatalf("resolved credentials = %#v", creds)
}
}
func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) { func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
cfg := sourceOpenConfig(config.Backend{ cfg := sourceOpenConfig(config.Backend{
Backend: config.BackendSSH, Backend: config.BackendSSH,

View File

@@ -44,7 +44,16 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
} }
summary := runSummary{dryRun: options.DryRun} summary := runSummary{dryRun: options.DryRun}
var failures runFailures var failures runFailures
backends := newBackendFactory() secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
if err != nil {
return err
}
if options.Stdout != nil {
if err := writeSecretConflictWarnings(options.Stdout, secretLoad.Conflicts); err != nil {
return err
}
}
backends := newBackendFactoryWithEnvironment(secretLoad.Environment)
transforms := newTransformRegistry() transforms := newTransformRegistry()
if options.Stdout != nil { if options.Stdout != nil {
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil { if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
@@ -205,6 +214,15 @@ func destinationSummary(destinations []config.Destination) string {
return strings.Join(ids, ",") return strings.Join(ids, ",")
} }
func writeSecretConflictWarnings(w io.Writer, conflicts []config.SecretConflict) error {
for _, conflict := range conflicts {
if _, err := fmt.Fprintf(w, "Warning: secret %s ignored because the real environment already has that variable\n", conflict.Name); err != nil {
return err
}
}
return nil
}
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error { func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff { if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil { if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil {

View File

@@ -47,6 +47,77 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
} }
} }
func TestRunLoadsSecretsBeforeOpeningBackends(t *testing.T) {
sourceRoot := filepath.Join(t.TempDir(), "missing-source")
destinationRoot := t.TempDir()
configPath := writeConfigFile(t, `
secrets:
directory: `+filepath.Join(t.TempDir(), "missing-secrets")+`
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err == nil {
t.Fatal("Run() error = nil, want secrets directory error")
}
if !strings.Contains(err.Error(), "load secrets directory") {
t.Fatalf("Run() error = %v, want secrets directory error", err)
}
if strings.Contains(err.Error(), "missing-source") {
t.Fatalf("Run() error = %v, opened source before loading secrets", err)
}
}
func TestRunPrintsSecretConflictWarningWithoutValues(t *testing.T) {
name := "DISTRIBUTOR_TEST_RUN_SECRET"
t.Setenv(name, "process-value")
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
secretsRoot := t.TempDir()
if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
t.Fatalf("write secret: %v", err)
}
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
configPath := writeConfigFile(t, `
secrets:
directory: `+secretsRoot+`
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
`)
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: configPath,
DryRun: true,
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
if !strings.Contains(output, "secret "+name+" ignored because the real environment already has that variable") {
t.Fatalf("stdout = %q, want secret conflict warning", output)
}
if strings.Contains(output, "process-value") || strings.Contains(output, "secret-value") {
t.Fatalf("stdout exposed secret values: %q", output)
}
}
func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) { func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
err := writeSSHWarnings(&stdout, config.Pipeline{ err := writeSSHWarnings(&stdout, config.Pipeline{

View File

@@ -1,9 +1,14 @@
package config package config
type Config struct { type Config struct {
Secrets Secrets `yaml:"secrets"`
Pipelines []Pipeline `yaml:"pipelines"` Pipelines []Pipeline `yaml:"pipelines"`
} }
type Secrets struct {
Directory string `yaml:"directory"`
}
type Pipeline struct { type Pipeline struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Source Backend `yaml:"source"` Source Backend `yaml:"source"`

View File

@@ -33,6 +33,29 @@ pipelines:
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want { if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
t.Fatalf("transfer default = %q, want %q", got, want) t.Fatalf("transfer default = %q, want %q", got, want)
} }
if cfg.Secrets.Directory != "" {
t.Fatalf("secrets.directory = %q, want empty", cfg.Secrets.Directory)
}
}
func TestLoadFileValidSecretsDirectoryConfig(t *testing.T) {
cfg := loadConfig(t, `
secrets:
directory: /run/secrets/distributor
pipelines:
- id: local-copy
source:
backend: local
path: /var/spool/reports
destinations:
- id: archive
backend: local
path: /srv/archive
`)
if got, want := cfg.Secrets.Directory, "/run/secrets/distributor"; got != want {
t.Fatalf("secrets.directory = %q, want %q", got, want)
}
} }
func TestLoadFileValidFanOutConfig(t *testing.T) { func TestLoadFileValidFanOutConfig(t *testing.T) {
@@ -351,6 +374,23 @@ pipelines:
`, "field surprise not found") `, "field surprise not found")
} }
func TestLoadFileRejectsUnknownSecretsFields(t *testing.T) {
assertLoadError(t, `
secrets:
directory: /run/secrets/distributor
surprise: true
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
`, "field surprise not found")
}
func TestExampleConfigsLoad(t *testing.T) { func TestExampleConfigsLoad(t *testing.T) {
for _, path := range []string{ for _, path := range []string{
"../../examples/local-to-local.yml", "../../examples/local-to-local.yml",

151
internal/config/secrets.go Normal file
View File

@@ -0,0 +1,151 @@
package config
import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
var secretNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type EnvLookup func(string) (string, bool)
type Environment struct {
values map[string]string
lookup EnvLookup
}
type SecretConflict struct {
Name string
}
type SecretLoadResult struct {
Environment Environment
Conflicts []SecretConflict
}
type ResolvedCredentials struct {
AccessKeyID string
SecretAccessKey string
}
func ProcessEnvironment() Environment {
return NewEnvironment(nil, os.LookupEnv)
}
func NewEnvironment(values map[string]string, lookup EnvLookup) Environment {
copied := make(map[string]string, len(values))
for key, value := range values {
copied[key] = value
}
if lookup == nil {
lookup = os.LookupEnv
}
return Environment{values: copied, lookup: lookup}
}
func (e Environment) Lookup(name string) (string, bool) {
if e.lookup != nil {
if value, ok := e.lookup(name); ok {
return value, true
}
}
value, ok := e.values[name]
return value, ok
}
func (e Environment) ResolveCredentials(creds Credentials) (ResolvedCredentials, error) {
var resolved ResolvedCredentials
var err error
if creds.AccessKeyIDEnv != "" {
resolved.AccessKeyID, err = e.required(creds.AccessKeyIDEnv)
if err != nil {
return ResolvedCredentials{}, err
}
}
if creds.SecretAccessKeyEnv != "" {
resolved.SecretAccessKey, err = e.required(creds.SecretAccessKeyEnv)
if err != nil {
return ResolvedCredentials{}, err
}
}
return resolved, nil
}
func (e Environment) required(name string) (string, error) {
value, ok := e.Lookup(name)
if !ok {
return "", fmt.Errorf("credential environment variable %s is not set", name)
}
return value, nil
}
func LoadSecretEnvironment(directory string, lookup EnvLookup) (SecretLoadResult, error) {
if directory == "" {
return SecretLoadResult{Environment: NewEnvironment(nil, lookup)}, nil
}
values, err := loadSecretValues(directory)
if err != nil {
return SecretLoadResult{}, err
}
var conflicts []SecretConflict
for name, value := range values {
if processValue, ok := lookupValue(lookup, name); ok && processValue != value {
conflicts = append(conflicts, SecretConflict{Name: name})
}
}
sort.Slice(conflicts, func(i, j int) bool {
return conflicts[i].Name < conflicts[j].Name
})
return SecretLoadResult{
Environment: NewEnvironment(values, lookup),
Conflicts: conflicts,
}, nil
}
func loadSecretValues(directory string) (map[string]string, error) {
entries, err := os.ReadDir(directory)
if err != nil {
return nil, fmt.Errorf("load secrets directory %q: %w", directory, err)
}
values := make(map[string]string)
for _, entry := range entries {
path := filepath.Join(directory, entry.Name())
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("inspect secret file %q: %w", entry.Name(), err)
}
if info.IsDir() || !info.Mode().IsRegular() {
continue
}
if !secretNamePattern.MatchString(entry.Name()) {
return nil, fmt.Errorf("secret filename %q is invalid", entry.Name())
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read secret file %q: %w", entry.Name(), err)
}
values[entry.Name()] = trimOneTrailingLineEnding(string(data))
}
return values, nil
}
func trimOneTrailingLineEnding(value string) string {
if strings.HasSuffix(value, "\r\n") {
return strings.TrimSuffix(value, "\r\n")
}
if strings.HasSuffix(value, "\n") {
return strings.TrimSuffix(value, "\n")
}
return value
}
func lookupValue(lookup EnvLookup, name string) (string, bool) {
if lookup == nil {
lookup = os.LookupEnv
}
return lookup(name)
}

View File

@@ -0,0 +1,237 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadSecretEnvironmentLoadsValidFiles(t *testing.T) {
directory := t.TempDir()
writeSecret(t, directory, "API_KEY", "value\n")
writeSecret(t, directory, "CRLF", "value\r\n")
writeSecret(t, directory, "MULTILINE", "value\n\n")
writeSecret(t, directory, "SPACES", " value \n")
writeSecret(t, directory, "CARRIAGE", "value\r")
result, err := LoadSecretEnvironment(directory, emptyLookup)
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
assertEnvValue(t, result.Environment, "API_KEY", "value")
assertEnvValue(t, result.Environment, "CRLF", "value")
assertEnvValue(t, result.Environment, "MULTILINE", "value\n")
assertEnvValue(t, result.Environment, "SPACES", " value ")
assertEnvValue(t, result.Environment, "CARRIAGE", "value\r")
}
func TestLoadSecretEnvironmentRejectsInvalidFilenames(t *testing.T) {
directory := t.TempDir()
secretValue := "do-not-print"
writeSecret(t, directory, "1INVALID", secretValue)
_, err := LoadSecretEnvironment(directory, emptyLookup)
if err == nil {
t.Fatal("LoadSecretEnvironment() error = nil, want error")
}
if !strings.Contains(err.Error(), "secret filename") {
t.Fatalf("LoadSecretEnvironment() error = %q, want filename error", err)
}
if strings.Contains(err.Error(), secretValue) {
t.Fatalf("LoadSecretEnvironment() error exposed secret value: %q", err)
}
}
func TestLoadSecretEnvironmentIgnoresDirectoriesAndFollowsSymlinks(t *testing.T) {
directory := t.TempDir()
if err := os.Mkdir(filepath.Join(directory, "IGNORED_DIR"), 0o700); err != nil {
t.Fatalf("mkdir ignored dir: %v", err)
}
targetFile := filepath.Join(t.TempDir(), "target")
if err := os.WriteFile(targetFile, []byte("linked\n"), 0o600); err != nil {
t.Fatalf("write target file: %v", err)
}
if err := os.Symlink(targetFile, filepath.Join(directory, "LINKED_SECRET")); err != nil {
t.Fatalf("symlink file: %v", err)
}
targetDir := t.TempDir()
if err := os.Symlink(targetDir, filepath.Join(directory, "LINKED_DIR")); err != nil {
t.Fatalf("symlink dir: %v", err)
}
result, err := LoadSecretEnvironment(directory, emptyLookup)
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
assertEnvValue(t, result.Environment, "LINKED_SECRET", "linked")
if _, ok := result.Environment.Lookup("IGNORED_DIR"); ok {
t.Fatal("directory appeared in environment")
}
if _, ok := result.Environment.Lookup("LINKED_DIR"); ok {
t.Fatal("directory symlink appeared in environment")
}
}
func TestLoadSecretEnvironmentMissingDirectoryFails(t *testing.T) {
missing := filepath.Join(t.TempDir(), "missing")
_, err := LoadSecretEnvironment(missing, emptyLookup)
if err == nil {
t.Fatal("LoadSecretEnvironment() error = nil, want error")
}
if !strings.Contains(err.Error(), "load secrets directory") || !strings.Contains(err.Error(), missing) {
t.Fatalf("LoadSecretEnvironment() error = %q, want directory context", err)
}
}
func TestEnvironmentPrefersProcessValuesAndReportsDifferingConflicts(t *testing.T) {
directory := t.TempDir()
secretValue := "secret-value"
processValue := "process-value"
writeSecret(t, directory, "TOKEN", secretValue)
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": processValue}))
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
assertEnvValue(t, result.Environment, "TOKEN", processValue)
if got, want := len(result.Conflicts), 1; got != want {
t.Fatalf("conflict count = %d, want %d", got, want)
}
if result.Conflicts[0].Name != "TOKEN" {
t.Fatalf("conflict name = %q, want TOKEN", result.Conflicts[0].Name)
}
if strings.Contains(result.Conflicts[0].Name, secretValue) || strings.Contains(result.Conflicts[0].Name, processValue) {
t.Fatalf("conflict exposed secret values: %#v", result.Conflicts[0])
}
}
func TestEnvironmentDoesNotWarnWhenProcessValueMatchesSecret(t *testing.T) {
directory := t.TempDir()
writeSecret(t, directory, "TOKEN", "same-value")
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": "same-value"}))
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
if len(result.Conflicts) != 0 {
t.Fatalf("conflicts = %#v, want none", result.Conflicts)
}
}
func TestEnvironmentDoesNotMutateProcessEnvironment(t *testing.T) {
name := "DISTRIBUTOR_TEST_SECRET_ONLY"
t.Setenv(name, "")
if err := os.Unsetenv(name); err != nil {
t.Fatalf("unset env: %v", err)
}
directory := t.TempDir()
writeSecret(t, directory, name, "secret")
if _, err := LoadSecretEnvironment(directory, nil); err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
if _, ok := os.LookupEnv(name); ok {
t.Fatalf("%s was added to process environment", name)
}
}
func TestResolveCredentialsUsesSecretsAwareEnvironment(t *testing.T) {
directory := t.TempDir()
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
writeSecret(t, directory, "SECRET_ACCESS_KEY", "secret-secret")
result, err := LoadSecretEnvironment(directory, emptyLookup)
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
creds, err := result.Environment.ResolveCredentials(Credentials{
AccessKeyIDEnv: "ACCESS_KEY_ID",
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
})
if err != nil {
t.Fatalf("ResolveCredentials() error = %v", err)
}
if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
t.Fatalf("resolved credentials = %#v", creds)
}
}
func TestResolveCredentialsPrefersProcessEnvironment(t *testing.T) {
directory := t.TempDir()
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"ACCESS_KEY_ID": "process-access"}))
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
creds, err := result.Environment.ResolveCredentials(Credentials{AccessKeyIDEnv: "ACCESS_KEY_ID"})
if err != nil {
t.Fatalf("ResolveCredentials() error = %v", err)
}
if creds.AccessKeyID != "process-access" {
t.Fatalf("access key = %q, want process-access", creds.AccessKeyID)
}
}
func TestResolveCredentialsDoesNotFeedProcessEnvironment(t *testing.T) {
name := "DISTRIBUTOR_TEST_SDK_CHAIN_VALUE"
t.Setenv(name, "")
if err := os.Unsetenv(name); err != nil {
t.Fatalf("unset env: %v", err)
}
directory := t.TempDir()
writeSecret(t, directory, name, "secret")
result, err := LoadSecretEnvironment(directory, nil)
if err != nil {
t.Fatalf("LoadSecretEnvironment() error = %v", err)
}
assertEnvValue(t, result.Environment, name, "secret")
if _, ok := os.LookupEnv(name); ok {
t.Fatalf("%s is visible to process environment", name)
}
}
func TestResolveCredentialsMissingReferenceFailsWithoutSecretValue(t *testing.T) {
env := NewEnvironment(map[string]string{"PRESENT": "do-not-print"}, emptyLookup)
_, err := env.ResolveCredentials(Credentials{AccessKeyIDEnv: "MISSING"})
if err == nil {
t.Fatal("ResolveCredentials() error = nil, want error")
}
if !strings.Contains(err.Error(), "MISSING") {
t.Fatalf("ResolveCredentials() error = %q, want missing variable name", err)
}
if strings.Contains(err.Error(), "do-not-print") {
t.Fatalf("ResolveCredentials() error exposed secret value: %q", err)
}
}
func writeSecret(t *testing.T, directory, name, value string) {
t.Helper()
if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil {
t.Fatalf("write secret %s: %v", name, err)
}
}
func assertEnvValue(t *testing.T, env Environment, name, want string) {
t.Helper()
got, ok := env.Lookup(name)
if !ok {
t.Fatalf("Lookup(%q) ok = false", name)
}
if got != want {
t.Fatalf("Lookup(%q) = %q, want %q", name, got, want)
}
}
func emptyLookup(string) (string, bool) {
return "", false
}
func mapLookup(values map[string]string) EnvLookup {
return func(name string) (string, bool) {
value, ok := values[name]
return value, ok
}
}