Add secrets directory credential resolver
This commit is contained in:
@@ -1,9 +1,14 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
Secrets Secrets `yaml:"secrets"`
|
||||
Pipelines []Pipeline `yaml:"pipelines"`
|
||||
}
|
||||
|
||||
type Secrets struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
|
||||
type Pipeline struct {
|
||||
ID string `yaml:"id"`
|
||||
Source Backend `yaml:"source"`
|
||||
|
||||
@@ -33,6 +33,29 @@ pipelines:
|
||||
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||
}
|
||||
if cfg.Secrets.Directory != "" {
|
||||
t.Fatalf("secrets.directory = %q, want empty", cfg.Secrets.Directory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidSecretsDirectoryConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
pipelines:
|
||||
- id: local-copy
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/reports
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/archive
|
||||
`)
|
||||
|
||||
if got, want := cfg.Secrets.Directory, "/run/secrets/distributor"; got != want {
|
||||
t.Fatalf("secrets.directory = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidFanOutConfig(t *testing.T) {
|
||||
@@ -351,6 +374,23 @@ pipelines:
|
||||
`, "field surprise not found")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsUnknownSecretsFields(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
surprise: true
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`, "field surprise not found")
|
||||
}
|
||||
|
||||
func TestExampleConfigsLoad(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"../../examples/local-to-local.yml",
|
||||
|
||||
151
internal/config/secrets.go
Normal file
151
internal/config/secrets.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var secretNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type EnvLookup func(string) (string, bool)
|
||||
|
||||
type Environment struct {
|
||||
values map[string]string
|
||||
lookup EnvLookup
|
||||
}
|
||||
|
||||
type SecretConflict struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
type SecretLoadResult struct {
|
||||
Environment Environment
|
||||
Conflicts []SecretConflict
|
||||
}
|
||||
|
||||
type ResolvedCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
func ProcessEnvironment() Environment {
|
||||
return NewEnvironment(nil, os.LookupEnv)
|
||||
}
|
||||
|
||||
func NewEnvironment(values map[string]string, lookup EnvLookup) Environment {
|
||||
copied := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
copied[key] = value
|
||||
}
|
||||
if lookup == nil {
|
||||
lookup = os.LookupEnv
|
||||
}
|
||||
return Environment{values: copied, lookup: lookup}
|
||||
}
|
||||
|
||||
func (e Environment) Lookup(name string) (string, bool) {
|
||||
if e.lookup != nil {
|
||||
if value, ok := e.lookup(name); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
value, ok := e.values[name]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (e Environment) ResolveCredentials(creds Credentials) (ResolvedCredentials, error) {
|
||||
var resolved ResolvedCredentials
|
||||
var err error
|
||||
if creds.AccessKeyIDEnv != "" {
|
||||
resolved.AccessKeyID, err = e.required(creds.AccessKeyIDEnv)
|
||||
if err != nil {
|
||||
return ResolvedCredentials{}, err
|
||||
}
|
||||
}
|
||||
if creds.SecretAccessKeyEnv != "" {
|
||||
resolved.SecretAccessKey, err = e.required(creds.SecretAccessKeyEnv)
|
||||
if err != nil {
|
||||
return ResolvedCredentials{}, err
|
||||
}
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (e Environment) required(name string) (string, error) {
|
||||
value, ok := e.Lookup(name)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("credential environment variable %s is not set", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func LoadSecretEnvironment(directory string, lookup EnvLookup) (SecretLoadResult, error) {
|
||||
if directory == "" {
|
||||
return SecretLoadResult{Environment: NewEnvironment(nil, lookup)}, nil
|
||||
}
|
||||
values, err := loadSecretValues(directory)
|
||||
if err != nil {
|
||||
return SecretLoadResult{}, err
|
||||
}
|
||||
var conflicts []SecretConflict
|
||||
for name, value := range values {
|
||||
if processValue, ok := lookupValue(lookup, name); ok && processValue != value {
|
||||
conflicts = append(conflicts, SecretConflict{Name: name})
|
||||
}
|
||||
}
|
||||
sort.Slice(conflicts, func(i, j int) bool {
|
||||
return conflicts[i].Name < conflicts[j].Name
|
||||
})
|
||||
return SecretLoadResult{
|
||||
Environment: NewEnvironment(values, lookup),
|
||||
Conflicts: conflicts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadSecretValues(directory string) (map[string]string, error) {
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load secrets directory %q: %w", directory, err)
|
||||
}
|
||||
values := make(map[string]string)
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(directory, entry.Name())
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect secret file %q: %w", entry.Name(), err)
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if !secretNamePattern.MatchString(entry.Name()) {
|
||||
return nil, fmt.Errorf("secret filename %q is invalid", entry.Name())
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read secret file %q: %w", entry.Name(), err)
|
||||
}
|
||||
values[entry.Name()] = trimOneTrailingLineEnding(string(data))
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func trimOneTrailingLineEnding(value string) string {
|
||||
if strings.HasSuffix(value, "\r\n") {
|
||||
return strings.TrimSuffix(value, "\r\n")
|
||||
}
|
||||
if strings.HasSuffix(value, "\n") {
|
||||
return strings.TrimSuffix(value, "\n")
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func lookupValue(lookup EnvLookup, name string) (string, bool) {
|
||||
if lookup == nil {
|
||||
lookup = os.LookupEnv
|
||||
}
|
||||
return lookup(name)
|
||||
}
|
||||
237
internal/config/secrets_test.go
Normal file
237
internal/config/secrets_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadSecretEnvironmentLoadsValidFiles(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "API_KEY", "value\n")
|
||||
writeSecret(t, directory, "CRLF", "value\r\n")
|
||||
writeSecret(t, directory, "MULTILINE", "value\n\n")
|
||||
writeSecret(t, directory, "SPACES", " value \n")
|
||||
writeSecret(t, directory, "CARRIAGE", "value\r")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "API_KEY", "value")
|
||||
assertEnvValue(t, result.Environment, "CRLF", "value")
|
||||
assertEnvValue(t, result.Environment, "MULTILINE", "value\n")
|
||||
assertEnvValue(t, result.Environment, "SPACES", " value ")
|
||||
assertEnvValue(t, result.Environment, "CARRIAGE", "value\r")
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentRejectsInvalidFilenames(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
secretValue := "do-not-print"
|
||||
writeSecret(t, directory, "1INVALID", secretValue)
|
||||
|
||||
_, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err == nil {
|
||||
t.Fatal("LoadSecretEnvironment() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "secret filename") {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %q, want filename error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), secretValue) {
|
||||
t.Fatalf("LoadSecretEnvironment() error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentIgnoresDirectoriesAndFollowsSymlinks(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(directory, "IGNORED_DIR"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir ignored dir: %v", err)
|
||||
}
|
||||
targetFile := filepath.Join(t.TempDir(), "target")
|
||||
if err := os.WriteFile(targetFile, []byte("linked\n"), 0o600); err != nil {
|
||||
t.Fatalf("write target file: %v", err)
|
||||
}
|
||||
if err := os.Symlink(targetFile, filepath.Join(directory, "LINKED_SECRET")); err != nil {
|
||||
t.Fatalf("symlink file: %v", err)
|
||||
}
|
||||
targetDir := t.TempDir()
|
||||
if err := os.Symlink(targetDir, filepath.Join(directory, "LINKED_DIR")); err != nil {
|
||||
t.Fatalf("symlink dir: %v", err)
|
||||
}
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "LINKED_SECRET", "linked")
|
||||
if _, ok := result.Environment.Lookup("IGNORED_DIR"); ok {
|
||||
t.Fatal("directory appeared in environment")
|
||||
}
|
||||
if _, ok := result.Environment.Lookup("LINKED_DIR"); ok {
|
||||
t.Fatal("directory symlink appeared in environment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentMissingDirectoryFails(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing")
|
||||
_, err := LoadSecretEnvironment(missing, emptyLookup)
|
||||
if err == nil {
|
||||
t.Fatal("LoadSecretEnvironment() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets directory") || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %q, want directory context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentPrefersProcessValuesAndReportsDifferingConflicts(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
secretValue := "secret-value"
|
||||
processValue := "process-value"
|
||||
writeSecret(t, directory, "TOKEN", secretValue)
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": processValue}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "TOKEN", processValue)
|
||||
if got, want := len(result.Conflicts), 1; got != want {
|
||||
t.Fatalf("conflict count = %d, want %d", got, want)
|
||||
}
|
||||
if result.Conflicts[0].Name != "TOKEN" {
|
||||
t.Fatalf("conflict name = %q, want TOKEN", result.Conflicts[0].Name)
|
||||
}
|
||||
if strings.Contains(result.Conflicts[0].Name, secretValue) || strings.Contains(result.Conflicts[0].Name, processValue) {
|
||||
t.Fatalf("conflict exposed secret values: %#v", result.Conflicts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentDoesNotWarnWhenProcessValueMatchesSecret(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "TOKEN", "same-value")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": "same-value"}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
if len(result.Conflicts) != 0 {
|
||||
t.Fatalf("conflicts = %#v, want none", result.Conflicts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentDoesNotMutateProcessEnvironment(t *testing.T) {
|
||||
name := "DISTRIBUTOR_TEST_SECRET_ONLY"
|
||||
t.Setenv(name, "")
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatalf("unset env: %v", err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, name, "secret")
|
||||
|
||||
if _, err := LoadSecretEnvironment(directory, nil); err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
if _, ok := os.LookupEnv(name); ok {
|
||||
t.Fatalf("%s was added to process environment", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsUsesSecretsAwareEnvironment(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
|
||||
writeSecret(t, directory, "SECRET_ACCESS_KEY", "secret-secret")
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
|
||||
creds, err := result.Environment.ResolveCredentials(Credentials{
|
||||
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveCredentials() error = %v", err)
|
||||
}
|
||||
if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
|
||||
t.Fatalf("resolved credentials = %#v", creds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsPrefersProcessEnvironment(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"ACCESS_KEY_ID": "process-access"}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
|
||||
creds, err := result.Environment.ResolveCredentials(Credentials{AccessKeyIDEnv: "ACCESS_KEY_ID"})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveCredentials() error = %v", err)
|
||||
}
|
||||
if creds.AccessKeyID != "process-access" {
|
||||
t.Fatalf("access key = %q, want process-access", creds.AccessKeyID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsDoesNotFeedProcessEnvironment(t *testing.T) {
|
||||
name := "DISTRIBUTOR_TEST_SDK_CHAIN_VALUE"
|
||||
t.Setenv(name, "")
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatalf("unset env: %v", err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, name, "secret")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, name, "secret")
|
||||
if _, ok := os.LookupEnv(name); ok {
|
||||
t.Fatalf("%s is visible to process environment", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsMissingReferenceFailsWithoutSecretValue(t *testing.T) {
|
||||
env := NewEnvironment(map[string]string{"PRESENT": "do-not-print"}, emptyLookup)
|
||||
_, err := env.ResolveCredentials(Credentials{AccessKeyIDEnv: "MISSING"})
|
||||
if err == nil {
|
||||
t.Fatal("ResolveCredentials() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "MISSING") {
|
||||
t.Fatalf("ResolveCredentials() error = %q, want missing variable name", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "do-not-print") {
|
||||
t.Fatalf("ResolveCredentials() error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeSecret(t *testing.T, directory, name, value string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil {
|
||||
t.Fatalf("write secret %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertEnvValue(t *testing.T, env Environment, name, want string) {
|
||||
t.Helper()
|
||||
got, ok := env.Lookup(name)
|
||||
if !ok {
|
||||
t.Fatalf("Lookup(%q) ok = false", name)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("Lookup(%q) = %q, want %q", name, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func emptyLookup(string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) EnvLookup {
|
||||
return func(name string) (string, bool) {
|
||||
value, ok := values[name]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user