From 62551d43a0fd42905fb7e842cbb2db90c43db33b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 16 May 2026 07:56:22 -0500 Subject: [PATCH] Added filesystem-based secrets loading configuration --- README.md | 12 ++ architecture.md | 10 ++ examples/pipeline.minimal.yml | 5 + internal/app/commands_test.go | 125 +++++++++++++++++++++ internal/app/plan.go | 6 + internal/app/plan_test.go | 47 ++++++++ internal/app/runner.go | 3 + internal/app/secrets_env.go | 88 +++++++++++++++ internal/app/secrets_env_test.go | 155 ++++++++++++++++++++++++++ internal/config/config.go | 6 + internal/config/load_validate_test.go | 40 +++++++ internal/config/validate.go | 13 +++ 12 files changed, 510 insertions(+) create mode 100644 internal/app/secrets_env.go create mode 100644 internal/app/secrets_env_test.go diff --git a/README.md b/README.md index 2534a30..c0b84e8 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,16 @@ Pipeline config lookup for CLI commands: - `/usr/local/etc/narratio/pipeline.yml` - `/etc/narratio/pipeline.yml` +Optional secrets-from-files config: + +- `pipeline.secrets.env_dir` may point to a directory of secret files +- each top-level file with an env-var-style name is loaded as an environment variable: + - file name = env var name + - file contents = env var value (trailing newline/CRLF trimmed) +- process environment wins: existing env vars are not overwritten +- if configured, Narratio fails fast when `env_dir` is missing/unreadable +- relative `env_dir` values resolve from Narratio’s current working directory + YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast. ## Canonical Stage Order @@ -199,6 +209,8 @@ Prompt IDs and profile IDs are configuration values. They are not hardcoded in a Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only. +If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs. + ## Scriptorium Runtime Behavior Narratio integrates with Scriptorium through the public CLI subprocess contract: diff --git a/architecture.md b/architecture.md index 22ebc8a..d30ac69 100644 --- a/architecture.md +++ b/architecture.md @@ -101,6 +101,15 @@ CLI pipeline config path resolution: - `/usr/local/etc/narratio/pipeline.yml` - `/etc/narratio/pipeline.yml` +Optional pipeline secrets directory: + +- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution +- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed) +- only env-var-style file names are considered; other entries are ignored +- existing process environment values are preserved (not overwritten) +- if configured, unreadable/missing `env_dir` fails command execution early +- relative `env_dir` values are resolved from current working directory + `pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work. `pipeline.trim` is optional. Existing pipelines without trim config continue to work. @@ -288,6 +297,7 @@ Current expected paths for `session_recap`: - do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata - if API-key integration is configured, pass env var names only (never raw key values) +- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs - avoid logging transcript content or rendered prompt content by default - treat generated artifacts and logs as potentially sensitive session material diff --git a/examples/pipeline.minimal.yml b/examples/pipeline.minimal.yml index 4e5b963..59926e0 100644 --- a/examples/pipeline.minimal.yml +++ b/examples/pipeline.minimal.yml @@ -4,6 +4,11 @@ workspace: storage: backend: local +secrets: + # Optional: load environment variables from files in this directory. + # File name = env var name; file contents = env var value. + env_dir: /var/local/narratio/secrets + whisperx: transcribe_url: "https://transcription.example.com/transcribe" language: "en" diff --git a/internal/app/commands_test.go b/internal/app/commands_test.go index 4cba931..2b4f7ea 100644 --- a/internal/app/commands_test.go +++ b/internal/app/commands_test.go @@ -170,6 +170,131 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) { } } +func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) { + workspaceRoot := t.TempDir() + configDir := t.TempDir() + sessionID := "2026-05-03" + secretsDir := filepath.Join(configDir, "secrets") + if err := os.MkdirAll(secretsDir, 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", secretsDir, err) + } + if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil { + t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err) + } + + seriatimBinary := writeSeriatimAppTestWrapper(t) + auditaBinary := writeAuditaAppTestWrapper(t) + t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1") + t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1") + + pipelinePath := filepath.Join(configDir, "pipeline.yml") + sessionPath := filepath.Join(configDir, "session.yml") + pipelineYAML := `workspace: + root: ` + workspaceRoot + ` +storage: + backend: local +secrets: + env_dir: ./secrets +whisperx: + transcribe_url: https://example.com/transcribe +seriatim: + binary: ` + seriatimBinary + ` +audita: + binary: ` + auditaBinary + ` + llm_api_key_env: OPENROUTER_API_KEY +analyzer: + timeout: 20m +notification: + timeout: 10s +` + sessionYAML := `session_id: ` + sessionID + ` +inputs: + audio_dir: ./audio + speakers_file: ./speakers.yml + autocorrect_file: ./autocorrect.yml + glossary_file: ./glossary.yml +` + if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { + t.Fatalf("write pipeline.yml: %v", err) + } + if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil { + t.Fatalf("write session.yml: %v", err) + } + + originalWD, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd(): %v", err) + } + if err := os.Chdir(configDir); err != nil { + t.Fatalf("Chdir(%q): %v", configDir, err) + } + t.Cleanup(func() { + _ = os.Chdir(originalWD) + }) + + workRoot := filepath.Join(workspaceRoot, "work", sessionID) + mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`) + mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n") + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr) + if code != 0 { + t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") { + t.Fatalf("stdout = %q, want polish execution", stdout.String()) + } +} + +func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) { + workspaceRoot := t.TempDir() + configDir := t.TempDir() + pipelinePath := filepath.Join(configDir, "pipeline.yml") + sessionPath := filepath.Join(configDir, "session.yml") + + pipelineYAML := `workspace: + root: ` + workspaceRoot + ` +storage: + backend: local +secrets: + env_dir: ./missing-secrets +whisperx: + transcribe_url: https://example.com/transcribe +seriatim: + binary: seriatim +audita: + binary: audita +analyzer: + timeout: 20m +notification: + timeout: 10s +` + sessionYAML := `session_id: 2026-05-03 +inputs: + audio_dir: ./audio + speakers_file: ./speakers.yml + autocorrect_file: ./autocorrect.yml + glossary_file: ./glossary.yml +` + if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { + t.Fatalf("write pipeline.yml: %v", err) + } + if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil { + t.Fatalf("write session.yml: %v", err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) + if code == 0 { + t.Fatal("exit code = 0, want non-zero") + } + if !strings.Contains(stderr.String(), "read secrets env_dir") { + t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String()) + } +} + func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) { workspaceRoot := t.TempDir() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/app/plan.go b/internal/app/plan.go index cdccb07..1d7e572 100644 --- a/internal/app/plan.go +++ b/internal/app/plan.go @@ -5,9 +5,12 @@ import ( "flag" "fmt" "io" + "log/slog" + "os" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" + "gitea.maximumdirect.net/eric/narratio/internal/logging" "gitea.maximumdirect.net/eric/narratio/internal/manifest" ) @@ -45,6 +48,9 @@ func Plan(ctx context.Context, args []string, out io.Writer) error { if err := config.Validate(cfg); err != nil { return fmt.Errorf("plan: %w", err) } + if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil { + return fmt.Errorf("plan: %w", err) + } store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root) paths, err := store.EnsureLayout(cfg.Session.SessionID) diff --git a/internal/app/plan_test.go b/internal/app/plan_test.go index b204b2c..e0004fe 100644 --- a/internal/app/plan_test.go +++ b/internal/app/plan_test.go @@ -89,6 +89,53 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) { } } +func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) { + workspaceRoot := t.TempDir() + configDir := t.TempDir() + pipelinePath := filepath.Join(configDir, "pipeline.yml") + sessionPath := filepath.Join(configDir, "session.yml") + + pipelineYAML := `workspace: + root: ` + workspaceRoot + ` +storage: + backend: local +secrets: + env_dir: ./missing-secrets +whisperx: + transcribe_url: https://example.com/transcribe +seriatim: + binary: seriatim +audita: + binary: audita +analyzer: + timeout: 20m +notification: + timeout: 10s +` + sessionYAML := `session_id: 2026-05-03 +inputs: + audio_dir: ./audio + speakers_file: ./speakers.yml + autocorrect_file: ./autocorrect.yml + glossary_file: ./glossary.yml +` + if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { + t.Fatalf("write pipeline.yml: %v", err) + } + if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil { + t.Fatalf("write session.yml: %v", err) + } + + var out bytes.Buffer + err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "read secrets env_dir") { + t.Fatalf("error = %q, want secrets read error context", err.Error()) + } +} + func assertDir(t *testing.T, path string) { t.Helper() info, err := os.Stat(path) diff --git a/internal/app/runner.go b/internal/app/runner.go index 644b0fd..76ffe97 100644 --- a/internal/app/runner.go +++ b/internal/app/runner.go @@ -51,6 +51,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage if env.Logger == nil { env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo) } + if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil { + return nil, fmt.Errorf("load secrets from files: %w", err) + } if env.WhisperX == nil { client, err := buildDefaultWhisperXClient(env.Config) if err != nil { diff --git a/internal/app/secrets_env.go b/internal/app/secrets_env.go new file mode 100644 index 0000000..de2c7a3 --- /dev/null +++ b/internal/app/secrets_env.go @@ -0,0 +1,88 @@ +package app + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +type secretsLoadStats struct { + Dir string + Loaded int + PreservedExisting int + Skipped int +} + +func loadSecretsFromConfig(cfg *config.Config, logger *slog.Logger) (*secretsLoadStats, error) { + if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Secrets == nil { + return nil, nil + } + + rawDir := strings.TrimSpace(cfg.Pipeline.Secrets.EnvDir) + if rawDir == "" { + return nil, nil + } + + resolvedDir := rawDir + if !filepath.IsAbs(resolvedDir) { + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("resolve secrets env_dir %q from current working directory: %w", rawDir, err) + } + resolvedDir = filepath.Join(cwd, resolvedDir) + } + resolvedDir = filepath.Clean(resolvedDir) + + entries, err := os.ReadDir(resolvedDir) + if err != nil { + return nil, fmt.Errorf("read secrets env_dir %q: %w", resolvedDir, err) + } + + stats := &secretsLoadStats{Dir: resolvedDir} + for _, entry := range entries { + name := entry.Name() + if !envVarNamePattern.MatchString(name) { + stats.Skipped++ + continue + } + if entry.IsDir() { + stats.Skipped++ + continue + } + + secretPath := filepath.Join(resolvedDir, name) + bytes, err := os.ReadFile(secretPath) + if err != nil { + return nil, fmt.Errorf("read secret file %q: %w", secretPath, err) + } + value := strings.TrimRight(string(bytes), "\r\n") + + if _, exists := os.LookupEnv(name); exists { + stats.PreservedExisting++ + continue + } + if err := os.Setenv(name, value); err != nil { + return nil, fmt.Errorf("set environment variable %q from %q: %w", name, secretPath, err) + } + stats.Loaded++ + } + + if logger != nil { + logger.Info( + "loaded secret environment variables from filesystem", + "secrets_env_dir", stats.Dir, + "loaded", stats.Loaded, + "preserved_existing", stats.PreservedExisting, + "skipped", stats.Skipped, + ) + } + + return stats, nil +} diff --git a/internal/app/secrets_env_test.go b/internal/app/secrets_env_test.go new file mode 100644 index 0000000..2821b3b --- /dev/null +++ b/internal/app/secrets_env_test.go @@ -0,0 +1,155 @@ +package app + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/narratio/internal/config" +) + +func TestLoadSecretsFromConfigLoadsValidFiles(t *testing.T) { + dir := t.TempDir() + mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_A"), "value-1\n") + mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_B"), "value-2\r\n") + mustWriteSecretFile(t, filepath.Join(dir, "not-valid-name.txt"), "ignored") + + cfg := &config.Config{ + Pipeline: &config.PipelineConfig{ + Secrets: &config.SecretsConfig{EnvDir: dir}, + }, + } + + stats, err := loadSecretsFromConfig(cfg, nil) + if err != nil { + t.Fatalf("loadSecretsFromConfig() error = %v", err) + } + if stats == nil { + t.Fatal("stats = nil, want non-nil") + } + if stats.Loaded != 2 { + t.Fatalf("Loaded = %d, want 2", stats.Loaded) + } + if stats.PreservedExisting != 0 { + t.Fatalf("PreservedExisting = %d, want 0", stats.PreservedExisting) + } + if stats.Skipped == 0 { + t.Fatalf("Skipped = %d, want > 0 for invalid filename", stats.Skipped) + } + if got := os.Getenv("NARRATIO_TEST_SECRET_A"); got != "value-1" { + t.Fatalf("NARRATIO_TEST_SECRET_A = %q, want value-1", got) + } + if got := os.Getenv("NARRATIO_TEST_SECRET_B"); got != "value-2" { + t.Fatalf("NARRATIO_TEST_SECRET_B = %q, want value-2", got) + } +} + +func TestLoadSecretsFromConfigPreservesExistingEnv(t *testing.T) { + t.Setenv("OBJECT_STORAGE_KEY", "existing") + + dir := t.TempDir() + mustWriteSecretFile(t, filepath.Join(dir, "OBJECT_STORAGE_KEY"), "from-file\n") + + cfg := &config.Config{ + Pipeline: &config.PipelineConfig{ + Secrets: &config.SecretsConfig{EnvDir: dir}, + }, + } + + stats, err := loadSecretsFromConfig(cfg, nil) + if err != nil { + t.Fatalf("loadSecretsFromConfig() error = %v", err) + } + if stats.PreservedExisting != 1 { + t.Fatalf("PreservedExisting = %d, want 1", stats.PreservedExisting) + } + if got := os.Getenv("OBJECT_STORAGE_KEY"); got != "existing" { + t.Fatalf("OBJECT_STORAGE_KEY = %q, want existing", got) + } +} + +func TestLoadSecretsFromConfigRelativeDirUsesCWD(t *testing.T) { + cwd := t.TempDir() + secretsDir := filepath.Join(cwd, "secrets") + if err := os.MkdirAll(secretsDir, 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", secretsDir, err) + } + mustWriteSecretFile(t, filepath.Join(secretsDir, "OBJECT_STORAGE_KEY_ID"), "id-123\n") + + originalWD, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd(): %v", err) + } + if err := os.Chdir(cwd); err != nil { + t.Fatalf("Chdir(%q): %v", cwd, err) + } + t.Cleanup(func() { + _ = os.Chdir(originalWD) + }) + + cfg := &config.Config{ + Pipeline: &config.PipelineConfig{ + Secrets: &config.SecretsConfig{EnvDir: "./secrets"}, + }, + } + if _, err := loadSecretsFromConfig(cfg, nil); err != nil { + t.Fatalf("loadSecretsFromConfig() error = %v", err) + } + if got := os.Getenv("OBJECT_STORAGE_KEY_ID"); got != "id-123" { + t.Fatalf("OBJECT_STORAGE_KEY_ID = %q, want id-123", got) + } +} + +func TestLoadSecretsFromConfigMissingDirFails(t *testing.T) { + cfg := &config.Config{ + Pipeline: &config.PipelineConfig{ + Secrets: &config.SecretsConfig{EnvDir: filepath.Join(t.TempDir(), "missing")}, + }, + } + + _, err := loadSecretsFromConfig(cfg, nil) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "read secrets env_dir") { + t.Fatalf("error = %q, want read-dir context", err.Error()) + } +} + +func TestLoadSecretsFromConfigUnreadableValidEntryFails(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink behavior differs on windows") + } + + dir := t.TempDir() + broken := filepath.Join(dir, "OPENROUTER_API_KEY") + if err := os.Symlink(filepath.Join(dir, "does-not-exist"), broken); err != nil { + t.Fatalf("Symlink(%q): %v", broken, err) + } + + cfg := &config.Config{ + Pipeline: &config.PipelineConfig{ + Secrets: &config.SecretsConfig{EnvDir: dir}, + }, + } + + _, err := loadSecretsFromConfig(cfg, nil) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "read secret file") { + t.Fatalf("error = %q, want read secret file context", err.Error()) + } +} + +func mustWriteSecretFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", path, err) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("WriteFile(%q): %v", path, err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index e0d685d..b6de178 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ type Config struct { type PipelineConfig struct { Workspace WorkspaceConfig `yaml:"workspace"` Storage StorageConfig `yaml:"storage"` + Secrets *SecretsConfig `yaml:"secrets"` WhisperX WhisperXConfig `yaml:"whisperx"` Seriatim SeriatimConfig `yaml:"seriatim"` Audita AuditaConfig `yaml:"audita"` @@ -36,6 +37,11 @@ type WorkspaceConfig struct { Root string `yaml:"root"` } +// SecretsConfig configures optional local filesystem secret loading. +type SecretsConfig struct { + EnvDir string `yaml:"env_dir"` +} + // StorageConfig configures storage backends and related parameters. type StorageConfig struct { Backend string `yaml:"backend"` diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index cc5ba86..f6a0c7f 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -72,6 +72,46 @@ inputs: `, wantLoadErr: "strict decode failed", }, + { + name: "unknown secrets field fails", + pipelineYAML: `workspace: + root: /tmp/narratio +whisperx: + transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe +seriatim: + binary: seriatim +secrets: + bogus: true +`, + sessionYAML: `session_id: 2026-05-03 +inputs: + audio_dir: ./audio + speakers_file: ./speakers.yml + autocorrect_file: ./autocorrect.yml + glossary_file: ./glossary.yml +`, + wantLoadErr: "strict decode failed", + }, + { + name: "empty secrets env_dir fails", + pipelineYAML: `workspace: + root: /tmp/narratio +whisperx: + transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe +seriatim: + binary: seriatim +secrets: + env_dir: " " +`, + sessionYAML: `session_id: 2026-05-03 +inputs: + audio_dir: ./audio + speakers_file: ./speakers.yml + autocorrect_file: ./autocorrect.yml + glossary_file: ./glossary.yml +`, + wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured", + }, { name: "unknown session field fails", pipelineYAML: `workspace: diff --git a/internal/config/validate.go b/internal/config/validate.go index 08fb099..0bcbdcd 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -33,6 +33,9 @@ func validatePipeline(cfg *PipelineConfig) error { if strings.TrimSpace(cfg.Workspace.Root) == "" { return fmt.Errorf("pipeline.workspace.root is required") } + if err := validateSecrets(cfg.Secrets); err != nil { + return err + } if err := validateWhisperX(cfg.WhisperX); err != nil { return err } @@ -61,6 +64,16 @@ func validatePipeline(cfg *PipelineConfig) error { return nil } +func validateSecrets(cfg *SecretsConfig) error { + if cfg == nil { + return nil + } + if strings.TrimSpace(cfg.EnvDir) == "" { + return fmt.Errorf("pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured") + } + return nil +} + func validateNormalize(cfg *NormalizeConfig) error { if cfg == nil { return nil