Added filesystem-based secrets loading configuration

README.md

@@ -35,6 +35,16 @@ Pipeline config lookup for CLI commands:
   - `/usr/local/etc/narratio/pipeline.yml`
   - `/etc/narratio/pipeline.yml`
 
+Optional secrets-from-files config:
+
+- `pipeline.secrets.env_dir` may point to a directory of secret files
+- each top-level file with an env-var-style name is loaded as an environment variable:
+  - file name = env var name
+  - file contents = env var value (trailing newline/CRLF trimmed)
+- process environment wins: existing env vars are not overwritten
+- if configured, Narratio fails fast when `env_dir` is missing/unreadable
+- relative `env_dir` values resolve from Narratio’s current working directory
+
 YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
 
 ## Canonical Stage Order
@@ -199,6 +209,8 @@ Prompt IDs and profile IDs are configuration values. They are not hardcoded in a
 
 Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
 
+If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs.
+
 ## Scriptorium Runtime Behavior
 
 Narratio integrates with Scriptorium through the public CLI subprocess contract:

architecture.md

@@ -101,6 +101,15 @@ CLI pipeline config path resolution:
   - `/usr/local/etc/narratio/pipeline.yml`
   - `/etc/narratio/pipeline.yml`
 
+Optional pipeline secrets directory:
+
+- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution
+- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed)
+- only env-var-style file names are considered; other entries are ignored
+- existing process environment values are preserved (not overwritten)
+- if configured, unreadable/missing `env_dir` fails command execution early
+- relative `env_dir` values are resolved from current working directory
+
 `pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
 
 `pipeline.trim` is optional. Existing pipelines without trim config continue to work.
@@ -288,6 +297,7 @@ Current expected paths for `session_recap`:
 
 - do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
 - if API-key integration is configured, pass env var names only (never raw key values)
+- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs
 - avoid logging transcript content or rendered prompt content by default
 - treat generated artifacts and logs as potentially sensitive session material
 

examples/pipeline.minimal.yml

@@ -4,6 +4,11 @@ workspace:
 storage:
   backend: local
 
+secrets:
+  # Optional: load environment variables from files in this directory.
+  # File name = env var name; file contents = env var value.
+  env_dir: /var/local/narratio/secrets
+
 whisperx:
   transcribe_url: "https://transcription.example.com/transcribe"
   language: "en"

internal/app/commands_test.go

@@ -170,6 +170,131 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
 	}
 }
 
+func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
+	workspaceRoot := t.TempDir()
+	configDir := t.TempDir()
+	sessionID := "2026-05-03"
+	secretsDir := filepath.Join(configDir, "secrets")
+	if err := os.MkdirAll(secretsDir, 0o755); err != nil {
+		t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
+	}
+	if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
+		t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err)
+	}
+
+	seriatimBinary := writeSeriatimAppTestWrapper(t)
+	auditaBinary := writeAuditaAppTestWrapper(t)
+	t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
+	t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
+
+	pipelinePath := filepath.Join(configDir, "pipeline.yml")
+	sessionPath := filepath.Join(configDir, "session.yml")
+	pipelineYAML := `workspace:
+  root: ` + workspaceRoot + `
+storage:
+  backend: local
+secrets:
+  env_dir: ./secrets
+whisperx:
+  transcribe_url: https://example.com/transcribe
+seriatim:
+  binary: ` + seriatimBinary + `
+audita:
+  binary: ` + auditaBinary + `
+  llm_api_key_env: OPENROUTER_API_KEY
+analyzer:
+  timeout: 20m
+notification:
+  timeout: 10s
+`
+	sessionYAML := `session_id: ` + sessionID + `
+inputs:
+  audio_dir: ./audio
+  speakers_file: ./speakers.yml
+  autocorrect_file: ./autocorrect.yml
+  glossary_file: ./glossary.yml
+`
+	if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
+		t.Fatalf("write pipeline.yml: %v", err)
+	}
+	if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
+		t.Fatalf("write session.yml: %v", err)
+	}
+
+	originalWD, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Getwd(): %v", err)
+	}
+	if err := os.Chdir(configDir); err != nil {
+		t.Fatalf("Chdir(%q): %v", configDir, err)
+	}
+	t.Cleanup(func() {
+		_ = os.Chdir(originalWD)
+	})
+
+	workRoot := filepath.Join(workspaceRoot, "work", sessionID)
+	mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
+	mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
+		t.Fatalf("stdout = %q, want polish execution", stdout.String())
+	}
+}
+
+func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
+	workspaceRoot := t.TempDir()
+	configDir := t.TempDir()
+	pipelinePath := filepath.Join(configDir, "pipeline.yml")
+	sessionPath := filepath.Join(configDir, "session.yml")
+
+	pipelineYAML := `workspace:
+  root: ` + workspaceRoot + `
+storage:
+  backend: local
+secrets:
+  env_dir: ./missing-secrets
+whisperx:
+  transcribe_url: https://example.com/transcribe
+seriatim:
+  binary: seriatim
+audita:
+  binary: audita
+analyzer:
+  timeout: 20m
+notification:
+  timeout: 10s
+`
+	sessionYAML := `session_id: 2026-05-03
+inputs:
+  audio_dir: ./audio
+  speakers_file: ./speakers.yml
+  autocorrect_file: ./autocorrect.yml
+  glossary_file: ./glossary.yml
+`
+	if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
+		t.Fatalf("write pipeline.yml: %v", err)
+	}
+	if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
+		t.Fatalf("write session.yml: %v", err)
+	}
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
+	if code == 0 {
+		t.Fatal("exit code = 0, want non-zero")
+	}
+	if !strings.Contains(stderr.String(), "read secrets env_dir") {
+		t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
+	}
+}
+
 func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
 	workspaceRoot := t.TempDir()
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

internal/app/plan.go

@@ -5,9 +5,12 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
+	"os"
 
 	"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
 	"gitea.maximumdirect.net/eric/narratio/internal/config"
+	"gitea.maximumdirect.net/eric/narratio/internal/logging"
 	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
 )
 
@@ -45,6 +48,9 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
 	if err := config.Validate(cfg); err != nil {
 		return fmt.Errorf("plan: %w", err)
 	}
+	if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
+		return fmt.Errorf("plan: %w", err)
+	}
 
 	store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
 	paths, err := store.EnsureLayout(cfg.Session.SessionID)

internal/app/plan_test.go

@@ -89,6 +89,53 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
 	}
 }
 
+func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
+	workspaceRoot := t.TempDir()
+	configDir := t.TempDir()
+	pipelinePath := filepath.Join(configDir, "pipeline.yml")
+	sessionPath := filepath.Join(configDir, "session.yml")
+
+	pipelineYAML := `workspace:
+  root: ` + workspaceRoot + `
+storage:
+  backend: local
+secrets:
+  env_dir: ./missing-secrets
+whisperx:
+  transcribe_url: https://example.com/transcribe
+seriatim:
+  binary: seriatim
+audita:
+  binary: audita
+analyzer:
+  timeout: 20m
+notification:
+  timeout: 10s
+`
+	sessionYAML := `session_id: 2026-05-03
+inputs:
+  audio_dir: ./audio
+  speakers_file: ./speakers.yml
+  autocorrect_file: ./autocorrect.yml
+  glossary_file: ./glossary.yml
+`
+	if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
+		t.Fatalf("write pipeline.yml: %v", err)
+	}
+	if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
+		t.Fatalf("write session.yml: %v", err)
+	}
+
+	var out bytes.Buffer
+	err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !strings.Contains(err.Error(), "read secrets env_dir") {
+		t.Fatalf("error = %q, want secrets read error context", err.Error())
+	}
+}
+
 func assertDir(t *testing.T, path string) {
 	t.Helper()
 	info, err := os.Stat(path)

internal/app/runner.go

@@ -51,6 +51,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 	if env.Logger == nil {
 		env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
 	}
+	if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
+		return nil, fmt.Errorf("load secrets from files: %w", err)
+	}
 	if env.WhisperX == nil {
 		client, err := buildDefaultWhisperXClient(env.Config)
 		if err != nil {

internal/app/secrets_env.go

@@ -0,0 +1,88 @@
+package app
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/config"
+)
+
+var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
+
+type secretsLoadStats struct {
+	Dir               string
+	Loaded            int
+	PreservedExisting int
+	Skipped           int
+}
+
+func loadSecretsFromConfig(cfg *config.Config, logger *slog.Logger) (*secretsLoadStats, error) {
+	if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Secrets == nil {
+		return nil, nil
+	}
+
+	rawDir := strings.TrimSpace(cfg.Pipeline.Secrets.EnvDir)
+	if rawDir == "" {
+		return nil, nil
+	}
+
+	resolvedDir := rawDir
+	if !filepath.IsAbs(resolvedDir) {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return nil, fmt.Errorf("resolve secrets env_dir %q from current working directory: %w", rawDir, err)
+		}
+		resolvedDir = filepath.Join(cwd, resolvedDir)
+	}
+	resolvedDir = filepath.Clean(resolvedDir)
+
+	entries, err := os.ReadDir(resolvedDir)
+	if err != nil {
+		return nil, fmt.Errorf("read secrets env_dir %q: %w", resolvedDir, err)
+	}
+
+	stats := &secretsLoadStats{Dir: resolvedDir}
+	for _, entry := range entries {
+		name := entry.Name()
+		if !envVarNamePattern.MatchString(name) {
+			stats.Skipped++
+			continue
+		}
+		if entry.IsDir() {
+			stats.Skipped++
+			continue
+		}
+
+		secretPath := filepath.Join(resolvedDir, name)
+		bytes, err := os.ReadFile(secretPath)
+		if err != nil {
+			return nil, fmt.Errorf("read secret file %q: %w", secretPath, err)
+		}
+		value := strings.TrimRight(string(bytes), "\r\n")
+
+		if _, exists := os.LookupEnv(name); exists {
+			stats.PreservedExisting++
+			continue
+		}
+		if err := os.Setenv(name, value); err != nil {
+			return nil, fmt.Errorf("set environment variable %q from %q: %w", name, secretPath, err)
+		}
+		stats.Loaded++
+	}
+
+	if logger != nil {
+		logger.Info(
+			"loaded secret environment variables from filesystem",
+			"secrets_env_dir", stats.Dir,
+			"loaded", stats.Loaded,
+			"preserved_existing", stats.PreservedExisting,
+			"skipped", stats.Skipped,
+		)
+	}
+
+	return stats, nil
+}

internal/app/secrets_env_test.go

@@ -0,0 +1,155 @@
+package app
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/config"
+)
+
+func TestLoadSecretsFromConfigLoadsValidFiles(t *testing.T) {
+	dir := t.TempDir()
+	mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_A"), "value-1\n")
+	mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_B"), "value-2\r\n")
+	mustWriteSecretFile(t, filepath.Join(dir, "not-valid-name.txt"), "ignored")
+
+	cfg := &config.Config{
+		Pipeline: &config.PipelineConfig{
+			Secrets: &config.SecretsConfig{EnvDir: dir},
+		},
+	}
+
+	stats, err := loadSecretsFromConfig(cfg, nil)
+	if err != nil {
+		t.Fatalf("loadSecretsFromConfig() error = %v", err)
+	}
+	if stats == nil {
+		t.Fatal("stats = nil, want non-nil")
+	}
+	if stats.Loaded != 2 {
+		t.Fatalf("Loaded = %d, want 2", stats.Loaded)
+	}
+	if stats.PreservedExisting != 0 {
+		t.Fatalf("PreservedExisting = %d, want 0", stats.PreservedExisting)
+	}
+	if stats.Skipped == 0 {
+		t.Fatalf("Skipped = %d, want > 0 for invalid filename", stats.Skipped)
+	}
+	if got := os.Getenv("NARRATIO_TEST_SECRET_A"); got != "value-1" {
+		t.Fatalf("NARRATIO_TEST_SECRET_A = %q, want value-1", got)
+	}
+	if got := os.Getenv("NARRATIO_TEST_SECRET_B"); got != "value-2" {
+		t.Fatalf("NARRATIO_TEST_SECRET_B = %q, want value-2", got)
+	}
+}
+
+func TestLoadSecretsFromConfigPreservesExistingEnv(t *testing.T) {
+	t.Setenv("OBJECT_STORAGE_KEY", "existing")
+
+	dir := t.TempDir()
+	mustWriteSecretFile(t, filepath.Join(dir, "OBJECT_STORAGE_KEY"), "from-file\n")
+
+	cfg := &config.Config{
+		Pipeline: &config.PipelineConfig{
+			Secrets: &config.SecretsConfig{EnvDir: dir},
+		},
+	}
+
+	stats, err := loadSecretsFromConfig(cfg, nil)
+	if err != nil {
+		t.Fatalf("loadSecretsFromConfig() error = %v", err)
+	}
+	if stats.PreservedExisting != 1 {
+		t.Fatalf("PreservedExisting = %d, want 1", stats.PreservedExisting)
+	}
+	if got := os.Getenv("OBJECT_STORAGE_KEY"); got != "existing" {
+		t.Fatalf("OBJECT_STORAGE_KEY = %q, want existing", got)
+	}
+}
+
+func TestLoadSecretsFromConfigRelativeDirUsesCWD(t *testing.T) {
+	cwd := t.TempDir()
+	secretsDir := filepath.Join(cwd, "secrets")
+	if err := os.MkdirAll(secretsDir, 0o755); err != nil {
+		t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
+	}
+	mustWriteSecretFile(t, filepath.Join(secretsDir, "OBJECT_STORAGE_KEY_ID"), "id-123\n")
+
+	originalWD, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Getwd(): %v", err)
+	}
+	if err := os.Chdir(cwd); err != nil {
+		t.Fatalf("Chdir(%q): %v", cwd, err)
+	}
+	t.Cleanup(func() {
+		_ = os.Chdir(originalWD)
+	})
+
+	cfg := &config.Config{
+		Pipeline: &config.PipelineConfig{
+			Secrets: &config.SecretsConfig{EnvDir: "./secrets"},
+		},
+	}
+	if _, err := loadSecretsFromConfig(cfg, nil); err != nil {
+		t.Fatalf("loadSecretsFromConfig() error = %v", err)
+	}
+	if got := os.Getenv("OBJECT_STORAGE_KEY_ID"); got != "id-123" {
+		t.Fatalf("OBJECT_STORAGE_KEY_ID = %q, want id-123", got)
+	}
+}
+
+func TestLoadSecretsFromConfigMissingDirFails(t *testing.T) {
+	cfg := &config.Config{
+		Pipeline: &config.PipelineConfig{
+			Secrets: &config.SecretsConfig{EnvDir: filepath.Join(t.TempDir(), "missing")},
+		},
+	}
+
+	_, err := loadSecretsFromConfig(cfg, nil)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !strings.Contains(err.Error(), "read secrets env_dir") {
+		t.Fatalf("error = %q, want read-dir context", err.Error())
+	}
+}
+
+func TestLoadSecretsFromConfigUnreadableValidEntryFails(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("symlink behavior differs on windows")
+	}
+
+	dir := t.TempDir()
+	broken := filepath.Join(dir, "OPENROUTER_API_KEY")
+	if err := os.Symlink(filepath.Join(dir, "does-not-exist"), broken); err != nil {
+		t.Fatalf("Symlink(%q): %v", broken, err)
+	}
+
+	cfg := &config.Config{
+		Pipeline: &config.PipelineConfig{
+			Secrets: &config.SecretsConfig{EnvDir: dir},
+		},
+	}
+
+	_, err := loadSecretsFromConfig(cfg, nil)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !strings.Contains(err.Error(), "read secret file") {
+		t.Fatalf("error = %q, want read secret file context", err.Error())
+	}
+}
+
+func mustWriteSecretFile(t *testing.T, path, contents string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("MkdirAll(%q): %v", path, err)
+	}
+	if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
+		t.Fatalf("WriteFile(%q): %v", path, err)
+	}
+}

internal/config/config.go

@@ -12,6 +12,7 @@ type Config struct {
 type PipelineConfig struct {
 	Workspace    WorkspaceConfig    `yaml:"workspace"`
 	Storage      StorageConfig      `yaml:"storage"`
+	Secrets      *SecretsConfig     `yaml:"secrets"`
 	WhisperX     WhisperXConfig     `yaml:"whisperx"`
 	Seriatim     SeriatimConfig     `yaml:"seriatim"`
 	Audita       AuditaConfig       `yaml:"audita"`
@@ -36,6 +37,11 @@ type WorkspaceConfig struct {
 	Root string `yaml:"root"`
 }
 
+// SecretsConfig configures optional local filesystem secret loading.
+type SecretsConfig struct {
+	EnvDir string `yaml:"env_dir"`
+}
+
 // StorageConfig configures storage backends and related parameters.
 type StorageConfig struct {
 	Backend string `yaml:"backend"`

internal/config/load_validate_test.go

@@ -72,6 +72,46 @@ inputs:
 `,
 			wantLoadErr: "strict decode failed",
 		},
+		{
+			name: "unknown secrets field fails",
+			pipelineYAML: `workspace:
+  root: /tmp/narratio
+whisperx:
+  transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
+seriatim:
+  binary: seriatim
+secrets:
+  bogus: true
+`,
+			sessionYAML: `session_id: 2026-05-03
+inputs:
+  audio_dir: ./audio
+  speakers_file: ./speakers.yml
+  autocorrect_file: ./autocorrect.yml
+  glossary_file: ./glossary.yml
+`,
+			wantLoadErr: "strict decode failed",
+		},
+		{
+			name: "empty secrets env_dir fails",
+			pipelineYAML: `workspace:
+  root: /tmp/narratio
+whisperx:
+  transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
+seriatim:
+  binary: seriatim
+secrets:
+  env_dir: "   "
+`,
+			sessionYAML: `session_id: 2026-05-03
+inputs:
+  audio_dir: ./audio
+  speakers_file: ./speakers.yml
+  autocorrect_file: ./autocorrect.yml
+  glossary_file: ./glossary.yml
+`,
+			wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured",
+		},
 		{
 			name: "unknown session field fails",
 			pipelineYAML: `workspace:

internal/config/validate.go

@@ -33,6 +33,9 @@ func validatePipeline(cfg *PipelineConfig) error {
 	if strings.TrimSpace(cfg.Workspace.Root) == "" {
 		return fmt.Errorf("pipeline.workspace.root is required")
 	}
+	if err := validateSecrets(cfg.Secrets); err != nil {
+		return err
+	}
 	if err := validateWhisperX(cfg.WhisperX); err != nil {
 		return err
 	}
@@ -61,6 +64,16 @@ func validatePipeline(cfg *PipelineConfig) error {
 	return nil
 }
 
+func validateSecrets(cfg *SecretsConfig) error {
+	if cfg == nil {
+		return nil
+	}
+	if strings.TrimSpace(cfg.EnvDir) == "" {
+		return fmt.Errorf("pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured")
+	}
+	return nil
+}
+
 func validateNormalize(cfg *NormalizeConfig) error {
 	if cfg == nil {
 		return nil