eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Resolve typed credentials safely

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-07 04:29:00 +00:00
parent d31ba5a410
commit 92432232ef
34 changed files with 572 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
internal/config/config.go
View File

@@ -1,5 +1,7 @@
package config
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
// Config is the resolved combined configuration from pipeline.yml,
// campaign.yml, and session.yml.
type Config struct {
@@ -98,13 +100,13 @@ const (
// StorageS3Config configures S3 storage coordinates.
type StorageS3Config struct {
Bucket string `yaml:"bucket" json:"bucket"`
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
Region string `yaml:"region" json:"region"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
AccessKeyIDEnv string `yaml:"access_key_id_env" json:"access_key_id_env"`
SecretKeyEnv string `yaml:"secret_access_key_env" json:"secret_access_key_env"`
Bucket string `yaml:"bucket" json:"bucket"`
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
Region string `yaml:"region" json:"region"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
AccessKeyID *configkit.Secret `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"`
SecretAccessKey *configkit.Secret `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"`
}
// SpoolConfig configures local spool storage for staged data.
@@ -163,21 +165,21 @@ type SeriatimEnvConfig struct {
// AuditaConfig configures audita adapter settings.
type AuditaConfig struct {
Binary string `yaml:"binary" json:"binary"`
Timeout string `yaml:"timeout" json:"timeout"`
LLMAPIKeyEnv string `yaml:"llm_api_key_env" json:"llm_api_key_env"`
Modules []string `yaml:"modules" json:"modules"`
BaseURL string `yaml:"base_url" json:"base_url"`
Model string `yaml:"model" json:"model"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency" json:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency" json:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model" json:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency" json:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
ConfigPath string `yaml:"config_path" json:"config_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention" json:"work_dir_retention"`
Report *bool `yaml:"report" json:"report"`
Binary string `yaml:"binary" json:"binary"`
Timeout string `yaml:"timeout" json:"timeout"`
LLMAPIKey *configkit.Secret `yaml:"llm_api_key,omitempty" json:"llm_api_key,omitempty"`
Modules []string `yaml:"modules" json:"modules"`
BaseURL string `yaml:"base_url" json:"base_url"`
Model string `yaml:"model" json:"model"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency" json:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency" json:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model" json:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency" json:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
ConfigPath string `yaml:"config_path" json:"config_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention" json:"work_dir_retention"`
Report *bool `yaml:"report" json:"report"`
}
// NormalizeConfig configures normalize-stage transcript schema/output behavior.

59
internal/config/credential_projection.go Normal file
View File

@@ -0,0 +1,59 @@
package config
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
// MarshalYAML exposes only public credential reference names. Resolved values
// remain available solely through ConfigKit's explicit trusted accessor.
func (cfg StorageS3Config) MarshalYAML() (any, error) {
type safeStorageS3 struct {
Bucket string `yaml:"bucket"`
RootPrefix string `yaml:"root_prefix"`
Region string `yaml:"region"`
Endpoint string `yaml:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style"`
AccessKeyID *string `yaml:"access_key_id,omitempty"`
SecretAccessKey *string `yaml:"secret_access_key,omitempty"`
}
return safeStorageS3{
Bucket: cfg.Bucket, RootPrefix: cfg.RootPrefix, Region: cfg.Region,
Endpoint: cfg.Endpoint, ForcePathStyle: cfg.ForcePathStyle,
AccessKeyID: secretReferenceName(cfg.AccessKeyID), SecretAccessKey: secretReferenceName(cfg.SecretAccessKey),
}, nil
}
// MarshalYAML exposes only the public Audita credential reference name.
func (cfg AuditaConfig) MarshalYAML() (any, error) {
type safeAudita struct {
Binary string `yaml:"binary"`
Timeout string `yaml:"timeout"`
LLMAPIKey *string `yaml:"llm_api_key,omitempty"`
Modules []string `yaml:"modules"`
BaseURL string `yaml:"base_url"`
Model string `yaml:"model"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description"`
ConfigPath string `yaml:"config_path"`
OutputSchema string `yaml:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention"`
Report *bool `yaml:"report"`
}
return safeAudita{
Binary: cfg.Binary, Timeout: cfg.Timeout, LLMAPIKey: secretReferenceName(cfg.LLMAPIKey),
Modules: cfg.Modules, BaseURL: cfg.BaseURL, Model: cfg.Model,
TotalLLMConcurrency: cfg.TotalLLMConcurrency, ProposalLLMConcurrency: cfg.ProposalLLMConcurrency,
ValidationModel: cfg.ValidationModel, ValidationLLMConcurrency: cfg.ValidationLLMConcurrency,
TranscriptDescription: cfg.TranscriptDescription, ConfigPath: cfg.ConfigPath,
OutputSchema: cfg.OutputSchema, WorkDirRetention: cfg.WorkDirRetention, Report: cfg.Report,
}, nil
}
func secretReferenceName(secret *configkit.Secret) *string {
if secret == nil {
return nil
}
name := secret.EnvName()
return &name
}

2
internal/config/defaults.go
View File

@@ -9,8 +9,6 @@ const (
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
DefaultS3SecretAccessKeyEnv = "OBJECT_STORAGE_KEY"
DefaultStorageS3RootPrefix = "dnd"
DefaultWorkspaceRoot = "/var/lib/narratio"
DefaultCampaignsRoot = "/usr/local/share/narratio/campaigns"

90
internal/config/load_validate_test.go
View File

@@ -1,6 +1,7 @@
package config
import (
"encoding/json"
"io/fs"
"os"
"path/filepath"
@@ -8,6 +9,8 @@ import (
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestRepresentativeLoadDefaultsAndValidation(t *testing.T) {
@@ -83,6 +86,78 @@ whisperx:
}
}
func TestCredentialValuesNeverEnterEffectiveProjectionOrDigest(t *testing.T) {
const firstCanary = "credential-canary-first"
const secondCanary = "credential-canary-second"
t.Setenv("NARRATIO_TEST_ACCESS_KEY", firstCanary)
t.Setenv("NARRATIO_TEST_SECRET_KEY", firstCanary)
t.Setenv("NARRATIO_TEST_AUDITA_KEY", firstCanary)
pipelineYAML := strings.Replace(testPipelineBaseYAML, "audita:\n binary: audita\n", "audita:\n binary: audita\n llm_api_key: NARRATIO_TEST_AUDITA_KEY\n", 1) + `
storage:
backend: s3
s3:
bucket: test-bucket
access_key_id: NARRATIO_TEST_ACCESS_KEY
secret_access_key: NARRATIO_TEST_SECRET_KEY
`
pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
first, err := LoadPipeline(pipelinePath)
if err != nil {
t.Fatal(err)
}
firstYAML, err := MarshalEffectivePipeline(first)
if err != nil {
t.Fatal(err)
}
firstValues, err := EffectivePipelineValues(first)
if err != nil {
t.Fatal(err)
}
serializedValues := ""
for _, value := range firstValues {
serializedValues += value.Path + "=" + value.Value + "\n"
}
directYAML, err := yaml.Marshal(first)
if err != nil {
t.Fatal(err)
}
directJSON, err := json.Marshal(first)
if err != nil {
t.Fatal(err)
}
for _, output := range []string{string(firstYAML), serializedValues, string(directYAML), string(directJSON)} {
if strings.Contains(output, firstCanary) {
t.Fatalf("effective output exposed credential value: %q", output)
}
for _, reference := range []string{"NARRATIO_TEST_ACCESS_KEY", "NARRATIO_TEST_SECRET_KEY", "NARRATIO_TEST_AUDITA_KEY"} {
if !strings.Contains(output, reference) {
t.Fatalf("effective output omitted safe reference %q: %q", reference, output)
}
}
}
t.Setenv("NARRATIO_TEST_ACCESS_KEY", secondCanary)
t.Setenv("NARRATIO_TEST_SECRET_KEY", secondCanary)
t.Setenv("NARRATIO_TEST_AUDITA_KEY", secondCanary)
second, err := LoadPipeline(pipelinePath)
if err != nil {
t.Fatal(err)
}
if EffectivePipelineDigest(first) != EffectivePipelineDigest(second) {
t.Fatal("credential value change altered effective digest")
}
t.Setenv("NARRATIO_TEST_AUDITA_KEY_ALTERNATE", secondCanary)
alternatePath, _ := writeConfigFiles(t, strings.Replace(pipelineYAML, "NARRATIO_TEST_AUDITA_KEY", "NARRATIO_TEST_AUDITA_KEY_ALTERNATE", 1), testSessionBaseYAML)
alternate, err := LoadPipeline(alternatePath)
if err != nil {
t.Fatal(err)
}
if EffectivePipelineDigest(first) == EffectivePipelineDigest(alternate) {
t.Fatal("credential reference-name change did not alter effective digest")
}
}
func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}}
nilValues, err := EffectivePipelineValues(base)
@@ -106,6 +181,13 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" {
t.Fatalf("exact integer value = %q", got)
}
encoded, err := MarshalEffectivePipeline(base)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(encoded), "9007199254740993") {
t.Fatalf("effective YAML lost exact integer: %q", encoded)
}
}
func effectiveValueForTest(t *testing.T, values []EffectivePipelineValueRecord, path string) string {
@@ -138,6 +220,9 @@ func TestValidateMissingAudioSource(t *testing.T) {
}
func TestExamplesLoadAndValidate(t *testing.T) {
t.Setenv("OBJECT_STORAGE_KEY_ID", "synthetic-example-access-key")
t.Setenv("OBJECT_STORAGE_KEY", "synthetic-example-secret-key")
t.Setenv("AUDITA_LLM_API_KEY", "synthetic-example-audita-key")
examplesDir := filepath.Join("..", "..", "examples")
if got, want := maintainedPipelineRoots(t, examplesDir), []string{
"pipeline.extraction-subset.yml",
@@ -226,7 +311,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
if cfg.Pipeline.Audita.Model != profile.model {
t.Fatalf("audita model = %q, want %q", cfg.Pipeline.Audita.Model, profile.model)
}
if cfg.Pipeline.Audita.LLMAPIKeyEnv != "" || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
if cfg.Pipeline.Audita.LLMAPIKey != nil || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
t.Fatalf("split bundle must remain offline and secret-free: audita=%#v storage=%#v", cfg.Pipeline.Audita, cfg.Pipeline.Storage)
}
if selectedProfile, ok := SelectedPipelineProfile(cfg.Pipeline); !ok || selectedProfile.Name != profile.name {
@@ -295,6 +380,9 @@ func maintainedPipelineRoots(t *testing.T, examplesDir string) []string {
}
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
t.Setenv("OBJECT_STORAGE_KEY_ID", "synthetic-example-access-key")
t.Setenv("OBJECT_STORAGE_KEY", "synthetic-example-secret-key")
t.Setenv("AUDITA_LLM_API_KEY", "synthetic-example-audita-key")
examplesDir := filepath.Join("..", "..", "examples")
full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml"))
if err != nil {

2
internal/config/pipeline_configkit.go
View File

@@ -163,8 +163,6 @@ func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution
if cfg.Storage.S3 != nil {
path := configkit.Keys("storage", "s3")
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference")
configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference")
}
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}

75
internal/config/storage_publish_test.go
View File

@@ -24,11 +24,8 @@ storage:
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != DefaultS3AccessKeyIDEnv {
t.Fatalf("storage.s3.access_key_id_env = %q, want %q", cfg.Pipeline.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != DefaultS3SecretAccessKeyEnv {
t.Fatalf("storage.s3.secret_access_key_env = %q, want %q", cfg.Pipeline.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv)
if cfg.Pipeline.Storage.S3.AccessKeyID != nil || cfg.Pipeline.Storage.S3.SecretAccessKey != nil {
t.Fatal("storage.s3 credentials should be optional without implicit references")
}
if cfg.Pipeline.Storage.S3.ForcePathStyle {
t.Fatalf("storage.s3.force_path_style = true, want false default")
@@ -82,14 +79,32 @@ storage:
}
}
func TestLegacyCredentialReferenceFieldsFailStrictDecode(t *testing.T) {
for _, field := range []string{"access_key_id_env", "secret_access_key_env", "llm_api_key_env"} {
t.Run(field, func(t *testing.T) {
pipelineYAML := strings.Replace(testPipelineBaseYAML, "audita:\n binary: audita\n", "audita:\n binary: audita\n "+field+": LEGACY_SECRET\n", 1)
if strings.HasPrefix(field, "access_") || strings.HasPrefix(field, "secret_access_") {
pipelineYAML = testPipelineBaseYAML + "storage:\n backend: s3\n s3:\n bucket: test-bucket\n " + field + ": LEGACY_SECRET\n"
}
pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := LoadPipeline(pipelinePath)
if err == nil || !strings.Contains(err.Error(), field) || !strings.Contains(err.Error(), "strict decode") {
t.Fatalf("LoadPipeline() error = %v, want strict rejection of %s", err, field)
}
})
}
}
func TestStorageS3CredentialEnvNamesLoadAndValidate(t *testing.T) {
t.Setenv("CUSTOM_KEY_ID", "synthetic-access-key")
t.Setenv("CUSTOM_SECRET", "synthetic-secret-key")
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: CUSTOM_KEY_ID
secret_access_key_env: CUSTOM_SECRET
access_key_id: CUSTOM_KEY_ID
secret_access_key: CUSTOM_SECRET
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
@@ -97,55 +112,69 @@ storage:
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != "CUSTOM_KEY_ID" {
t.Fatalf("storage.s3.access_key_id_env = %q, want CUSTOM_KEY_ID", cfg.Pipeline.Storage.S3.AccessKeyIDEnv)
if cfg.Pipeline.Storage.S3.AccessKeyID == nil || cfg.Pipeline.Storage.S3.AccessKeyID.EnvName() != "CUSTOM_KEY_ID" || cfg.Pipeline.Storage.S3.AccessKeyID.Value() != "synthetic-access-key" {
t.Fatal("storage.s3.access_key_id did not retain its resolved typed reference")
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != "CUSTOM_SECRET" {
t.Fatalf("storage.s3.secret_access_key_env = %q, want CUSTOM_SECRET", cfg.Pipeline.Storage.S3.SecretKeyEnv)
if cfg.Pipeline.Storage.S3.SecretAccessKey == nil || cfg.Pipeline.Storage.S3.SecretAccessKey.EnvName() != "CUSTOM_SECRET" || cfg.Pipeline.Storage.S3.SecretAccessKey.Value() != "synthetic-secret-key" {
t.Fatal("storage.s3.secret_access_key did not retain its resolved typed reference")
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvValidation(t *testing.T) {
func TestStorageS3CredentialReferenceValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "invalid access key env name",
name: "invalid access key reference",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: "123BAD"
access_key_id: "123BAD"
secret_access_key: CUSTOM_SECRET
`,
wantErr: "pipeline.storage.s3.access_key_id_env must be a valid environment variable name",
wantErr: "secret.invalid_name",
},
{
name: "invalid secret key env name",
name: "partial pair",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
secret_access_key_env: "bad-name"
access_key_id: CUSTOM_KEY_ID
`,
wantErr: "pipeline.storage.s3.secret_access_key_env must be a valid environment variable name",
wantErr: "must be configured together",
},
{
name: "missing effective reference",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id: MISSING_CUSTOM_KEY_ID
secret_access_key: MISSING_CUSTOM_SECRET
`,
wantErr: "secret.unset",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
pipelinePath, _ := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
t.Setenv("CUSTOM_KEY_ID", "synthetic-access-key")
t.Setenv("CUSTOM_SECRET", "synthetic-secret-key")
cfg, err := LoadPipeline(pipelinePath)
if err == nil {
err = ValidatePipelineConfig(cfg)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}

12
internal/config/validate_helpers.go
View File

@@ -10,7 +10,6 @@ import (
)
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
func sortedStringKeys[V any](values map[string]V) []string {
keys := make([]string, 0, len(values))
@@ -21,17 +20,6 @@ func sortedStringKeys[V any](values map[string]V) []string {
return keys
}
func validateEnvVarNameField(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if !envVarNameRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a valid environment variable name", fieldName)
}
return nil
}
func validateRelativeSafePath(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {

7
internal/config/validate_publish.go
View File

@@ -33,11 +33,8 @@ func validateStorage(cfg StorageConfig) error {
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
}
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
return err
}
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
return err
if (cfg.S3.AccessKeyID == nil) != (cfg.S3.SecretAccessKey == nil) {
return fmt.Errorf("pipeline.storage.s3.access_key_id and pipeline.storage.s3.secret_access_key must be configured together")
}
return nil
}