diff --git a/internal/framework/llm/secrets.go b/internal/framework/llm/secrets.go new file mode 100644 index 0000000..9322404 --- /dev/null +++ b/internal/framework/llm/secrets.go @@ -0,0 +1,17 @@ +package llm + +import "gitea.maximumdirect.net/eric/audita/internal/core/config" + +// ConfiguredSecrets returns all configured LLM API-key values that should be +// redacted from diagnostics and surfaced error payloads. +func ConfiguredSecrets(cfg *config.Config) []string { + if cfg == nil { + return nil + } + effectiveValidation := cfg.EffectiveValidationLLMConfig() + return []string{ + cfg.PrimaryLLM.APIKey, + cfg.ValidationLLM.APIKey, + effectiveValidation.APIKey, + } +} diff --git a/internal/framework/llm/secrets_test.go b/internal/framework/llm/secrets_test.go new file mode 100644 index 0000000..c7be874 --- /dev/null +++ b/internal/framework/llm/secrets_test.go @@ -0,0 +1,38 @@ +package llm + +import ( + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/core/config" +) + +func TestConfiguredSecretsNilConfig(t *testing.T) { + if got := ConfiguredSecrets(nil); got != nil { + t.Fatalf("expected nil secrets for nil config, got %v", got) + } +} + +func TestConfiguredSecretsWithValidationOverride(t *testing.T) { + cfg := config.Default() + cfg.PrimaryLLM.APIKey = "primary-secret" + cfg.ValidationLLM.APIKey = "validation-secret" + + got := ConfiguredSecrets(&cfg) + want := []string{"primary-secret", "validation-secret", "validation-secret"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected secrets: got=%v want=%v", got, want) + } +} + +func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) { + cfg := config.Default() + cfg.PrimaryLLM.APIKey = "primary-secret" + cfg.ValidationLLM.APIKey = "" + + got := ConfiguredSecrets(&cfg) + want := []string{"primary-secret", "", "primary-secret"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want) + } +} diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 56e2bdf..b71fae9 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -108,7 +108,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { writer = diagnosticsWriterAdapter{ writer: llm.NewDiagnosticsWriter( filepath.Join(req.DiagnosticsDir, req.ModuleInstance), - proposalGenerationSecrets(req.Config), + llm.ConfiguredSecrets(req.Config), ), } } @@ -213,17 +213,6 @@ func resolveModel(cfg *config.Config, override string) string { return llm.ResolvePrimaryConfig(*cfg).Model } -func proposalGenerationSecrets(cfg *config.Config) []string { - if cfg == nil { - return nil - } - return []string{ - cfg.PrimaryLLM.APIKey, - cfg.ValidationLLM.APIKey, - cfg.EffectiveValidationLLMConfig().APIKey, - } -} - func errPayload(err error) any { if err == nil { return nil diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index d2c734c..0ce8d37 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -361,20 +361,22 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { } func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { - secret := "proposal-secret" + primarySecret := "proposal-primary-secret" + validationSecret := "proposal-validation-secret" client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}}, }, } cfg := config.Default() - cfg.PrimaryLLM.APIKey = secret + cfg.PrimaryLLM.APIKey = primarySecret + cfg.ValidationLLM.APIKey = validationSecret req := defaultRequest(t) req.Config = &cfg req.LLMClient = client req.DiagnosticsDir = t.TempDir() req.Messages = []contracts.LLMMessage{ - {Role: "system", Content: "include secret " + secret}, + {Role: "system", Content: "include secret " + primarySecret}, {Role: "user", Content: "fix it"}, } @@ -391,8 +393,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { if readErr != nil { t.Fatalf("read artifact %q: %v", path, readErr) } - if strings.Contains(string(raw), secret) { - t.Fatalf("artifact leaked secret %q: %s", path, string(raw)) + if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { + t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw)) } if !strings.Contains(string(raw), "[REDACTED]") { t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw)) diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index e787e4e..0ab6dbf 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -527,7 +527,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida diagnosticsWriter = &llmDiagnosticsWriterAdapter{ writer: llm.NewDiagnosticsWriter( filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), - validatorSecrets(input.Config), + llm.ConfiguredSecrets(input.Config), ), } } @@ -744,15 +744,3 @@ func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMeta ErrorPayloadPath: art.ErrorPayloadPath, }, nil } - -func validatorSecrets(cfg *config.Config) []string { - if cfg == nil { - return nil - } - effective := cfg.EffectiveValidationLLMConfig() - return []string{ - cfg.PrimaryLLM.APIKey, - effective.APIKey, - cfg.ValidationLLM.APIKey, - } -} diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 5d3948d..559bde5 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -1307,12 +1307,13 @@ func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) { } func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { - secret := "super-secret-key" - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}} + primarySecret := "runner-primary-secret" + validationSecret := "runner-validation-secret" + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") cfg := config.Default() - cfg.PrimaryLLM.APIKey = secret - cfg.ValidationLLM.APIKey = secret + cfg.PrimaryLLM.APIKey = primarySecret + cfg.ValidationLLM.APIKey = validationSecret diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { @@ -1321,7 +1322,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, ValidationDiagnosticsDir: diagDir, @@ -1332,12 +1333,26 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" { t.Fatalf("expected diagnostic artifact path on decision") } + matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json")) + if globErr != nil { + t.Fatalf("glob diagnostics: %v", globErr) + } + if len(matches) == 0 { + t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m")) + } + for _, path := range matches { + raw, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatalf("read diagnostic %q: %v", path, readErr) + } + if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { + t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw)) + } + } + raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath) if readErr != nil { - t.Fatalf("read diagnostic: %v", readErr) - } - if strings.Contains(string(raw), secret) { - t.Fatalf("secret leaked in diagnostics: %s", string(raw)) + t.Fatalf("read decision diagnostic: %v", readErr) } if !strings.Contains(string(raw), "[REDACTED]") { t.Fatalf("expected redaction marker in diagnostics")