eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Share configured LLM secret extraction across diagnostics paths

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-23 18:11:36 +00:00
parent 99391cd18b
commit 222222f449
6 changed files with 89 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/framework/llm/secrets.go Normal file
View File

@@ -0,0 +1,17 @@
package llm
import "gitea.maximumdirect.net/eric/audita/internal/core/config"
// ConfiguredSecrets returns all configured LLM API-key values that should be
// redacted from diagnostics and surfaced error payloads.
func ConfiguredSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effectiveValidation := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
cfg.ValidationLLM.APIKey,
effectiveValidation.APIKey,
}
}

38
internal/framework/llm/secrets_test.go Normal file
View File

@@ -0,0 +1,38 @@
package llm
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestConfiguredSecretsNilConfig(t *testing.T) {
if got := ConfiguredSecrets(nil); got != nil {
t.Fatalf("expected nil secrets for nil config, got %v", got)
}
}
func TestConfiguredSecretsWithValidationOverride(t *testing.T) {
cfg := config.Default()
cfg.PrimaryLLM.APIKey = "primary-secret"
cfg.ValidationLLM.APIKey = "validation-secret"
got := ConfiguredSecrets(&cfg)
want := []string{"primary-secret", "validation-secret", "validation-secret"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected secrets: got=%v want=%v", got, want)
}
}
func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) {
cfg := config.Default()
cfg.PrimaryLLM.APIKey = "primary-secret"
cfg.ValidationLLM.APIKey = ""
got := ConfiguredSecrets(&cfg)
want := []string{"primary-secret", "", "primary-secret"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want)
}
}

13
internal/framework/proposal_generation/generate.go
View File

@@ -108,7 +108,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
proposalGenerationSecrets(req.Config),
llm.ConfiguredSecrets(req.Config),
),
}
}
@@ -213,17 +213,6 @@ func resolveModel(cfg *config.Config, override string) string {
return llm.ResolvePrimaryConfig(*cfg).Model
}
func proposalGenerationSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
return []string{
cfg.PrimaryLLM.APIKey,
cfg.ValidationLLM.APIKey,
cfg.EffectiveValidationLLMConfig().APIKey,
}
}
func errPayload(err error) any {
if err == nil {
return nil

14
internal/framework/proposal_generation/generate_test.go
View File

@@ -361,20 +361,22 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
}
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
secret := "proposal-secret"
primarySecret := "proposal-primary-secret"
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
cfg.PrimaryLLM.APIKey = primarySecret
cfg.ValidationLLM.APIKey = validationSecret
req := defaultRequest(t)
req.Config = &cfg
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Messages = []contracts.LLMMessage{
{Role: "system", Content: "include secret " + secret},
{Role: "system", Content: "include secret " + primarySecret},
{Role: "user", Content: "fix it"},
}
@@ -391,8 +393,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))

14
internal/framework/runner/runner.go
View File

@@ -527,7 +527,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
validatorSecrets(input.Config),
llm.ConfiguredSecrets(input.Config),
),
}
}
@@ -744,15 +744,3 @@ func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMeta
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}
func validatorSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effective := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
effective.APIKey,
cfg.ValidationLLM.APIKey,
}
}

33
internal/framework/runner/runner_test.go
View File

@@ -1307,12 +1307,13 @@ func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
}
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
secret := "super-secret-key"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
cfg.ValidationLLM.APIKey = secret
cfg.PrimaryLLM.APIKey = primarySecret
cfg.ValidationLLM.APIKey = validationSecret
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1321,7 +1322,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ValidationDiagnosticsDir: diagDir,
@@ -1332,12 +1333,26 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("expected diagnostic artifact path on decision")
}
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(matches) == 0 {
t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m"))
}
for _, path := range matches {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
}
}
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
if readErr != nil {
t.Fatalf("read diagnostic: %v", readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
t.Fatalf("read decision diagnostic: %v", readErr)
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in diagnostics")