From 3e8d19cccdb825171ea0aa6992175855080df14d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 11 May 2026 13:58:13 +0000 Subject: [PATCH] Add redacted run metadata artifacts --- internal/cli/run.go | 14 +++ internal/cli/run_test.go | 131 +++++++++++++++++++++++++++ internal/core/diagnostics/run_dir.go | 53 +++++++++++ 3 files changed, 198 insertions(+) diff --git a/internal/cli/run.go b/internal/cli/run.go index caff920..a8b549d 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -36,6 +36,20 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio return nil, nil, runDir, fmt.Errorf("%s: %w", phase, err) } + if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{ + Operation: "process", + TranscriptPath: inv.TranscriptPath, + GlossaryPath: inv.GlossaryPath, + OutputPath: inv.OutputPath, + ReportJSONPath: inv.ReportJSONPath, + Modules: append([]string(nil), inv.Config.Modules...), + }); err != nil { + _ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err)) + } + if err := runDir.WriteEffectiveConfig(inv.Config); err != nil { + _ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err)) + } + transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript") if err != nil { return fail("transcript_read", err) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 368c2f0..6d27613 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -515,6 +515,137 @@ func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) { } } +func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + workDir := t.TempDir() + outputPath := filepath.Join(t.TempDir(), "normalized.json") + reportPath := filepath.Join(t.TempDir(), "report.json") + primaryKey := "super-secret-primary-key" + validationKey := "super-secret-validation-key" + t.Setenv("AUDITA_LLM_API_KEY", primaryKey) + t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", validationKey) + + exitCode := Run([]string{ + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--output", + outputPath, + "--report-json", + reportPath, + "--work-dir", + workDir, + "--work-dir-retention", + "always", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout when --output is provided, got %q", stdout.String()) + } + + // Existing report-json behavior should still work. + report := readProcessReport(t, reportPath) + if report.Status != "success" { + t.Fatalf("expected success report status, got %q", report.Status) + } + + runPath := onlyRunDir(t, workDir) + invocationPath := filepath.Join(runPath, "invocation.json") + configPath := filepath.Join(runPath, "effective-config.json") + + if _, err := os.Stat(invocationPath); err != nil { + t.Fatalf("expected invocation metadata artifact: %v", err) + } + if _, err := os.Stat(configPath); err != nil { + t.Fatalf("expected effective config metadata artifact: %v", err) + } + + invocationBytes := readFile(t, invocationPath) + configBytes := readFile(t, configPath) + if strings.Contains(string(invocationBytes), primaryKey) || strings.Contains(string(invocationBytes), validationKey) { + t.Fatalf("invocation artifact leaked API key material") + } + if strings.Contains(string(configBytes), primaryKey) || strings.Contains(string(configBytes), validationKey) { + t.Fatalf("effective config artifact leaked API key material") + } + + var invocation struct { + Operation string `json:"operation"` + TranscriptPath string `json:"transcript_path"` + GlossaryPath string `json:"glossary_path"` + OutputPath string `json:"output_path"` + ReportJSONPath string `json:"report_json_path"` + Modules []string `json:"modules"` + RunID string `json:"run_id"` + StartedAt string `json:"started_at"` + } + if err := json.Unmarshal(invocationBytes, &invocation); err != nil { + t.Fatalf("failed to parse invocation metadata: %v", err) + } + if invocation.Operation != "process" { + t.Fatalf("expected operation=process, got %q", invocation.Operation) + } + if invocation.TranscriptPath != fixturePath("tiny_transcript.json") { + t.Fatalf("unexpected transcript_path: %q", invocation.TranscriptPath) + } + if invocation.GlossaryPath != fixturePath("tiny_glossary.yaml") { + t.Fatalf("unexpected glossary_path: %q", invocation.GlossaryPath) + } + if invocation.OutputPath != outputPath { + t.Fatalf("unexpected output_path: %q", invocation.OutputPath) + } + if invocation.ReportJSONPath != reportPath { + t.Fatalf("unexpected report_json_path: %q", invocation.ReportJSONPath) + } + if len(invocation.Modules) == 0 { + t.Fatalf("expected non-empty modules list in invocation metadata") + } + if invocation.RunID == "" { + t.Fatalf("expected non-empty run_id in invocation metadata") + } + if invocation.StartedAt == "" { + t.Fatalf("expected non-empty started_at in invocation metadata") + } +} + +func TestRunProcessStdoutTranscriptWhenNoOutputWithRunMetadataArtifacts(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + workDir := t.TempDir() + + exitCode := Run([]string{ + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--work-dir", + workDir, + "--work-dir-retention", + "always", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + + if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { + t.Fatalf("expected stdout to contain only transcript JSON, got parse error: %v", err) + } + + runPath := onlyRunDir(t, workDir) + if _, err := os.Stat(filepath.Join(runPath, "invocation.json")); err != nil { + t.Fatalf("expected invocation metadata artifact: %v", err) + } + if _, err := os.Stat(filepath.Join(runPath, "effective-config.json")); err != nil { + t.Fatalf("expected effective config metadata artifact: %v", err) + } +} + func TestRunProcessTargetSectionsThroughCLI(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/internal/core/diagnostics/run_dir.go b/internal/core/diagnostics/run_dir.go index ae0fc7d..4bead05 100644 --- a/internal/core/diagnostics/run_dir.go +++ b/internal/core/diagnostics/run_dir.go @@ -8,6 +8,7 @@ import ( "time" "gitea.maximumdirect.net/eric/audita/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/normalization" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/schema" @@ -20,6 +21,18 @@ type RunDirectory struct { createdAt time.Time } +// InvocationMetadata captures non-secret invocation details for diagnostics. +type InvocationMetadata struct { + Operation string `json:"operation"` + TranscriptPath string `json:"transcript_path"` + GlossaryPath string `json:"glossary_path"` + OutputPath string `json:"output_path,omitempty"` + ReportJSONPath string `json:"report_json_path,omitempty"` + Modules []string `json:"modules"` + RunID string `json:"run_id"` + StartedAt time.Time `json:"started_at"` +} + // NewRunDirectory creates a new run directory under the configured work dir func NewRunDirectory(workDir, retention string) (*RunDirectory, error) { if workDir == "" { @@ -51,6 +64,46 @@ func (r *RunDirectory) Path() string { return r.path } +func (r *RunDirectory) runID() string { + return filepath.Base(r.path) +} + +// WriteInvocationMetadata writes invocation metadata for this run. +func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error { + if metadata.RunID == "" { + metadata.RunID = r.runID() + } + if metadata.StartedAt.IsZero() { + metadata.StartedAt = r.createdAt + } + + path := filepath.Join(r.path, "invocation.json") + bytes, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal invocation metadata: %w", err) + } + bytes = append(bytes, '\n') + if err := os.WriteFile(path, bytes, 0o644); err != nil { + return fmt.Errorf("failed to write invocation metadata: %w", err) + } + return nil +} + +// WriteEffectiveConfig writes redacted effective config metadata for this run. +func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error { + path := filepath.Join(r.path, "effective-config.json") + redacted := cfg.Redacted() + bytes, err := json.MarshalIndent(redacted, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal effective config: %w", err) + } + bytes = append(bytes, '\n') + if err := os.WriteFile(path, bytes, 0o644); err != nil { + return fmt.Errorf("failed to write effective config: %w", err) + } + return nil +} + // WriteSourceTranscript writes the source transcript artifact func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error { // Write raw source for reference