eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Add version two LLM reports and safe execution diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:31:37 +00:00
parent 4bfc5f3456
commit 9561e80efb
16 changed files with 428 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/cli/run.go
View File

@@ -3,6 +3,7 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"strings"
@@ -34,6 +35,7 @@ type processInvocation struct {
LLMRuntime *llm.Runtime
ConfigPath string
ConfigSource string
Overlay string
ConfigVersion *int
ExplicitModules bool
}
@@ -52,7 +54,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
}
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %s", phase, llm.RedactError(err, inv.Config)))
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
}
@@ -64,6 +66,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
ReportJSONPath: inv.ReportJSONPath,
ConfigPath: inv.ConfigPath,
ConfigSource: inv.ConfigSource,
Overlay: inv.Overlay,
ConfigVersion: inv.ConfigVersion,
TranscriptDescription: inv.Config.Context.Description,
Modules: append([]string(nil), inv.Config.Pipeline.Modules...),
@@ -78,6 +81,14 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
}
if origins, err := llm.ExplanationYAML(inv.ConfigResult); err == nil {
_ = runDir.WriteArtifact(diagnostics.ArtifactConfigOrigins, origins)
}
}
if inv.LLMRuntime != nil {
if profiles, err := inv.LLMRuntime.SelectedProfilesYAML(); err == nil {
_ = runDir.WriteArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
}
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
@@ -194,6 +205,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
ValidationDiagnosticsDir: runDir.Path(),
})
runnerResult.Observations = observations.Snapshot()
runnerResult.ObservationsAvailable = true
runnerResult.LLMSummary = llm.Summarize(runnerResult.Observations)
runOutput = &runnerResult
if runErr != nil {
@@ -320,6 +332,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
LLMRuntime: llmRuntime,
ConfigPath: selected.Path,
ConfigSource: selected.Source,
Overlay: input.Overlay,
ConfigVersion: &version,
ExplicitModules: explicitModules,
}
@@ -329,6 +342,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
if runErr != nil {
if runDir != nil && runOutput != nil {
writeExecutionArtifacts(runDir, runOutput, inv.Config)
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
@@ -336,7 +350,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}))
}
errorPhase, errorMessage := extractErrorPhase(runErr)
errorMessage = llm.RedactError(fmt.Errorf("%s", errorMessage), inv.Config)
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
report = redactProcessReport(report, inv.Config)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -351,7 +367,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
})
}
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(runErr, inv.Config))
if runDir != nil {
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
}
@@ -359,6 +375,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
if runDir != nil && runOutput != nil {
writeExecutionArtifacts(runDir, runOutput, inv.Config)
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
@@ -367,6 +384,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
report = redactProcessReport(report, inv.Config)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -395,6 +413,32 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 0
}
func writeExecutionArtifacts(runDir *diagnostics.RunDirectory, output *runner.RunOutput, cfg config.Config) {
if !output.ObservationsAvailable {
return
}
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations))
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, cfg.Backends))
}
func redactProcessReport(report reporting.ProcessReport, cfg config.Config) reporting.ProcessReport {
raw, err := json.Marshal(report)
if err != nil {
return report
}
content := string(raw)
for _, secret := range llm.ConfiguredSecrets(&cfg) {
content = strings.ReplaceAll(content, secret, "[REDACTED]")
encoded, _ := json.Marshal(secret)
content = strings.ReplaceAll(content, string(encoded[1:len(encoded)-1]), "[REDACTED]")
}
var safe reporting.ProcessReport
if json.Unmarshal([]byte(content), &safe) != nil {
return report
}
return safe
}
func runConfig(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || isHelpCommand(args) || args[0] == "--help" || args[0] == "-h" {
writeConfigUsage(stdout)
@@ -514,6 +558,7 @@ func processReportInput(status string, inv processInvocation, runDir *diagnostic
NormalizationSummary: normalizationSummary,
ChunkingSummary: chunkingSummary,
RunOutput: runOutput,
Backends: inv.Config.Backends,
}
}