eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add utilization diagnostics and correction ledger

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 19:03:37 +00:00
parent 037121e9ce
commit ff4ed82239
15 changed files with 969 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
internal/cli/run_test.go
View File

@@ -1239,6 +1239,9 @@ func TestRunProcessReportJSONBestEffortOnFailure(t *testing.T) {
if report.Diagnostics.DirectoryPath == "" || report.Diagnostics.ErrorLogPath == "" {
t.Fatalf("expected diagnostics directory and error log references in failed report: %+v", report.Diagnostics)
}
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
t.Fatalf("expected review artifact paths in failed report diagnostics: %+v", report.Diagnostics)
}
if _, err := os.Stat(report.Diagnostics.ErrorLogPath); err != nil {
t.Fatalf("expected error log file at reported path: %v", err)
}
@@ -1312,6 +1315,153 @@ func TestRunProcessReportJSONAndRunDirReportShareDiagnosticsMetadata(t *testing.
}
}
func TestRunProcessWritesUtilizationAndCorrectionLedgerArtifacts(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.Diagnostics == nil {
t.Fatalf("expected diagnostics metadata")
}
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
t.Fatalf("expected utilization and correction ledger paths, got %+v", report.Diagnostics)
}
if _, err := os.Stat(report.Diagnostics.UtilizationSummaryPath); err != nil {
t.Fatalf("expected utilization diagnostics artifact: %v", err)
}
if _, err := os.Stat(report.Diagnostics.CorrectionLedgerPath); err != nil {
t.Fatalf("expected correction ledger artifact: %v", err)
}
var utilization struct {
EffectiveConcurrency struct {
TotalLLM int `json:"total_llm"`
ProposalLLM int `json:"proposal_llm"`
ValidationLLM int `json:"validation_llm"`
} `json:"effective_concurrency"`
Modules []struct {
ModuleInstance string `json:"module_instance"`
} `json:"modules"`
LLMCalls struct {
TotalProposalCalls int `json:"total_proposal_calls"`
TotalValidationCalls int `json:"total_validation_calls"`
} `json:"llm_calls"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &utilization); err != nil {
t.Fatalf("unmarshal utilization artifact: %v", err)
}
if utilization.EffectiveConcurrency.TotalLLM <= 0 || utilization.EffectiveConcurrency.ProposalLLM <= 0 || utilization.EffectiveConcurrency.ValidationLLM <= 0 {
t.Fatalf("expected positive effective concurrency limits, got %+v", utilization.EffectiveConcurrency)
}
if len(utilization.Modules) == 0 || utilization.Modules[0].ModuleInstance == "" {
t.Fatalf("expected module-level timing summaries, got %+v", utilization.Modules)
}
if utilization.LLMCalls.TotalProposalCalls == 0 {
t.Fatalf("expected proposal LLM calls to be recorded")
}
var ledger []map[string]any
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &ledger); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
}
func TestRunProcessCorrectionLedgerKeepsValidatorRejectionDistinctFromSkip(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{
TargetSegmentID: 1,
OriginalText: "hello",
CorrectedText: "hi",
Confidence: 0.99,
},
},
},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: false, Confidence: 0.1, Reason: "reject"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
exitCode := Run([]string{
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
var ledger []struct {
Disposition string `json:"disposition"`
DispositionReasonCode string `json:"disposition_reason_code"`
ModuleInstance string `json:"module_instance"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &ledger); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
foundRejected := false
for _, entry := range ledger {
if entry.ModuleInstance == "grammar" && entry.Disposition == "rejected" {
foundRejected = true
if entry.DispositionReasonCode == "" {
t.Fatalf("expected rejection reason code in ledger entry")
}
}
}
if !foundRejected {
t.Fatalf("expected rejected ledger entry, got %+v", ledger)
}
}
func TestRunProcessReportJSONIncludesConfigVersionWhenConfigFileUsed(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -2547,6 +2697,49 @@ func TestRunProcessDefaultFullPipelineAggregatesSkipsAndAutoRetentionKeepsRunDir
if runDirReport.ModulesSummary == nil || runDirReport.ModulesSummary.TotalSkippedChanges != 2 {
t.Fatalf("expected skipped totals in retained run-dir report, got %+v", runDirReport.ModulesSummary)
}
if runDirReport.Diagnostics == nil {
t.Fatalf("expected diagnostics metadata in run-dir report")
}
var utilization struct {
RunTiming struct {
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
} `json:"run_timing"`
Validators []struct {
ValidatorKey string `json:"validator_key"`
} `json:"validators"`
}
if err := json.Unmarshal(readFile(t, runDirReport.Diagnostics.UtilizationSummaryPath), &utilization); err != nil {
t.Fatalf("unmarshal utilization diagnostics: %v", err)
}
if utilization.RunTiming.LLMExecutionTimeMS < 0 || utilization.RunTiming.SchedulerQueueWaitMS < 0 || utilization.RunTiming.DeterministicValidationMS < 0 {
t.Fatalf("expected non-negative run timing fields, got %+v", utilization.RunTiming)
}
if len(utilization.Validators) == 0 {
t.Fatalf("expected per-validator timing summaries")
}
var ledger []struct {
Disposition string `json:"disposition"`
}
if err := json.Unmarshal(readFile(t, runDirReport.Diagnostics.CorrectionLedgerPath), &ledger); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
foundSkipped := false
foundRejected := false
for _, entry := range ledger {
if entry.Disposition == "skipped" {
foundSkipped = true
}
if entry.Disposition == "rejected" {
foundRejected = true
}
}
if !foundSkipped || !foundRejected {
t.Fatalf("expected skipped and rejected ledger dispositions, got %+v", ledger)
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics payloads: %v", globErr)