Moved report/ledger assembly to a new processreport module
This commit is contained in:
@@ -1,46 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
)
|
||||
|
||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{
|
||||
ProposalIndex: 3,
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||
}
|
||||
entry := ledger[0]
|
||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||
}
|
||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||
}
|
||||
}
|
||||
@@ -23,10 +23,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
@@ -531,10 +531,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
||||
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
@@ -560,10 +563,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
|
||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
||||
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
@@ -578,21 +584,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
}
|
||||
|
||||
hasSkippedCorrections := false
|
||||
if runOutput != nil {
|
||||
for _, mr := range runOutput.ModuleResults {
|
||||
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
||||
hasSkippedCorrections = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: true,
|
||||
HasSkippedCorrections: hasSkippedCorrections,
|
||||
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||
}); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||
return 1
|
||||
@@ -710,130 +706,28 @@ func extractErrorPhase(err error) (phase string, message string) {
|
||||
return "", msg
|
||||
}
|
||||
|
||||
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
|
||||
report := reporting.ProcessReport{
|
||||
ReportMetadata: reporting.ReportMetadata{
|
||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: inv.Config.OutputSchema,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: status,
|
||||
Operation: "process",
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
Modules: append([]string(nil), inv.Config.Modules...),
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
ErrorPhase: errorPhase,
|
||||
}
|
||||
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||
runDirectoryPath := ""
|
||||
if runDir != nil {
|
||||
runSucceeded := status == "success"
|
||||
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
||||
report.Diagnostics = &metadata
|
||||
runDirectoryPath = runDir.Path()
|
||||
}
|
||||
if errorMessage != "" {
|
||||
report.ErrorMessage = errorMessage
|
||||
return processreport.BuildInput{
|
||||
Status: status,
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
Modules: inv.Config.Modules,
|
||||
OutputSchema: inv.Config.OutputSchema,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorMessage: errorMessage,
|
||||
ErrorPhase: errorPhase,
|
||||
RunDirectoryPath: runDirectoryPath,
|
||||
NormalizationSummary: normalizationSummary,
|
||||
ChunkingSummary: chunkingSummary,
|
||||
RunOutput: runOutput,
|
||||
}
|
||||
if normalizationSummary != nil {
|
||||
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
|
||||
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
|
||||
report.NormalizationMerges = &normalizationSummary.MergesPerformed
|
||||
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
|
||||
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
|
||||
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
|
||||
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
||||
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||
}
|
||||
if chunkingSummary != nil {
|
||||
report.Chunking = &reporting.ChunkingSummary{
|
||||
ChunkCount: chunkingSummary.ChunkCount,
|
||||
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
||||
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
||||
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
||||
TargetSections: chunkingSummary.TargetSections,
|
||||
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
||||
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
|
||||
return report
|
||||
}
|
||||
|
||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||
for _, r := range runOutput.ModuleResults {
|
||||
startedAt := r.StartedAt
|
||||
completedAt := r.CompletedAt
|
||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||
ModuleKey: r.ModuleKey,
|
||||
ModuleInstance: r.ModuleInstance,
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
SkippedChanges: r.SkippedChanges,
|
||||
ErrorMessage: r.ErrorMessage,
|
||||
StartedAt: &startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
})
|
||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||
summary.FailedModuleInstance = r.ModuleInstance
|
||||
}
|
||||
}
|
||||
|
||||
return summary, moduleReports
|
||||
}
|
||||
|
||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorDecisionReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorRejectedReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
ModuleKey: d.ModuleKey,
|
||||
ModuleInstance: d.ModuleInstance,
|
||||
TargetSegmentID: d.TargetSegmentID,
|
||||
OriginalText: d.OriginalText,
|
||||
CorrectedText: d.CorrectedText,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type processFlags struct {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package cli
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
@@ -15,7 +15,12 @@ const (
|
||||
correctionDispositionFailed = "failed"
|
||||
)
|
||||
|
||||
type correctionLedgerEntry struct {
|
||||
type CorrectionLedgerInput struct {
|
||||
RunDirectoryPath string
|
||||
RunOutput *runner.RunOutput
|
||||
}
|
||||
|
||||
type CorrectionLedgerEntry struct {
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
@@ -28,27 +33,28 @@ type correctionLedgerEntry struct {
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
||||
DispositionMessage string `json:"disposition_message,omitempty"`
|
||||
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||
}
|
||||
|
||||
type ledgerValidatorDecisionRecord struct {
|
||||
type LedgerValidatorDecisionRecord struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
|
||||
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
|
||||
runOutput := input.RunOutput
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil
|
||||
}
|
||||
runID := ""
|
||||
if runDirPath != "" {
|
||||
runID = filepath.Base(runDirPath)
|
||||
if input.RunDirectoryPath != "" {
|
||||
runID = filepath.Base(input.RunDirectoryPath)
|
||||
}
|
||||
|
||||
entries := make([]correctionLedgerEntry, 0)
|
||||
entries := make([]CorrectionLedgerEntry, 0)
|
||||
for _, module := range runOutput.ModuleResults {
|
||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
@@ -56,7 +62,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
}
|
||||
|
||||
for _, change := range module.AppliedChanges {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
@@ -72,7 +78,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
})
|
||||
}
|
||||
for _, change := range module.SkippedChanges {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
@@ -89,7 +95,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
})
|
||||
}
|
||||
for _, rejection := range module.ValidatorRejected {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
@@ -106,7 +112,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
})
|
||||
}
|
||||
if module.Status == runner.ModuleStatusFailed {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
@@ -130,17 +136,29 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
return entries
|
||||
}
|
||||
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
||||
func HasSkippedCorrections(runOutput *runner.RunOutput) bool {
|
||||
if runOutput == nil {
|
||||
return false
|
||||
}
|
||||
for _, mr := range runOutput.ModuleResults {
|
||||
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
||||
out := make([]LedgerValidatorDecisionRecord, 0, len(in))
|
||||
for _, decision := range in {
|
||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||
if isLLMBacked != wantLLM {
|
||||
continue
|
||||
}
|
||||
out = append(out, ledgerValidatorDecisionRecord{
|
||||
out = append(out, LedgerValidatorDecisionRecord{
|
||||
ValidatorKey: decision.ValidatorName,
|
||||
Approved: decision.Approved,
|
||||
ReasonCode: decision.ReasonCode,
|
||||
128
internal/framework/processreport/correction_ledger_test.go
Normal file
128
internal/framework/processreport/correction_ledger_test.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
)
|
||||
|
||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{
|
||||
ProposalIndex: 3,
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{
|
||||
RunDirectoryPath: "/tmp/audita-run-id",
|
||||
RunOutput: output,
|
||||
})
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||
}
|
||||
entry := ledger[0]
|
||||
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected applied ledger entry: %+v", entry)
|
||||
}
|
||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||
}
|
||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Status: runner.ModuleStatusSuccess,
|
||||
SkippedChanges: []proposals.SkippedChange{
|
||||
{
|
||||
ProposalIndex: 2,
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "old",
|
||||
CorrectedText: "new",
|
||||
SkipReason: proposals.SkipReasonAmbiguousOriginal,
|
||||
Message: "ambiguous",
|
||||
},
|
||||
},
|
||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||
{
|
||||
ProposalIndex: 3,
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "before",
|
||||
CorrectedText: "after",
|
||||
ReasonCode: "protected_term",
|
||||
Message: "blocked",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ModuleKey: "capitalization",
|
||||
ModuleInstance: "capitalization",
|
||||
Status: runner.ModuleStatusFailed,
|
||||
ErrorMessage: "failed",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
if len(ledger) != 3 {
|
||||
t.Fatalf("expected skipped, rejected, and failed entries, got %+v", ledger)
|
||||
}
|
||||
|
||||
byDisposition := make(map[string]CorrectionLedgerEntry)
|
||||
for _, entry := range ledger {
|
||||
byDisposition[entry.Disposition] = entry
|
||||
}
|
||||
if byDisposition["skipped"].DispositionReasonCode != string(proposals.SkipReasonAmbiguousOriginal) ||
|
||||
byDisposition["skipped"].DispositionMessage != "ambiguous" {
|
||||
t.Fatalf("unexpected skipped ledger entry: %+v", byDisposition["skipped"])
|
||||
}
|
||||
if byDisposition["rejected"].DispositionReasonCode != "protected_term" ||
|
||||
byDisposition["rejected"].ProposedCorrectedText != "after" {
|
||||
t.Fatalf("unexpected rejected ledger entry: %+v", byDisposition["rejected"])
|
||||
}
|
||||
if byDisposition["failed"].DispositionReasonCode != "module_failed" ||
|
||||
byDisposition["failed"].DispositionMessage != "failed" {
|
||||
t.Fatalf("unexpected failed ledger entry: %+v", byDisposition["failed"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) {
|
||||
if HasSkippedCorrections(nil) {
|
||||
t.Fatal("nil output should not have skipped corrections")
|
||||
}
|
||||
if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) {
|
||||
t.Fatal("applied-only output should not have skipped corrections")
|
||||
}
|
||||
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) {
|
||||
t.Fatal("application skips should count as skipped corrections")
|
||||
}
|
||||
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) {
|
||||
t.Fatal("validator rejections should count as skipped corrections")
|
||||
}
|
||||
}
|
||||
158
internal/framework/processreport/report.go
Normal file
158
internal/framework/processreport/report.go
Normal file
@@ -0,0 +1,158 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// BuildInput contains already-computed process execution facts for report assembly.
|
||||
type BuildInput struct {
|
||||
Status string
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
ConfigVersion *int
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
ErrorMessage string
|
||||
ErrorPhase string
|
||||
RunDirectoryPath string
|
||||
NormalizationSummary *normalization.NormalizationSummary
|
||||
ChunkingSummary *chunking.Summary
|
||||
RunOutput *runner.RunOutput
|
||||
}
|
||||
|
||||
// Build creates the public process report without owning command parsing or config loading.
|
||||
func Build(input BuildInput) reporting.ProcessReport {
|
||||
report := reporting.ProcessReport{
|
||||
ReportMetadata: reporting.ReportMetadata{
|
||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: input.OutputSchema,
|
||||
ConfigVersion: input.ConfigVersion,
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: input.Status,
|
||||
Operation: "process",
|
||||
TranscriptPath: input.TranscriptPath,
|
||||
GlossaryPath: input.GlossaryPath,
|
||||
OutputPath: input.OutputPath,
|
||||
Modules: append([]string(nil), input.Modules...),
|
||||
StartedAt: input.StartedAt,
|
||||
CompletedAt: &input.CompletedAt,
|
||||
ErrorPhase: input.ErrorPhase,
|
||||
}
|
||||
if input.RunDirectoryPath != "" {
|
||||
runSucceeded := input.Status == "success"
|
||||
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||
report.Diagnostics = &metadata
|
||||
}
|
||||
if input.ErrorMessage != "" {
|
||||
report.ErrorMessage = input.ErrorMessage
|
||||
}
|
||||
if input.NormalizationSummary != nil {
|
||||
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||
}
|
||||
if input.ChunkingSummary != nil {
|
||||
report.Chunking = &reporting.ChunkingSummary{
|
||||
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||
TargetSections: input.ChunkingSummary.TargetSections,
|
||||
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||
return report
|
||||
}
|
||||
|
||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||
for _, r := range runOutput.ModuleResults {
|
||||
startedAt := r.StartedAt
|
||||
completedAt := r.CompletedAt
|
||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||
ModuleKey: r.ModuleKey,
|
||||
ModuleInstance: r.ModuleInstance,
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
SkippedChanges: r.SkippedChanges,
|
||||
ErrorMessage: r.ErrorMessage,
|
||||
StartedAt: &startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
})
|
||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||
summary.FailedModuleInstance = r.ModuleInstance
|
||||
}
|
||||
}
|
||||
|
||||
return summary, moduleReports
|
||||
}
|
||||
|
||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorDecisionReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorRejectedReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
ModuleKey: d.ModuleKey,
|
||||
ModuleInstance: d.ModuleInstance,
|
||||
TargetSegmentID: d.TargetSegmentID,
|
||||
OriginalText: d.OriginalText,
|
||||
CorrectedText: d.CorrectedText,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
180
internal/framework/processreport/report_test.go
Normal file
180
internal/framework/processreport/report_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
)
|
||||
|
||||
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
configVersion := 4
|
||||
targetSections := 2
|
||||
inputSegments := 5
|
||||
outputSegments := 4
|
||||
merges := 1
|
||||
reassigned := 2
|
||||
differentSpeakers := 3
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "success",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
OutputPath: "out.json",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
ConfigVersion: &configVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
RunDirectoryPath: filepath.Join(
|
||||
"tmp",
|
||||
"audita-run",
|
||||
),
|
||||
NormalizationSummary: &normalization.NormalizationSummary{
|
||||
InputSegmentCount: inputSegments,
|
||||
OutputSegmentCount: outputSegments,
|
||||
MergesPerformed: merges,
|
||||
IDsReassigned: reassigned,
|
||||
SkippedMerges: struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
}{
|
||||
DifferentSpeakers: differentSpeakers,
|
||||
},
|
||||
},
|
||||
ChunkingSummary: &chunking.Summary{
|
||||
ChunkCount: 3,
|
||||
MinEstimatedTokens: 10,
|
||||
MaxEstimatedTokens: 20,
|
||||
TotalEstimatedTokens: 45,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: 200,
|
||||
MinSectionTokens: 50,
|
||||
},
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Status: runner.ModuleStatusSuccess,
|
||||
ProposalCount: 2,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{
|
||||
ValidatorName: "proposal_shape",
|
||||
ProposalIndex: 1,
|
||||
Approved: true,
|
||||
ReasonCode: "approved",
|
||||
Message: "ok",
|
||||
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||
},
|
||||
},
|
||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||
{
|
||||
ValidatorName: "protected_term",
|
||||
ProposalIndex: 2,
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "old",
|
||||
CorrectedText: "new",
|
||||
ReasonCode: "protected_term",
|
||||
Message: "blocked",
|
||||
},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||
},
|
||||
SkippedChanges: []proposals.SkippedChange{
|
||||
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||
},
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||
report.ReportMetadata.OutputSchema != "default" ||
|
||||
report.ReportMetadata.ConfigVersion == nil ||
|
||||
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||
}
|
||||
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||
}
|
||||
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||
}
|
||||
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||
}
|
||||
if report.ModulesSummary == nil ||
|
||||
report.ModulesSummary.ModuleCount != 1 ||
|
||||
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "failed",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorPhase: "module",
|
||||
ErrorMessage: "module failed",
|
||||
RunDirectoryPath: "run-dir",
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
Status: runner.ModuleStatusFailed,
|
||||
ErrorMessage: "module failed",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||
t.Fatalf("unexpected failure fields: %+v", report)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||
}
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user