Moved report/ledger assembly to a new processreport module
This commit is contained in:
@@ -1,46 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
|
||||||
output := &runner.RunOutput{
|
|
||||||
ModuleResults: []runner.ModuleResult{
|
|
||||||
{
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
|
||||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
|
||||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
},
|
|
||||||
AppliedChanges: []proposals.AppliedChange{
|
|
||||||
{
|
|
||||||
ProposalIndex: 3,
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
TargetSegmentID: 1,
|
|
||||||
OriginalText: "gestures",
|
|
||||||
CorrectedText: "Jesters",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
|
||||||
if len(ledger) != 1 {
|
|
||||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
|
||||||
}
|
|
||||||
entry := ledger[0]
|
|
||||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
|
||||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
|
||||||
}
|
|
||||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
|
||||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -23,10 +23,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type noOpStructuredLLMClient struct{}
|
type noOpStructuredLLMClient struct{}
|
||||||
@@ -531,10 +531,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -560,10 +563,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -578,21 +584,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hasSkippedCorrections := false
|
|
||||||
if runOutput != nil {
|
|
||||||
for _, mr := range runOutput.ModuleResults {
|
|
||||||
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
|
||||||
hasSkippedCorrections = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
_ = runDir.WriteReport(report)
|
_ = runDir.WriteReport(report)
|
||||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||||
RunSucceeded: true,
|
RunSucceeded: true,
|
||||||
HasSkippedCorrections: hasSkippedCorrections,
|
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
@@ -710,130 +706,28 @@ func extractErrorPhase(err error) (phase string, message string) {
|
|||||||
return "", msg
|
return "", msg
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
|
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||||
report := reporting.ProcessReport{
|
runDirectoryPath := ""
|
||||||
ReportMetadata: reporting.ReportMetadata{
|
|
||||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
||||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
||||||
OutputSchema: inv.Config.OutputSchema,
|
|
||||||
ConfigVersion: inv.ConfigVersion,
|
|
||||||
},
|
|
||||||
Phase: "default_pipeline",
|
|
||||||
Status: status,
|
|
||||||
Operation: "process",
|
|
||||||
TranscriptPath: inv.TranscriptPath,
|
|
||||||
GlossaryPath: inv.GlossaryPath,
|
|
||||||
OutputPath: inv.OutputPath,
|
|
||||||
Modules: append([]string(nil), inv.Config.Modules...),
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
ErrorPhase: errorPhase,
|
|
||||||
}
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
runSucceeded := status == "success"
|
runDirectoryPath = runDir.Path()
|
||||||
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
|
||||||
report.Diagnostics = &metadata
|
|
||||||
}
|
}
|
||||||
if errorMessage != "" {
|
return processreport.BuildInput{
|
||||||
report.ErrorMessage = errorMessage
|
Status: status,
|
||||||
|
TranscriptPath: inv.TranscriptPath,
|
||||||
|
GlossaryPath: inv.GlossaryPath,
|
||||||
|
OutputPath: inv.OutputPath,
|
||||||
|
Modules: inv.Config.Modules,
|
||||||
|
OutputSchema: inv.Config.OutputSchema,
|
||||||
|
ConfigVersion: inv.ConfigVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorMessage: errorMessage,
|
||||||
|
ErrorPhase: errorPhase,
|
||||||
|
RunDirectoryPath: runDirectoryPath,
|
||||||
|
NormalizationSummary: normalizationSummary,
|
||||||
|
ChunkingSummary: chunkingSummary,
|
||||||
|
RunOutput: runOutput,
|
||||||
}
|
}
|
||||||
if normalizationSummary != nil {
|
|
||||||
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
|
|
||||||
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
|
|
||||||
report.NormalizationMerges = &normalizationSummary.MergesPerformed
|
|
||||||
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
|
|
||||||
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
|
|
||||||
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
|
|
||||||
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
|
||||||
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
|
||||||
}
|
|
||||||
if chunkingSummary != nil {
|
|
||||||
report.Chunking = &reporting.ChunkingSummary{
|
|
||||||
ChunkCount: chunkingSummary.ChunkCount,
|
|
||||||
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
|
||||||
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
|
||||||
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
|
||||||
TargetSections: chunkingSummary.TargetSections,
|
|
||||||
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
|
||||||
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
|
||||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
|
||||||
for _, r := range runOutput.ModuleResults {
|
|
||||||
startedAt := r.StartedAt
|
|
||||||
completedAt := r.CompletedAt
|
|
||||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
|
||||||
ModuleKey: r.ModuleKey,
|
|
||||||
ModuleInstance: r.ModuleInstance,
|
|
||||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
|
||||||
Status: r.Status,
|
|
||||||
ProposalCount: r.ProposalCount,
|
|
||||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
|
||||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
|
||||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
|
||||||
AppliedChanges: r.AppliedChanges,
|
|
||||||
SkippedChanges: r.SkippedChanges,
|
|
||||||
ErrorMessage: r.ErrorMessage,
|
|
||||||
StartedAt: &startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
})
|
|
||||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
|
||||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
|
||||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
|
||||||
summary.FailedModuleInstance = r.ModuleInstance
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary, moduleReports
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorDecisionReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
Approved: d.Approved,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorRejectedReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
ModuleKey: d.ModuleKey,
|
|
||||||
ModuleInstance: d.ModuleInstance,
|
|
||||||
TargetSegmentID: d.TargetSegmentID,
|
|
||||||
OriginalText: d.OriginalText,
|
|
||||||
CorrectedText: d.CorrectedText,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type processFlags struct {
|
type processFlags struct {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package cli
|
package processreport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -15,7 +15,12 @@ const (
|
|||||||
correctionDispositionFailed = "failed"
|
correctionDispositionFailed = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
type correctionLedgerEntry struct {
|
type CorrectionLedgerInput struct {
|
||||||
|
RunDirectoryPath string
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
type CorrectionLedgerEntry struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
@@ -28,27 +33,28 @@ type correctionLedgerEntry struct {
|
|||||||
Disposition string `json:"disposition"`
|
Disposition string `json:"disposition"`
|
||||||
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
||||||
DispositionMessage string `json:"disposition_message,omitempty"`
|
DispositionMessage string `json:"disposition_message,omitempty"`
|
||||||
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||||
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ledgerValidatorDecisionRecord struct {
|
type LedgerValidatorDecisionRecord struct {
|
||||||
ValidatorKey string `json:"validator_key"`
|
ValidatorKey string `json:"validator_key"`
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
|
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
|
||||||
|
runOutput := input.RunOutput
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
runID := ""
|
runID := ""
|
||||||
if runDirPath != "" {
|
if input.RunDirectoryPath != "" {
|
||||||
runID = filepath.Base(runDirPath)
|
runID = filepath.Base(input.RunDirectoryPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
entries := make([]correctionLedgerEntry, 0)
|
entries := make([]CorrectionLedgerEntry, 0)
|
||||||
for _, module := range runOutput.ModuleResults {
|
for _, module := range runOutput.ModuleResults {
|
||||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||||
for _, decision := range module.ValidatorDecisions {
|
for _, decision := range module.ValidatorDecisions {
|
||||||
@@ -56,7 +62,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, change := range module.AppliedChanges {
|
for _, change := range module.AppliedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -72,7 +78,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, change := range module.SkippedChanges {
|
for _, change := range module.SkippedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -89,7 +95,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, rejection := range module.ValidatorRejected {
|
for _, rejection := range module.ValidatorRejected {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -106,7 +112,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if module.Status == runner.ModuleStatusFailed {
|
if module.Status == runner.ModuleStatusFailed {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -130,17 +136,29 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
func HasSkippedCorrections(runOutput *runner.RunOutput) bool {
|
||||||
|
if runOutput == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, mr := range runOutput.ModuleResults {
|
||||||
|
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
|
||||||
if len(in) == 0 {
|
if len(in) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
out := make([]LedgerValidatorDecisionRecord, 0, len(in))
|
||||||
for _, decision := range in {
|
for _, decision := range in {
|
||||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||||
if isLLMBacked != wantLLM {
|
if isLLMBacked != wantLLM {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ledgerValidatorDecisionRecord{
|
out = append(out, LedgerValidatorDecisionRecord{
|
||||||
ValidatorKey: decision.ValidatorName,
|
ValidatorKey: decision.ValidatorName,
|
||||||
Approved: decision.Approved,
|
Approved: decision.Approved,
|
||||||
ReasonCode: decision.ReasonCode,
|
ReasonCode: decision.ReasonCode,
|
||||||
128
internal/framework/processreport/correction_ledger_test.go
Normal file
128
internal/framework/processreport/correction_ledger_test.go
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
TargetSegmentID: 1,
|
||||||
|
OriginalText: "gestures",
|
||||||
|
CorrectedText: "Jesters",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: "/tmp/audita-run-id",
|
||||||
|
RunOutput: output,
|
||||||
|
})
|
||||||
|
if len(ledger) != 1 {
|
||||||
|
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||||
|
}
|
||||||
|
entry := ledger[0]
|
||||||
|
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText != "Jesters" {
|
||||||
|
t.Fatalf("unexpected applied ledger entry: %+v", entry)
|
||||||
|
}
|
||||||
|
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||||
|
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||||
|
}
|
||||||
|
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||||
|
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 2,
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
SkipReason: proposals.SkipReasonAmbiguousOriginal,
|
||||||
|
Message: "ambiguous",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
TargetSegmentID: 8,
|
||||||
|
OriginalText: "before",
|
||||||
|
CorrectedText: "after",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ModuleKey: "capitalization",
|
||||||
|
ModuleInstance: "capitalization",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "failed",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("expected skipped, rejected, and failed entries, got %+v", ledger)
|
||||||
|
}
|
||||||
|
|
||||||
|
byDisposition := make(map[string]CorrectionLedgerEntry)
|
||||||
|
for _, entry := range ledger {
|
||||||
|
byDisposition[entry.Disposition] = entry
|
||||||
|
}
|
||||||
|
if byDisposition["skipped"].DispositionReasonCode != string(proposals.SkipReasonAmbiguousOriginal) ||
|
||||||
|
byDisposition["skipped"].DispositionMessage != "ambiguous" {
|
||||||
|
t.Fatalf("unexpected skipped ledger entry: %+v", byDisposition["skipped"])
|
||||||
|
}
|
||||||
|
if byDisposition["rejected"].DispositionReasonCode != "protected_term" ||
|
||||||
|
byDisposition["rejected"].ProposedCorrectedText != "after" {
|
||||||
|
t.Fatalf("unexpected rejected ledger entry: %+v", byDisposition["rejected"])
|
||||||
|
}
|
||||||
|
if byDisposition["failed"].DispositionReasonCode != "module_failed" ||
|
||||||
|
byDisposition["failed"].DispositionMessage != "failed" {
|
||||||
|
t.Fatalf("unexpected failed ledger entry: %+v", byDisposition["failed"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) {
|
||||||
|
if HasSkippedCorrections(nil) {
|
||||||
|
t.Fatal("nil output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("applied-only output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("application skips should count as skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("validator rejections should count as skipped corrections")
|
||||||
|
}
|
||||||
|
}
|
||||||
158
internal/framework/processreport/report.go
Normal file
158
internal/framework/processreport/report.go
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BuildInput contains already-computed process execution facts for report assembly.
|
||||||
|
type BuildInput struct {
|
||||||
|
Status string
|
||||||
|
TranscriptPath string
|
||||||
|
GlossaryPath string
|
||||||
|
OutputPath string
|
||||||
|
Modules []string
|
||||||
|
OutputSchema string
|
||||||
|
ConfigVersion *int
|
||||||
|
StartedAt time.Time
|
||||||
|
CompletedAt time.Time
|
||||||
|
ErrorMessage string
|
||||||
|
ErrorPhase string
|
||||||
|
RunDirectoryPath string
|
||||||
|
NormalizationSummary *normalization.NormalizationSummary
|
||||||
|
ChunkingSummary *chunking.Summary
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build creates the public process report without owning command parsing or config loading.
|
||||||
|
func Build(input BuildInput) reporting.ProcessReport {
|
||||||
|
report := reporting.ProcessReport{
|
||||||
|
ReportMetadata: reporting.ReportMetadata{
|
||||||
|
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||||
|
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||||
|
OutputSchema: input.OutputSchema,
|
||||||
|
ConfigVersion: input.ConfigVersion,
|
||||||
|
},
|
||||||
|
Phase: "default_pipeline",
|
||||||
|
Status: input.Status,
|
||||||
|
Operation: "process",
|
||||||
|
TranscriptPath: input.TranscriptPath,
|
||||||
|
GlossaryPath: input.GlossaryPath,
|
||||||
|
OutputPath: input.OutputPath,
|
||||||
|
Modules: append([]string(nil), input.Modules...),
|
||||||
|
StartedAt: input.StartedAt,
|
||||||
|
CompletedAt: &input.CompletedAt,
|
||||||
|
ErrorPhase: input.ErrorPhase,
|
||||||
|
}
|
||||||
|
if input.RunDirectoryPath != "" {
|
||||||
|
runSucceeded := input.Status == "success"
|
||||||
|
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||||
|
report.Diagnostics = &metadata
|
||||||
|
}
|
||||||
|
if input.ErrorMessage != "" {
|
||||||
|
report.ErrorMessage = input.ErrorMessage
|
||||||
|
}
|
||||||
|
if input.NormalizationSummary != nil {
|
||||||
|
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||||
|
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||||
|
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||||
|
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||||
|
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||||
|
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||||
|
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||||
|
}
|
||||||
|
if input.ChunkingSummary != nil {
|
||||||
|
report.Chunking = &reporting.ChunkingSummary{
|
||||||
|
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||||
|
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||||
|
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||||
|
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||||
|
TargetSections: input.ChunkingSummary.TargetSections,
|
||||||
|
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||||
|
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||||
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||||
|
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||||
|
for _, r := range runOutput.ModuleResults {
|
||||||
|
startedAt := r.StartedAt
|
||||||
|
completedAt := r.CompletedAt
|
||||||
|
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||||
|
ModuleKey: r.ModuleKey,
|
||||||
|
ModuleInstance: r.ModuleInstance,
|
||||||
|
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||||
|
Status: r.Status,
|
||||||
|
ProposalCount: r.ProposalCount,
|
||||||
|
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||||
|
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||||
|
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||||
|
AppliedChanges: r.AppliedChanges,
|
||||||
|
SkippedChanges: r.SkippedChanges,
|
||||||
|
ErrorMessage: r.ErrorMessage,
|
||||||
|
StartedAt: &startedAt,
|
||||||
|
CompletedAt: &completedAt,
|
||||||
|
})
|
||||||
|
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||||
|
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||||
|
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||||
|
summary.FailedModuleInstance = r.ModuleInstance
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary, moduleReports
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorDecisionReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
Approved: d.Approved,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorRejectedReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
ModuleKey: d.ModuleKey,
|
||||||
|
ModuleInstance: d.ModuleInstance,
|
||||||
|
TargetSegmentID: d.TargetSegmentID,
|
||||||
|
OriginalText: d.OriginalText,
|
||||||
|
CorrectedText: d.CorrectedText,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
180
internal/framework/processreport/report_test.go
Normal file
180
internal/framework/processreport/report_test.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
configVersion := 4
|
||||||
|
targetSections := 2
|
||||||
|
inputSegments := 5
|
||||||
|
outputSegments := 4
|
||||||
|
merges := 1
|
||||||
|
reassigned := 2
|
||||||
|
differentSpeakers := 3
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "success",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
OutputPath: "out.json",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
ConfigVersion: &configVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
RunDirectoryPath: filepath.Join(
|
||||||
|
"tmp",
|
||||||
|
"audita-run",
|
||||||
|
),
|
||||||
|
NormalizationSummary: &normalization.NormalizationSummary{
|
||||||
|
InputSegmentCount: inputSegments,
|
||||||
|
OutputSegmentCount: outputSegments,
|
||||||
|
MergesPerformed: merges,
|
||||||
|
IDsReassigned: reassigned,
|
||||||
|
SkippedMerges: struct {
|
||||||
|
DifferentSpeakers int `json:"different_speakers"`
|
||||||
|
GapTooLarge int `json:"gap_too_large"`
|
||||||
|
DurationExceeded int `json:"duration_exceeded"`
|
||||||
|
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||||
|
}{
|
||||||
|
DifferentSpeakers: differentSpeakers,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ChunkingSummary: &chunking.Summary{
|
||||||
|
ChunkCount: 3,
|
||||||
|
MinEstimatedTokens: 10,
|
||||||
|
MaxEstimatedTokens: 20,
|
||||||
|
TotalEstimatedTokens: 45,
|
||||||
|
TargetSections: &targetSections,
|
||||||
|
MaxSectionTokens: 200,
|
||||||
|
MinSectionTokens: 50,
|
||||||
|
},
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
ProposalCount: 2,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{
|
||||||
|
ValidatorName: "proposal_shape",
|
||||||
|
ProposalIndex: 1,
|
||||||
|
Approved: true,
|
||||||
|
ReasonCode: "approved",
|
||||||
|
Message: "ok",
|
||||||
|
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ValidatorName: "protected_term",
|
||||||
|
ProposalIndex: 2,
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||||
|
},
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||||
|
},
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||||
|
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||||
|
report.ReportMetadata.OutputSchema != "default" ||
|
||||||
|
report.ReportMetadata.ConfigVersion == nil ||
|
||||||
|
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||||
|
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||||
|
}
|
||||||
|
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||||
|
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||||
|
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||||
|
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||||
|
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||||
|
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil ||
|
||||||
|
report.ModulesSummary.ModuleCount != 1 ||
|
||||||
|
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||||
|
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||||
|
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
if len(report.ModuleResults) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||||
|
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "failed",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorPhase: "module",
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
RunDirectoryPath: "run-dir",
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||||
|
t.Fatalf("unexpected failure fields: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||||
|
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||||
|
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user