Map processing outcomes to versioned YAML reports
This commit is contained in:
@@ -45,13 +45,8 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
|
||||
}
|
||||
|
||||
func TestReportSummaryAvailability(t *testing.T) {
|
||||
input := BuildInput{Status: "failed", StartedAt: time.Now(), CompletedAt: time.Now()}
|
||||
if got := Build(input).LLMSummary; got != nil {
|
||||
t.Fatalf("unavailable summary: %+v", got)
|
||||
}
|
||||
input.RunOutput = &runner.RunOutput{ObservationsAvailable: true, LLMSummary: llm.Summarize(llm.ObservationSnapshot{})}
|
||||
got := Build(input).LLMSummary
|
||||
if got == nil || got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||
got := mapLLMSummary(llm.Summarize(llm.ObservationSnapshot{}))
|
||||
if got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||
t.Fatalf("initialized empty summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,164 +1,252 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// BuildInput contains already-computed process execution facts for report assembly.
|
||||
// RunFailure describes a fatal or output failure without carrying its raw cause.
|
||||
type RunFailure struct {
|
||||
Phase string
|
||||
Reason outcomes.ReasonCode
|
||||
OutputPath string
|
||||
Origin *outcomes.Reference
|
||||
StageIndex *int
|
||||
}
|
||||
|
||||
type BuildInput struct {
|
||||
Status string
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
ConfigVersion *int
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
ErrorMessage string
|
||||
ErrorPhase string
|
||||
RunDirectoryPath string
|
||||
NormalizationSummary *normalization.NormalizationSummary
|
||||
ChunkingSummary *chunking.Summary
|
||||
RunOutput *runner.RunOutput
|
||||
Backends map[string]config.Backend
|
||||
RunID string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
RunOutput *runner.RunOutput
|
||||
RunFailures []RunFailure
|
||||
}
|
||||
|
||||
// Build creates the public process report without owning command parsing or config loading.
|
||||
func Build(input BuildInput) reporting.ProcessReport {
|
||||
report := reporting.ProcessReport{
|
||||
ReportMetadata: reporting.ReportMetadata{
|
||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: input.OutputSchema,
|
||||
ConfigVersion: input.ConfigVersion,
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: input.Status,
|
||||
Operation: "process",
|
||||
TranscriptPath: input.TranscriptPath,
|
||||
GlossaryPath: input.GlossaryPath,
|
||||
OutputPath: input.OutputPath,
|
||||
Modules: append([]string(nil), input.Modules...),
|
||||
StartedAt: input.StartedAt,
|
||||
CompletedAt: &input.CompletedAt,
|
||||
ErrorPhase: input.ErrorPhase,
|
||||
// Build maps processing facts once into an immutable presentation snapshot.
|
||||
func Build(input BuildInput) reporting.Snapshot {
|
||||
snapshot := reporting.Snapshot{
|
||||
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
|
||||
TranscriptPath: input.TranscriptPath, GlossaryPath: input.GlossaryPath, OutputPath: input.OutputPath,
|
||||
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
|
||||
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
|
||||
}
|
||||
if input.RunDirectoryPath != "" {
|
||||
runSucceeded := input.Status == "success"
|
||||
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||
report.Diagnostics = &metadata
|
||||
for i, spec := range input.ModuleSpecs {
|
||||
snapshot.StageNames[i] = spec.InstanceName
|
||||
}
|
||||
if input.ErrorMessage != "" {
|
||||
report.ErrorMessage = input.ErrorMessage
|
||||
}
|
||||
if input.NormalizationSummary != nil {
|
||||
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||
}
|
||||
if input.ChunkingSummary != nil {
|
||||
report.Chunking = &reporting.ChunkingSummary{
|
||||
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||
TargetSections: input.ChunkingSummary.TargetSections,
|
||||
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||
if input.RunOutput != nil && input.RunOutput.ObservationsAvailable {
|
||||
summary := mapLLMSummary(input.RunOutput.LLMSummary)
|
||||
report.LLMSummary = &summary
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||
for _, r := range runOutput.ModuleResults {
|
||||
startedAt := r.StartedAt
|
||||
completedAt := r.CompletedAt
|
||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||
ModuleKey: r.ModuleKey,
|
||||
ModuleInstance: r.ModuleInstance,
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
SkippedChanges: r.SkippedChanges,
|
||||
ErrorMessage: r.ErrorMessage,
|
||||
StartedAt: &startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
if input.RunOutput != nil {
|
||||
units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...)
|
||||
sort.SliceStable(units, func(i, j int) bool {
|
||||
if units[i].ModuleIndex != units[j].ModuleIndex {
|
||||
return units[i].ModuleIndex < units[j].ModuleIndex
|
||||
}
|
||||
return units[i].Section.Index < units[j].Section.Index
|
||||
})
|
||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||
summary.FailedModuleInstance = r.ModuleInstance
|
||||
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
|
||||
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
|
||||
for _, unit := range units {
|
||||
if unit.State == outcomes.WorkNotStarted {
|
||||
notStarted++
|
||||
} else {
|
||||
started++
|
||||
}
|
||||
if unit.State == outcomes.WorkCompleted {
|
||||
completed++
|
||||
}
|
||||
if unit.State == outcomes.WorkInterrupted {
|
||||
interrupted++
|
||||
}
|
||||
corrections += unit.CorrectionsApplied
|
||||
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied}
|
||||
operations := append([]outcomes.Operation(nil), unit.Operations...)
|
||||
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
|
||||
failures := make([]reporting.Failure, 0)
|
||||
for _, operation := range operations {
|
||||
if operation.Reference.Kind != outcomes.OperationValidationInput {
|
||||
llmOperations++
|
||||
}
|
||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||
proposalResponses++
|
||||
}
|
||||
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
|
||||
recoveryCount++
|
||||
row.RecoveryCount++
|
||||
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
|
||||
filteringResponses++
|
||||
}
|
||||
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
||||
}
|
||||
if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) {
|
||||
failures = append(failures, mapFailure(operation))
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
withErrors++
|
||||
row.ErrorCount = 1
|
||||
stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied
|
||||
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage,
|
||||
ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: &chunk,
|
||||
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
|
||||
}
|
||||
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
|
||||
}
|
||||
runErrors := len(input.RunFailures)
|
||||
snapshot.Summary = reporting.Summary{
|
||||
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
|
||||
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors,
|
||||
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
|
||||
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
|
||||
ProposalResponsesWithFiltering: &filteringResponses,
|
||||
}
|
||||
} else if len(input.RunFailures) > 0 {
|
||||
runErrors := len(input.RunFailures)
|
||||
snapshot.Summary.RunErrors = &runErrors
|
||||
}
|
||||
|
||||
return summary, moduleReports
|
||||
for _, failure := range sortedRunFailures(input.RunFailures) {
|
||||
record := reporting.ErrorRecord{Scope: "run", Phase: failure.Phase, Reason: string(failure.Reason),
|
||||
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
|
||||
if failure.Origin != nil {
|
||||
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
|
||||
chunk := failure.Origin.SectionIndex
|
||||
record.ChunkIndex = &chunk
|
||||
record.Operation = string(failure.Origin.Kind)
|
||||
record.Validator = failure.Origin.ValidatorName
|
||||
record.BatchIndex = failure.Origin.BatchIndex
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
func mapFailure(operation outcomes.Operation) reporting.Failure {
|
||||
indices := append([]int(nil), operation.ProposalIndices...)
|
||||
sort.Ints(indices)
|
||||
return reporting.Failure{Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName,
|
||||
BatchIndex: operation.Reference.BatchIndex, Reason: string(operation.FailureReason), Message: safeMessage(operation.FailureReason),
|
||||
ProposalIndices: indices, ExecutionCounts: mapCounts(operation.Counts)}
|
||||
}
|
||||
|
||||
func mapRecovery(unit runner.WorkUnitOutcome, operation outcomes.Operation) reporting.RecoveryRecord {
|
||||
reasons := make([]string, len(operation.RecoveryReasons))
|
||||
for i, reason := range operation.RecoveryReasons {
|
||||
reasons[i] = string(reason)
|
||||
}
|
||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorDecisionReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorRejectedReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
ModuleKey: d.ModuleKey,
|
||||
ModuleInstance: d.ModuleInstance,
|
||||
TargetSegmentID: d.TargetSegmentID,
|
||||
OriginalText: d.OriginalText,
|
||||
CorrectedText: d.CorrectedText,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
sort.Strings(reasons)
|
||||
filtered := make([]reporting.FilteredProposal, len(operation.Filtered))
|
||||
for i, item := range operation.Filtered {
|
||||
filtered[i] = reporting.FilteredProposal{ProposalIndex: item.ProposalIndex, Reason: string(item.Reason), Count: item.Count}
|
||||
}
|
||||
sort.Slice(filtered, func(i, j int) bool { return filtered[i].ProposalIndex < filtered[j].ProposalIndex })
|
||||
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||
Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName, BatchIndex: operation.Reference.BatchIndex,
|
||||
Reasons: reasons, Message: "Model operation succeeded after recovery or proposal filtering.",
|
||||
ExecutionCounts: mapCounts(operation.Counts), FilteredProposals: filtered}
|
||||
}
|
||||
|
||||
func mapCounts(counts outcomes.ExecutionCounts) reporting.ExecutionCounts {
|
||||
if !counts.Available {
|
||||
return reporting.ExecutionCounts{}
|
||||
}
|
||||
attempts, retries, repairs, exhausted := counts.ProviderAttempts, counts.TransientRetries, counts.StructuralRepairs, counts.RetriesExhausted
|
||||
return reporting.ExecutionCounts{ProviderAttempts: &attempts, TransientRetries: &retries,
|
||||
StructuralRepairs: &repairs, RetriesExhausted: &exhausted, HTTPStatus: counts.HTTPStatus}
|
||||
}
|
||||
|
||||
func isRecoverableReason(reason outcomes.ReasonCode) bool {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout, outcomes.ReasonCapacityExceeded, outcomes.ReasonTransportFailed,
|
||||
outcomes.ReasonProviderResponseInvalid, outcomes.ReasonProviderFailed,
|
||||
outcomes.ReasonInvalidOutputExhausted, outcomes.ReasonInvalidOutput,
|
||||
outcomes.ReasonValidationInputTooLarge, outcomes.ReasonValidationDecisionsInvalid:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func safeMessage(reason outcomes.ReasonCode) string {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout:
|
||||
return "Model operation timed out."
|
||||
case outcomes.ReasonCapacityExceeded:
|
||||
return "Model capacity was unavailable."
|
||||
case outcomes.ReasonTransportFailed:
|
||||
return "Model transport failed."
|
||||
case outcomes.ReasonProviderResponseInvalid:
|
||||
return "Provider response was invalid."
|
||||
case outcomes.ReasonProviderFailed:
|
||||
return "Provider request failed."
|
||||
case outcomes.ReasonInvalidOutputExhausted:
|
||||
return "Model output remained invalid after repair attempts."
|
||||
case outcomes.ReasonInvalidOutput:
|
||||
return "Model output was invalid."
|
||||
case outcomes.ReasonValidationInputTooLarge:
|
||||
return "Validation input exceeded its size limit."
|
||||
case outcomes.ReasonValidationDecisionsInvalid:
|
||||
return "Validation decisions were unusable."
|
||||
case outcomes.ReasonCanceled:
|
||||
return "Processing was canceled."
|
||||
case outcomes.ReasonSetupFailure:
|
||||
return "Processing setup failed."
|
||||
default:
|
||||
return "Processing failed."
|
||||
}
|
||||
}
|
||||
|
||||
func operationLess(a, b outcomes.Operation) bool {
|
||||
rank := func(kind outcomes.OperationKind) int {
|
||||
switch kind {
|
||||
case outcomes.OperationProposal:
|
||||
return 0
|
||||
case outcomes.OperationValidationInput:
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
|
||||
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
|
||||
}
|
||||
order := func(p *int) int {
|
||||
if p == nil {
|
||||
return -1
|
||||
}
|
||||
return *p
|
||||
}
|
||||
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
|
||||
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
|
||||
}
|
||||
return rank(a.Reference.Kind) < rank(b.Reference.Kind) || (rank(a.Reference.Kind) == rank(b.Reference.Kind) && order(a.Reference.BatchIndex) < order(b.Reference.BatchIndex))
|
||||
}
|
||||
|
||||
func sortedRunFailures(in []RunFailure) []RunFailure {
|
||||
out := append([]RunFailure(nil), in...)
|
||||
phase := map[string]int{"configuration": 0, "destination": 1, "initialization": 2, "input": 3, "normalization": 4, "chunking": 5, "processing": 6, "encoding": 7, "transcript_write": 8, "report_write": 9, "debug": 10}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
a, aok := phase[out[i].Phase]
|
||||
b, bok := phase[out[j].Phase]
|
||||
if !aok {
|
||||
a = 100
|
||||
}
|
||||
if !bok {
|
||||
b = 100
|
||||
}
|
||||
if a != b {
|
||||
return a < b
|
||||
}
|
||||
return out[i].OutputPath < out[j].OutputPath
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,180 +1,78 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
configVersion := 4
|
||||
targetSections := 2
|
||||
inputSegments := 5
|
||||
outputSegments := 4
|
||||
merges := 1
|
||||
reassigned := 2
|
||||
differentSpeakers := 3
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "success",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
OutputPath: "out.json",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
ConfigVersion: &configVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
RunDirectoryPath: filepath.Join(
|
||||
"tmp",
|
||||
"audita-run",
|
||||
),
|
||||
NormalizationSummary: &normalization.NormalizationSummary{
|
||||
InputSegmentCount: inputSegments,
|
||||
OutputSegmentCount: outputSegments,
|
||||
MergesPerformed: merges,
|
||||
IDsReassigned: reassigned,
|
||||
SkippedMerges: struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
}{
|
||||
DifferentSpeakers: differentSpeakers,
|
||||
},
|
||||
},
|
||||
ChunkingSummary: &chunking.Summary{
|
||||
ChunkCount: 3,
|
||||
MinEstimatedTokens: 10,
|
||||
MaxEstimatedTokens: 20,
|
||||
TotalEstimatedTokens: 45,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: 200,
|
||||
MinSectionTokens: 50,
|
||||
},
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Status: runner.ModuleStatusSuccess,
|
||||
ProposalCount: 2,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{
|
||||
ValidatorName: "proposal_shape",
|
||||
ProposalIndex: 1,
|
||||
Approved: true,
|
||||
ReasonCode: "approved",
|
||||
Message: "ok",
|
||||
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||
},
|
||||
},
|
||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||
{
|
||||
ValidatorName: "protected_term",
|
||||
ProposalIndex: 2,
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "old",
|
||||
CorrectedText: "new",
|
||||
ReasonCode: "protected_term",
|
||||
Message: "blocked",
|
||||
},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||
},
|
||||
SkippedChanges: []proposals.SkippedChange{
|
||||
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||
},
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||
report.ReportMetadata.OutputSchema != "default" ||
|
||||
report.ReportMetadata.ConfigVersion == nil ||
|
||||
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||
func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
|
||||
started := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
|
||||
completed := started.Add(time.Minute)
|
||||
proposal := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
|
||||
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, TransientRetries: 1, StructuralRepairs: 1},
|
||||
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
|
||||
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
|
||||
batch := 1
|
||||
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||
output := &runner.RunOutput{WorkUnits: []runner.WorkUnitOutcome{
|
||||
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
|
||||
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
|
||||
}}
|
||||
snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output})
|
||||
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
|
||||
t.Fatalf("summary: %+v", snapshot.Summary)
|
||||
}
|
||||
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 2 || snapshot.Errors[0].Failures[0].Operation != "validation_input" || !reflect.DeepEqual(snapshot.Errors[0].Failures[1].ProposalIndices, []int{3, 9}) {
|
||||
t.Fatalf("failure ordering: %+v", snapshot.Errors)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"output_repaired", "proposals_filtered", "service_recovered"}) || snapshot.Recoveries[0].FilteredProposals[0].ProposalIndex != 2 {
|
||||
t.Fatalf("recovery: %+v", snapshot.Recoveries)
|
||||
}
|
||||
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||
if snapshot.WorkUnits[0].ModuleInstance != "grammar_1" || snapshot.WorkUnits[1].State != "not_started" {
|
||||
t.Fatalf("unit order: %+v", snapshot.WorkUnits)
|
||||
}
|
||||
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ModulesSummary == nil ||
|
||||
report.ModulesSummary.ModuleCount != 1 ||
|
||||
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||
var decoded map[string]any
|
||||
if err := yaml.Unmarshal(errorsYAML, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||
if decoded["version"] != 1 || decoded["schema"] != "audita-errors" || decoded["status"] != "completed" {
|
||||
t.Fatalf("envelope: %#v", decoded)
|
||||
}
|
||||
if strings.Contains(string(errorsYAML), "provider_attempts: null") || !strings.Contains(string(errorsYAML), "errors:") {
|
||||
t.Fatal(string(errorsYAML))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "failed",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorPhase: "module",
|
||||
ErrorMessage: "module failed",
|
||||
RunDirectoryPath: "run-dir",
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
Status: runner.ModuleStatusFailed,
|
||||
ErrorMessage: "module failed",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||
t.Fatalf("unexpected failure fields: %+v", report)
|
||||
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})
|
||||
initialDoc := initial.ErrorsDocument()
|
||||
if initialDoc.CompletedAt != nil || initialDoc.Summary.WorkUnitsTotal != nil || initialDoc.Errors == nil {
|
||||
t.Fatalf("initial: %+v", initialDoc)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||
finished := Build(BuildInput{RunID: "r", Status: reporting.StatusCompleted, StartedAt: start, RunOutput: &runner.RunOutput{}})
|
||||
if finished.Summary.WorkUnitsTotal == nil || *finished.Summary.WorkUnitsTotal != 0 || finished.RecoveriesDocument().Recoveries == nil {
|
||||
t.Fatalf("empty: %+v", finished)
|
||||
}
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||
failed := Build(BuildInput{RunID: "r", Status: reporting.StatusFailed, StartedAt: start,
|
||||
RunFailures: []RunFailure{{Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "z.yml"}, {Phase: "processing", Reason: outcomes.ReasonCanceled}, {Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "a.yml"}}})
|
||||
if len(failed.Errors) != 3 || failed.Errors[0].Phase != "processing" || failed.Errors[1].OutputPath != "a.yml" || failed.Errors[2].OutputPath != "z.yml" || failed.Summary.WorkUnitsTotal != nil || *failed.Summary.RunErrors != 3 {
|
||||
t.Fatalf("run failures: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(value int) *int { return &value }
|
||||
|
||||
Reference in New Issue
Block a user