Map processing outcomes to versioned YAML reports
This commit is contained in:
@@ -1,180 +1,78 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
configVersion := 4
|
||||
targetSections := 2
|
||||
inputSegments := 5
|
||||
outputSegments := 4
|
||||
merges := 1
|
||||
reassigned := 2
|
||||
differentSpeakers := 3
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "success",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
OutputPath: "out.json",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
ConfigVersion: &configVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
RunDirectoryPath: filepath.Join(
|
||||
"tmp",
|
||||
"audita-run",
|
||||
),
|
||||
NormalizationSummary: &normalization.NormalizationSummary{
|
||||
InputSegmentCount: inputSegments,
|
||||
OutputSegmentCount: outputSegments,
|
||||
MergesPerformed: merges,
|
||||
IDsReassigned: reassigned,
|
||||
SkippedMerges: struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
}{
|
||||
DifferentSpeakers: differentSpeakers,
|
||||
},
|
||||
},
|
||||
ChunkingSummary: &chunking.Summary{
|
||||
ChunkCount: 3,
|
||||
MinEstimatedTokens: 10,
|
||||
MaxEstimatedTokens: 20,
|
||||
TotalEstimatedTokens: 45,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: 200,
|
||||
MinSectionTokens: 50,
|
||||
},
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Status: runner.ModuleStatusSuccess,
|
||||
ProposalCount: 2,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{
|
||||
ValidatorName: "proposal_shape",
|
||||
ProposalIndex: 1,
|
||||
Approved: true,
|
||||
ReasonCode: "approved",
|
||||
Message: "ok",
|
||||
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||
},
|
||||
},
|
||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||
{
|
||||
ValidatorName: "protected_term",
|
||||
ProposalIndex: 2,
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "old",
|
||||
CorrectedText: "new",
|
||||
ReasonCode: "protected_term",
|
||||
Message: "blocked",
|
||||
},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||
},
|
||||
SkippedChanges: []proposals.SkippedChange{
|
||||
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||
},
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||
report.ReportMetadata.OutputSchema != "default" ||
|
||||
report.ReportMetadata.ConfigVersion == nil ||
|
||||
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||
func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
|
||||
started := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
|
||||
completed := started.Add(time.Minute)
|
||||
proposal := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
|
||||
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, TransientRetries: 1, StructuralRepairs: 1},
|
||||
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
|
||||
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
|
||||
batch := 1
|
||||
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||
output := &runner.RunOutput{WorkUnits: []runner.WorkUnitOutcome{
|
||||
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
|
||||
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
|
||||
}}
|
||||
snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output})
|
||||
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
|
||||
t.Fatalf("summary: %+v", snapshot.Summary)
|
||||
}
|
||||
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 2 || snapshot.Errors[0].Failures[0].Operation != "validation_input" || !reflect.DeepEqual(snapshot.Errors[0].Failures[1].ProposalIndices, []int{3, 9}) {
|
||||
t.Fatalf("failure ordering: %+v", snapshot.Errors)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"output_repaired", "proposals_filtered", "service_recovered"}) || snapshot.Recoveries[0].FilteredProposals[0].ProposalIndex != 2 {
|
||||
t.Fatalf("recovery: %+v", snapshot.Recoveries)
|
||||
}
|
||||
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||
if snapshot.WorkUnits[0].ModuleInstance != "grammar_1" || snapshot.WorkUnits[1].State != "not_started" {
|
||||
t.Fatalf("unit order: %+v", snapshot.WorkUnits)
|
||||
}
|
||||
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ModulesSummary == nil ||
|
||||
report.ModulesSummary.ModuleCount != 1 ||
|
||||
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||
var decoded map[string]any
|
||||
if err := yaml.Unmarshal(errorsYAML, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||
if decoded["version"] != 1 || decoded["schema"] != "audita-errors" || decoded["status"] != "completed" {
|
||||
t.Fatalf("envelope: %#v", decoded)
|
||||
}
|
||||
if strings.Contains(string(errorsYAML), "provider_attempts: null") || !strings.Contains(string(errorsYAML), "errors:") {
|
||||
t.Fatal(string(errorsYAML))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "failed",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorPhase: "module",
|
||||
ErrorMessage: "module failed",
|
||||
RunDirectoryPath: "run-dir",
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
Status: runner.ModuleStatusFailed,
|
||||
ErrorMessage: "module failed",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||
t.Fatalf("unexpected failure fields: %+v", report)
|
||||
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})
|
||||
initialDoc := initial.ErrorsDocument()
|
||||
if initialDoc.CompletedAt != nil || initialDoc.Summary.WorkUnitsTotal != nil || initialDoc.Errors == nil {
|
||||
t.Fatalf("initial: %+v", initialDoc)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||
finished := Build(BuildInput{RunID: "r", Status: reporting.StatusCompleted, StartedAt: start, RunOutput: &runner.RunOutput{}})
|
||||
if finished.Summary.WorkUnitsTotal == nil || *finished.Summary.WorkUnitsTotal != 0 || finished.RecoveriesDocument().Recoveries == nil {
|
||||
t.Fatalf("empty: %+v", finished)
|
||||
}
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||
failed := Build(BuildInput{RunID: "r", Status: reporting.StatusFailed, StartedAt: start,
|
||||
RunFailures: []RunFailure{{Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "z.yml"}, {Phase: "processing", Reason: outcomes.ReasonCanceled}, {Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "a.yml"}}})
|
||||
if len(failed.Errors) != 3 || failed.Errors[0].Phase != "processing" || failed.Errors[1].OutputPath != "a.yml" || failed.Errors[2].OutputPath != "z.yml" || failed.Summary.WorkUnitsTotal != nil || *failed.Summary.RunErrors != 3 {
|
||||
t.Fatalf("run failures: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(value int) *int { return &value }
|
||||
|
||||
Reference in New Issue
Block a user