Map processing outcomes to versioned YAML reports
This commit is contained in:
@@ -619,6 +619,16 @@ filesystem requirements for disabled features.
|
||||
|
||||
## Stage 8 — Map Outcomes to Versioned YAML Reports
|
||||
|
||||
**Completed 2026-09-24.** The old ordinary JSON report DTO was replaced with
|
||||
versioned errors/recoveries YAML DTOs and one processing snapshot. Mapping derives
|
||||
nullable summary counts, grouped recoverable work-unit failures, run failures,
|
||||
combined recovery reasons, and deterministic semantic order without raw causes or
|
||||
transcript payloads. Tests cover partial and empty work, multiple failures,
|
||||
initialization unknowns, ordered run errors, safe redaction, and YAML scalar
|
||||
types. `go test ./internal/core/reporting ./internal/framework/processreport`
|
||||
and `git diff --check` passed. CLI and diagnostics consumers of the removed JSON
|
||||
schema remain to be reconnected in later stages.
|
||||
|
||||
**Work:** Replace ordinary report DTOs and assembly in `core/reporting` and
|
||||
`framework/processreport` with F. Map actual runner state once into an immutable
|
||||
report snapshot consumed by all presentations. Preserve debug-specific ledger and
|
||||
|
||||
@@ -1,138 +1,180 @@
|
||||
// Package reporting owns the public report schema, separate from processing.
|
||||
package reporting
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type ProcessReport struct {
|
||||
ReportMetadata ReportMetadata `json:"report_metadata"`
|
||||
Phase string `json:"phase"`
|
||||
Status string `json:"status"`
|
||||
Operation string `json:"operation"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
GlossaryPath string `json:"glossary_path"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
Modules []string `json:"modules"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
||||
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
||||
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
||||
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
||||
NormalizationSkipped NormalizationSkipped `json:"normalization_skipped,omitempty"`
|
||||
Chunking *ChunkingSummary `json:"chunking,omitempty"`
|
||||
Diagnostics *DiagnosticsMetadata `json:"diagnostics,omitempty"`
|
||||
LLMSummary *LLMSummary `json:"llm_summary,omitempty"`
|
||||
ModulesSummary *ModulesSummary `json:"modules_summary,omitempty"`
|
||||
ModuleResults []ModuleReport `json:"module_results,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultProcessReportSchemaName = "audita-process-report"
|
||||
DefaultProcessReportSchemaVersion = "v2"
|
||||
StatusInProgress = "in_progress"
|
||||
StatusCompleted = "completed"
|
||||
StatusFailed = "failed"
|
||||
)
|
||||
|
||||
type ReportMetadata struct {
|
||||
ReportSchemaName string `json:"report_schema_name"`
|
||||
ReportSchemaVersion string `json:"report_schema_version"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
ConfigVersion *int `json:"config_version,omitempty"`
|
||||
// Nil counts mean unavailable; a pointer to zero means a known empty count.
|
||||
type Summary struct {
|
||||
WorkUnitsTotal *int `yaml:"work_units_total"`
|
||||
WorkUnitsStarted *int `yaml:"work_units_started"`
|
||||
WorkUnitsCompleted *int `yaml:"work_units_completed"`
|
||||
WorkUnitsInterrupted *int `yaml:"work_units_interrupted"`
|
||||
WorkUnitsNotStarted *int `yaml:"work_units_not_started"`
|
||||
WorkUnitsWithErrors *int `yaml:"work_units_with_errors"`
|
||||
RunErrors *int `yaml:"run_errors"`
|
||||
CorrectionsApplied *int `yaml:"corrections_applied"`
|
||||
LLMOperations *int `yaml:"llm_operations"`
|
||||
ProposalResponses *int `yaml:"proposal_responses"`
|
||||
Recoveries *int `yaml:"recoveries"`
|
||||
ProposalResponsesWithFiltering *int `yaml:"proposal_responses_with_filtering"`
|
||||
}
|
||||
|
||||
type ModuleReport struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
type ExecutionCounts struct {
|
||||
ProviderAttempts *int `yaml:"provider_attempts,omitempty"`
|
||||
TransientRetries *int `yaml:"transient_retries,omitempty"`
|
||||
StructuralRepairs *int `yaml:"structural_repairs,omitempty"`
|
||||
RetriesExhausted *bool `yaml:"retries_exhausted,omitempty"`
|
||||
HTTPStatus *int `yaml:"http_status,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionReport struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
type Failure struct {
|
||||
Operation string `yaml:"operation"`
|
||||
Validator string `yaml:"validator,omitempty"`
|
||||
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||
Reason string `yaml:"reason"`
|
||||
Message string `yaml:"message"`
|
||||
ProposalIndices []int `yaml:"proposal_indices,omitempty"`
|
||||
ExecutionCounts `yaml:",inline"`
|
||||
}
|
||||
|
||||
type ValidatorRejectedReport struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
TargetSegmentID int `json:"target_segment_id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
type ErrorRecord struct {
|
||||
Scope string `yaml:"scope"`
|
||||
StageIndex *int `yaml:"stage_index,omitempty"`
|
||||
ModuleKey string `yaml:"module_key,omitempty"`
|
||||
ModuleInstance string `yaml:"module_instance,omitempty"`
|
||||
ChunkIndex *int `yaml:"chunk_index,omitempty"`
|
||||
StartSegmentID *int `yaml:"start_segment_id,omitempty"`
|
||||
EndSegmentID *int `yaml:"end_segment_id,omitempty"`
|
||||
CorrectionsApplied *int `yaml:"corrections_applied,omitempty"`
|
||||
Failures []Failure `yaml:"failures,omitempty"`
|
||||
Phase string `yaml:"phase,omitempty"`
|
||||
Reason string `yaml:"reason,omitempty"`
|
||||
Message string `yaml:"message,omitempty"`
|
||||
Operation string `yaml:"operation,omitempty"`
|
||||
Validator string `yaml:"validator,omitempty"`
|
||||
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||
OutputPath string `yaml:"output_path,omitempty"`
|
||||
}
|
||||
|
||||
type ModulesSummary struct {
|
||||
ModuleCount int `json:"module_count"`
|
||||
TotalAppliedChanges int `json:"total_applied_changes"`
|
||||
TotalSkippedChanges int `json:"total_skipped_changes"`
|
||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
||||
type FilteredProposal struct {
|
||||
ProposalIndex int `yaml:"proposal_index"`
|
||||
Reason string `yaml:"reason"`
|
||||
Count int `yaml:"count"`
|
||||
}
|
||||
|
||||
type DiagnosticsMetadata struct {
|
||||
DirectoryPath string `json:"directory_path,omitempty"`
|
||||
SourceTranscriptPath string `json:"source_transcript_path,omitempty"`
|
||||
ParsedSourceTranscriptPath string `json:"parsed_source_transcript_path,omitempty"`
|
||||
NormalizedTranscriptPath string `json:"normalized_transcript_path,omitempty"`
|
||||
NormalizationSummaryPath string `json:"normalization_summary_path,omitempty"`
|
||||
ChunkingSummaryPath string `json:"chunking_summary_path,omitempty"`
|
||||
UtilizationSummaryPath string `json:"utilization_summary_path,omitempty"`
|
||||
LLMExecutionPath string `json:"llm_execution_path,omitempty"`
|
||||
SelectedProfilesPath string `json:"selected_profiles_path,omitempty"`
|
||||
CorrectionLedgerPath string `json:"correction_ledger_path,omitempty"`
|
||||
InvocationMetadataPath string `json:"invocation_metadata_path,omitempty"`
|
||||
RedactedEffectiveConfigPath string `json:"redacted_effective_config_path,omitempty"`
|
||||
ErrorLogPath string `json:"error_log_path,omitempty"`
|
||||
type RecoveryRecord struct {
|
||||
StageIndex int `yaml:"stage_index"`
|
||||
ModuleKey string `yaml:"module_key"`
|
||||
ModuleInstance string `yaml:"module_instance"`
|
||||
ChunkIndex int `yaml:"chunk_index"`
|
||||
StartSegmentID int `yaml:"start_segment_id"`
|
||||
EndSegmentID int `yaml:"end_segment_id"`
|
||||
Operation string `yaml:"operation"`
|
||||
Validator string `yaml:"validator,omitempty"`
|
||||
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||
Reasons []string `yaml:"reasons"`
|
||||
Message string `yaml:"message"`
|
||||
ExecutionCounts `yaml:",inline"`
|
||||
FilteredProposals []FilteredProposal `yaml:"filtered_proposals,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizationSkipped struct {
|
||||
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
||||
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
||||
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
||||
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
||||
type ErrorsDocument struct {
|
||||
Schema string `yaml:"schema"`
|
||||
Version int `yaml:"version"`
|
||||
RunID string `yaml:"run_id"`
|
||||
Status string `yaml:"status"`
|
||||
StartedAt string `yaml:"started_at"`
|
||||
CompletedAt *string `yaml:"completed_at"`
|
||||
Summary Summary `yaml:"summary"`
|
||||
Errors []ErrorRecord `yaml:"errors"`
|
||||
}
|
||||
|
||||
type ChunkingSummary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
type RecoveriesDocument struct {
|
||||
Schema string `yaml:"schema"`
|
||||
Version int `yaml:"version"`
|
||||
RunID string `yaml:"run_id"`
|
||||
Status string `yaml:"status"`
|
||||
StartedAt string `yaml:"started_at"`
|
||||
CompletedAt *string `yaml:"completed_at"`
|
||||
Summary Summary `yaml:"summary"`
|
||||
Recoveries []RecoveryRecord `yaml:"recoveries"`
|
||||
}
|
||||
|
||||
func WriteProcessReport(path string, report ProcessReport, secrets ...string) error {
|
||||
payload, err := redaction.New(secrets).JSON(report)
|
||||
// WorkUnit is payload-free matrix input for report.md.
|
||||
type WorkUnit struct {
|
||||
StageIndex int
|
||||
ModuleKey string
|
||||
ModuleInstance string
|
||||
ChunkIndex int
|
||||
StartSegmentID int
|
||||
EndSegmentID int
|
||||
State string
|
||||
CorrectionsApplied int
|
||||
ErrorCount int
|
||||
RecoveryCount int
|
||||
}
|
||||
|
||||
// Snapshot is built once from processing state and shared by presentations.
|
||||
type Snapshot struct {
|
||||
RunID string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
StageNames []string
|
||||
WorkUnits []WorkUnit
|
||||
Summary Summary
|
||||
Errors []ErrorRecord
|
||||
Recoveries []RecoveryRecord
|
||||
}
|
||||
|
||||
func (s Snapshot) ErrorsDocument() ErrorsDocument {
|
||||
errors := s.Errors
|
||||
if errors == nil {
|
||||
errors = []ErrorRecord{}
|
||||
}
|
||||
return ErrorsDocument{Schema: "audita-errors", Version: 1, RunID: s.RunID, Status: s.Status, StartedAt: s.StartedAt.UTC().Format(time.RFC3339), CompletedAt: completionTime(s.CompletedAt), Summary: s.Summary, Errors: errors}
|
||||
}
|
||||
|
||||
func (s Snapshot) RecoveriesDocument() RecoveriesDocument {
|
||||
recoveries := s.Recoveries
|
||||
if recoveries == nil {
|
||||
recoveries = []RecoveryRecord{}
|
||||
}
|
||||
return RecoveriesDocument{Schema: "audita-recoveries", Version: 1, RunID: s.RunID, Status: s.Status, StartedAt: s.StartedAt.UTC().Format(time.RFC3339), CompletedAt: completionTime(s.CompletedAt), Summary: s.Summary, Recoveries: recoveries}
|
||||
}
|
||||
|
||||
func completionTime(t *time.Time) *string {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
value := t.UTC().Format(time.RFC3339)
|
||||
return &value
|
||||
}
|
||||
|
||||
// YAML redacts only string nodes after typed serialization.
|
||||
func YAML(document any, secrets ...string) ([]byte, error) {
|
||||
raw, err := yaml.Marshal(document)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal process report JSON: %w", err)
|
||||
return nil, fmt.Errorf("encode report YAML: %w", err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write report JSON file %q: %w", path, err)
|
||||
safe, err := redaction.New(secrets).YAML(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redact report YAML: %w", err)
|
||||
}
|
||||
return nil
|
||||
return safe, nil
|
||||
}
|
||||
|
||||
@@ -1,130 +1,41 @@
|
||||
package reporting
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
report := ProcessReport{
|
||||
ReportMetadata: ReportMetadata{
|
||||
ReportSchemaName: DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: "bare-segments",
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: "success",
|
||||
ModuleResults: []ModuleReport{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar_1",
|
||||
ReplacementPolicy: "require_unique",
|
||||
Status: "success",
|
||||
ProposalCount: 2,
|
||||
ValidatorDecisions: []ValidatorDecisionReport{{
|
||||
ValidatorName: "confidence_threshold",
|
||||
ProposalIndex: 0,
|
||||
Approved: true,
|
||||
ReasonCode: "approved",
|
||||
Message: "approved",
|
||||
}},
|
||||
ValidatorRejected: []ValidatorRejectedReport{{
|
||||
ValidatorName: "confidence_threshold",
|
||||
ProposalIndex: 1,
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar_1",
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "bad",
|
||||
CorrectedText: "worse",
|
||||
ReasonCode: "low_confidence",
|
||||
Message: "rejected",
|
||||
}},
|
||||
AppliedChanges: []proposals.AppliedChange{{
|
||||
ProposalIndex: 0,
|
||||
ModuleKey: "grammar",
|
||||
}},
|
||||
SkippedChanges: []proposals.SkippedChange{{
|
||||
ProposalIndex: 1,
|
||||
ModuleKey: "grammar",
|
||||
SkipReason: proposals.SkipReasonMissingOriginalText,
|
||||
}},
|
||||
StartedAt: &now,
|
||||
CompletedAt: &now,
|
||||
},
|
||||
},
|
||||
ModulesSummary: &ModulesSummary{ModuleCount: 1, TotalAppliedChanges: 1, TotalSkippedChanges: 1},
|
||||
func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
|
||||
zero, one := 0, 1
|
||||
falseValue := false
|
||||
snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC),
|
||||
Summary: Summary{RunErrors: &zero, CorrectionsApplied: &one},
|
||||
Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}},
|
||||
Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}}
|
||||
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
|
||||
payload, err := YAML(document, `secret\"key`, "1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(payload), `secret\"key`) {
|
||||
t.Fatalf("secret leaked: %s", payload)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := yaml.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
|
||||
t.Fatalf("wrong envelope types: %#v", decoded)
|
||||
}
|
||||
summary := decoded["summary"].(map[string]any)
|
||||
if summary["corrections_applied"] != 1 || summary["work_units_total"] != nil {
|
||||
t.Fatalf("changed count types: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal report: %v", err)
|
||||
}
|
||||
if !json.Valid(b) {
|
||||
t.Fatalf("expected valid json")
|
||||
}
|
||||
|
||||
var parsed ProcessReport
|
||||
if err := json.Unmarshal(b, &parsed); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if len(parsed.ModuleResults) != 1 {
|
||||
t.Fatalf("expected one module result, got %d", len(parsed.ModuleResults))
|
||||
}
|
||||
if len(parsed.ModuleResults[0].ValidatorDecisions) != 1 || len(parsed.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("expected validator result payloads in module report")
|
||||
}
|
||||
if parsed.ModulesSummary == nil || parsed.ModulesSummary.TotalSkippedChanges != 1 {
|
||||
t.Fatalf("unexpected module summary: %+v", parsed.ModulesSummary)
|
||||
}
|
||||
if parsed.ReportMetadata.ReportSchemaName != DefaultProcessReportSchemaName ||
|
||||
parsed.ReportMetadata.ReportSchemaVersion != DefaultProcessReportSchemaVersion ||
|
||||
parsed.ReportMetadata.OutputSchema != "bare-segments" {
|
||||
t.Fatalf("unexpected report metadata after roundtrip: %+v", parsed.ReportMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
report := ProcessReport{
|
||||
ReportMetadata: ReportMetadata{
|
||||
ReportSchemaName: DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: "audita-v1",
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: "failed",
|
||||
ModuleResults: []ModuleReport{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar_1",
|
||||
Status: "failed",
|
||||
ErrorMessage: "boom",
|
||||
StartedAt: &now,
|
||||
CompletedAt: &now,
|
||||
},
|
||||
},
|
||||
ModulesSummary: &ModulesSummary{ModuleCount: 1, FailedModuleInstance: "grammar_1"},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal report: %v", err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal map: %v", err)
|
||||
}
|
||||
if _, ok := decoded["module_results"]; !ok {
|
||||
t.Fatalf("expected module_results field")
|
||||
}
|
||||
if _, ok := decoded["report_metadata"]; !ok {
|
||||
t.Fatalf("expected report_metadata field")
|
||||
}
|
||||
if _, ok := decoded["modules_summary"]; !ok {
|
||||
t.Fatalf("expected modules_summary field")
|
||||
if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
|
||||
t.Fatalf("boolean changed: %s %v", payload, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,13 +45,8 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
|
||||
}
|
||||
|
||||
func TestReportSummaryAvailability(t *testing.T) {
|
||||
input := BuildInput{Status: "failed", StartedAt: time.Now(), CompletedAt: time.Now()}
|
||||
if got := Build(input).LLMSummary; got != nil {
|
||||
t.Fatalf("unavailable summary: %+v", got)
|
||||
}
|
||||
input.RunOutput = &runner.RunOutput{ObservationsAvailable: true, LLMSummary: llm.Summarize(llm.ObservationSnapshot{})}
|
||||
got := Build(input).LLMSummary
|
||||
if got == nil || got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||
got := mapLLMSummary(llm.Summarize(llm.ObservationSnapshot{}))
|
||||
if got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||
t.Fatalf("initialized empty summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,164 +1,252 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// BuildInput contains already-computed process execution facts for report assembly.
|
||||
// RunFailure describes a fatal or output failure without carrying its raw cause.
|
||||
type RunFailure struct {
|
||||
Phase string
|
||||
Reason outcomes.ReasonCode
|
||||
OutputPath string
|
||||
Origin *outcomes.Reference
|
||||
StageIndex *int
|
||||
}
|
||||
|
||||
type BuildInput struct {
|
||||
Status string
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
ConfigVersion *int
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
ErrorMessage string
|
||||
ErrorPhase string
|
||||
RunDirectoryPath string
|
||||
NormalizationSummary *normalization.NormalizationSummary
|
||||
ChunkingSummary *chunking.Summary
|
||||
RunOutput *runner.RunOutput
|
||||
Backends map[string]config.Backend
|
||||
RunID string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
RunOutput *runner.RunOutput
|
||||
RunFailures []RunFailure
|
||||
}
|
||||
|
||||
// Build creates the public process report without owning command parsing or config loading.
|
||||
func Build(input BuildInput) reporting.ProcessReport {
|
||||
report := reporting.ProcessReport{
|
||||
ReportMetadata: reporting.ReportMetadata{
|
||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
OutputSchema: input.OutputSchema,
|
||||
ConfigVersion: input.ConfigVersion,
|
||||
},
|
||||
Phase: "default_pipeline",
|
||||
Status: input.Status,
|
||||
Operation: "process",
|
||||
TranscriptPath: input.TranscriptPath,
|
||||
GlossaryPath: input.GlossaryPath,
|
||||
OutputPath: input.OutputPath,
|
||||
Modules: append([]string(nil), input.Modules...),
|
||||
StartedAt: input.StartedAt,
|
||||
CompletedAt: &input.CompletedAt,
|
||||
ErrorPhase: input.ErrorPhase,
|
||||
// Build maps processing facts once into an immutable presentation snapshot.
|
||||
func Build(input BuildInput) reporting.Snapshot {
|
||||
snapshot := reporting.Snapshot{
|
||||
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
|
||||
TranscriptPath: input.TranscriptPath, GlossaryPath: input.GlossaryPath, OutputPath: input.OutputPath,
|
||||
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
|
||||
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
|
||||
}
|
||||
if input.RunDirectoryPath != "" {
|
||||
runSucceeded := input.Status == "success"
|
||||
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||
report.Diagnostics = &metadata
|
||||
for i, spec := range input.ModuleSpecs {
|
||||
snapshot.StageNames[i] = spec.InstanceName
|
||||
}
|
||||
if input.ErrorMessage != "" {
|
||||
report.ErrorMessage = input.ErrorMessage
|
||||
}
|
||||
if input.NormalizationSummary != nil {
|
||||
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||
}
|
||||
if input.ChunkingSummary != nil {
|
||||
report.Chunking = &reporting.ChunkingSummary{
|
||||
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||
TargetSections: input.ChunkingSummary.TargetSections,
|
||||
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||
if input.RunOutput != nil && input.RunOutput.ObservationsAvailable {
|
||||
summary := mapLLMSummary(input.RunOutput.LLMSummary)
|
||||
report.LLMSummary = &summary
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||
for _, r := range runOutput.ModuleResults {
|
||||
startedAt := r.StartedAt
|
||||
completedAt := r.CompletedAt
|
||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||
ModuleKey: r.ModuleKey,
|
||||
ModuleInstance: r.ModuleInstance,
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
SkippedChanges: r.SkippedChanges,
|
||||
ErrorMessage: r.ErrorMessage,
|
||||
StartedAt: &startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
if input.RunOutput != nil {
|
||||
units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...)
|
||||
sort.SliceStable(units, func(i, j int) bool {
|
||||
if units[i].ModuleIndex != units[j].ModuleIndex {
|
||||
return units[i].ModuleIndex < units[j].ModuleIndex
|
||||
}
|
||||
return units[i].Section.Index < units[j].Section.Index
|
||||
})
|
||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||
summary.FailedModuleInstance = r.ModuleInstance
|
||||
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
|
||||
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
|
||||
for _, unit := range units {
|
||||
if unit.State == outcomes.WorkNotStarted {
|
||||
notStarted++
|
||||
} else {
|
||||
started++
|
||||
}
|
||||
if unit.State == outcomes.WorkCompleted {
|
||||
completed++
|
||||
}
|
||||
if unit.State == outcomes.WorkInterrupted {
|
||||
interrupted++
|
||||
}
|
||||
corrections += unit.CorrectionsApplied
|
||||
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied}
|
||||
operations := append([]outcomes.Operation(nil), unit.Operations...)
|
||||
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
|
||||
failures := make([]reporting.Failure, 0)
|
||||
for _, operation := range operations {
|
||||
if operation.Reference.Kind != outcomes.OperationValidationInput {
|
||||
llmOperations++
|
||||
}
|
||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||
proposalResponses++
|
||||
}
|
||||
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
|
||||
recoveryCount++
|
||||
row.RecoveryCount++
|
||||
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
|
||||
filteringResponses++
|
||||
}
|
||||
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
||||
}
|
||||
if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) {
|
||||
failures = append(failures, mapFailure(operation))
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
withErrors++
|
||||
row.ErrorCount = 1
|
||||
stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied
|
||||
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage,
|
||||
ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: &chunk,
|
||||
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
|
||||
}
|
||||
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
|
||||
}
|
||||
runErrors := len(input.RunFailures)
|
||||
snapshot.Summary = reporting.Summary{
|
||||
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
|
||||
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors,
|
||||
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
|
||||
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
|
||||
ProposalResponsesWithFiltering: &filteringResponses,
|
||||
}
|
||||
} else if len(input.RunFailures) > 0 {
|
||||
runErrors := len(input.RunFailures)
|
||||
snapshot.Summary.RunErrors = &runErrors
|
||||
}
|
||||
|
||||
return summary, moduleReports
|
||||
for _, failure := range sortedRunFailures(input.RunFailures) {
|
||||
record := reporting.ErrorRecord{Scope: "run", Phase: failure.Phase, Reason: string(failure.Reason),
|
||||
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
|
||||
if failure.Origin != nil {
|
||||
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
|
||||
chunk := failure.Origin.SectionIndex
|
||||
record.ChunkIndex = &chunk
|
||||
record.Operation = string(failure.Origin.Kind)
|
||||
record.Validator = failure.Origin.ValidatorName
|
||||
record.BatchIndex = failure.Origin.BatchIndex
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
func mapFailure(operation outcomes.Operation) reporting.Failure {
|
||||
indices := append([]int(nil), operation.ProposalIndices...)
|
||||
sort.Ints(indices)
|
||||
return reporting.Failure{Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName,
|
||||
BatchIndex: operation.Reference.BatchIndex, Reason: string(operation.FailureReason), Message: safeMessage(operation.FailureReason),
|
||||
ProposalIndices: indices, ExecutionCounts: mapCounts(operation.Counts)}
|
||||
}
|
||||
|
||||
func mapRecovery(unit runner.WorkUnitOutcome, operation outcomes.Operation) reporting.RecoveryRecord {
|
||||
reasons := make([]string, len(operation.RecoveryReasons))
|
||||
for i, reason := range operation.RecoveryReasons {
|
||||
reasons[i] = string(reason)
|
||||
}
|
||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorDecisionReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||
for i, d := range in {
|
||||
out[i] = reporting.ValidatorRejectedReport{
|
||||
ValidatorName: d.ValidatorName,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
ModuleKey: d.ModuleKey,
|
||||
ModuleInstance: d.ModuleInstance,
|
||||
TargetSegmentID: d.TargetSegmentID,
|
||||
OriginalText: d.OriginalText,
|
||||
CorrectedText: d.CorrectedText,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
sort.Strings(reasons)
|
||||
filtered := make([]reporting.FilteredProposal, len(operation.Filtered))
|
||||
for i, item := range operation.Filtered {
|
||||
filtered[i] = reporting.FilteredProposal{ProposalIndex: item.ProposalIndex, Reason: string(item.Reason), Count: item.Count}
|
||||
}
|
||||
sort.Slice(filtered, func(i, j int) bool { return filtered[i].ProposalIndex < filtered[j].ProposalIndex })
|
||||
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||
Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName, BatchIndex: operation.Reference.BatchIndex,
|
||||
Reasons: reasons, Message: "Model operation succeeded after recovery or proposal filtering.",
|
||||
ExecutionCounts: mapCounts(operation.Counts), FilteredProposals: filtered}
|
||||
}
|
||||
|
||||
func mapCounts(counts outcomes.ExecutionCounts) reporting.ExecutionCounts {
|
||||
if !counts.Available {
|
||||
return reporting.ExecutionCounts{}
|
||||
}
|
||||
attempts, retries, repairs, exhausted := counts.ProviderAttempts, counts.TransientRetries, counts.StructuralRepairs, counts.RetriesExhausted
|
||||
return reporting.ExecutionCounts{ProviderAttempts: &attempts, TransientRetries: &retries,
|
||||
StructuralRepairs: &repairs, RetriesExhausted: &exhausted, HTTPStatus: counts.HTTPStatus}
|
||||
}
|
||||
|
||||
func isRecoverableReason(reason outcomes.ReasonCode) bool {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout, outcomes.ReasonCapacityExceeded, outcomes.ReasonTransportFailed,
|
||||
outcomes.ReasonProviderResponseInvalid, outcomes.ReasonProviderFailed,
|
||||
outcomes.ReasonInvalidOutputExhausted, outcomes.ReasonInvalidOutput,
|
||||
outcomes.ReasonValidationInputTooLarge, outcomes.ReasonValidationDecisionsInvalid:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func safeMessage(reason outcomes.ReasonCode) string {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout:
|
||||
return "Model operation timed out."
|
||||
case outcomes.ReasonCapacityExceeded:
|
||||
return "Model capacity was unavailable."
|
||||
case outcomes.ReasonTransportFailed:
|
||||
return "Model transport failed."
|
||||
case outcomes.ReasonProviderResponseInvalid:
|
||||
return "Provider response was invalid."
|
||||
case outcomes.ReasonProviderFailed:
|
||||
return "Provider request failed."
|
||||
case outcomes.ReasonInvalidOutputExhausted:
|
||||
return "Model output remained invalid after repair attempts."
|
||||
case outcomes.ReasonInvalidOutput:
|
||||
return "Model output was invalid."
|
||||
case outcomes.ReasonValidationInputTooLarge:
|
||||
return "Validation input exceeded its size limit."
|
||||
case outcomes.ReasonValidationDecisionsInvalid:
|
||||
return "Validation decisions were unusable."
|
||||
case outcomes.ReasonCanceled:
|
||||
return "Processing was canceled."
|
||||
case outcomes.ReasonSetupFailure:
|
||||
return "Processing setup failed."
|
||||
default:
|
||||
return "Processing failed."
|
||||
}
|
||||
}
|
||||
|
||||
func operationLess(a, b outcomes.Operation) bool {
|
||||
rank := func(kind outcomes.OperationKind) int {
|
||||
switch kind {
|
||||
case outcomes.OperationProposal:
|
||||
return 0
|
||||
case outcomes.OperationValidationInput:
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
|
||||
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
|
||||
}
|
||||
order := func(p *int) int {
|
||||
if p == nil {
|
||||
return -1
|
||||
}
|
||||
return *p
|
||||
}
|
||||
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
|
||||
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
|
||||
}
|
||||
return rank(a.Reference.Kind) < rank(b.Reference.Kind) || (rank(a.Reference.Kind) == rank(b.Reference.Kind) && order(a.Reference.BatchIndex) < order(b.Reference.BatchIndex))
|
||||
}
|
||||
|
||||
func sortedRunFailures(in []RunFailure) []RunFailure {
|
||||
out := append([]RunFailure(nil), in...)
|
||||
phase := map[string]int{"configuration": 0, "destination": 1, "initialization": 2, "input": 3, "normalization": 4, "chunking": 5, "processing": 6, "encoding": 7, "transcript_write": 8, "report_write": 9, "debug": 10}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
a, aok := phase[out[i].Phase]
|
||||
b, bok := phase[out[j].Phase]
|
||||
if !aok {
|
||||
a = 100
|
||||
}
|
||||
if !bok {
|
||||
b = 100
|
||||
}
|
||||
if a != b {
|
||||
return a < b
|
||||
}
|
||||
return out[i].OutputPath < out[j].OutputPath
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,180 +1,78 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
configVersion := 4
|
||||
targetSections := 2
|
||||
inputSegments := 5
|
||||
outputSegments := 4
|
||||
merges := 1
|
||||
reassigned := 2
|
||||
differentSpeakers := 3
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "success",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
OutputPath: "out.json",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
ConfigVersion: &configVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
RunDirectoryPath: filepath.Join(
|
||||
"tmp",
|
||||
"audita-run",
|
||||
),
|
||||
NormalizationSummary: &normalization.NormalizationSummary{
|
||||
InputSegmentCount: inputSegments,
|
||||
OutputSegmentCount: outputSegments,
|
||||
MergesPerformed: merges,
|
||||
IDsReassigned: reassigned,
|
||||
SkippedMerges: struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
}{
|
||||
DifferentSpeakers: differentSpeakers,
|
||||
},
|
||||
},
|
||||
ChunkingSummary: &chunking.Summary{
|
||||
ChunkCount: 3,
|
||||
MinEstimatedTokens: 10,
|
||||
MaxEstimatedTokens: 20,
|
||||
TotalEstimatedTokens: 45,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: 200,
|
||||
MinSectionTokens: 50,
|
||||
},
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Status: runner.ModuleStatusSuccess,
|
||||
ProposalCount: 2,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{
|
||||
ValidatorName: "proposal_shape",
|
||||
ProposalIndex: 1,
|
||||
Approved: true,
|
||||
ReasonCode: "approved",
|
||||
Message: "ok",
|
||||
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||
},
|
||||
},
|
||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||
{
|
||||
ValidatorName: "protected_term",
|
||||
ProposalIndex: 2,
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "old",
|
||||
CorrectedText: "new",
|
||||
ReasonCode: "protected_term",
|
||||
Message: "blocked",
|
||||
},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||
},
|
||||
SkippedChanges: []proposals.SkippedChange{
|
||||
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||
},
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||
report.ReportMetadata.OutputSchema != "default" ||
|
||||
report.ReportMetadata.ConfigVersion == nil ||
|
||||
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||
func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
|
||||
started := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
|
||||
completed := started.Add(time.Minute)
|
||||
proposal := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
|
||||
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, TransientRetries: 1, StructuralRepairs: 1},
|
||||
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
|
||||
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
|
||||
batch := 1
|
||||
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||
output := &runner.RunOutput{WorkUnits: []runner.WorkUnitOutcome{
|
||||
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
|
||||
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
|
||||
}}
|
||||
snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output})
|
||||
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
|
||||
t.Fatalf("summary: %+v", snapshot.Summary)
|
||||
}
|
||||
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 2 || snapshot.Errors[0].Failures[0].Operation != "validation_input" || !reflect.DeepEqual(snapshot.Errors[0].Failures[1].ProposalIndices, []int{3, 9}) {
|
||||
t.Fatalf("failure ordering: %+v", snapshot.Errors)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"output_repaired", "proposals_filtered", "service_recovered"}) || snapshot.Recoveries[0].FilteredProposals[0].ProposalIndex != 2 {
|
||||
t.Fatalf("recovery: %+v", snapshot.Recoveries)
|
||||
}
|
||||
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||
if snapshot.WorkUnits[0].ModuleInstance != "grammar_1" || snapshot.WorkUnits[1].State != "not_started" {
|
||||
t.Fatalf("unit order: %+v", snapshot.WorkUnits)
|
||||
}
|
||||
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ModulesSummary == nil ||
|
||||
report.ModulesSummary.ModuleCount != 1 ||
|
||||
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||
var decoded map[string]any
|
||||
if err := yaml.Unmarshal(errorsYAML, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||
if decoded["version"] != 1 || decoded["schema"] != "audita-errors" || decoded["status"] != "completed" {
|
||||
t.Fatalf("envelope: %#v", decoded)
|
||||
}
|
||||
if strings.Contains(string(errorsYAML), "provider_attempts: null") || !strings.Contains(string(errorsYAML), "errors:") {
|
||||
t.Fatal(string(errorsYAML))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Second)
|
||||
|
||||
report := Build(BuildInput{
|
||||
Status: "failed",
|
||||
TranscriptPath: "transcript.json",
|
||||
GlossaryPath: "glossary.yaml",
|
||||
Modules: []string{"grammar"},
|
||||
OutputSchema: "default",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorPhase: "module",
|
||||
ErrorMessage: "module failed",
|
||||
RunDirectoryPath: "run-dir",
|
||||
RunOutput: &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "grammar",
|
||||
ModuleInstance: "grammar",
|
||||
Status: runner.ModuleStatusFailed,
|
||||
ErrorMessage: "module failed",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||
t.Fatalf("unexpected failure fields: %+v", report)
|
||||
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})
|
||||
initialDoc := initial.ErrorsDocument()
|
||||
if initialDoc.CompletedAt != nil || initialDoc.Summary.WorkUnitsTotal != nil || initialDoc.Errors == nil {
|
||||
t.Fatalf("initial: %+v", initialDoc)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||
finished := Build(BuildInput{RunID: "r", Status: reporting.StatusCompleted, StartedAt: start, RunOutput: &runner.RunOutput{}})
|
||||
if finished.Summary.WorkUnitsTotal == nil || *finished.Summary.WorkUnitsTotal != 0 || finished.RecoveriesDocument().Recoveries == nil {
|
||||
t.Fatalf("empty: %+v", finished)
|
||||
}
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||
failed := Build(BuildInput{RunID: "r", Status: reporting.StatusFailed, StartedAt: start,
|
||||
RunFailures: []RunFailure{{Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "z.yml"}, {Phase: "processing", Reason: outcomes.ReasonCanceled}, {Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "a.yml"}}})
|
||||
if len(failed.Errors) != 3 || failed.Errors[0].Phase != "processing" || failed.Errors[1].OutputPath != "a.yml" || failed.Errors[2].OutputPath != "z.yml" || failed.Summary.WorkUnitsTotal != nil || *failed.Summary.RunErrors != 3 {
|
||||
t.Fatalf("run failures: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(value int) *int { return &value }
|
||||
|
||||
Reference in New Issue
Block a user