Map processing outcomes to versioned YAML reports
This commit is contained in:
@@ -619,6 +619,16 @@ filesystem requirements for disabled features.
|
|||||||
|
|
||||||
## Stage 8 — Map Outcomes to Versioned YAML Reports
|
## Stage 8 — Map Outcomes to Versioned YAML Reports
|
||||||
|
|
||||||
|
**Completed 2026-09-24.** The old ordinary JSON report DTO was replaced with
|
||||||
|
versioned errors/recoveries YAML DTOs and one processing snapshot. Mapping derives
|
||||||
|
nullable summary counts, grouped recoverable work-unit failures, run failures,
|
||||||
|
combined recovery reasons, and deterministic semantic order without raw causes or
|
||||||
|
transcript payloads. Tests cover partial and empty work, multiple failures,
|
||||||
|
initialization unknowns, ordered run errors, safe redaction, and YAML scalar
|
||||||
|
types. `go test ./internal/core/reporting ./internal/framework/processreport`
|
||||||
|
and `git diff --check` passed. CLI and diagnostics consumers of the removed JSON
|
||||||
|
schema remain to be reconnected in later stages.
|
||||||
|
|
||||||
**Work:** Replace ordinary report DTOs and assembly in `core/reporting` and
|
**Work:** Replace ordinary report DTOs and assembly in `core/reporting` and
|
||||||
`framework/processreport` with F. Map actual runner state once into an immutable
|
`framework/processreport` with F. Map actual runner state once into an immutable
|
||||||
report snapshot consumed by all presentations. Preserve debug-specific ledger and
|
report snapshot consumed by all presentations. Preserve debug-specific ledger and
|
||||||
|
|||||||
@@ -1,138 +1,180 @@
|
|||||||
|
// Package reporting owns the public report schema, separate from processing.
|
||||||
package reporting
|
package reporting
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gopkg.in/yaml.v3"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ProcessReport struct {
|
|
||||||
ReportMetadata ReportMetadata `json:"report_metadata"`
|
|
||||||
Phase string `json:"phase"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
Operation string `json:"operation"`
|
|
||||||
TranscriptPath string `json:"transcript_path"`
|
|
||||||
GlossaryPath string `json:"glossary_path"`
|
|
||||||
OutputPath string `json:"output_path,omitempty"`
|
|
||||||
Modules []string `json:"modules"`
|
|
||||||
StartedAt time.Time `json:"started_at"`
|
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
|
||||||
ErrorMessage string `json:"error_message,omitempty"`
|
|
||||||
ErrorPhase string `json:"error_phase,omitempty"`
|
|
||||||
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
|
||||||
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
|
||||||
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
|
||||||
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
|
||||||
NormalizationSkipped NormalizationSkipped `json:"normalization_skipped,omitempty"`
|
|
||||||
Chunking *ChunkingSummary `json:"chunking,omitempty"`
|
|
||||||
Diagnostics *DiagnosticsMetadata `json:"diagnostics,omitempty"`
|
|
||||||
LLMSummary *LLMSummary `json:"llm_summary,omitempty"`
|
|
||||||
ModulesSummary *ModulesSummary `json:"modules_summary,omitempty"`
|
|
||||||
ModuleResults []ModuleReport `json:"module_results,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultProcessReportSchemaName = "audita-process-report"
|
StatusInProgress = "in_progress"
|
||||||
DefaultProcessReportSchemaVersion = "v2"
|
StatusCompleted = "completed"
|
||||||
|
StatusFailed = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ReportMetadata struct {
|
// Nil counts mean unavailable; a pointer to zero means a known empty count.
|
||||||
ReportSchemaName string `json:"report_schema_name"`
|
type Summary struct {
|
||||||
ReportSchemaVersion string `json:"report_schema_version"`
|
WorkUnitsTotal *int `yaml:"work_units_total"`
|
||||||
OutputSchema string `json:"output_schema"`
|
WorkUnitsStarted *int `yaml:"work_units_started"`
|
||||||
ConfigVersion *int `json:"config_version,omitempty"`
|
WorkUnitsCompleted *int `yaml:"work_units_completed"`
|
||||||
|
WorkUnitsInterrupted *int `yaml:"work_units_interrupted"`
|
||||||
|
WorkUnitsNotStarted *int `yaml:"work_units_not_started"`
|
||||||
|
WorkUnitsWithErrors *int `yaml:"work_units_with_errors"`
|
||||||
|
RunErrors *int `yaml:"run_errors"`
|
||||||
|
CorrectionsApplied *int `yaml:"corrections_applied"`
|
||||||
|
LLMOperations *int `yaml:"llm_operations"`
|
||||||
|
ProposalResponses *int `yaml:"proposal_responses"`
|
||||||
|
Recoveries *int `yaml:"recoveries"`
|
||||||
|
ProposalResponsesWithFiltering *int `yaml:"proposal_responses_with_filtering"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ModuleReport struct {
|
type ExecutionCounts struct {
|
||||||
ModuleKey string `json:"module_key"`
|
ProviderAttempts *int `yaml:"provider_attempts,omitempty"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
TransientRetries *int `yaml:"transient_retries,omitempty"`
|
||||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
StructuralRepairs *int `yaml:"structural_repairs,omitempty"`
|
||||||
Status string `json:"status"`
|
RetriesExhausted *bool `yaml:"retries_exhausted,omitempty"`
|
||||||
ProposalCount int `json:"proposal_count"`
|
HTTPStatus *int `yaml:"http_status,omitempty"`
|
||||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
|
||||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
|
||||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
|
||||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
|
||||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
|
||||||
ErrorMessage string `json:"error_message,omitempty"`
|
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ValidatorDecisionReport struct {
|
type Failure struct {
|
||||||
ValidatorName string `json:"validator_name"`
|
Operation string `yaml:"operation"`
|
||||||
ProposalIndex int `json:"proposal_index"`
|
Validator string `yaml:"validator,omitempty"`
|
||||||
Approved bool `json:"approved"`
|
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||||
ReasonCode string `json:"reason_code"`
|
Reason string `yaml:"reason"`
|
||||||
Message string `json:"message"`
|
Message string `yaml:"message"`
|
||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
ProposalIndices []int `yaml:"proposal_indices,omitempty"`
|
||||||
|
ExecutionCounts `yaml:",inline"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ValidatorRejectedReport struct {
|
type ErrorRecord struct {
|
||||||
ValidatorName string `json:"validator_name"`
|
Scope string `yaml:"scope"`
|
||||||
ProposalIndex int `json:"proposal_index"`
|
StageIndex *int `yaml:"stage_index,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `yaml:"module_key,omitempty"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `yaml:"module_instance,omitempty"`
|
||||||
TargetSegmentID int `json:"target_segment_id"`
|
ChunkIndex *int `yaml:"chunk_index,omitempty"`
|
||||||
OriginalText string `json:"original_text"`
|
StartSegmentID *int `yaml:"start_segment_id,omitempty"`
|
||||||
CorrectedText string `json:"corrected_text"`
|
EndSegmentID *int `yaml:"end_segment_id,omitempty"`
|
||||||
ReasonCode string `json:"reason_code"`
|
CorrectionsApplied *int `yaml:"corrections_applied,omitempty"`
|
||||||
Message string `json:"message"`
|
Failures []Failure `yaml:"failures,omitempty"`
|
||||||
|
Phase string `yaml:"phase,omitempty"`
|
||||||
|
Reason string `yaml:"reason,omitempty"`
|
||||||
|
Message string `yaml:"message,omitempty"`
|
||||||
|
Operation string `yaml:"operation,omitempty"`
|
||||||
|
Validator string `yaml:"validator,omitempty"`
|
||||||
|
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||||
|
OutputPath string `yaml:"output_path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ModulesSummary struct {
|
type FilteredProposal struct {
|
||||||
ModuleCount int `json:"module_count"`
|
ProposalIndex int `yaml:"proposal_index"`
|
||||||
TotalAppliedChanges int `json:"total_applied_changes"`
|
Reason string `yaml:"reason"`
|
||||||
TotalSkippedChanges int `json:"total_skipped_changes"`
|
Count int `yaml:"count"`
|
||||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type DiagnosticsMetadata struct {
|
type RecoveryRecord struct {
|
||||||
DirectoryPath string `json:"directory_path,omitempty"`
|
StageIndex int `yaml:"stage_index"`
|
||||||
SourceTranscriptPath string `json:"source_transcript_path,omitempty"`
|
ModuleKey string `yaml:"module_key"`
|
||||||
ParsedSourceTranscriptPath string `json:"parsed_source_transcript_path,omitempty"`
|
ModuleInstance string `yaml:"module_instance"`
|
||||||
NormalizedTranscriptPath string `json:"normalized_transcript_path,omitempty"`
|
ChunkIndex int `yaml:"chunk_index"`
|
||||||
NormalizationSummaryPath string `json:"normalization_summary_path,omitempty"`
|
StartSegmentID int `yaml:"start_segment_id"`
|
||||||
ChunkingSummaryPath string `json:"chunking_summary_path,omitempty"`
|
EndSegmentID int `yaml:"end_segment_id"`
|
||||||
UtilizationSummaryPath string `json:"utilization_summary_path,omitempty"`
|
Operation string `yaml:"operation"`
|
||||||
LLMExecutionPath string `json:"llm_execution_path,omitempty"`
|
Validator string `yaml:"validator,omitempty"`
|
||||||
SelectedProfilesPath string `json:"selected_profiles_path,omitempty"`
|
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||||
CorrectionLedgerPath string `json:"correction_ledger_path,omitempty"`
|
Reasons []string `yaml:"reasons"`
|
||||||
InvocationMetadataPath string `json:"invocation_metadata_path,omitempty"`
|
Message string `yaml:"message"`
|
||||||
RedactedEffectiveConfigPath string `json:"redacted_effective_config_path,omitempty"`
|
ExecutionCounts `yaml:",inline"`
|
||||||
ErrorLogPath string `json:"error_log_path,omitempty"`
|
FilteredProposals []FilteredProposal `yaml:"filtered_proposals,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type NormalizationSkipped struct {
|
type ErrorsDocument struct {
|
||||||
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
Schema string `yaml:"schema"`
|
||||||
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
Version int `yaml:"version"`
|
||||||
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
RunID string `yaml:"run_id"`
|
||||||
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
Status string `yaml:"status"`
|
||||||
|
StartedAt string `yaml:"started_at"`
|
||||||
|
CompletedAt *string `yaml:"completed_at"`
|
||||||
|
Summary Summary `yaml:"summary"`
|
||||||
|
Errors []ErrorRecord `yaml:"errors"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkingSummary struct {
|
type RecoveriesDocument struct {
|
||||||
ChunkCount int `json:"chunk_count"`
|
Schema string `yaml:"schema"`
|
||||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
Version int `yaml:"version"`
|
||||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
RunID string `yaml:"run_id"`
|
||||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
Status string `yaml:"status"`
|
||||||
TargetSections *int `json:"target_sections,omitempty"`
|
StartedAt string `yaml:"started_at"`
|
||||||
MaxSectionTokens int `json:"max_section_tokens"`
|
CompletedAt *string `yaml:"completed_at"`
|
||||||
MinSectionTokens int `json:"min_section_tokens"`
|
Summary Summary `yaml:"summary"`
|
||||||
|
Recoveries []RecoveryRecord `yaml:"recoveries"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteProcessReport(path string, report ProcessReport, secrets ...string) error {
|
// WorkUnit is payload-free matrix input for report.md.
|
||||||
payload, err := redaction.New(secrets).JSON(report)
|
type WorkUnit struct {
|
||||||
|
StageIndex int
|
||||||
|
ModuleKey string
|
||||||
|
ModuleInstance string
|
||||||
|
ChunkIndex int
|
||||||
|
StartSegmentID int
|
||||||
|
EndSegmentID int
|
||||||
|
State string
|
||||||
|
CorrectionsApplied int
|
||||||
|
ErrorCount int
|
||||||
|
RecoveryCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot is built once from processing state and shared by presentations.
|
||||||
|
type Snapshot struct {
|
||||||
|
RunID string
|
||||||
|
Status string
|
||||||
|
StartedAt time.Time
|
||||||
|
CompletedAt *time.Time
|
||||||
|
TranscriptPath string
|
||||||
|
GlossaryPath string
|
||||||
|
OutputPath string
|
||||||
|
StageNames []string
|
||||||
|
WorkUnits []WorkUnit
|
||||||
|
Summary Summary
|
||||||
|
Errors []ErrorRecord
|
||||||
|
Recoveries []RecoveryRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Snapshot) ErrorsDocument() ErrorsDocument {
|
||||||
|
errors := s.Errors
|
||||||
|
if errors == nil {
|
||||||
|
errors = []ErrorRecord{}
|
||||||
|
}
|
||||||
|
return ErrorsDocument{Schema: "audita-errors", Version: 1, RunID: s.RunID, Status: s.Status, StartedAt: s.StartedAt.UTC().Format(time.RFC3339), CompletedAt: completionTime(s.CompletedAt), Summary: s.Summary, Errors: errors}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Snapshot) RecoveriesDocument() RecoveriesDocument {
|
||||||
|
recoveries := s.Recoveries
|
||||||
|
if recoveries == nil {
|
||||||
|
recoveries = []RecoveryRecord{}
|
||||||
|
}
|
||||||
|
return RecoveriesDocument{Schema: "audita-recoveries", Version: 1, RunID: s.RunID, Status: s.Status, StartedAt: s.StartedAt.UTC().Format(time.RFC3339), CompletedAt: completionTime(s.CompletedAt), Summary: s.Summary, Recoveries: recoveries}
|
||||||
|
}
|
||||||
|
|
||||||
|
func completionTime(t *time.Time) *string {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value := t.UTC().Format(time.RFC3339)
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
|
// YAML redacts only string nodes after typed serialization.
|
||||||
|
func YAML(document any, secrets ...string) ([]byte, error) {
|
||||||
|
raw, err := yaml.Marshal(document)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal process report JSON: %w", err)
|
return nil, fmt.Errorf("encode report YAML: %w", err)
|
||||||
}
|
}
|
||||||
payload = append(payload, '\n')
|
safe, err := redaction.New(secrets).YAML(raw)
|
||||||
|
if err != nil {
|
||||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
return nil, fmt.Errorf("redact report YAML: %w", err)
|
||||||
return fmt.Errorf("failed to write report JSON file %q: %w", path, err)
|
|
||||||
}
|
}
|
||||||
return nil
|
return safe, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,130 +1,41 @@
|
|||||||
package reporting
|
package reporting
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
|
func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
|
||||||
now := time.Now().UTC()
|
zero, one := 0, 1
|
||||||
report := ProcessReport{
|
falseValue := false
|
||||||
ReportMetadata: ReportMetadata{
|
snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC),
|
||||||
ReportSchemaName: DefaultProcessReportSchemaName,
|
Summary: Summary{RunErrors: &zero, CorrectionsApplied: &one},
|
||||||
ReportSchemaVersion: DefaultProcessReportSchemaVersion,
|
Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}},
|
||||||
OutputSchema: "bare-segments",
|
Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}}
|
||||||
},
|
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
|
||||||
Phase: "default_pipeline",
|
payload, err := YAML(document, `secret\"key`, "1")
|
||||||
Status: "success",
|
if err != nil {
|
||||||
ModuleResults: []ModuleReport{
|
t.Fatal(err)
|
||||||
{
|
}
|
||||||
ModuleKey: "grammar",
|
if strings.Contains(string(payload), `secret\"key`) {
|
||||||
ModuleInstance: "grammar_1",
|
t.Fatalf("secret leaked: %s", payload)
|
||||||
ReplacementPolicy: "require_unique",
|
}
|
||||||
Status: "success",
|
var decoded map[string]any
|
||||||
ProposalCount: 2,
|
if err := yaml.Unmarshal(payload, &decoded); err != nil {
|
||||||
ValidatorDecisions: []ValidatorDecisionReport{{
|
t.Fatal(err)
|
||||||
ValidatorName: "confidence_threshold",
|
}
|
||||||
ProposalIndex: 0,
|
if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
|
||||||
Approved: true,
|
t.Fatalf("wrong envelope types: %#v", decoded)
|
||||||
ReasonCode: "approved",
|
}
|
||||||
Message: "approved",
|
summary := decoded["summary"].(map[string]any)
|
||||||
}},
|
if summary["corrections_applied"] != 1 || summary["work_units_total"] != nil {
|
||||||
ValidatorRejected: []ValidatorRejectedReport{{
|
t.Fatalf("changed count types: %#v", summary)
|
||||||
ValidatorName: "confidence_threshold",
|
}
|
||||||
ProposalIndex: 1,
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
ModuleInstance: "grammar_1",
|
|
||||||
TargetSegmentID: 1,
|
|
||||||
OriginalText: "bad",
|
|
||||||
CorrectedText: "worse",
|
|
||||||
ReasonCode: "low_confidence",
|
|
||||||
Message: "rejected",
|
|
||||||
}},
|
|
||||||
AppliedChanges: []proposals.AppliedChange{{
|
|
||||||
ProposalIndex: 0,
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
}},
|
|
||||||
SkippedChanges: []proposals.SkippedChange{{
|
|
||||||
ProposalIndex: 1,
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
SkipReason: proposals.SkipReasonMissingOriginalText,
|
|
||||||
}},
|
|
||||||
StartedAt: &now,
|
|
||||||
CompletedAt: &now,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ModulesSummary: &ModulesSummary{ModuleCount: 1, TotalAppliedChanges: 1, TotalSkippedChanges: 1},
|
|
||||||
}
|
}
|
||||||
|
if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
|
||||||
b, err := json.Marshal(report)
|
t.Fatalf("boolean changed: %s %v", payload, err)
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal report: %v", err)
|
|
||||||
}
|
|
||||||
if !json.Valid(b) {
|
|
||||||
t.Fatalf("expected valid json")
|
|
||||||
}
|
|
||||||
|
|
||||||
var parsed ProcessReport
|
|
||||||
if err := json.Unmarshal(b, &parsed); err != nil {
|
|
||||||
t.Fatalf("unmarshal report: %v", err)
|
|
||||||
}
|
|
||||||
if len(parsed.ModuleResults) != 1 {
|
|
||||||
t.Fatalf("expected one module result, got %d", len(parsed.ModuleResults))
|
|
||||||
}
|
|
||||||
if len(parsed.ModuleResults[0].ValidatorDecisions) != 1 || len(parsed.ModuleResults[0].ValidatorRejected) != 1 {
|
|
||||||
t.Fatalf("expected validator result payloads in module report")
|
|
||||||
}
|
|
||||||
if parsed.ModulesSummary == nil || parsed.ModulesSummary.TotalSkippedChanges != 1 {
|
|
||||||
t.Fatalf("unexpected module summary: %+v", parsed.ModulesSummary)
|
|
||||||
}
|
|
||||||
if parsed.ReportMetadata.ReportSchemaName != DefaultProcessReportSchemaName ||
|
|
||||||
parsed.ReportMetadata.ReportSchemaVersion != DefaultProcessReportSchemaVersion ||
|
|
||||||
parsed.ReportMetadata.OutputSchema != "bare-segments" {
|
|
||||||
t.Fatalf("unexpected report metadata after roundtrip: %+v", parsed.ReportMetadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
report := ProcessReport{
|
|
||||||
ReportMetadata: ReportMetadata{
|
|
||||||
ReportSchemaName: DefaultProcessReportSchemaName,
|
|
||||||
ReportSchemaVersion: DefaultProcessReportSchemaVersion,
|
|
||||||
OutputSchema: "audita-v1",
|
|
||||||
},
|
|
||||||
Phase: "default_pipeline",
|
|
||||||
Status: "failed",
|
|
||||||
ModuleResults: []ModuleReport{
|
|
||||||
{
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
ModuleInstance: "grammar_1",
|
|
||||||
Status: "failed",
|
|
||||||
ErrorMessage: "boom",
|
|
||||||
StartedAt: &now,
|
|
||||||
CompletedAt: &now,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ModulesSummary: &ModulesSummary{ModuleCount: 1, FailedModuleInstance: "grammar_1"},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(report)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal report: %v", err)
|
|
||||||
}
|
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal map: %v", err)
|
|
||||||
}
|
|
||||||
if _, ok := decoded["module_results"]; !ok {
|
|
||||||
t.Fatalf("expected module_results field")
|
|
||||||
}
|
|
||||||
if _, ok := decoded["report_metadata"]; !ok {
|
|
||||||
t.Fatalf("expected report_metadata field")
|
|
||||||
}
|
|
||||||
if _, ok := decoded["modules_summary"]; !ok {
|
|
||||||
t.Fatalf("expected modules_summary field")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,13 +45,8 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReportSummaryAvailability(t *testing.T) {
|
func TestReportSummaryAvailability(t *testing.T) {
|
||||||
input := BuildInput{Status: "failed", StartedAt: time.Now(), CompletedAt: time.Now()}
|
got := mapLLMSummary(llm.Summarize(llm.ObservationSnapshot{}))
|
||||||
if got := Build(input).LLMSummary; got != nil {
|
if got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||||
t.Fatalf("unavailable summary: %+v", got)
|
|
||||||
}
|
|
||||||
input.RunOutput = &runner.RunOutput{ObservationsAvailable: true, LLMSummary: llm.Summarize(llm.ObservationSnapshot{})}
|
|
||||||
got := Build(input).LLMSummary
|
|
||||||
if got == nil || got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
|
||||||
t.Fatalf("initialized empty summary: %+v", got)
|
t.Fatalf("initialized empty summary: %+v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,164 +1,252 @@
|
|||||||
package processreport
|
package processreport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// BuildInput contains already-computed process execution facts for report assembly.
|
// RunFailure describes a fatal or output failure without carrying its raw cause.
|
||||||
|
type RunFailure struct {
|
||||||
|
Phase string
|
||||||
|
Reason outcomes.ReasonCode
|
||||||
|
OutputPath string
|
||||||
|
Origin *outcomes.Reference
|
||||||
|
StageIndex *int
|
||||||
|
}
|
||||||
|
|
||||||
type BuildInput struct {
|
type BuildInput struct {
|
||||||
Status string
|
RunID string
|
||||||
TranscriptPath string
|
Status string
|
||||||
GlossaryPath string
|
StartedAt time.Time
|
||||||
OutputPath string
|
CompletedAt *time.Time
|
||||||
Modules []string
|
TranscriptPath string
|
||||||
OutputSchema string
|
GlossaryPath string
|
||||||
ConfigVersion *int
|
OutputPath string
|
||||||
StartedAt time.Time
|
ModuleSpecs []contracts.ModuleRunSpec
|
||||||
CompletedAt time.Time
|
RunOutput *runner.RunOutput
|
||||||
ErrorMessage string
|
RunFailures []RunFailure
|
||||||
ErrorPhase string
|
|
||||||
RunDirectoryPath string
|
|
||||||
NormalizationSummary *normalization.NormalizationSummary
|
|
||||||
ChunkingSummary *chunking.Summary
|
|
||||||
RunOutput *runner.RunOutput
|
|
||||||
Backends map[string]config.Backend
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build creates the public process report without owning command parsing or config loading.
|
// Build maps processing facts once into an immutable presentation snapshot.
|
||||||
func Build(input BuildInput) reporting.ProcessReport {
|
func Build(input BuildInput) reporting.Snapshot {
|
||||||
report := reporting.ProcessReport{
|
snapshot := reporting.Snapshot{
|
||||||
ReportMetadata: reporting.ReportMetadata{
|
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
|
||||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
TranscriptPath: input.TranscriptPath, GlossaryPath: input.GlossaryPath, OutputPath: input.OutputPath,
|
||||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
|
||||||
OutputSchema: input.OutputSchema,
|
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
|
||||||
ConfigVersion: input.ConfigVersion,
|
|
||||||
},
|
|
||||||
Phase: "default_pipeline",
|
|
||||||
Status: input.Status,
|
|
||||||
Operation: "process",
|
|
||||||
TranscriptPath: input.TranscriptPath,
|
|
||||||
GlossaryPath: input.GlossaryPath,
|
|
||||||
OutputPath: input.OutputPath,
|
|
||||||
Modules: append([]string(nil), input.Modules...),
|
|
||||||
StartedAt: input.StartedAt,
|
|
||||||
CompletedAt: &input.CompletedAt,
|
|
||||||
ErrorPhase: input.ErrorPhase,
|
|
||||||
}
|
}
|
||||||
if input.RunDirectoryPath != "" {
|
for i, spec := range input.ModuleSpecs {
|
||||||
runSucceeded := input.Status == "success"
|
snapshot.StageNames[i] = spec.InstanceName
|
||||||
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
|
||||||
report.Diagnostics = &metadata
|
|
||||||
}
|
}
|
||||||
if input.ErrorMessage != "" {
|
if input.RunOutput != nil {
|
||||||
report.ErrorMessage = input.ErrorMessage
|
units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...)
|
||||||
}
|
sort.SliceStable(units, func(i, j int) bool {
|
||||||
if input.NormalizationSummary != nil {
|
if units[i].ModuleIndex != units[j].ModuleIndex {
|
||||||
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
return units[i].ModuleIndex < units[j].ModuleIndex
|
||||||
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
}
|
||||||
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
return units[i].Section.Index < units[j].Section.Index
|
||||||
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
|
||||||
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
|
||||||
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
|
||||||
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
|
||||||
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
|
||||||
}
|
|
||||||
if input.ChunkingSummary != nil {
|
|
||||||
report.Chunking = &reporting.ChunkingSummary{
|
|
||||||
ChunkCount: input.ChunkingSummary.ChunkCount,
|
|
||||||
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
|
||||||
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
|
||||||
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
|
||||||
TargetSections: input.ChunkingSummary.TargetSections,
|
|
||||||
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
|
||||||
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
|
||||||
if input.RunOutput != nil && input.RunOutput.ObservationsAvailable {
|
|
||||||
summary := mapLLMSummary(input.RunOutput.LLMSummary)
|
|
||||||
report.LLMSummary = &summary
|
|
||||||
}
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
|
||||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
|
||||||
for _, r := range runOutput.ModuleResults {
|
|
||||||
startedAt := r.StartedAt
|
|
||||||
completedAt := r.CompletedAt
|
|
||||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
|
||||||
ModuleKey: r.ModuleKey,
|
|
||||||
ModuleInstance: r.ModuleInstance,
|
|
||||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
|
||||||
Status: r.Status,
|
|
||||||
ProposalCount: r.ProposalCount,
|
|
||||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
|
||||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
|
||||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
|
||||||
AppliedChanges: r.AppliedChanges,
|
|
||||||
SkippedChanges: r.SkippedChanges,
|
|
||||||
ErrorMessage: r.ErrorMessage,
|
|
||||||
StartedAt: &startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
})
|
})
|
||||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
|
||||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
|
||||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
for _, unit := range units {
|
||||||
summary.FailedModuleInstance = r.ModuleInstance
|
if unit.State == outcomes.WorkNotStarted {
|
||||||
|
notStarted++
|
||||||
|
} else {
|
||||||
|
started++
|
||||||
|
}
|
||||||
|
if unit.State == outcomes.WorkCompleted {
|
||||||
|
completed++
|
||||||
|
}
|
||||||
|
if unit.State == outcomes.WorkInterrupted {
|
||||||
|
interrupted++
|
||||||
|
}
|
||||||
|
corrections += unit.CorrectionsApplied
|
||||||
|
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||||
|
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||||
|
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied}
|
||||||
|
operations := append([]outcomes.Operation(nil), unit.Operations...)
|
||||||
|
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
|
||||||
|
failures := make([]reporting.Failure, 0)
|
||||||
|
for _, operation := range operations {
|
||||||
|
if operation.Reference.Kind != outcomes.OperationValidationInput {
|
||||||
|
llmOperations++
|
||||||
|
}
|
||||||
|
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||||
|
proposalResponses++
|
||||||
|
}
|
||||||
|
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
|
||||||
|
recoveryCount++
|
||||||
|
row.RecoveryCount++
|
||||||
|
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
|
||||||
|
filteringResponses++
|
||||||
|
}
|
||||||
|
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
||||||
|
}
|
||||||
|
if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) {
|
||||||
|
failures = append(failures, mapFailure(operation))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failures) > 0 {
|
||||||
|
withErrors++
|
||||||
|
row.ErrorCount = 1
|
||||||
|
stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied
|
||||||
|
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage,
|
||||||
|
ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: &chunk,
|
||||||
|
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
|
||||||
|
}
|
||||||
|
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
|
||||||
}
|
}
|
||||||
|
runErrors := len(input.RunFailures)
|
||||||
|
snapshot.Summary = reporting.Summary{
|
||||||
|
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
|
||||||
|
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors,
|
||||||
|
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
|
||||||
|
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
|
||||||
|
ProposalResponsesWithFiltering: &filteringResponses,
|
||||||
|
}
|
||||||
|
} else if len(input.RunFailures) > 0 {
|
||||||
|
runErrors := len(input.RunFailures)
|
||||||
|
snapshot.Summary.RunErrors = &runErrors
|
||||||
}
|
}
|
||||||
|
for _, failure := range sortedRunFailures(input.RunFailures) {
|
||||||
return summary, moduleReports
|
record := reporting.ErrorRecord{Scope: "run", Phase: failure.Phase, Reason: string(failure.Reason),
|
||||||
|
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
|
||||||
|
if failure.Origin != nil {
|
||||||
|
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
|
||||||
|
chunk := failure.Origin.SectionIndex
|
||||||
|
record.ChunkIndex = &chunk
|
||||||
|
record.Operation = string(failure.Origin.Kind)
|
||||||
|
record.Validator = failure.Origin.ValidatorName
|
||||||
|
record.BatchIndex = failure.Origin.BatchIndex
|
||||||
|
}
|
||||||
|
snapshot.Errors = append(snapshot.Errors, record)
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
func mapFailure(operation outcomes.Operation) reporting.Failure {
|
||||||
if len(in) == 0 {
|
indices := append([]int(nil), operation.ProposalIndices...)
|
||||||
return nil
|
sort.Ints(indices)
|
||||||
|
return reporting.Failure{Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName,
|
||||||
|
BatchIndex: operation.Reference.BatchIndex, Reason: string(operation.FailureReason), Message: safeMessage(operation.FailureReason),
|
||||||
|
ProposalIndices: indices, ExecutionCounts: mapCounts(operation.Counts)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapRecovery(unit runner.WorkUnitOutcome, operation outcomes.Operation) reporting.RecoveryRecord {
|
||||||
|
reasons := make([]string, len(operation.RecoveryReasons))
|
||||||
|
for i, reason := range operation.RecoveryReasons {
|
||||||
|
reasons[i] = string(reason)
|
||||||
}
|
}
|
||||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
sort.Strings(reasons)
|
||||||
for i, d := range in {
|
filtered := make([]reporting.FilteredProposal, len(operation.Filtered))
|
||||||
out[i] = reporting.ValidatorDecisionReport{
|
for i, item := range operation.Filtered {
|
||||||
ValidatorName: d.ValidatorName,
|
filtered[i] = reporting.FilteredProposal{ProposalIndex: item.ProposalIndex, Reason: string(item.Reason), Count: item.Count}
|
||||||
ProposalIndex: d.ProposalIndex,
|
}
|
||||||
Approved: d.Approved,
|
sort.Slice(filtered, func(i, j int) bool { return filtered[i].ProposalIndex < filtered[j].ProposalIndex })
|
||||||
ReasonCode: d.ReasonCode,
|
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||||
Message: d.Message,
|
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName, BatchIndex: operation.Reference.BatchIndex,
|
||||||
}
|
Reasons: reasons, Message: "Model operation succeeded after recovery or proposal filtering.",
|
||||||
}
|
ExecutionCounts: mapCounts(operation.Counts), FilteredProposals: filtered}
|
||||||
return out
|
}
|
||||||
}
|
|
||||||
|
func mapCounts(counts outcomes.ExecutionCounts) reporting.ExecutionCounts {
|
||||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
if !counts.Available {
|
||||||
if len(in) == 0 {
|
return reporting.ExecutionCounts{}
|
||||||
return nil
|
}
|
||||||
}
|
attempts, retries, repairs, exhausted := counts.ProviderAttempts, counts.TransientRetries, counts.StructuralRepairs, counts.RetriesExhausted
|
||||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
return reporting.ExecutionCounts{ProviderAttempts: &attempts, TransientRetries: &retries,
|
||||||
for i, d := range in {
|
StructuralRepairs: &repairs, RetriesExhausted: &exhausted, HTTPStatus: counts.HTTPStatus}
|
||||||
out[i] = reporting.ValidatorRejectedReport{
|
}
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
func isRecoverableReason(reason outcomes.ReasonCode) bool {
|
||||||
ModuleKey: d.ModuleKey,
|
switch reason {
|
||||||
ModuleInstance: d.ModuleInstance,
|
case outcomes.ReasonOperationTimeout, outcomes.ReasonCapacityExceeded, outcomes.ReasonTransportFailed,
|
||||||
TargetSegmentID: d.TargetSegmentID,
|
outcomes.ReasonProviderResponseInvalid, outcomes.ReasonProviderFailed,
|
||||||
OriginalText: d.OriginalText,
|
outcomes.ReasonInvalidOutputExhausted, outcomes.ReasonInvalidOutput,
|
||||||
CorrectedText: d.CorrectedText,
|
outcomes.ReasonValidationInputTooLarge, outcomes.ReasonValidationDecisionsInvalid:
|
||||||
ReasonCode: d.ReasonCode,
|
return true
|
||||||
Message: d.Message,
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeMessage(reason outcomes.ReasonCode) string {
|
||||||
|
switch reason {
|
||||||
|
case outcomes.ReasonOperationTimeout:
|
||||||
|
return "Model operation timed out."
|
||||||
|
case outcomes.ReasonCapacityExceeded:
|
||||||
|
return "Model capacity was unavailable."
|
||||||
|
case outcomes.ReasonTransportFailed:
|
||||||
|
return "Model transport failed."
|
||||||
|
case outcomes.ReasonProviderResponseInvalid:
|
||||||
|
return "Provider response was invalid."
|
||||||
|
case outcomes.ReasonProviderFailed:
|
||||||
|
return "Provider request failed."
|
||||||
|
case outcomes.ReasonInvalidOutputExhausted:
|
||||||
|
return "Model output remained invalid after repair attempts."
|
||||||
|
case outcomes.ReasonInvalidOutput:
|
||||||
|
return "Model output was invalid."
|
||||||
|
case outcomes.ReasonValidationInputTooLarge:
|
||||||
|
return "Validation input exceeded its size limit."
|
||||||
|
case outcomes.ReasonValidationDecisionsInvalid:
|
||||||
|
return "Validation decisions were unusable."
|
||||||
|
case outcomes.ReasonCanceled:
|
||||||
|
return "Processing was canceled."
|
||||||
|
case outcomes.ReasonSetupFailure:
|
||||||
|
return "Processing setup failed."
|
||||||
|
default:
|
||||||
|
return "Processing failed."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func operationLess(a, b outcomes.Operation) bool {
|
||||||
|
rank := func(kind outcomes.OperationKind) int {
|
||||||
|
switch kind {
|
||||||
|
case outcomes.OperationProposal:
|
||||||
|
return 0
|
||||||
|
case outcomes.OperationValidationInput:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
|
||||||
|
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
|
||||||
|
}
|
||||||
|
order := func(p *int) int {
|
||||||
|
if p == nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return *p
|
||||||
|
}
|
||||||
|
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
|
||||||
|
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
|
||||||
|
}
|
||||||
|
return rank(a.Reference.Kind) < rank(b.Reference.Kind) || (rank(a.Reference.Kind) == rank(b.Reference.Kind) && order(a.Reference.BatchIndex) < order(b.Reference.BatchIndex))
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedRunFailures(in []RunFailure) []RunFailure {
|
||||||
|
out := append([]RunFailure(nil), in...)
|
||||||
|
phase := map[string]int{"configuration": 0, "destination": 1, "initialization": 2, "input": 3, "normalization": 4, "chunking": 5, "processing": 6, "encoding": 7, "transcript_write": 8, "report_write": 9, "debug": 10}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool {
|
||||||
|
a, aok := phase[out[i].Phase]
|
||||||
|
b, bok := phase[out[j].Phase]
|
||||||
|
if !aok {
|
||||||
|
a = 100
|
||||||
|
}
|
||||||
|
if !bok {
|
||||||
|
b = 100
|
||||||
|
}
|
||||||
|
if a != b {
|
||||||
|
return a < b
|
||||||
|
}
|
||||||
|
return out[i].OutputPath < out[j].OutputPath
|
||||||
|
})
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,180 +1,78 @@
|
|||||||
package processreport
|
package processreport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
|
||||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
started := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
|
||||||
completedAt := startedAt.Add(time.Second)
|
completed := started.Add(time.Minute)
|
||||||
configVersion := 4
|
proposal := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
|
||||||
targetSections := 2
|
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, TransientRetries: 1, StructuralRepairs: 1},
|
||||||
inputSegments := 5
|
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
|
||||||
outputSegments := 4
|
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
|
||||||
merges := 1
|
batch := 1
|
||||||
reassigned := 2
|
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||||
differentSpeakers := 3
|
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||||
|
output := &runner.RunOutput{WorkUnits: []runner.WorkUnitOutcome{
|
||||||
report := Build(BuildInput{
|
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
|
||||||
Status: "success",
|
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
|
||||||
TranscriptPath: "transcript.json",
|
}}
|
||||||
GlossaryPath: "glossary.yaml",
|
snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
|
||||||
OutputPath: "out.json",
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output})
|
||||||
Modules: []string{"grammar"},
|
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
|
||||||
OutputSchema: "default",
|
t.Fatalf("summary: %+v", snapshot.Summary)
|
||||||
ConfigVersion: &configVersion,
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: completedAt,
|
|
||||||
RunDirectoryPath: filepath.Join(
|
|
||||||
"tmp",
|
|
||||||
"audita-run",
|
|
||||||
),
|
|
||||||
NormalizationSummary: &normalization.NormalizationSummary{
|
|
||||||
InputSegmentCount: inputSegments,
|
|
||||||
OutputSegmentCount: outputSegments,
|
|
||||||
MergesPerformed: merges,
|
|
||||||
IDsReassigned: reassigned,
|
|
||||||
SkippedMerges: struct {
|
|
||||||
DifferentSpeakers int `json:"different_speakers"`
|
|
||||||
GapTooLarge int `json:"gap_too_large"`
|
|
||||||
DurationExceeded int `json:"duration_exceeded"`
|
|
||||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
|
||||||
}{
|
|
||||||
DifferentSpeakers: differentSpeakers,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ChunkingSummary: &chunking.Summary{
|
|
||||||
ChunkCount: 3,
|
|
||||||
MinEstimatedTokens: 10,
|
|
||||||
MaxEstimatedTokens: 20,
|
|
||||||
TotalEstimatedTokens: 45,
|
|
||||||
TargetSections: &targetSections,
|
|
||||||
MaxSectionTokens: 200,
|
|
||||||
MinSectionTokens: 50,
|
|
||||||
},
|
|
||||||
RunOutput: &runner.RunOutput{
|
|
||||||
ModuleResults: []runner.ModuleResult{
|
|
||||||
{
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
ModuleInstance: "grammar",
|
|
||||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
|
||||||
Status: runner.ModuleStatusSuccess,
|
|
||||||
ProposalCount: 2,
|
|
||||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
|
||||||
{
|
|
||||||
ValidatorName: "proposal_shape",
|
|
||||||
ProposalIndex: 1,
|
|
||||||
Approved: true,
|
|
||||||
ReasonCode: "approved",
|
|
||||||
Message: "ok",
|
|
||||||
DiagnosticArtifactPath: "diagnostics/validator.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ValidatorRejected: []runner.ValidatorRejectedChange{
|
|
||||||
{
|
|
||||||
ValidatorName: "protected_term",
|
|
||||||
ProposalIndex: 2,
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
ModuleInstance: "grammar",
|
|
||||||
TargetSegmentID: 7,
|
|
||||||
OriginalText: "old",
|
|
||||||
CorrectedText: "new",
|
|
||||||
ReasonCode: "protected_term",
|
|
||||||
Message: "blocked",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
AppliedChanges: []proposals.AppliedChange{
|
|
||||||
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
|
||||||
},
|
|
||||||
SkippedChanges: []proposals.SkippedChange{
|
|
||||||
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
|
||||||
},
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: completedAt,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
|
||||||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
|
||||||
report.ReportMetadata.OutputSchema != "default" ||
|
|
||||||
report.ReportMetadata.ConfigVersion == nil ||
|
|
||||||
*report.ReportMetadata.ConfigVersion != configVersion {
|
|
||||||
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
|
||||||
}
|
}
|
||||||
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 2 || snapshot.Errors[0].Failures[0].Operation != "validation_input" || !reflect.DeepEqual(snapshot.Errors[0].Failures[1].ProposalIndices, []int{3, 9}) {
|
||||||
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
t.Fatalf("failure ordering: %+v", snapshot.Errors)
|
||||||
}
|
}
|
||||||
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"output_repaired", "proposals_filtered", "service_recovered"}) || snapshot.Recoveries[0].FilteredProposals[0].ProposalIndex != 2 {
|
||||||
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
t.Fatalf("recovery: %+v", snapshot.Recoveries)
|
||||||
}
|
}
|
||||||
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
if snapshot.WorkUnits[0].ModuleInstance != "grammar_1" || snapshot.WorkUnits[1].State != "not_started" {
|
||||||
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
t.Fatalf("unit order: %+v", snapshot.WorkUnits)
|
||||||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
|
||||||
t.Fatalf("unexpected normalization summary: %+v", report)
|
|
||||||
}
|
}
|
||||||
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument())
|
||||||
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if report.ModulesSummary == nil ||
|
var decoded map[string]any
|
||||||
report.ModulesSummary.ModuleCount != 1 ||
|
if err := yaml.Unmarshal(errorsYAML, &decoded); err != nil {
|
||||||
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
t.Fatal(err)
|
||||||
report.ModulesSummary.TotalSkippedChanges != 2 {
|
|
||||||
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
|
||||||
}
|
}
|
||||||
if len(report.ModuleResults) != 1 ||
|
if decoded["version"] != 1 || decoded["schema"] != "audita-errors" || decoded["status"] != "completed" {
|
||||||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
t.Fatalf("envelope: %#v", decoded)
|
||||||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
}
|
||||||
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
if strings.Contains(string(errorsYAML), "provider_attempts: null") || !strings.Contains(string(errorsYAML), "errors:") {
|
||||||
|
t.Fatal(string(errorsYAML))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
|
||||||
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
start := time.Now().UTC()
|
||||||
completedAt := startedAt.Add(time.Second)
|
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})
|
||||||
|
initialDoc := initial.ErrorsDocument()
|
||||||
report := Build(BuildInput{
|
if initialDoc.CompletedAt != nil || initialDoc.Summary.WorkUnitsTotal != nil || initialDoc.Errors == nil {
|
||||||
Status: "failed",
|
t.Fatalf("initial: %+v", initialDoc)
|
||||||
TranscriptPath: "transcript.json",
|
|
||||||
GlossaryPath: "glossary.yaml",
|
|
||||||
Modules: []string{"grammar"},
|
|
||||||
OutputSchema: "default",
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: completedAt,
|
|
||||||
ErrorPhase: "module",
|
|
||||||
ErrorMessage: "module failed",
|
|
||||||
RunDirectoryPath: "run-dir",
|
|
||||||
RunOutput: &runner.RunOutput{
|
|
||||||
ModuleResults: []runner.ModuleResult{
|
|
||||||
{
|
|
||||||
ModuleKey: "grammar",
|
|
||||||
ModuleInstance: "grammar",
|
|
||||||
Status: runner.ModuleStatusFailed,
|
|
||||||
ErrorMessage: "module failed",
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: completedAt,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
|
||||||
t.Fatalf("unexpected failure fields: %+v", report)
|
|
||||||
}
|
}
|
||||||
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
finished := Build(BuildInput{RunID: "r", Status: reporting.StatusCompleted, StartedAt: start, RunOutput: &runner.RunOutput{}})
|
||||||
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
if finished.Summary.WorkUnitsTotal == nil || *finished.Summary.WorkUnitsTotal != 0 || finished.RecoveriesDocument().Recoveries == nil {
|
||||||
|
t.Fatalf("empty: %+v", finished)
|
||||||
}
|
}
|
||||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
failed := Build(BuildInput{RunID: "r", Status: reporting.StatusFailed, StartedAt: start,
|
||||||
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
RunFailures: []RunFailure{{Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "z.yml"}, {Phase: "processing", Reason: outcomes.ReasonCanceled}, {Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "a.yml"}}})
|
||||||
|
if len(failed.Errors) != 3 || failed.Errors[0].Phase != "processing" || failed.Errors[1].OutputPath != "a.yml" || failed.Errors[2].OutputPath != "z.yml" || failed.Summary.WorkUnitsTotal != nil || *failed.Summary.RunErrors != 3 {
|
||||||
|
t.Fatalf("run failures: %+v", failed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func intPtr(value int) *int { return &value }
|
||||||
|
|||||||
Reference in New Issue
Block a user