eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Report schema validation and normalization results

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:16:50 +00:00
parent e2ae7f77d8
commit aeb9c4f062
3 changed files with 234 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
internal/cli/run.go
View File

@@ -23,27 +23,27 @@ type processInvocation struct {
Config config.Config
}
var processRunner = func(inv processInvocation, stdout io.Writer) error {
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, error) {
transcriptBytes, err := io.ReadRequiredFile(inv.TranscriptPath, "transcript")
if err != nil {
return err
return nil, fmt.Errorf("transcript_read: %w", err)
}
glossaryBytes, err := io.ReadRequiredFile(inv.GlossaryPath, "glossary")
if err != nil {
return err
return nil, fmt.Errorf("glossary_read: %w", err)
}
// Parse and validate transcript using typed schema
transcript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil {
return fmt.Errorf("invalid transcript: %w", err)
return nil, fmt.Errorf("transcript_schema: %w", err)
}
// Parse and validate glossary using typed schema
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
if err != nil {
return fmt.Errorf("invalid glossary: %w", err)
return nil, fmt.Errorf("glossary_schema: %w", err)
}
// Convert to canonical transcript format for normalization
@@ -73,25 +73,25 @@ var processRunner = func(inv processInvocation, stdout io.Writer) error {
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, _ = normalizer.Normalize(normalizedTranscript)
normalizedTranscript, normalizationSummary := normalizer.Normalize(normalizedTranscript)
// Serialize normalized transcript to JSON
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
return fmt.Errorf("failed to serialize transcript: %w", err)
return nil, fmt.Errorf("failed to serialize transcript: %w", err)
}
if strings.TrimSpace(inv.OutputPath) != "" {
if err := io.WriteFile(inv.OutputPath, outputBytes); err != nil {
return err
return nil, err
}
return nil
return normalizationSummary, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fmt.Errorf("failed to write transcript to stdout: %w", err)
return nil, fmt.Errorf("failed to write transcript to stdout: %w", err)
}
return nil
return normalizationSummary, nil
}
// Run executes the Audita CLI with the provided arguments and streams.
@@ -233,17 +233,47 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
Config: cfg,
}
if err := processRunner(inv, stdout); err != nil {
if normalizationSummary, err := processRunner(inv, stdout); err != nil {
completedAt := time.Now().UTC()
// Extract error phase from error message if present
errorPhase := ""
errorMsg := err.Error()
if strings.Contains(errorMsg, ": ") {
parts := strings.SplitN(errorMsg, ": ", 2)
if len(parts) == 2 {
errorPhase = parts[0]
errorMsg = parts[1]
}
}
if strings.TrimSpace(inv.ReportJSONPath) != "" {
report := buildProcessReport("failed", inv, startedAt, completedAt, err.Error())
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMsg, errorPhase, nil)
if reportErr := reporting.WriteProcessReport(inv.ReportJSONPath, report); reportErr != nil {
fmt.Fprintf(stderr, "audita process: %v\n", reportErr)
return 1
}
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
completedAt := time.Now().UTC()
if strings.TrimSpace(inv.ReportJSONPath) != "" {
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normalizationSummary)
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
}
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
if strings.TrimSpace(inv.ReportJSONPath) != "" {
completedAt := time.Now().UTC()
@@ -257,9 +287,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 0
}
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string) reporting.ProcessReport {
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase1-minimal",
Phase: "phase2-normalization",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,
@@ -268,10 +298,21 @@ func buildProcessReport(status string, inv processInvocation, startedAt, complet
Modules: append([]string(nil), inv.Config.Modules...),
StartedAt: startedAt,
CompletedAt: &completedAt,
ErrorPhase: errorPhase,
}
if errorMessage != "" {
report.ErrorMessage = errorMessage
}
if normalizationSummary != nil {
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
report.NormalizationMerges = &normalizationSummary.MergesPerformed
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
}
return report
}

167
internal/cli/run_test.go
View File

@@ -673,11 +673,11 @@ func TestRunProcessNoStdoutWithOutput(t *testing.T) {
}
}
func TestRunProcessNormalizationConfigOverride(t *testing.T) {
func TestRunProcessReportIncludesNormalizationSummary(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
// Create segments that would normally merge but set very small max gap
// Create a test transcript with segments that will be merged
transcriptJSON := `[
{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"},
{"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"}
@@ -685,24 +685,171 @@ func TestRunProcessNormalizationConfigOverride(t *testing.T) {
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
reportPath := filepath.Join(t.TempDir(), "report.json")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
// Set max segment gap to 0.1 (smaller than the 0.5 gap between segments)
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--normalize-max-segment-gap", "0.1"}, &stdout, &stderr)
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--report-json", reportPath}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout when --output is not used but --report-json is provided, got %q", stdout.String())
}
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
// Read and parse the report
reportBytes := readFile(t, reportPath)
var report reporting.ProcessReport
if err := json.Unmarshal(reportBytes, &report); err != nil {
t.Fatalf("failed to parse report JSON: %v (raw: %s)", err, string(reportBytes))
}
// Verify report includes normalization summary
if report.Status != "success" {
t.Fatalf("expected success status, got %q", report.Status)
}
if report.InputSegmentCount == nil || *report.InputSegmentCount != 2 {
t.Fatalf("expected input segment count 2, got %v", report.InputSegmentCount)
}
if report.NormalizedSegmentCount == nil || *report.NormalizedSegmentCount != 1 {
t.Fatalf("expected normalized segment count 1, got %v", report.NormalizedSegmentCount)
}
if report.NormalizationMerges == nil || *report.NormalizationMerges != 1 {
t.Fatalf("expected 1 merge, got %v", report.NormalizationMerges)
}
if report.NormalizationIDReassignments == nil || *report.NormalizationIDReassignments != 0 {
t.Fatalf("expected 0 ID reassignments (already sequential), got %v", report.NormalizationIDReassignments)
}
}
func TestRunProcessReportFailedTranscriptSchema(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
fixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for invalid transcript")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
// Read and parse the report
reportBytes := readFile(t, reportPath)
var report reporting.ProcessReport
if err := json.Unmarshal(reportBytes, &report); err != nil {
t.Fatalf("failed to parse report JSON: %v (raw: %s)", err, string(reportBytes))
}
// Verify failed report structure
if report.Status != "failed" {
t.Fatalf("expected failed status, got %q", report.Status)
}
if report.ErrorPhase != "transcript_schema" {
t.Fatalf("expected error phase 'transcript_schema', got %q", report.ErrorPhase)
}
if report.ErrorMessage == "" {
t.Fatalf("expected error message in failed report")
}
if !strings.Contains(report.ErrorMessage, "speaker") {
t.Fatalf("expected speaker validation error, got %q", report.ErrorMessage)
}
}
func TestRunProcessReportFailedGlossarySchema(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
"../core/schema/testdata/glossary_missing_fields.yaml",
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for invalid glossary")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
// Read and parse the report
reportBytes := readFile(t, reportPath)
var report reporting.ProcessReport
if err := json.Unmarshal(reportBytes, &report); err != nil {
t.Fatalf("failed to parse report JSON: %v (raw: %s)", err, string(reportBytes))
}
// Verify failed report structure
if report.Status != "failed" {
t.Fatalf("expected failed status, got %q", report.Status)
}
if report.ErrorPhase != "glossary_schema" {
t.Fatalf("expected error phase 'glossary_schema', got %q", report.ErrorPhase)
}
if report.ErrorMessage == "" {
t.Fatalf("expected error message in failed report")
}
}
func TestRunProcessReportJSONNotInStdout(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[
{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}
]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
reportPath := filepath.Join(t.TempDir(), "report.json")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--report-json", reportPath}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Should NOT merge due to small max gap setting
var outputTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil {
t.Fatalf("failed to parse output transcript: %v", err)
// Read the report to get its content
reportBytes := readFile(t, reportPath)
// Verify report JSON is not in stdout
if bytes.Contains(stdout.Bytes(), reportBytes) {
t.Fatalf("stdout was polluted with report JSON")
}
if len(outputTranscript.Segments) != 2 {
t.Fatalf("expected 2 unmerged segments due to small max gap, got %d", len(outputTranscript.Segments))
// Verify stdout only contains transcript JSON
if !json.Valid(stdout.Bytes()) {
t.Fatalf("expected valid JSON in stdout, got %q", stdout.String())
}
// Parse stdout to verify it's a transcript, not a report
var stdoutContent map[string]interface{}
if err := json.Unmarshal(stdout.Bytes(), &stdoutContent); err != nil {
t.Fatalf("failed to parse stdout as JSON: %v", err)
}
// Transcript should be an array at top level
if _, isArray := stdoutContent["segments"]; !isArray && stdoutContent["segments"] != nil {
t.Fatalf("expected transcript in stdout, got report structure")
}
}

31
internal/core/reporting/report.go
View File

@@ -8,16 +8,27 @@ import (
)
type ProcessReport struct {
Phase string `json:"phase"`
Status string `json:"status"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
Modules []string `json:"modules"`
StartedAt time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
Phase string `json:"phase"`
Status string `json:"status"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
Modules []string `json:"modules"`
StartedAt time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
InputSegmentCount *int `json:"input_segment_count,omitempty"`
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
NormalizationMerges *int `json:"normalization_merges,omitempty"`
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
NormalizationSkipped struct {
DifferentSpeakers *int `json:"different_speakers,omitempty"`
GapTooLarge *int `json:"gap_too_large,omitempty"`
DurationExceeded *int `json:"duration_exceeded,omitempty"`
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
} `json:"normalization_skipped,omitempty"`
}
func WriteProcessReport(path string, report ProcessReport) error {