eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Phase 2 normalization diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:19:28 +00:00
parent aeb9c4f062
commit 14e51698c2
3 changed files with 488 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/cli/run.go
View File

@@ -9,6 +9,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
@@ -24,25 +25,49 @@ type processInvocation struct {
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, error) {
// Create run directory early to capture artifacts
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, fmt.Errorf("run_dir_creation: %w", err)
}
defer func() {
// Apply retention policy - failed runs are always kept
// For now, we consider any run that saved source as successful
if runDir != nil {
_ = runDir.ApplyRetention()
}
}()
transcriptBytes, err := io.ReadRequiredFile(inv.TranscriptPath, "transcript")
if err != nil {
// Best effort to write error log on failure
_ = runDir.WriteErrorLog(fmt.Sprintf("transcript_read: %v", err))
return nil, fmt.Errorf("transcript_read: %w", err)
}
glossaryBytes, err := io.ReadRequiredFile(inv.GlossaryPath, "glossary")
if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("glossary_read: %v", err))
return nil, fmt.Errorf("glossary_read: %w", err)
}
// Parse and validate transcript using typed schema
transcript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("transcript_schema: %v", err))
return nil, fmt.Errorf("transcript_schema: %w", err)
}
// Write source transcript artifacts
if err := runDir.WriteSourceTranscript(transcript, transcriptBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
// Continue without failing the whole process
}
// Parse and validate glossary using typed schema
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("glossary_schema: %v", err))
return nil, fmt.Errorf("glossary_schema: %w", err)
}
@@ -75,21 +100,36 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
normalizedTranscript, normalizationSummary := normalizer.Normalize(normalizedTranscript)
// Write normalized transcript artifact
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
// Continue without failing the whole process
}
// Write normalization summary artifact
if err := runDir.WriteNormalizationSummary(normalizationSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
// Continue without failing the whole process
}
// Serialize normalized transcript to JSON
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
return nil, fmt.Errorf("failed to serialize transcript: %w", err)
_ = runDir.WriteErrorLog(fmt.Sprintf("serialization: %v", err))
return nil, fmt.Errorf("serialization: %w", err)
}
if strings.TrimSpace(inv.OutputPath) != "" {
if err := io.WriteFile(inv.OutputPath, outputBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("output_write: %v", err))
return nil, err
}
return normalizationSummary, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return nil, fmt.Errorf("failed to write transcript to stdout: %w", err)
_ = runDir.WriteErrorLog(fmt.Sprintf("stdout_write: %v", err))
return nil, fmt.Errorf("stdout_write: %w", err)
}
return normalizationSummary, nil
}

323
internal/cli/run_test.go
View File

@@ -808,48 +808,327 @@ func TestRunProcessReportFailedGlossarySchema(t *testing.T) {
}
}
func TestRunProcessReportJSONNotInStdout(t *testing.T) {
func TestRunProcessRunDirectoryCreation(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[
{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}
]`
transcriptJSON := `[{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
reportPath := filepath.Join(t.TempDir(), "report.json")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--report-json", reportPath}, &stdout, &stderr)
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--work-dir", workDir}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Read the report to get its content
reportBytes := readFile(t, reportPath)
// Verify report JSON is not in stdout
if bytes.Contains(stdout.Bytes(), reportBytes) {
t.Fatalf("stdout was polluted with report JSON")
// Check that run directory was created
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
if len(runDirs) != 1 {
t.Fatalf("expected 1 run directory, got %d", len(runDirs))
}
// Verify stdout only contains transcript JSON
if !json.Valid(stdout.Bytes()) {
t.Fatalf("expected valid JSON in stdout, got %q", stdout.String())
runPath := filepath.Join(workDir, runDirs[0].Name())
// Check that source transcript artifact exists
if _, err := os.Stat(filepath.Join(runPath, "source-transcript.json")); err != nil {
t.Fatalf("expected source transcript artifact: %v", err)
}
// Parse stdout to verify it's a transcript, not a report
var stdoutContent map[string]interface{}
if err := json.Unmarshal(stdout.Bytes(), &stdoutContent); err != nil {
t.Fatalf("failed to parse stdout as JSON: %v", err)
// Check that normalized transcript artifact exists
if _, err := os.Stat(filepath.Join(runPath, "normalized-transcript.json")); err != nil {
t.Fatalf("expected normalized transcript artifact: %v", err)
}
// Transcript should be an array at top level
if _, isArray := stdoutContent["segments"]; !isArray && stdoutContent["segments"] != nil {
t.Fatalf("expected transcript in stdout, got report structure")
// Check that normalization summary artifact exists
if _, err := os.Stat(filepath.Join(runPath, "normalization-summary.json")); err != nil {
t.Fatalf("expected normalization summary artifact: %v", err)
}
// Check that report artifact exists
if _, err := os.Stat(filepath.Join(runPath, "report.json")); err != nil {
t.Fatalf("expected report artifact: %v", err)
}
}
func TestRunProcessSourceTranscriptArtifact(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--work-dir", workDir}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
runPath := filepath.Join(workDir, runDirs[0].Name())
// Read and verify source transcript artifact
sourceBytes := readFile(t, filepath.Join(runPath, "source-transcript.json"))
if !bytes.Equal(sourceBytes, []byte(transcriptJSON)) {
t.Fatalf("source transcript artifact doesn't match input")
}
}
func TestRunProcessNormalizedTranscriptArtifact(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[
{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"},
{"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"}
]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--work-dir", workDir}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
runPath := filepath.Join(workDir, runDirs[0].Name())
// Read and verify normalized transcript artifact
normalizedBytes := readFile(t, filepath.Join(runPath, "normalized-transcript.json"))
var normalizedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(normalizedBytes); err != nil {
t.Fatalf("failed to parse normalized transcript artifact: %v", err)
}
// Should be merged (same speaker, small gap)
if len(normalizedTranscript.Segments) != 1 {
t.Fatalf("expected 1 merged segment in normalized artifact, got %d", len(normalizedTranscript.Segments))
}
}
func TestRunProcessNormalizationSummaryArtifact(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[
{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"},
{"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"}
]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--work-dir", workDir}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
runPath := filepath.Join(workDir, runDirs[0].Name())
// Read and verify normalization summary artifact
summaryBytes := readFile(t, filepath.Join(runPath, "normalization-summary.json"))
var summary normalization.NormalizationSummary
if err := json.Unmarshal(summaryBytes, &summary); err != nil {
t.Fatalf("failed to parse normalization summary: %v", err)
}
// Verify summary content
if summary.InputSegmentCount != 2 {
t.Fatalf("expected input count 2, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 1 {
t.Fatalf("expected output count 1, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 1 {
t.Fatalf("expected 1 merge, got %d", summary.MergesPerformed)
}
}
func TestRunProcessReportArtifact(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--work-dir", workDir}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
runPath := filepath.Join(workDir, runDirs[0].Name())
// Verify report artifact exists and is valid
reportBytes := readFile(t, filepath.Join(runPath, "report.json"))
var report reporting.ProcessReport
if err := json.Unmarshal(reportBytes, &report); err != nil {
t.Fatalf("failed to parse report: %v", err)
}
if report.Status != "success" {
t.Fatalf("expected success status, got %q", report.Status)
}
}
func TestRunProcessErrorLogOnFailure(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := filepath.Join(t.TempDir(), "work")
exitCode := Run([]string{
"process",
fixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--work-dir",
workDir,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for invalid transcript")
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
if len(runDirs) != 1 {
t.Fatalf("expected 1 run directory on failure, got %d", len(runDirs))
}
runPath := filepath.Join(workDir, runDirs[0].Name())
// Verify error.log exists
if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil {
t.Fatalf("expected error.log on failure: %v", err)
}
// Verify error.log contains useful information
errorBytes := readFile(t, filepath.Join(runPath, "error.log"))
if !strings.Contains(string(errorBytes), "transcript_schema") {
t.Fatalf("expected error phase in error.log, got %q", string(errorBytes))
}
}
func TestRunProcessFailedRunsRetained(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := filepath.Join(t.TempDir(), "work")
exitCode := Run([]string{
"process",
fixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--work-dir",
workDir,
"--work-dir-retention",
"never", // Even with "never", failed runs should be retained
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for invalid transcript")
}
// Find the run directory
runDirs, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work directory: %v", err)
}
if len(runDirs) != 1 {
t.Fatalf("expected failed run to be retained, got %d run directories", len(runDirs))
}
// Failed runs should always be retained regardless of retention policy
runPath := filepath.Join(workDir, runDirs[0].Name())
if _, err := os.Stat(runPath); err != nil {
t.Fatalf("failed run directory should be retained: %v", err)
}
}
func TestRunProcessSuccessfulRunRetention(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptJSON := `[{"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}]`
transcriptPath := filepath.Join(t.TempDir(), "transcript.json")
glossaryPath := fixturePath("tiny_glossary.yaml")
workDir := filepath.Join(t.TempDir(), "work")
if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil {
t.Fatalf("failed to create test transcript: %v", err)
}
// Test with "never" retention - successful runs should be removed
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
glossaryPath,
"--work-dir",
workDir,
"--work-dir-retention",
"never",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// With "never" retention, successful runs should be removed
runDirs, err := os.ReadDir(workDir)
if err == nil && len(runDirs) > 0 {
t.Fatalf("expected successful run to be removed with 'never' retention, found %d directories", len(runDirs))
}
}