eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 2 Go normalization foundation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:41:29 +00:00
parent d847168ecd
commit 10377876e4
11 changed files with 480 additions and 1241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
README.md
View File

@@ -1,26 +1,29 @@
# Audita (Go Rewrite) # Audita (Go Rewrite)
Audita is a transcript polishing CLI. The Go implementation at the repository root is the canonical implementation going forward, with a compatibility-first rewrite strategy. Audita is a transcript polishing CLI. The Go implementation at the repository root is canonical going forward, and is being delivered compatibility-first against the frozen Python reference in `python/`.
## Current Status: Phase 1 Skeleton ## Current Status: Phase 2 Foundation
The current Go `audita process` command is a passthrough skeleton: The current `audita process` implementation now includes deterministic input handling and normalization foundations:
- loads transcript and glossary files; - typed transcript parsing for both accepted top-level forms (`[]` and `{ "segments": [...] }`);
- validates transcript as well-formed JSON; - typed glossary YAML parsing;
- writes transcript JSON back unchanged to `--output` or stdout; - transcript and glossary schema validation with actionable errors;
- writes an early minimal `--report-json` report when requested; - deterministic same-speaker normalization;
- preserves subprocess-safe stdout/stderr behavior. - sequential normalized segment IDs starting at `1`;
- normalized transcript JSON output;
- minimal structured `--report-json` output including normalization summary fields;
- minimal normalization diagnostics artifacts in per-run work directories;
- subprocess-safe stdout/stderr behavior.
Not implemented yet in Phase 1: Still not implemented in Phase 2:
- transcript/glossary schema validation; - transcript-polishing correction modules;
- normalization; - chunking/proposal generation and application;
- chunking/proposals; - deterministic and LLM validators;
- validators; - structured LLM client execution;
- LLM calls; - module pipeline execution;
- module execution; - full long-term diagnostics retention/reporting model.
- full diagnostics retention/report model.
## Quick Start (Go) ## Quick Start (Go)
@@ -37,17 +40,17 @@ go run ./cmd/audita --help
go run ./cmd/audita process --help go run ./cmd/audita process --help
``` ```
Run minimal passthrough: Run Phase 2 normalization flow:
```sh ```sh
go run ./cmd/audita process transcript.json --glossary glossary.yaml --output corrected.json go run ./cmd/audita process transcript.json --glossary glossary.yaml --output corrected.json
``` ```
Without `--output`, corrected transcript JSON is written to stdout. Without `--output`, normalized transcript JSON is written to stdout.
## Repository Notes ## Repository Notes
- `python/` contains the frozen Python implementation retained as behavioral reference during the port. - `python/` contains the frozen Python implementation used as behavioral reference during the Go port.
- Go architecture and rewrite guidance: - Architecture and rewrite guidance:
- `docs/architecture.md` - `docs/architecture.md`
- `docs/rewrite-notes.md` - `docs/rewrite-notes.md`

204
internal/cli/run.go
View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics" "gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/internal/core/io" coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization" "gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -24,63 +24,84 @@ type processInvocation struct {
Config config.Config Config config.Config
} }
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, error) { var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
// Create run directory early to capture artifacts
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention)) runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil { if err != nil {
return nil, fmt.Errorf("run_dir_creation: %w", err) return nil, nil, fmt.Errorf("run_dir_creation: %w", err)
} }
defer func() {
// Apply retention policy - failed runs are always kept
// For now, we consider any run that saved source as successful
if runDir != nil {
_ = runDir.ApplyRetention()
}
}()
transcriptBytes, err := io.ReadRequiredFile(inv.TranscriptPath, "transcript") fail := func(phase string, err error) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, runDir, fmt.Errorf("%s: %w", phase, err)
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
if err != nil { if err != nil {
// Best effort to write error log on failure return fail("transcript_read", err)
_ = runDir.WriteErrorLog(fmt.Sprintf("transcript_read: %v", err))
return nil, fmt.Errorf("transcript_read: %w", err)
} }
glossaryBytes, err := io.ReadRequiredFile(inv.GlossaryPath, "glossary") glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
if err != nil { if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("glossary_read: %v", err)) return fail("glossary_read", err)
return nil, fmt.Errorf("glossary_read: %w", err)
} }
// Parse and validate transcript using typed schema sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
transcript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil { if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("transcript_schema: %v", err)) return fail("transcript_schema", err)
return nil, fmt.Errorf("transcript_schema: %w", err)
} }
// Write source transcript artifacts if _, err := schema.ParseGlossaryYAML(glossaryBytes); err != nil {
if err := runDir.WriteSourceTranscript(transcript, transcriptBytes); err != nil { return fail("glossary_schema", err)
}
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err)) _ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
// Continue without failing the whole process
} }
// Parse and validate glossary using typed schema canonical := sourceToCanonicalTranscript(sourceTranscript)
glossary, err := schema.ParseGlossaryYAML(glossaryBytes) normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
EllipsisGap: inv.Config.Normalization.EllipsisGap,
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, summary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(summary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil { if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("glossary_schema: %v", err)) return fail("serialization", err)
return nil, fmt.Errorf("glossary_schema: %w", err)
} }
// Convert to canonical transcript format for normalization if strings.TrimSpace(inv.OutputPath) != "" {
normalizedTranscript := &schema.Transcript{ if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
Segments: make([]schema.Segment, len(transcript.Segments)), return fail("output_write", err)
}
return summary, runDir, nil
} }
for i, s := range transcript.Segments {
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err)
}
return summary, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
segments := make([]schema.Segment, len(source.Segments))
for i, s := range source.Segments {
id := i + 1 id := i + 1
if s.ID != nil { if s.ID != nil {
id = *s.ID id = *s.ID
} }
normalizedTranscript.Segments[i] = schema.Segment{ segments[i] = schema.Segment{
ID: id, ID: id,
Speaker: s.Speaker, Speaker: s.Speaker,
Start: s.Start, Start: s.Start,
@@ -89,49 +110,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
Categories: s.Categories, Categories: s.Categories,
} }
} }
return &schema.Transcript{Segments: segments}
// Apply deterministic normalization
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
EllipsisGap: inv.Config.Normalization.EllipsisGap,
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, normalizationSummary := normalizer.Normalize(normalizedTranscript)
// Write normalized transcript artifact
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
// Continue without failing the whole process
}
// Write normalization summary artifact
if err := runDir.WriteNormalizationSummary(normalizationSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
// Continue without failing the whole process
}
// Serialize normalized transcript to JSON
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("serialization: %v", err))
return nil, fmt.Errorf("serialization: %w", err)
}
if strings.TrimSpace(inv.OutputPath) != "" {
if err := io.WriteFile(inv.OutputPath, outputBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("output_write: %v", err))
return nil, err
}
return normalizationSummary, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("stdout_write: %v", err))
return nil, fmt.Errorf("stdout_write: %w", err)
}
return normalizationSummary, nil
} }
// Run executes the Audita CLI with the provided arguments and streams. // Run executes the Audita CLI with the provided arguments and streams.
@@ -273,53 +252,45 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
Config: cfg, Config: cfg,
} }
if normalizationSummary, err := processRunner(inv, stdout); err != nil { summary, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC() completedAt := time.Now().UTC()
// Extract error phase from error message if present if runErr != nil {
errorPhase := "" errorPhase, errorMessage := extractErrorPhase(runErr)
errorMsg := err.Error() report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil)
if strings.Contains(errorMsg, ": ") {
parts := strings.SplitN(errorMsg, ": ", 2)
if len(parts) == 2 {
errorPhase = parts[0]
errorMsg = parts[1]
}
}
if strings.TrimSpace(inv.ReportJSONPath) != "" { if strings.TrimSpace(inv.ReportJSONPath) != "" {
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMsg, errorPhase, nil) if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
if reportErr := reporting.WriteProcessReport(inv.ReportJSONPath, report); reportErr != nil { fmt.Fprintf(stderr, "audita process: %v\n", err)
fmt.Fprintf(stderr, "audita process: %v\n", reportErr)
return 1
} }
} }
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1 if runDir != nil {
} _ = runDir.WriteReport(report)
_ = runDir.ApplyRetention(false)
} }
fmt.Fprintf(stderr, "audita process: %v\n", err)
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
return 1 return 1
} }
completedAt := time.Now().UTC() report := buildProcessReport("success", inv, startedAt, completedAt, "", "", summary)
if strings.TrimSpace(inv.ReportJSONPath) != "" { if strings.TrimSpace(inv.ReportJSONPath) != "" {
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normalizationSummary)
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil { if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
if runDir != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
_ = runDir.ApplyRetention(false)
}
fmt.Fprintf(stderr, "audita process: %v\n", err) fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1 return 1
} }
} }
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
if strings.TrimSpace(inv.ReportJSONPath) != "" { if runDir != nil {
completedAt := time.Now().UTC() _ = runDir.WriteReport(report)
report := buildProcessReport("success", inv, startedAt, completedAt, "") if err := runDir.ApplyRetention(true); err != nil {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil { fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1 return 1
} }
} }
@@ -327,9 +298,20 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 0 return 0
} }
func extractErrorPhase(err error) (phase string, message string) {
msg := err.Error()
if strings.Contains(msg, ": ") {
parts := strings.SplitN(msg, ": ", 2)
if len(parts) == 2 {
return parts[0], parts[1]
}
}
return "", msg
}
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport { func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
report := reporting.ProcessReport{ report := reporting.ProcessReport{
Phase: "phase2-normalization", Phase: "phase2-foundation",
Status: status, Status: status,
Operation: "process", Operation: "process",
TranscriptPath: inv.TranscriptPath, TranscriptPath: inv.TranscriptPath,

1195
internal/cli/run_test.go
View File

File diff suppressed because it is too large Load Diff

9
internal/cli/testdata/tiny_glossary.yaml vendored
View File

@@ -1,3 +1,6 @@
terms: glossary:
- source: audita - name: Audita
target: Audita aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

48
internal/core/diagnostics/run_dir.go
View File

@@ -14,10 +14,9 @@ import (
// RunDirectory represents a per-run diagnostics directory // RunDirectory represents a per-run diagnostics directory
type RunDirectory struct { type RunDirectory struct {
path string path string
retention string retention string
createdAt time.Time createdAt time.Time
sourceSaved bool
} }
// NewRunDirectory creates a new run directory under the configured work dir // NewRunDirectory creates a new run directory under the configured work dir
@@ -31,9 +30,8 @@ func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err) return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
} }
// Create unique run directory with timestamp // Create a unique run directory identifier.
timestamp := time.Now().UTC().Format("2006-01-02T15-04-05Z") runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
runID := fmt.Sprintf("run-%s", timestamp)
runPath := filepath.Join(workDir, runID) runPath := filepath.Join(workDir, runID)
if err := os.Mkdir(runPath, 0o755); err != nil { if err := os.Mkdir(runPath, 0o755); err != nil {
@@ -71,7 +69,6 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
return fmt.Errorf("failed to write parsed source transcript: %w", err) return fmt.Errorf("failed to write parsed source transcript: %w", err)
} }
r.sourceSaved = true
return nil return nil
} }
@@ -122,24 +119,21 @@ func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644) return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
} }
// ApplyRetention applies the retention policy // ApplyRetention applies a minimal phase-2 retention policy.
// TODO: In later phases, implement full retention logic including: // TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
// - auto mode should keep runs with skipped corrections func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
// - Add proper cleanup of old runs if !runSucceeded {
// - Consider disk space limits // Failed runs are always retained.
func (r *RunDirectory) ApplyRetention() error { return nil
// For Phase 2, implement minimal retention: }
// - always: keep everything (do nothing)
// - never: remove successful runs switch r.retention {
// - auto: remove successful runs (same as never for now) case "always":
// - failed runs are always kept (handled by caller) return nil
case "never", "auto":
if r.retention == "never" || r.retention == "auto" { return os.RemoveAll(r.path)
// Remove successful runs default:
if r.sourceSaved { // Simple heuristic: if we saved source, it was successful // Retain by default for unknown modes; config validation should prevent this.
return os.RemoveAll(r.path) return nil
}
} }
// always: keep everything, or failed runs: keep everything
return nil
} }

36
internal/core/normalization/normalize.go
View File

@@ -2,7 +2,6 @@ package normalization
import ( import (
"sort" "sort"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
) )
@@ -63,13 +62,11 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Merge same-speaker adjacent segments // Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment var normalizedSegments []schema.Segment
var currentSegment schema.Segment var currentSegment schema.Segment
var sourceIDs []int
for i, segment := range sortedSegments { for i, segment := range sortedSegments {
if i == 0 { if i == 0 {
// Initialize with first segment // Initialize with first segment
currentSegment = segment currentSegment = segment
sourceIDs = []int{segment.ID}
continue continue
} }
@@ -79,7 +76,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
if merge { if merge {
summary.MergesPerformed++ summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap) currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
sourceIDs = append(sourceIDs, segment.ID)
} else { } else {
// Track the reason for not merging // Track the reason for not merging
switch reason { switch reason {
@@ -95,7 +91,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Finalize current segment and start new one // Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment) normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment currentSegment = segment
sourceIDs = []int{segment.ID}
} }
} }
@@ -155,6 +150,37 @@ func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
return true return true
} }
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment { func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text mergedText := current.Text
if gap >= ellipsisGap { if gap >= ellipsisGap {

131
internal/core/normalization/normalize_test.go
View File

@@ -1,6 +1,8 @@
package normalization package normalization
import ( import (
"os"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -155,7 +157,7 @@ func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 3, // Very small token limit MaxSegmentTokens: 1, // Very small token limit
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -273,10 +275,10 @@ func TestNormalizationCategoryPreservation(t *testing.T) {
func TestNormalizationGoldenBareArrayTranscript(t *testing.T) { func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -316,20 +318,20 @@ func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) { func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -369,20 +371,20 @@ func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) { func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -422,20 +424,20 @@ func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) { func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -475,20 +477,20 @@ func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) { func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -528,20 +530,20 @@ func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenEllipsisMerge(t *testing.T) { func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -581,20 +583,20 @@ func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) { func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
config := NormalizationConfig{ config := NormalizationConfig{
MaxSegmentGap: 2.0, MaxSegmentGap: 2.0,
EllipsisGap: 1.0, EllipsisGap: 1.0,
MaxSegmentDuration: 60.0, MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100, MaxSegmentTokens: 100,
} }
normalizer := NewNormalizer(config) normalizer := NewNormalizer(config)
@@ -634,12 +636,12 @@ func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err) t.Fatalf("failed to read golden fixture: %v", err)
} }
var expectedTranscript schema.Transcript expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil { if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err) t.Fatalf("failed to parse golden transcript: %v", err)
} }
assertTranscriptsEqual(t, normalized, &expectedTranscript) assertTranscriptsEqual(t, normalized, expectedTranscript)
} }
func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) { func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
@@ -679,33 +681,4 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
} }
} }
} }
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
{ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"},
},
}
_, summary := normalizer.Normalize(transcript)
// Verify summary counts
if summary.InputSegmentCount != 3 {
t.Errorf("expected input count 3, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 1 {
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
}
if summary.IDsReassigned != 3 {
t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
} }

4
internal/core/normalization/testdata/transcript_same_speaker_merge.golden.json vendored
View File

@@ -3,7 +3,7 @@
"id": 1, "id": 1,
"speaker": "Alice", "speaker": "Alice",
"start": 0.0, "start": 0.0,
"end": 2.5, "end": 2.2,
"text": "Hello world" "text": "Hello world"
} }
] ]

20
internal/core/schema/glossary_test.go
View File

@@ -54,8 +54,8 @@ func TestParseGlossaryYAML_MalformedYAML(t *testing.T) {
t.Fatal("expected error for malformed YAML, got nil") t.Fatal("expected error for malformed YAML, got nil")
} }
var parseErr *ParseError parseErr, ok := err.(*ParseError)
if parseErr, ok := err.(*ParseError); !ok { if !ok {
t.Errorf("expected *ParseError, got %T", err) t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message == "" { } else if parseErr.Message == "" {
t.Error("expected non-empty parse error message") t.Error("expected non-empty parse error message")
@@ -73,8 +73,8 @@ func TestParseGlossaryYAML_MissingRequiredFields(t *testing.T) {
t.Fatal("expected error for missing required fields, got nil") t.Fatal("expected error for missing required fields, got nil")
} }
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Field == "" { } else if valErr.Field == "" {
t.Error("expected non-empty validation error field") t.Error("expected non-empty validation error field")
@@ -89,8 +89,8 @@ func TestParseGlossaryYAML_EmptyGlossary(t *testing.T) {
t.Fatal("expected error for empty glossary, got nil") t.Fatal("expected error for empty glossary, got nil")
} }
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must contain at least one entry" { } else if valErr.Message != "must contain at least one entry" {
t.Errorf("expected 'must contain at least one entry', got %q", valErr.Message) t.Errorf("expected 'must contain at least one entry', got %q", valErr.Message)
@@ -110,8 +110,8 @@ glossary:
t.Fatal("expected error for empty name, got nil") t.Fatal("expected error for empty name, got nil")
} }
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must not be empty" { } else if valErr.Message != "must not be empty" {
t.Errorf("expected 'must not be empty', got %q", valErr.Message) t.Errorf("expected 'must not be empty', got %q", valErr.Message)
@@ -133,8 +133,8 @@ glossary:
t.Fatal("expected error for empty alias, got nil") t.Fatal("expected error for empty alias, got nil")
} }
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must not be empty" { } else if valErr.Message != "must not be empty" {
t.Errorf("expected 'must not be empty', got %q", valErr.Message) t.Errorf("expected 'must not be empty', got %q", valErr.Message)

7
internal/core/schema/transcript.go
View File

@@ -47,14 +47,13 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
return nil, err return nil, err
} }
if len(segmentsRaw) == 0 {
return nil, &ParseError{Message: "transcript must contain at least one segment"}
}
var segments []SourceSegment var segments []SourceSegment
if err := json.Unmarshal(segmentsRaw, &segments); err != nil { if err := json.Unmarshal(segmentsRaw, &segments); err != nil {
return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)} return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)}
} }
if len(segments) == 0 {
return nil, &ParseError{Message: "transcript must contain at least one segment"}
}
if err := validateSourceSegments(segments); err != nil { if err := validateSourceSegments(segments); err != nil {
return nil, err return nil, err

24
internal/core/schema/transcript_test.go
View File

@@ -80,8 +80,8 @@ func TestParseSourceTranscriptJSON_MalformedJSON(t *testing.T) {
t.Fatal("expected error for malformed JSON, got nil") t.Fatal("expected error for malformed JSON, got nil")
} }
var parseErr *ParseError parseErr, ok := err.(*ParseError)
if parseErr, ok := err.(*ParseError); !ok { if !ok {
t.Errorf("expected *ParseError, got %T", err) t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript is not valid JSON" { } else if parseErr.Message != "transcript is not valid JSON" {
t.Errorf("expected 'transcript is not valid JSON', got %q", parseErr.Message) t.Errorf("expected 'transcript is not valid JSON', got %q", parseErr.Message)
@@ -155,8 +155,8 @@ func TestParseSourceTranscriptJSON_EmptyArray(t *testing.T) {
t.Fatal("expected error for empty array, got nil") t.Fatal("expected error for empty array, got nil")
} }
var parseErr *ParseError parseErr, ok := err.(*ParseError)
if parseErr, ok := err.(*ParseError); !ok { if !ok {
t.Errorf("expected *ParseError, got %T", err) t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript must contain at least one segment" { } else if parseErr.Message != "transcript must contain at least one segment" {
t.Errorf("expected 'transcript must contain at least one segment', got %q", parseErr.Message) t.Errorf("expected 'transcript must contain at least one segment', got %q", parseErr.Message)
@@ -208,8 +208,8 @@ func TestParseSourceTranscriptJSON_UnsupportedTopLevelShape(t *testing.T) {
t.Fatal("expected error for unsupported top-level shape, got nil") t.Fatal("expected error for unsupported top-level shape, got nil")
} }
var parseErr *ParseError parseErr, ok := err.(*ParseError)
if parseErr, ok := err.(*ParseError); !ok { if !ok {
t.Errorf("expected *ParseError, got %T", err) t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript must be a JSON array or an object with a segments array" { } else if parseErr.Message != "transcript must be a JSON array or an object with a segments array" {
t.Errorf("expected unsupported shape message, got %q", parseErr.Message) t.Errorf("expected unsupported shape message, got %q", parseErr.Message)
@@ -224,8 +224,8 @@ func TestParseSourceTranscriptJSON_ObjectWithoutSegments(t *testing.T) {
t.Fatal("expected error for object without segments, got nil") t.Fatal("expected error for object without segments, got nil")
} }
var parseErr *ParseError parseErr, ok := err.(*ParseError)
if parseErr, ok := err.(*ParseError); !ok { if !ok {
t.Errorf("expected *ParseError, got %T", err) t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript object must contain a segments array" { } else if parseErr.Message != "transcript object must contain a segments array" {
t.Errorf("expected 'transcript object must contain a segments array', got %q", parseErr.Message) t.Errorf("expected 'transcript object must contain a segments array', got %q", parseErr.Message)
@@ -265,8 +265,8 @@ func TestTranscriptToJSON(t *testing.T) {
func assertValidationError(t *testing.T, err error, fieldContains, messageContains string) { func assertValidationError(t *testing.T, err error, fieldContains, messageContains string) {
t.Helper() t.Helper()
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
return return
} }
@@ -282,8 +282,8 @@ func assertValidationError(t *testing.T, err error, fieldContains, messageContai
func assertValidationErrorContains(t *testing.T, err error, fieldContains, messageContains string) { func assertValidationErrorContains(t *testing.T, err error, fieldContains, messageContains string) {
t.Helper() t.Helper()
var valErr *ValidationError valErr, ok := err.(*ValidationError)
if valErr, ok := err.(*ValidationError); !ok { if !ok {
t.Errorf("expected *ValidationError, got %T", err) t.Errorf("expected *ValidationError, got %T", err)
return return
} }