eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 2 Go normalization foundation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:41:29 +00:00
parent d847168ecd
commit 10377876e4
11 changed files with 480 additions and 1241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
internal/core/diagnostics/run_dir.go
View File

@@ -14,10 +14,9 @@ import (
// RunDirectory represents a per-run diagnostics directory
type RunDirectory struct {
path string
retention string
createdAt time.Time
sourceSaved bool
path string
retention string
createdAt time.Time
}
// NewRunDirectory creates a new run directory under the configured work dir
@@ -31,9 +30,8 @@ func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
}
// Create unique run directory with timestamp
timestamp := time.Now().UTC().Format("2006-01-02T15-04-05Z")
runID := fmt.Sprintf("run-%s", timestamp)
// Create a unique run directory identifier.
runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
runPath := filepath.Join(workDir, runID)
if err := os.Mkdir(runPath, 0o755); err != nil {
@@ -71,7 +69,6 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
return fmt.Errorf("failed to write parsed source transcript: %w", err)
}
r.sourceSaved = true
return nil
}
@@ -122,24 +119,21 @@ func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
}
// ApplyRetention applies the retention policy
// TODO: In later phases, implement full retention logic including:
// - auto mode should keep runs with skipped corrections
// - Add proper cleanup of old runs
// - Consider disk space limits
func (r *RunDirectory) ApplyRetention() error {
// For Phase 2, implement minimal retention:
// - always: keep everything (do nothing)
// - never: remove successful runs
// - auto: remove successful runs (same as never for now)
// - failed runs are always kept (handled by caller)
if r.retention == "never" || r.retention == "auto" {
// Remove successful runs
if r.sourceSaved { // Simple heuristic: if we saved source, it was successful
return os.RemoveAll(r.path)
}
// ApplyRetention applies a minimal phase-2 retention policy.
// TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
if !runSucceeded {
// Failed runs are always retained.
return nil
}
switch r.retention {
case "always":
return nil
case "never", "auto":
return os.RemoveAll(r.path)
default:
// Retain by default for unknown modes; config validation should prevent this.
return nil
}
// always: keep everything, or failed runs: keep everything
return nil
}

36
internal/core/normalization/normalize.go
View File

@@ -2,7 +2,6 @@ package normalization
import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
@@ -63,13 +62,11 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
var sourceIDs []int
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
sourceIDs = []int{segment.ID}
continue
}
@@ -79,7 +76,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
sourceIDs = append(sourceIDs, segment.ID)
} else {
// Track the reason for not merging
switch reason {
@@ -95,7 +91,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
sourceIDs = []int{segment.ID}
}
}
@@ -155,6 +150,37 @@ func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {

131
internal/core/normalization/normalize_test.go
View File

@@ -1,6 +1,8 @@
package normalization
import (
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -155,7 +157,7 @@ func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 3, // Very small token limit
MaxSegmentTokens: 1, // Very small token limit
}
normalizer := NewNormalizer(config)
@@ -273,10 +275,10 @@ func TestNormalizationCategoryPreservation(t *testing.T) {
func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -316,20 +318,20 @@ func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -369,20 +371,20 @@ func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -422,20 +424,20 @@ func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -475,20 +477,20 @@ func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -528,20 +530,20 @@ func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -581,20 +583,20 @@ func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -634,12 +636,12 @@ func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
@@ -679,33 +681,4 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
}
}
}
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
{ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"},
},
}
_, summary := normalizer.Normalize(transcript)
// Verify summary counts
if summary.InputSegmentCount != 3 {
t.Errorf("expected input count 3, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 1 {
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
}
if summary.IDsReassigned != 3 {
t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
}

4
internal/core/normalization/testdata/transcript_same_speaker_merge.golden.json vendored
View File

@@ -3,7 +3,7 @@
"id": 1,
"speaker": "Alice",
"start": 0.0,
"end": 2.5,
"end": 2.2,
"text": "Hello world"
}
]
]

20
internal/core/schema/glossary_test.go
View File

@@ -54,8 +54,8 @@ func TestParseGlossaryYAML_MalformedYAML(t *testing.T) {
t.Fatal("expected error for malformed YAML, got nil")
}
var parseErr *ParseError
if parseErr, ok := err.(*ParseError); !ok {
parseErr, ok := err.(*ParseError)
if !ok {
t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message == "" {
t.Error("expected non-empty parse error message")
@@ -73,8 +73,8 @@ func TestParseGlossaryYAML_MissingRequiredFields(t *testing.T) {
t.Fatal("expected error for missing required fields, got nil")
}
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Field == "" {
t.Error("expected non-empty validation error field")
@@ -89,8 +89,8 @@ func TestParseGlossaryYAML_EmptyGlossary(t *testing.T) {
t.Fatal("expected error for empty glossary, got nil")
}
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must contain at least one entry" {
t.Errorf("expected 'must contain at least one entry', got %q", valErr.Message)
@@ -110,8 +110,8 @@ glossary:
t.Fatal("expected error for empty name, got nil")
}
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must not be empty" {
t.Errorf("expected 'must not be empty', got %q", valErr.Message)
@@ -133,8 +133,8 @@ glossary:
t.Fatal("expected error for empty alias, got nil")
}
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
} else if valErr.Message != "must not be empty" {
t.Errorf("expected 'must not be empty', got %q", valErr.Message)

7
internal/core/schema/transcript.go
View File

@@ -47,14 +47,13 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
return nil, err
}
if len(segmentsRaw) == 0 {
return nil, &ParseError{Message: "transcript must contain at least one segment"}
}
var segments []SourceSegment
if err := json.Unmarshal(segmentsRaw, &segments); err != nil {
return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)}
}
if len(segments) == 0 {
return nil, &ParseError{Message: "transcript must contain at least one segment"}
}
if err := validateSourceSegments(segments); err != nil {
return nil, err

24
internal/core/schema/transcript_test.go
View File

@@ -80,8 +80,8 @@ func TestParseSourceTranscriptJSON_MalformedJSON(t *testing.T) {
t.Fatal("expected error for malformed JSON, got nil")
}
var parseErr *ParseError
if parseErr, ok := err.(*ParseError); !ok {
parseErr, ok := err.(*ParseError)
if !ok {
t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript is not valid JSON" {
t.Errorf("expected 'transcript is not valid JSON', got %q", parseErr.Message)
@@ -155,8 +155,8 @@ func TestParseSourceTranscriptJSON_EmptyArray(t *testing.T) {
t.Fatal("expected error for empty array, got nil")
}
var parseErr *ParseError
if parseErr, ok := err.(*ParseError); !ok {
parseErr, ok := err.(*ParseError)
if !ok {
t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript must contain at least one segment" {
t.Errorf("expected 'transcript must contain at least one segment', got %q", parseErr.Message)
@@ -208,8 +208,8 @@ func TestParseSourceTranscriptJSON_UnsupportedTopLevelShape(t *testing.T) {
t.Fatal("expected error for unsupported top-level shape, got nil")
}
var parseErr *ParseError
if parseErr, ok := err.(*ParseError); !ok {
parseErr, ok := err.(*ParseError)
if !ok {
t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript must be a JSON array or an object with a segments array" {
t.Errorf("expected unsupported shape message, got %q", parseErr.Message)
@@ -224,8 +224,8 @@ func TestParseSourceTranscriptJSON_ObjectWithoutSegments(t *testing.T) {
t.Fatal("expected error for object without segments, got nil")
}
var parseErr *ParseError
if parseErr, ok := err.(*ParseError); !ok {
parseErr, ok := err.(*ParseError)
if !ok {
t.Errorf("expected *ParseError, got %T", err)
} else if parseErr.Message != "transcript object must contain a segments array" {
t.Errorf("expected 'transcript object must contain a segments array', got %q", parseErr.Message)
@@ -265,8 +265,8 @@ func TestTranscriptToJSON(t *testing.T) {
func assertValidationError(t *testing.T, err error, fieldContains, messageContains string) {
t.Helper()
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
return
}
@@ -282,8 +282,8 @@ func assertValidationError(t *testing.T, err error, fieldContains, messageContai
func assertValidationErrorContains(t *testing.T, err error, fieldContains, messageContains string) {
t.Helper()
var valErr *ValidationError
if valErr, ok := err.(*ValidationError); !ok {
valErr, ok := err.(*ValidationError)
if !ok {
t.Errorf("expected *ValidationError, got %T", err)
return
}