eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Record chunking metadata in process runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 12:47:31 +00:00
parent 12fd541669
commit b997e7c97c
6 changed files with 598 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/cli/run.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
@@ -24,15 +25,15 @@ type processInvocation struct {
Config config.Config
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, nil, fmt.Errorf("run_dir_creation: %w", err)
return nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
}
fail := func(phase string, err error) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
fail := func(phase string, err error) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, runDir, fmt.Errorf("%s: %w", phase, err)
return nil, nil, runDir, fmt.Errorf("%s: %w", phase, err)
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
@@ -66,15 +67,39 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, summary := normalizer.Normalize(canonical)
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(summary); err != nil {
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
// Compute chunks after normalization
chunker := chunking.NewChunker(chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
})
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
if chunkErr != nil {
return fail("chunking", chunkErr)
}
chunkConfig := chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
}
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
}
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
return fail("serialization", err)
@@ -84,14 +109,14 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
return fail("output_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
@@ -252,12 +277,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
Config: cfg,
}
summary, runDir, runErr := processRunner(inv, stdout)
normSummary, chunkSummary, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC()
if runErr != nil {
errorPhase, errorMessage := extractErrorPhase(runErr)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil, nil)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -274,7 +299,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 1
}
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", summary)
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normSummary, chunkSummary)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -309,9 +334,9 @@ func extractErrorPhase(err error) (phase string, message string) {
return "", msg
}
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase2-foundation",
Phase: "phase3-chunking",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,
@@ -335,6 +360,17 @@ func buildProcessReport(status string, inv processInvocation, startedAt, complet
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
}
if chunkingSummary != nil {
report.Chunking = &reporting.ChunkingSummary{
ChunkCount: chunkingSummary.ChunkCount,
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
TargetSections: chunkingSummary.TargetSections,
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
MinSectionTokens: chunkingSummary.MinSectionTokens,
}
}
return report
}

268
internal/cli/run_test.go
View File

@@ -413,6 +413,274 @@ func TestRunProcessFailedRunRetainedWithNeverRetention(t *testing.T) {
}
}
func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world this is a test"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another segment here with more text"},
{"id":3,"speaker":"Alice","start":4.0,"end":5.0,"text":"Final segment text"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "normalized.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
report := readProcessReport(t, reportPath)
if report.Chunking == nil {
t.Fatalf("expected chunking summary in report")
}
if report.Chunking.ChunkCount == 0 {
t.Errorf("expected non-zero chunk count")
}
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase3-chunking" {
t.Errorf("expected phase 'phase3-chunking', got %q", report.Phase)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := t.TempDir()
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
]`)
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
runPath := onlyRunDir(t, workDir)
// Verify chunking-summary.json exists
chunkingSummaryPath := filepath.Join(runPath, "chunking-summary.json")
if _, err := os.Stat(chunkingSummaryPath); err != nil {
t.Fatalf("expected chunking-summary.json artifact: %v", err)
}
// Verify it contains valid JSON with expected fields
content := readFile(t, chunkingSummaryPath)
var summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
Chunks []struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
} `json:"chunks"`
}
if err := json.Unmarshal(content, &summary); err != nil {
t.Fatalf("failed to parse chunking summary: %v", err)
}
if summary.ChunkCount == 0 {
t.Errorf("expected non-zero chunk count in summary")
}
if len(summary.Chunks) != summary.ChunkCount {
t.Errorf("expected %d chunks, got %d", summary.ChunkCount, len(summary.Chunks))
}
}
func TestRunProcessTargetSectionsThroughCLI(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another test"},
{"id":3,"speaker":"Charlie","start":4.0,"end":5.0,"text":"Third segment"},
{"id":4,"speaker":"Alice","start":6.0,"end":7.0,"text":"Final segment"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--target-sections",
"2",
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.Chunking == nil {
t.Fatalf("expected chunking summary in report")
}
if report.Chunking.ChunkCount != 2 {
t.Errorf("expected 2 chunks with --target-sections 2, got %d", report.Chunking.ChunkCount)
}
if report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != 2 {
t.Errorf("expected target_sections=2 in report")
}
}
func TestRunProcessImpossibleTargetSectionsFailsCleanly(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
// Create transcript with 2 segments
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
workDir := t.TempDir()
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--target-sections",
"5",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for impossible target-sections")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "target_sections") {
t.Errorf("expected target_sections error in stderr, got %q", stderr.String())
}
// Verify report was written with failure info
report := readProcessReport(t, reportPath)
if report.Status != "failed" {
t.Errorf("expected failed status, got %q", report.Status)
}
if report.ErrorPhase != "chunking" {
t.Errorf("expected error phase 'chunking', got %q", report.ErrorPhase)
}
// Verify run dir was retained with error.log
runPath := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil {
t.Errorf("expected error.log in retained failed run: %v", err)
}
}
func TestRunProcessOutputTranscriptStillNormalizedOnly(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"}
]`)
outputPath := filepath.Join(t.TempDir(), "normalized.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
// Verify output is the normalized transcript (same as before)
output := readFile(t, outputPath)
transcript, err := schema.ParseTranscriptJSON(output)
if err != nil {
t.Fatalf("expected valid transcript JSON: %v", err)
}
if len(transcript.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(transcript.Segments))
}
// Text should be normalized (merged with space)
if transcript.Segments[0].Text != "Hello world" {
t.Errorf("expected merged text 'Hello world', got %q", transcript.Segments[0].Text)
}
}
func TestRunProcessReportJSONNotPrintedToStdout(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Verify stdout contains only transcript JSON, not report JSON
stdoutStr := stdout.String()
if strings.Contains(stdoutStr, "chunk_count") {
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
}
if strings.Contains(stdoutStr, "normalization_merges") {
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
}
// Verify transcript is valid JSON
if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil {
t.Errorf("stdout should contain valid transcript JSON: %v", err)
}
}
func fixturePath(name string) string {
return filepath.Join("testdata", name)
}