Record chunking metadata in process runs
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
||||||
@@ -24,15 +25,15 @@ type processInvocation struct {
|
|||||||
Config config.Config
|
Config config.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
|
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
|
||||||
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
return nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fail := func(phase string, err error) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
|
fail := func(phase string, err error) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
|
||||||
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
||||||
return nil, runDir, fmt.Errorf("%s: %w", phase, err)
|
return nil, nil, runDir, fmt.Errorf("%s: %w", phase, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
||||||
@@ -66,15 +67,39 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
|
|||||||
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
||||||
})
|
})
|
||||||
|
|
||||||
normalizedTranscript, summary := normalizer.Normalize(canonical)
|
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
|
||||||
|
|
||||||
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
||||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
||||||
}
|
}
|
||||||
if err := runDir.WriteNormalizationSummary(summary); err != nil {
|
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
|
||||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compute chunks after normalization
|
||||||
|
chunker := chunking.NewChunker(chunking.ChunkingConfig{
|
||||||
|
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||||
|
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||||
|
TargetSections: inv.Config.TargetSections,
|
||||||
|
})
|
||||||
|
|
||||||
|
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
|
||||||
|
if chunkErr != nil {
|
||||||
|
return fail("chunking", chunkErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkConfig := chunking.ChunkingConfig{
|
||||||
|
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||||
|
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||||
|
TargetSections: inv.Config.TargetSections,
|
||||||
|
}
|
||||||
|
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
|
||||||
|
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
|
||||||
|
|
||||||
|
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
|
||||||
|
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
|
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail("serialization", err)
|
return fail("serialization", err)
|
||||||
@@ -84,14 +109,14 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
|
|||||||
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
||||||
return fail("output_write", err)
|
return fail("output_write", err)
|
||||||
}
|
}
|
||||||
return summary, runDir, nil
|
return normSummary, &chunkSummary, runDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := stdout.Write(outputBytes); err != nil {
|
if _, err := stdout.Write(outputBytes); err != nil {
|
||||||
return fail("stdout_write", err)
|
return fail("stdout_write", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return summary, runDir, nil
|
return normSummary, &chunkSummary, runDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
||||||
@@ -252,12 +277,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
Config: cfg,
|
Config: cfg,
|
||||||
}
|
}
|
||||||
|
|
||||||
summary, runDir, runErr := processRunner(inv, stdout)
|
normSummary, chunkSummary, runDir, runErr := processRunner(inv, stdout)
|
||||||
completedAt := time.Now().UTC()
|
completedAt := time.Now().UTC()
|
||||||
|
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||||
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil)
|
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil, nil)
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -274,7 +299,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", summary)
|
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normSummary, chunkSummary)
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -309,9 +334,9 @@ func extractErrorPhase(err error) (phase string, message string) {
|
|||||||
return "", msg
|
return "", msg
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
|
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary) reporting.ProcessReport {
|
||||||
report := reporting.ProcessReport{
|
report := reporting.ProcessReport{
|
||||||
Phase: "phase2-foundation",
|
Phase: "phase3-chunking",
|
||||||
Status: status,
|
Status: status,
|
||||||
Operation: "process",
|
Operation: "process",
|
||||||
TranscriptPath: inv.TranscriptPath,
|
TranscriptPath: inv.TranscriptPath,
|
||||||
@@ -335,6 +360,17 @@ func buildProcessReport(status string, inv processInvocation, startedAt, complet
|
|||||||
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
||||||
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||||
}
|
}
|
||||||
|
if chunkingSummary != nil {
|
||||||
|
report.Chunking = &reporting.ChunkingSummary{
|
||||||
|
ChunkCount: chunkingSummary.ChunkCount,
|
||||||
|
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
||||||
|
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
||||||
|
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
||||||
|
TargetSections: chunkingSummary.TargetSections,
|
||||||
|
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
||||||
|
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -413,6 +413,274 @@ func TestRunProcessFailedRunRetainedWithNeverRetention(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world this is a test"},
|
||||||
|
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another segment here with more text"},
|
||||||
|
{"id":3,"speaker":"Alice","start":4.0,"end":5.0,"text":"Final segment text"}
|
||||||
|
]`)
|
||||||
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
outputPath := filepath.Join(t.TempDir(), "normalized.json")
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--output",
|
||||||
|
outputPath,
|
||||||
|
"--report-json",
|
||||||
|
reportPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
report := readProcessReport(t, reportPath)
|
||||||
|
if report.Chunking == nil {
|
||||||
|
t.Fatalf("expected chunking summary in report")
|
||||||
|
}
|
||||||
|
if report.Chunking.ChunkCount == 0 {
|
||||||
|
t.Errorf("expected non-zero chunk count")
|
||||||
|
}
|
||||||
|
if report.Chunking.MaxSectionTokens == 0 {
|
||||||
|
t.Errorf("expected max_section_tokens in report")
|
||||||
|
}
|
||||||
|
if report.Phase != "phase3-chunking" {
|
||||||
|
t.Errorf("expected phase 'phase3-chunking', got %q", report.Phase)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
workDir := t.TempDir()
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
|
||||||
|
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
|
||||||
|
]`)
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--work-dir",
|
||||||
|
workDir,
|
||||||
|
"--work-dir-retention",
|
||||||
|
"always",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
runPath := onlyRunDir(t, workDir)
|
||||||
|
|
||||||
|
// Verify chunking-summary.json exists
|
||||||
|
chunkingSummaryPath := filepath.Join(runPath, "chunking-summary.json")
|
||||||
|
if _, err := os.Stat(chunkingSummaryPath); err != nil {
|
||||||
|
t.Fatalf("expected chunking-summary.json artifact: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify it contains valid JSON with expected fields
|
||||||
|
content := readFile(t, chunkingSummaryPath)
|
||||||
|
var summary struct {
|
||||||
|
ChunkCount int `json:"chunk_count"`
|
||||||
|
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||||
|
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||||
|
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||||
|
Chunks []struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
StartSegmentID int `json:"start_segment_id"`
|
||||||
|
EndSegmentID int `json:"end_segment_id"`
|
||||||
|
EstimatedTokens int `json:"estimated_tokens"`
|
||||||
|
SegmentCount int `json:"segment_count"`
|
||||||
|
} `json:"chunks"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(content, &summary); err != nil {
|
||||||
|
t.Fatalf("failed to parse chunking summary: %v", err)
|
||||||
|
}
|
||||||
|
if summary.ChunkCount == 0 {
|
||||||
|
t.Errorf("expected non-zero chunk count in summary")
|
||||||
|
}
|
||||||
|
if len(summary.Chunks) != summary.ChunkCount {
|
||||||
|
t.Errorf("expected %d chunks, got %d", summary.ChunkCount, len(summary.Chunks))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProcessTargetSectionsThroughCLI(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"},
|
||||||
|
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another test"},
|
||||||
|
{"id":3,"speaker":"Charlie","start":4.0,"end":5.0,"text":"Third segment"},
|
||||||
|
{"id":4,"speaker":"Alice","start":6.0,"end":7.0,"text":"Final segment"}
|
||||||
|
]`)
|
||||||
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--target-sections",
|
||||||
|
"2",
|
||||||
|
"--report-json",
|
||||||
|
reportPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
report := readProcessReport(t, reportPath)
|
||||||
|
if report.Chunking == nil {
|
||||||
|
t.Fatalf("expected chunking summary in report")
|
||||||
|
}
|
||||||
|
if report.Chunking.ChunkCount != 2 {
|
||||||
|
t.Errorf("expected 2 chunks with --target-sections 2, got %d", report.Chunking.ChunkCount)
|
||||||
|
}
|
||||||
|
if report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != 2 {
|
||||||
|
t.Errorf("expected target_sections=2 in report")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProcessImpossibleTargetSectionsFailsCleanly(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
// Create transcript with 2 segments
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
|
||||||
|
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
|
||||||
|
]`)
|
||||||
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
workDir := t.TempDir()
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--target-sections",
|
||||||
|
"5",
|
||||||
|
"--report-json",
|
||||||
|
reportPath,
|
||||||
|
"--work-dir",
|
||||||
|
workDir,
|
||||||
|
"--work-dir-retention",
|
||||||
|
"always",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode == 0 {
|
||||||
|
t.Fatalf("expected nonzero exit code for impossible target-sections")
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "target_sections") {
|
||||||
|
t.Errorf("expected target_sections error in stderr, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify report was written with failure info
|
||||||
|
report := readProcessReport(t, reportPath)
|
||||||
|
if report.Status != "failed" {
|
||||||
|
t.Errorf("expected failed status, got %q", report.Status)
|
||||||
|
}
|
||||||
|
if report.ErrorPhase != "chunking" {
|
||||||
|
t.Errorf("expected error phase 'chunking', got %q", report.ErrorPhase)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify run dir was retained with error.log
|
||||||
|
runPath := onlyRunDir(t, workDir)
|
||||||
|
if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil {
|
||||||
|
t.Errorf("expected error.log in retained failed run: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProcessOutputTranscriptStillNormalizedOnly(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
|
||||||
|
{"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"}
|
||||||
|
]`)
|
||||||
|
outputPath := filepath.Join(t.TempDir(), "normalized.json")
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--output",
|
||||||
|
outputPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output is the normalized transcript (same as before)
|
||||||
|
output := readFile(t, outputPath)
|
||||||
|
transcript, err := schema.ParseTranscriptJSON(output)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected valid transcript JSON: %v", err)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 1 {
|
||||||
|
t.Errorf("expected 1 merged segment, got %d", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
// Text should be normalized (merged with space)
|
||||||
|
if transcript.Segments[0].Text != "Hello world" {
|
||||||
|
t.Errorf("expected merged text 'Hello world', got %q", transcript.Segments[0].Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunProcessReportJSONNotPrintedToStdout(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
|
||||||
|
]`)
|
||||||
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
|
||||||
|
exitCode := Run([]string{
|
||||||
|
"process",
|
||||||
|
transcriptPath,
|
||||||
|
"--glossary",
|
||||||
|
fixturePath("tiny_glossary.yaml"),
|
||||||
|
"--report-json",
|
||||||
|
reportPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify stdout contains only transcript JSON, not report JSON
|
||||||
|
stdoutStr := stdout.String()
|
||||||
|
if strings.Contains(stdoutStr, "chunk_count") {
|
||||||
|
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
|
||||||
|
}
|
||||||
|
if strings.Contains(stdoutStr, "normalization_merges") {
|
||||||
|
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify transcript is valid JSON
|
||||||
|
if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil {
|
||||||
|
t.Errorf("stdout should contain valid transcript JSON: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func fixturePath(name string) string {
|
func fixturePath(name string) string {
|
||||||
return filepath.Join("testdata", name)
|
return filepath.Join("testdata", name)
|
||||||
}
|
}
|
||||||
|
|||||||
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package chunking
|
||||||
|
|
||||||
|
// Summary provides a concise overview of chunking results for reports
|
||||||
|
type Summary struct {
|
||||||
|
ChunkCount int `json:"chunk_count"`
|
||||||
|
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||||
|
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||||
|
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||||
|
TargetSections *int `json:"target_sections,omitempty"`
|
||||||
|
MaxSectionTokens int `json:"max_section_tokens"`
|
||||||
|
MinSectionTokens int `json:"min_section_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||||
|
type ChunkSummary struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
StartSegmentID int `json:"start_segment_id"`
|
||||||
|
EndSegmentID int `json:"end_segment_id"`
|
||||||
|
EstimatedTokens int `json:"estimated_tokens"`
|
||||||
|
SegmentCount int `json:"segment_count"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetailedSummary provides per-chunk details for diagnostics
|
||||||
|
type DetailedSummary struct {
|
||||||
|
Summary `json:",inline"`
|
||||||
|
Chunks []ChunkSummary `json:"chunks"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComputeSummary creates a Summary from sections and config
|
||||||
|
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||||
|
if len(sections) == 0 {
|
||||||
|
return Summary{
|
||||||
|
ChunkCount: 0,
|
||||||
|
MaxSectionTokens: config.MaxSectionTokens,
|
||||||
|
MinSectionTokens: config.MinSectionTokens,
|
||||||
|
TargetSections: config.TargetSections,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
minTokens := sections[0].EstimatedTokens
|
||||||
|
maxTokens := sections[0].EstimatedTokens
|
||||||
|
totalTokens := 0
|
||||||
|
|
||||||
|
for _, sec := range sections {
|
||||||
|
if sec.EstimatedTokens < minTokens {
|
||||||
|
minTokens = sec.EstimatedTokens
|
||||||
|
}
|
||||||
|
if sec.EstimatedTokens > maxTokens {
|
||||||
|
maxTokens = sec.EstimatedTokens
|
||||||
|
}
|
||||||
|
totalTokens += sec.EstimatedTokens
|
||||||
|
}
|
||||||
|
|
||||||
|
return Summary{
|
||||||
|
ChunkCount: len(sections),
|
||||||
|
MinEstimatedTokens: minTokens,
|
||||||
|
MaxEstimatedTokens: maxTokens,
|
||||||
|
TotalEstimatedTokens: totalTokens,
|
||||||
|
TargetSections: config.TargetSections,
|
||||||
|
MaxSectionTokens: config.MaxSectionTokens,
|
||||||
|
MinSectionTokens: config.MinSectionTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||||
|
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||||
|
summary := ComputeSummary(sections, config)
|
||||||
|
|
||||||
|
chunks := make([]ChunkSummary, len(sections))
|
||||||
|
for i, sec := range sections {
|
||||||
|
chunks[i] = ChunkSummary{
|
||||||
|
Index: sec.Index,
|
||||||
|
StartSegmentID: sec.StartSegmentID,
|
||||||
|
EndSegmentID: sec.EndSegmentID,
|
||||||
|
EstimatedTokens: sec.EstimatedTokens,
|
||||||
|
SegmentCount: len(sec.Segments),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return DetailedSummary{
|
||||||
|
Summary: summary,
|
||||||
|
Chunks: chunks,
|
||||||
|
}
|
||||||
|
}
|
||||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
package chunking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestComputeSummary(t *testing.T) {
|
||||||
|
config := ChunkingConfig{
|
||||||
|
MaxSectionTokens: 100,
|
||||||
|
MinSectionTokens: 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := []Section{
|
||||||
|
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||||
|
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||||
|
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := ComputeSummary(sections, config)
|
||||||
|
|
||||||
|
if summary.ChunkCount != 3 {
|
||||||
|
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||||
|
}
|
||||||
|
if summary.MinEstimatedTokens != 20 {
|
||||||
|
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||||
|
}
|
||||||
|
if summary.MaxEstimatedTokens != 50 {
|
||||||
|
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||||
|
}
|
||||||
|
if summary.TotalEstimatedTokens != 100 {
|
||||||
|
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||||
|
}
|
||||||
|
if summary.MaxSectionTokens != 100 {
|
||||||
|
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||||
|
}
|
||||||
|
if summary.MinSectionTokens != 10 {
|
||||||
|
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||||
|
}
|
||||||
|
if summary.TargetSections != nil {
|
||||||
|
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||||
|
target := 5
|
||||||
|
config := ChunkingConfig{
|
||||||
|
MaxSectionTokens: 100,
|
||||||
|
MinSectionTokens: 10,
|
||||||
|
TargetSections: &target,
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := []Section{
|
||||||
|
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := ComputeSummary(sections, config)
|
||||||
|
|
||||||
|
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||||
|
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||||
|
config := ChunkingConfig{
|
||||||
|
MaxSectionTokens: 100,
|
||||||
|
MinSectionTokens: 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := []Section{}
|
||||||
|
|
||||||
|
summary := ComputeSummary(sections, config)
|
||||||
|
|
||||||
|
if summary.ChunkCount != 0 {
|
||||||
|
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||||
|
}
|
||||||
|
if summary.MinEstimatedTokens != 0 {
|
||||||
|
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||||
|
}
|
||||||
|
if summary.MaxSectionTokens != 100 {
|
||||||
|
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComputeDetailedSummary(t *testing.T) {
|
||||||
|
config := ChunkingConfig{
|
||||||
|
MaxSectionTokens: 100,
|
||||||
|
MinSectionTokens: 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := []Section{
|
||||||
|
{
|
||||||
|
Index: 0,
|
||||||
|
EstimatedTokens: 30,
|
||||||
|
StartSegmentID: 1,
|
||||||
|
EndSegmentID: 2,
|
||||||
|
Segments: make([]schema.Segment, 2), // 2 segments
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Index: 1,
|
||||||
|
EstimatedTokens: 50,
|
||||||
|
StartSegmentID: 3,
|
||||||
|
EndSegmentID: 5,
|
||||||
|
Segments: make([]schema.Segment, 3), // 3 segments
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
detailed := ComputeDetailedSummary(sections, config)
|
||||||
|
|
||||||
|
if detailed.ChunkCount != 2 {
|
||||||
|
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||||
|
}
|
||||||
|
if len(detailed.Chunks) != 2 {
|
||||||
|
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check first chunk
|
||||||
|
if detailed.Chunks[0].Index != 0 {
|
||||||
|
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||||
|
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||||
|
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||||
|
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[0].SegmentCount != 2 {
|
||||||
|
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check second chunk
|
||||||
|
if detailed.Chunks[1].Index != 1 {
|
||||||
|
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||||
|
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||||
|
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||||
|
}
|
||||||
|
if detailed.Chunks[1].SegmentCount != 3 {
|
||||||
|
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
@@ -119,7 +120,20 @@ func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
|||||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ApplyRetention applies a minimal phase-2 retention policy.
|
// WriteChunkingSummary writes the chunking summary artifact
|
||||||
|
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||||
|
summaryPath := filepath.Join(r.path, "chunking-summary.json")
|
||||||
|
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||||
|
}
|
||||||
|
bytes = append(bytes, '\n')
|
||||||
|
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("failed to write chunking summary: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
|
// TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
|
||||||
func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
|
func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
|
||||||
if !runSucceeded {
|
if !runSucceeded {
|
||||||
|
|||||||
@@ -8,27 +8,40 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type ProcessReport struct {
|
type ProcessReport struct {
|
||||||
Phase string `json:"phase"`
|
Phase string `json:"phase"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
Operation string `json:"operation"`
|
Operation string `json:"operation"`
|
||||||
TranscriptPath string `json:"transcript_path"`
|
TranscriptPath string `json:"transcript_path"`
|
||||||
GlossaryPath string `json:"glossary_path"`
|
GlossaryPath string `json:"glossary_path"`
|
||||||
OutputPath string `json:"output_path,omitempty"`
|
OutputPath string `json:"output_path,omitempty"`
|
||||||
Modules []string `json:"modules"`
|
Modules []string `json:"modules"`
|
||||||
StartedAt time.Time `json:"started_at"`
|
StartedAt time.Time `json:"started_at"`
|
||||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
ErrorMessage string `json:"error_message,omitempty"`
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
ErrorPhase string `json:"error_phase,omitempty"`
|
ErrorPhase string `json:"error_phase,omitempty"`
|
||||||
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
||||||
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
||||||
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
||||||
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
||||||
NormalizationSkipped struct {
|
NormalizationSkipped NormalizationSkipped `json:"normalization_skipped,omitempty"`
|
||||||
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
Chunking *ChunkingSummary `json:"chunking,omitempty"`
|
||||||
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
}
|
||||||
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
|
||||||
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
type NormalizationSkipped struct {
|
||||||
} `json:"normalization_skipped,omitempty"`
|
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
||||||
|
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
||||||
|
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
||||||
|
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkingSummary struct {
|
||||||
|
ChunkCount int `json:"chunk_count"`
|
||||||
|
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||||
|
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||||
|
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||||
|
TargetSections *int `json:"target_sections,omitempty"`
|
||||||
|
MaxSectionTokens int `json:"max_section_tokens"`
|
||||||
|
MinSectionTokens int `json:"min_section_tokens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteProcessReport(path string, report ProcessReport) error {
|
func WriteProcessReport(path string, report ProcessReport) error {
|
||||||
|
|||||||
Reference in New Issue
Block a user