eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Record chunking metadata in process runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 12:47:31 +00:00
parent 12fd541669
commit b997e7c97c
6 changed files with 598 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/cli/run.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
@@ -24,15 +25,15 @@ type processInvocation struct {
Config config.Config
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, nil, fmt.Errorf("run_dir_creation: %w", err)
return nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
}
fail := func(phase string, err error) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
fail := func(phase string, err error) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, runDir, fmt.Errorf("%s: %w", phase, err)
return nil, nil, runDir, fmt.Errorf("%s: %w", phase, err)
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
@@ -66,15 +67,39 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, summary := normalizer.Normalize(canonical)
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(summary); err != nil {
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
// Compute chunks after normalization
chunker := chunking.NewChunker(chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
})
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
if chunkErr != nil {
return fail("chunking", chunkErr)
}
chunkConfig := chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
}
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
}
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
return fail("serialization", err)
@@ -84,14 +109,14 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
return fail("output_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
@@ -252,12 +277,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
Config: cfg,
}
summary, runDir, runErr := processRunner(inv, stdout)
normSummary, chunkSummary, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC()
if runErr != nil {
errorPhase, errorMessage := extractErrorPhase(runErr)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil, nil)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -274,7 +299,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 1
}
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", summary)
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normSummary, chunkSummary)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -309,9 +334,9 @@ func extractErrorPhase(err error) (phase string, message string) {
return "", msg
}
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase2-foundation",
Phase: "phase3-chunking",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,
@@ -335,6 +360,17 @@ func buildProcessReport(status string, inv processInvocation, startedAt, complet
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
}
if chunkingSummary != nil {
report.Chunking = &reporting.ChunkingSummary{
ChunkCount: chunkingSummary.ChunkCount,
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
TargetSections: chunkingSummary.TargetSections,
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
MinSectionTokens: chunkingSummary.MinSectionTokens,
}
}
return report
}

268
internal/cli/run_test.go
View File

@@ -413,6 +413,274 @@ func TestRunProcessFailedRunRetainedWithNeverRetention(t *testing.T) {
}
}
func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world this is a test"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another segment here with more text"},
{"id":3,"speaker":"Alice","start":4.0,"end":5.0,"text":"Final segment text"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "normalized.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
report := readProcessReport(t, reportPath)
if report.Chunking == nil {
t.Fatalf("expected chunking summary in report")
}
if report.Chunking.ChunkCount == 0 {
t.Errorf("expected non-zero chunk count")
}
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase3-chunking" {
t.Errorf("expected phase 'phase3-chunking', got %q", report.Phase)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
workDir := t.TempDir()
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
]`)
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
runPath := onlyRunDir(t, workDir)
// Verify chunking-summary.json exists
chunkingSummaryPath := filepath.Join(runPath, "chunking-summary.json")
if _, err := os.Stat(chunkingSummaryPath); err != nil {
t.Fatalf("expected chunking-summary.json artifact: %v", err)
}
// Verify it contains valid JSON with expected fields
content := readFile(t, chunkingSummaryPath)
var summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
Chunks []struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
} `json:"chunks"`
}
if err := json.Unmarshal(content, &summary); err != nil {
t.Fatalf("failed to parse chunking summary: %v", err)
}
if summary.ChunkCount == 0 {
t.Errorf("expected non-zero chunk count in summary")
}
if len(summary.Chunks) != summary.ChunkCount {
t.Errorf("expected %d chunks, got %d", summary.ChunkCount, len(summary.Chunks))
}
}
func TestRunProcessTargetSectionsThroughCLI(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another test"},
{"id":3,"speaker":"Charlie","start":4.0,"end":5.0,"text":"Third segment"},
{"id":4,"speaker":"Alice","start":6.0,"end":7.0,"text":"Final segment"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--target-sections",
"2",
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.Chunking == nil {
t.Fatalf("expected chunking summary in report")
}
if report.Chunking.ChunkCount != 2 {
t.Errorf("expected 2 chunks with --target-sections 2, got %d", report.Chunking.ChunkCount)
}
if report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != 2 {
t.Errorf("expected target_sections=2 in report")
}
}
func TestRunProcessImpossibleTargetSectionsFailsCleanly(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
// Create transcript with 2 segments
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
workDir := t.TempDir()
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--target-sections",
"5",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for impossible target-sections")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "target_sections") {
t.Errorf("expected target_sections error in stderr, got %q", stderr.String())
}
// Verify report was written with failure info
report := readProcessReport(t, reportPath)
if report.Status != "failed" {
t.Errorf("expected failed status, got %q", report.Status)
}
if report.ErrorPhase != "chunking" {
t.Errorf("expected error phase 'chunking', got %q", report.ErrorPhase)
}
// Verify run dir was retained with error.log
runPath := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil {
t.Errorf("expected error.log in retained failed run: %v", err)
}
}
func TestRunProcessOutputTranscriptStillNormalizedOnly(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"},
{"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"}
]`)
outputPath := filepath.Join(t.TempDir(), "normalized.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
// Verify output is the normalized transcript (same as before)
output := readFile(t, outputPath)
transcript, err := schema.ParseTranscriptJSON(output)
if err != nil {
t.Fatalf("expected valid transcript JSON: %v", err)
}
if len(transcript.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(transcript.Segments))
}
// Text should be normalized (merged with space)
if transcript.Segments[0].Text != "Hello world" {
t.Errorf("expected merged text 'Hello world', got %q", transcript.Segments[0].Text)
}
}
func TestRunProcessReportJSONNotPrintedToStdout(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
reportPath := filepath.Join(t.TempDir(), "report.json")
exitCode := Run([]string{
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String())
}
// Verify stdout contains only transcript JSON, not report JSON
stdoutStr := stdout.String()
if strings.Contains(stdoutStr, "chunk_count") {
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
}
if strings.Contains(stdoutStr, "normalization_merges") {
t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr)
}
// Verify transcript is valid JSON
if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil {
t.Errorf("stdout should contain valid transcript JSON: %v", err)
}
}
func fixturePath(name string) string {
return filepath.Join("testdata", name)
}

84
internal/core/chunking/summary.go Normal file
View File

@@ -0,0 +1,84 @@
package chunking
// Summary provides a concise overview of chunking results for reports
type Summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
// ChunkSummary represents a single chunk's metadata for diagnostics
type ChunkSummary struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
}
// DetailedSummary provides per-chunk details for diagnostics
type DetailedSummary struct {
Summary `json:",inline"`
Chunks []ChunkSummary `json:"chunks"`
}
// ComputeSummary creates a Summary from sections and config
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
if len(sections) == 0 {
return Summary{
ChunkCount: 0,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
TargetSections: config.TargetSections,
}
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
totalTokens := 0
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
totalTokens += sec.EstimatedTokens
}
return Summary{
ChunkCount: len(sections),
MinEstimatedTokens: minTokens,
MaxEstimatedTokens: maxTokens,
TotalEstimatedTokens: totalTokens,
TargetSections: config.TargetSections,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
}
}
// ComputeDetailedSummary creates a DetailedSummary from sections and config
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
summary := ComputeSummary(sections, config)
chunks := make([]ChunkSummary, len(sections))
for i, sec := range sections {
chunks[i] = ChunkSummary{
Index: sec.Index,
StartSegmentID: sec.StartSegmentID,
EndSegmentID: sec.EndSegmentID,
EstimatedTokens: sec.EstimatedTokens,
SegmentCount: len(sec.Segments),
}
}
return DetailedSummary{
Summary: summary,
Chunks: chunks,
}
}

148
internal/core/chunking/summary_test.go Normal file
View File

@@ -0,0 +1,148 @@
package chunking
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestComputeSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 3 {
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 20 {
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
}
if summary.MaxEstimatedTokens != 50 {
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
}
if summary.TotalEstimatedTokens != 100 {
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
}
if summary.MinSectionTokens != 10 {
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
}
if summary.TargetSections != nil {
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
}
}
func TestComputeSummaryWithTarget(t *testing.T) {
target := 5
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
TargetSections: &target,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
}
summary := ComputeSummary(sections, config)
if summary.TargetSections == nil || *summary.TargetSections != 5 {
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
}
}
func TestComputeSummaryEmptySections(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 0 {
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 0 {
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
}
}
func TestComputeDetailedSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{
Index: 0,
EstimatedTokens: 30,
StartSegmentID: 1,
EndSegmentID: 2,
Segments: make([]schema.Segment, 2), // 2 segments
},
{
Index: 1,
EstimatedTokens: 50,
StartSegmentID: 3,
EndSegmentID: 5,
Segments: make([]schema.Segment, 3), // 3 segments
},
}
detailed := ComputeDetailedSummary(sections, config)
if detailed.ChunkCount != 2 {
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
}
if len(detailed.Chunks) != 2 {
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
}
// Check first chunk
if detailed.Chunks[0].Index != 0 {
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
}
if detailed.Chunks[0].StartSegmentID != 1 {
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
}
if detailed.Chunks[0].EndSegmentID != 2 {
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
}
if detailed.Chunks[0].EstimatedTokens != 30 {
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
}
if detailed.Chunks[0].SegmentCount != 2 {
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
}
// Check second chunk
if detailed.Chunks[1].Index != 1 {
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
}
if detailed.Chunks[1].StartSegmentID != 3 {
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
}
if detailed.Chunks[1].EndSegmentID != 5 {
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
}
if detailed.Chunks[1].SegmentCount != 3 {
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
}
}

16
internal/core/diagnostics/run_dir.go
View File

@@ -7,6 +7,7 @@ import (
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -119,7 +120,20 @@ func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
}
// ApplyRetention applies a minimal phase-2 retention policy.
// WriteChunkingSummary writes the chunking summary artifact
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
summaryPath := filepath.Join(r.path, "chunking-summary.json")
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal chunking summary: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write chunking summary: %w", err)
}
return nil
}
// TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
if !runSucceeded {

55
internal/core/reporting/report.go
View File

@@ -8,27 +8,40 @@ import (
)
type ProcessReport struct {
Phase string `json:"phase"`
Status string `json:"status"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
Modules []string `json:"modules"`
StartedAt time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
InputSegmentCount *int `json:"input_segment_count,omitempty"`
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
NormalizationMerges *int `json:"normalization_merges,omitempty"`
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
NormalizationSkipped struct {
DifferentSpeakers *int `json:"different_speakers,omitempty"`
GapTooLarge *int `json:"gap_too_large,omitempty"`
DurationExceeded *int `json:"duration_exceeded,omitempty"`
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
} `json:"normalization_skipped,omitempty"`
Phase string `json:"phase"`
Status string `json:"status"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
Modules []string `json:"modules"`
StartedAt time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
InputSegmentCount *int `json:"input_segment_count,omitempty"`
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
NormalizationMerges *int `json:"normalization_merges,omitempty"`
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
NormalizationSkipped NormalizationSkipped `json:"normalization_skipped,omitempty"`
Chunking *ChunkingSummary `json:"chunking,omitempty"`
}
type NormalizationSkipped struct {
DifferentSpeakers *int `json:"different_speakers,omitempty"`
GapTooLarge *int `json:"gap_too_large,omitempty"`
DurationExceeded *int `json:"duration_exceeded,omitempty"`
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
}
type ChunkingSummary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
func WriteProcessReport(path string, report ProcessReport) error {