Record chunking metadata in process runs
This commit is contained in:
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package chunking
|
||||
|
||||
// Summary provides a concise overview of chunking results for reports
|
||||
type Summary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||
type ChunkSummary struct {
|
||||
Index int `json:"index"`
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
SegmentCount int `json:"segment_count"`
|
||||
}
|
||||
|
||||
// DetailedSummary provides per-chunk details for diagnostics
|
||||
type DetailedSummary struct {
|
||||
Summary `json:",inline"`
|
||||
Chunks []ChunkSummary `json:"chunks"`
|
||||
}
|
||||
|
||||
// ComputeSummary creates a Summary from sections and config
|
||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||
if len(sections) == 0 {
|
||||
return Summary{
|
||||
ChunkCount: 0,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
}
|
||||
}
|
||||
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
totalTokens := 0
|
||||
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
totalTokens += sec.EstimatedTokens
|
||||
}
|
||||
|
||||
return Summary{
|
||||
ChunkCount: len(sections),
|
||||
MinEstimatedTokens: minTokens,
|
||||
MaxEstimatedTokens: maxTokens,
|
||||
TotalEstimatedTokens: totalTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
chunks := make([]ChunkSummary, len(sections))
|
||||
for i, sec := range sections {
|
||||
chunks[i] = ChunkSummary{
|
||||
Index: sec.Index,
|
||||
StartSegmentID: sec.StartSegmentID,
|
||||
EndSegmentID: sec.EndSegmentID,
|
||||
EstimatedTokens: sec.EstimatedTokens,
|
||||
SegmentCount: len(sec.Segments),
|
||||
}
|
||||
}
|
||||
|
||||
return DetailedSummary{
|
||||
Summary: summary,
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestComputeSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 3 {
|
||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 20 {
|
||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxEstimatedTokens != 50 {
|
||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||
}
|
||||
if summary.TotalEstimatedTokens != 100 {
|
||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
if summary.MinSectionTokens != 10 {
|
||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||
}
|
||||
if summary.TargetSections != nil {
|
||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||
target := 5
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
TargetSections: &target,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 0 {
|
||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 0 {
|
||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDetailedSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{
|
||||
Index: 0,
|
||||
EstimatedTokens: 30,
|
||||
StartSegmentID: 1,
|
||||
EndSegmentID: 2,
|
||||
Segments: make([]schema.Segment, 2), // 2 segments
|
||||
},
|
||||
{
|
||||
Index: 1,
|
||||
EstimatedTokens: 50,
|
||||
StartSegmentID: 3,
|
||||
EndSegmentID: 5,
|
||||
Segments: make([]schema.Segment, 3), // 3 segments
|
||||
},
|
||||
}
|
||||
|
||||
detailed := ComputeDetailedSummary(sections, config)
|
||||
|
||||
if detailed.ChunkCount != 2 {
|
||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||
}
|
||||
if len(detailed.Chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||
}
|
||||
|
||||
// Check first chunk
|
||||
if detailed.Chunks[0].Index != 0 {
|
||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||
}
|
||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||
}
|
||||
if detailed.Chunks[0].SegmentCount != 2 {
|
||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||
}
|
||||
|
||||
// Check second chunk
|
||||
if detailed.Chunks[1].Index != 1 {
|
||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||
}
|
||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].SegmentCount != 3 {
|
||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
@@ -119,7 +120,20 @@ func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// ApplyRetention applies a minimal phase-2 retention policy.
|
||||
// WriteChunkingSummary writes the chunking summary artifact
|
||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||
summaryPath := filepath.Join(r.path, "chunking-summary.json")
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write chunking summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: In later phases, implement full "auto" semantics based on skip/report outcomes.
|
||||
func (r *RunDirectory) ApplyRetention(runSucceeded bool) error {
|
||||
if !runSucceeded {
|
||||
|
||||
@@ -8,27 +8,40 @@ import (
|
||||
)
|
||||
|
||||
type ProcessReport struct {
|
||||
Phase string `json:"phase"`
|
||||
Status string `json:"status"`
|
||||
Operation string `json:"operation"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
GlossaryPath string `json:"glossary_path"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
Modules []string `json:"modules"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
||||
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
||||
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
||||
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
||||
NormalizationSkipped struct {
|
||||
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
||||
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
||||
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
||||
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
||||
} `json:"normalization_skipped,omitempty"`
|
||||
Phase string `json:"phase"`
|
||||
Status string `json:"status"`
|
||||
Operation string `json:"operation"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
GlossaryPath string `json:"glossary_path"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
Modules []string `json:"modules"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
InputSegmentCount *int `json:"input_segment_count,omitempty"`
|
||||
NormalizedSegmentCount *int `json:"normalized_segment_count,omitempty"`
|
||||
NormalizationMerges *int `json:"normalization_merges,omitempty"`
|
||||
NormalizationIDReassignments *int `json:"normalization_id_reassignments,omitempty"`
|
||||
NormalizationSkipped NormalizationSkipped `json:"normalization_skipped,omitempty"`
|
||||
Chunking *ChunkingSummary `json:"chunking,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizationSkipped struct {
|
||||
DifferentSpeakers *int `json:"different_speakers,omitempty"`
|
||||
GapTooLarge *int `json:"gap_too_large,omitempty"`
|
||||
DurationExceeded *int `json:"duration_exceeded,omitempty"`
|
||||
TokenLimitExceeded *int `json:"token_limit_exceeded,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkingSummary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
func WriteProcessReport(path string, report ProcessReport) error {
|
||||
|
||||
Reference in New Issue
Block a user