85 lines
2.5 KiB
Go
85 lines
2.5 KiB
Go
package chunking
|
|
|
|
// Summary provides a concise overview of chunking results for reports
|
|
type Summary struct {
|
|
ChunkCount int `json:"chunk_count"`
|
|
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
|
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
|
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
|
TargetSections *int `json:"target_sections,omitempty"`
|
|
MaxSectionTokens int `json:"max_section_tokens"`
|
|
MinSectionTokens int `json:"min_section_tokens"`
|
|
}
|
|
|
|
// ChunkSummary represents a single chunk's metadata for diagnostics
|
|
type ChunkSummary struct {
|
|
Index int `json:"index"`
|
|
StartSegmentID int `json:"start_segment_id"`
|
|
EndSegmentID int `json:"end_segment_id"`
|
|
EstimatedTokens int `json:"estimated_tokens"`
|
|
SegmentCount int `json:"segment_count"`
|
|
}
|
|
|
|
// DetailedSummary provides per-chunk details for diagnostics
|
|
type DetailedSummary struct {
|
|
Summary `json:",inline"`
|
|
Chunks []ChunkSummary `json:"chunks"`
|
|
}
|
|
|
|
// ComputeSummary creates a Summary from sections and config
|
|
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
|
if len(sections) == 0 {
|
|
return Summary{
|
|
ChunkCount: 0,
|
|
MaxSectionTokens: config.MaxSectionTokens,
|
|
MinSectionTokens: config.MinSectionTokens,
|
|
TargetSections: config.TargetSections,
|
|
}
|
|
}
|
|
|
|
minTokens := sections[0].EstimatedTokens
|
|
maxTokens := sections[0].EstimatedTokens
|
|
totalTokens := 0
|
|
|
|
for _, sec := range sections {
|
|
if sec.EstimatedTokens < minTokens {
|
|
minTokens = sec.EstimatedTokens
|
|
}
|
|
if sec.EstimatedTokens > maxTokens {
|
|
maxTokens = sec.EstimatedTokens
|
|
}
|
|
totalTokens += sec.EstimatedTokens
|
|
}
|
|
|
|
return Summary{
|
|
ChunkCount: len(sections),
|
|
MinEstimatedTokens: minTokens,
|
|
MaxEstimatedTokens: maxTokens,
|
|
TotalEstimatedTokens: totalTokens,
|
|
TargetSections: config.TargetSections,
|
|
MaxSectionTokens: config.MaxSectionTokens,
|
|
MinSectionTokens: config.MinSectionTokens,
|
|
}
|
|
}
|
|
|
|
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
|
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
|
summary := ComputeSummary(sections, config)
|
|
|
|
chunks := make([]ChunkSummary, len(sections))
|
|
for i, sec := range sections {
|
|
chunks[i] = ChunkSummary{
|
|
Index: sec.Index,
|
|
StartSegmentID: sec.StartSegmentID,
|
|
EndSegmentID: sec.EndSegmentID,
|
|
EstimatedTokens: sec.EstimatedTokens,
|
|
SegmentCount: len(sec.Segments),
|
|
}
|
|
}
|
|
|
|
return DetailedSummary{
|
|
Summary: summary,
|
|
Chunks: chunks,
|
|
}
|
|
}
|