eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Record chunking metadata in process runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 12:47:31 +00:00
parent 12fd541669
commit b997e7c97c
6 changed files with 598 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/cli/run.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
@@ -24,15 +25,15 @@ type processInvocation struct {
Config config.Config
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, nil, fmt.Errorf("run_dir_creation: %w", err)
return nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
}
fail := func(phase string, err error) (*normalization.NormalizationSummary, *diagnostics.RunDirectory, error) {
fail := func(phase string, err error) (*normalization.NormalizationSummary, *chunking.Summary, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, runDir, fmt.Errorf("%s: %w", phase, err)
return nil, nil, runDir, fmt.Errorf("%s: %w", phase, err)
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
@@ -66,15 +67,39 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, summary := normalizer.Normalize(canonical)
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(summary); err != nil {
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
// Compute chunks after normalization
chunker := chunking.NewChunker(chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
})
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
if chunkErr != nil {
return fail("chunking", chunkErr)
}
chunkConfig := chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
}
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
}
outputBytes, err := schema.TranscriptToJSON(normalizedTranscript)
if err != nil {
return fail("serialization", err)
@@ -84,14 +109,14 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
return fail("output_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err)
}
return summary, runDir, nil
return normSummary, &chunkSummary, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
@@ -252,12 +277,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
Config: cfg,
}
summary, runDir, runErr := processRunner(inv, stdout)
normSummary, chunkSummary, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC()
if runErr != nil {
errorPhase, errorMessage := extractErrorPhase(runErr)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil)
report := buildProcessReport("failed", inv, startedAt, completedAt, errorMessage, errorPhase, nil, nil)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -274,7 +299,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 1
}
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", summary)
report := buildProcessReport("success", inv, startedAt, completedAt, "", "", normSummary, chunkSummary)
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -309,9 +334,9 @@ func extractErrorPhase(err error) (phase string, message string) {
return "", msg
}
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary) reporting.ProcessReport {
func buildProcessReport(status string, inv processInvocation, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase2-foundation",
Phase: "phase3-chunking",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,
@@ -335,6 +360,17 @@ func buildProcessReport(status string, inv processInvocation, startedAt, complet
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
}
if chunkingSummary != nil {
report.Chunking = &reporting.ChunkingSummary{
ChunkCount: chunkingSummary.ChunkCount,
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
TargetSections: chunkingSummary.TargetSections,
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
MinSectionTokens: chunkingSummary.MinSectionTokens,
}
}
return report
}