834 lines
31 KiB
Go
834 lines
31 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
|
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
|
)
|
|
|
|
type noOpStructuredLLMClient struct{}
|
|
|
|
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
_ = ctx
|
|
_ = req
|
|
switch target := out.(type) {
|
|
case *validators.LLMValidationResponse:
|
|
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
|
|
case *proposal_generation.StructuredCorrectionSet:
|
|
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
|
}
|
|
return contracts.StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
func shouldUseNoOpLLMClientForTests() bool {
|
|
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test")
|
|
}
|
|
|
|
type processInvocation struct {
|
|
TranscriptPath string
|
|
GlossaryPath string
|
|
OutputPath string
|
|
ReportJSONPath string
|
|
Config config.Config
|
|
ExplicitModules bool
|
|
}
|
|
|
|
var processModuleFactory runner.ModuleFactory
|
|
var processProposalLLMClient contracts.StructuredLLMClient
|
|
var processProposalLLMScheduler runner.ValidationScheduler
|
|
var processValidationLLMClient contracts.StructuredLLMClient
|
|
var processValidationLLMScheduler runner.ValidationScheduler
|
|
var processRunnerContext = func() (context.Context, context.CancelFunc) {
|
|
return context.Background(), func() {}
|
|
}
|
|
|
|
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
|
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
|
if err != nil {
|
|
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
|
}
|
|
|
|
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
|
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
|
|
}
|
|
|
|
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
|
Operation: "process",
|
|
TranscriptPath: inv.TranscriptPath,
|
|
GlossaryPath: inv.GlossaryPath,
|
|
OutputPath: inv.OutputPath,
|
|
ReportJSONPath: inv.ReportJSONPath,
|
|
Modules: append([]string(nil), inv.Config.Modules...),
|
|
}); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
|
|
}
|
|
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
|
|
}
|
|
|
|
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
|
if err != nil {
|
|
return fail("transcript_read", err, nil)
|
|
}
|
|
|
|
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
|
|
if err != nil {
|
|
return fail("glossary_read", err, nil)
|
|
}
|
|
|
|
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
|
if err != nil {
|
|
return fail("transcript_schema", err, nil)
|
|
}
|
|
|
|
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
|
if err != nil {
|
|
return fail("glossary_schema", err, nil)
|
|
}
|
|
|
|
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
|
|
}
|
|
|
|
canonical := sourceToCanonicalTranscript(sourceTranscript)
|
|
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
|
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
|
|
EllipsisGap: inv.Config.Normalization.EllipsisGap,
|
|
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
|
|
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
|
})
|
|
|
|
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
|
|
|
|
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
|
}
|
|
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
|
}
|
|
|
|
// Compute chunks after normalization
|
|
chunker := chunking.NewChunker(chunking.ChunkingConfig{
|
|
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
|
MinSectionTokens: inv.Config.MinSectionTokens,
|
|
TargetSections: inv.Config.TargetSections,
|
|
})
|
|
|
|
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
|
|
if chunkErr != nil {
|
|
return fail("chunking", chunkErr, nil)
|
|
}
|
|
|
|
chunkConfig := chunking.ChunkingConfig{
|
|
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
|
MinSectionTokens: inv.Config.MinSectionTokens,
|
|
TargetSections: inv.Config.TargetSections,
|
|
}
|
|
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
|
|
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
|
|
|
|
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
|
|
}
|
|
|
|
workingTranscript := normalizedTranscript
|
|
var runOutput *runner.RunOutput
|
|
moduleFactory := processModuleFactory
|
|
if moduleFactory == nil {
|
|
moduleFactory = modules.NewFactory(modules.Dependencies{
|
|
Config: &inv.Config,
|
|
Glossary: glossary,
|
|
DiagnosticsDir: runDir.Path(),
|
|
})
|
|
}
|
|
if moduleFactory != nil {
|
|
proposalLLMClient := processProposalLLMClient
|
|
validationLLMClient := processValidationLLMClient
|
|
proposalScheduler := processProposalLLMScheduler
|
|
validationScheduler := processValidationLLMScheduler
|
|
|
|
if processModuleFactory == nil {
|
|
// Production runtime path: construct clients/schedulers from config.
|
|
if proposalLLMClient == nil {
|
|
if shouldUseNoOpLLMClientForTests() {
|
|
proposalLLMClient = noOpStructuredLLMClient{}
|
|
} else {
|
|
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
|
|
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
|
|
if clientErr != nil {
|
|
return fail("runner_setup", clientErr, nil)
|
|
}
|
|
proposalLLMClient = client
|
|
}
|
|
}
|
|
if validationLLMClient == nil {
|
|
if shouldUseNoOpLLMClientForTests() {
|
|
validationLLMClient = noOpStructuredLLMClient{}
|
|
} else {
|
|
validationCfg := llm.ResolveValidationConfig(inv.Config)
|
|
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
|
|
if clientErr != nil {
|
|
return fail("runner_setup", clientErr, nil)
|
|
}
|
|
validationLLMClient = client
|
|
}
|
|
}
|
|
globalScheduler := proposalScheduler
|
|
if globalScheduler == nil {
|
|
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
|
|
if sErr != nil {
|
|
return fail("runner_setup", sErr, nil)
|
|
}
|
|
globalScheduler = s
|
|
}
|
|
if proposalScheduler == nil {
|
|
proposalScheduler = globalScheduler
|
|
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
|
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
|
|
if sErr != nil {
|
|
return fail("runner_setup", sErr, nil)
|
|
}
|
|
proposalScheduler = composeSchedulers(globalScheduler, s)
|
|
}
|
|
}
|
|
if validationScheduler == nil {
|
|
validationScheduler = globalScheduler
|
|
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
|
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
|
|
if sErr != nil {
|
|
return fail("runner_setup", sErr, nil)
|
|
}
|
|
validationScheduler = composeSchedulers(globalScheduler, s)
|
|
}
|
|
}
|
|
}
|
|
|
|
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
|
|
if err != nil {
|
|
return fail("runner_setup", err, nil)
|
|
}
|
|
|
|
runCtx, cancelRun := processRunnerContext()
|
|
defer cancelRun()
|
|
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
|
|
Config: &inv.Config,
|
|
Transcript: normalizedTranscript,
|
|
Glossary: glossary,
|
|
ModuleSpecs: moduleSpecs,
|
|
ProposalLLMClient: proposalLLMClient,
|
|
ProposalLLMScheduler: proposalScheduler,
|
|
ProposalDiagnosticsDir: runDir.Path(),
|
|
ValidationLLMClient: validationLLMClient,
|
|
ValidationLLMScheduler: validationScheduler,
|
|
ValidationDiagnosticsDir: runDir.Path(),
|
|
})
|
|
runOutput = &runnerResult
|
|
if runErr != nil {
|
|
return fail("runner_execution", runErr, runOutput)
|
|
}
|
|
workingTranscript = runnerResult.FinalTranscript
|
|
}
|
|
|
|
outputBytes, err := schema.TranscriptToJSON(workingTranscript)
|
|
if err != nil {
|
|
return fail("serialization", err, runOutput)
|
|
}
|
|
|
|
if strings.TrimSpace(inv.OutputPath) != "" {
|
|
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
|
return fail("output_write", err, runOutput)
|
|
}
|
|
return normSummary, &chunkSummary, runOutput, runDir, nil
|
|
}
|
|
|
|
if _, err := stdout.Write(outputBytes); err != nil {
|
|
return fail("stdout_write", err, runOutput)
|
|
}
|
|
|
|
return normSummary, &chunkSummary, runOutput, runDir, nil
|
|
}
|
|
|
|
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
|
segments := make([]schema.Segment, len(source.Segments))
|
|
for i, s := range source.Segments {
|
|
id := i + 1
|
|
if s.ID != nil {
|
|
id = *s.ID
|
|
}
|
|
segments[i] = schema.Segment{
|
|
ID: id,
|
|
Speaker: s.Speaker,
|
|
Start: s.Start,
|
|
End: s.End,
|
|
Text: s.Text,
|
|
Categories: s.Categories,
|
|
}
|
|
}
|
|
return &schema.Transcript{Segments: segments}
|
|
}
|
|
|
|
type chainedScheduler struct {
|
|
schedulers []runner.ValidationScheduler
|
|
}
|
|
|
|
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
|
if len(s.schedulers) == 0 {
|
|
return fn(ctx)
|
|
}
|
|
run := fn
|
|
for i := len(s.schedulers) - 1; i >= 0; i-- {
|
|
scheduler := s.schedulers[i]
|
|
next := run
|
|
run = func(callCtx context.Context) error {
|
|
return scheduler.Run(callCtx, next)
|
|
}
|
|
}
|
|
return run(ctx)
|
|
}
|
|
|
|
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
|
|
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
|
|
for _, scheduler := range schedulers {
|
|
if scheduler != nil {
|
|
filtered = append(filtered, scheduler)
|
|
}
|
|
}
|
|
switch len(filtered) {
|
|
case 0:
|
|
return nil
|
|
case 1:
|
|
return filtered[0]
|
|
default:
|
|
return chainedScheduler{schedulers: filtered}
|
|
}
|
|
}
|
|
|
|
// Run executes the Audita CLI with the provided arguments and streams.
|
|
func Run(args []string, stdout, stderr io.Writer) int {
|
|
if len(args) == 0 {
|
|
writeRootUsage(stdout)
|
|
return 0
|
|
}
|
|
|
|
if isHelpCommand(args) {
|
|
writeRootUsage(stdout)
|
|
return 0
|
|
}
|
|
|
|
if args[0] == "process" {
|
|
return runProcess(args[1:], stdout, stderr)
|
|
}
|
|
|
|
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
|
|
writeRootUsage(stderr)
|
|
return 2
|
|
}
|
|
|
|
func runProcess(args []string, stdout, stderr io.Writer) int {
|
|
startedAt := time.Now().UTC()
|
|
|
|
cfg, err := config.LoadFromEnv()
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", err)
|
|
return 2
|
|
}
|
|
|
|
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
|
|
|
if isHelpCommand(args) || hasHelpFlag(args) {
|
|
writeProcessUsage(stdout, fs)
|
|
return 0
|
|
}
|
|
|
|
parseArgs := args
|
|
transcriptFromFront := ""
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
transcriptFromFront = args[0]
|
|
parseArgs = args[1:]
|
|
}
|
|
|
|
if err := fs.Parse(parseArgs); err != nil {
|
|
if errors.Is(err, flag.ErrHelp) {
|
|
writeProcessUsage(stdout, fs)
|
|
return 0
|
|
}
|
|
return 2
|
|
}
|
|
|
|
overrides := config.CLIOverrides{}
|
|
explicitModules := false
|
|
fs.Visit(func(f *flag.Flag) {
|
|
switch f.Name {
|
|
case "modules":
|
|
explicitModules = true
|
|
overrides.ModulesCSV = pFlags.modules
|
|
case "llm-api-key":
|
|
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
|
|
case "validation-llm-api-key":
|
|
overrides.ValidationLLMAPIKey = pFlags.validationLLMAPIKey
|
|
case "model":
|
|
overrides.PrimaryModel = pFlags.model
|
|
case "validation-model":
|
|
overrides.ValidationModel = pFlags.validationModel
|
|
case "base-url":
|
|
overrides.PrimaryBaseURL = pFlags.baseURL
|
|
case "validation-base-url":
|
|
overrides.ValidationBaseURL = pFlags.validationBaseURL
|
|
case "llm-timeout-seconds":
|
|
overrides.PrimaryLLMTimeoutSeconds = pFlags.llmTimeoutSeconds
|
|
case "total-llm-concurrency":
|
|
overrides.TotalLLMConcurrency = pFlags.totalLLMConcurrency
|
|
case "proposal-llm-concurrency":
|
|
overrides.ProposalLLMConcurrency = pFlags.proposalLLMConcurrency
|
|
case "llm-concurrency":
|
|
overrides.PrimaryLLMConcurrency = pFlags.llmConcurrency
|
|
case "validation-llm-timeout-seconds":
|
|
overrides.ValidationLLMTimeoutSeconds = pFlags.validationLLMTimeoutSeconds
|
|
case "max-retries":
|
|
overrides.MaxRetries = pFlags.maxRetries
|
|
case "validation-max-retries":
|
|
overrides.ValidationMaxRetries = pFlags.validationMaxRetries
|
|
case "validation-llm-concurrency":
|
|
overrides.ValidationLLMConcurrency = pFlags.validationLLMConcurrency
|
|
case "validation-max-prompt-tokens":
|
|
overrides.ValidationMaxPromptTokens = pFlags.validationMaxPromptTokens
|
|
case "max-section-tokens":
|
|
overrides.MaxSectionTokens = pFlags.maxSectionTokens
|
|
case "min-section-tokens":
|
|
overrides.MinSectionTokens = pFlags.minSectionTokens
|
|
case "target-sections":
|
|
overrides.TargetSections = pFlags.targetSections
|
|
case "glossary-confidence-threshold":
|
|
overrides.GlossaryConfidenceThreshold = pFlags.glossaryConfidenceThreshold
|
|
case "grammar-confidence-threshold":
|
|
overrides.GrammarConfidenceThreshold = pFlags.grammarConfidenceThreshold
|
|
case "homophones-confidence-threshold":
|
|
overrides.HomophonesConfidenceThreshold = pFlags.homophonesConfidenceThreshold
|
|
case "spoken-word-confidence-threshold":
|
|
overrides.SpokenWordConfidenceThreshold = pFlags.spokenWordConfidenceThreshold
|
|
case "normalize-max-segment-gap":
|
|
overrides.NormalizeMaxSegmentGap = pFlags.normalizeMaxSegmentGap
|
|
case "normalize-ellipsis-gap":
|
|
overrides.NormalizeEllipsisGap = pFlags.normalizeEllipsisGap
|
|
case "normalize-max-segment-duration":
|
|
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
|
|
case "normalize-max-segment-tokens":
|
|
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
|
|
case "work-dir":
|
|
overrides.WorkDir = pFlags.workDir
|
|
case "work-dir-retention":
|
|
overrides.WorkDirRetention = pFlags.workDirRetention
|
|
}
|
|
})
|
|
|
|
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
|
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
|
return 2
|
|
}
|
|
|
|
remaining := fs.Args()
|
|
positional := make([]string, 0, len(remaining)+1)
|
|
if transcriptFromFront != "" {
|
|
positional = append(positional, transcriptFromFront)
|
|
}
|
|
positional = append(positional, remaining...)
|
|
|
|
if len(positional) != 1 {
|
|
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
|
|
return 2
|
|
}
|
|
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
|
|
fmt.Fprintln(stderr, "audita process: --glossary is required")
|
|
return 2
|
|
}
|
|
|
|
inv := processInvocation{
|
|
TranscriptPath: positional[0],
|
|
GlossaryPath: *pFlags.glossaryPath,
|
|
OutputPath: *pFlags.outputPath,
|
|
ReportJSONPath: *pFlags.reportJSONPath,
|
|
Config: cfg,
|
|
ExplicitModules: explicitModules,
|
|
}
|
|
|
|
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
|
|
completedAt := time.Now().UTC()
|
|
|
|
if runErr != nil {
|
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
|
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
|
|
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
|
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
|
}
|
|
}
|
|
|
|
if runDir != nil {
|
|
_ = runDir.WriteReport(report)
|
|
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
|
RunSucceeded: false,
|
|
})
|
|
}
|
|
|
|
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
|
|
if runDir != nil {
|
|
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
|
|
}
|
|
return 1
|
|
}
|
|
|
|
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
|
|
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
|
if runDir != nil {
|
|
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
|
|
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
|
RunSucceeded: false,
|
|
})
|
|
}
|
|
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
|
return 1
|
|
}
|
|
}
|
|
|
|
hasSkippedCorrections := false
|
|
if runOutput != nil {
|
|
for _, mr := range runOutput.ModuleResults {
|
|
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
|
hasSkippedCorrections = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if runDir != nil {
|
|
_ = runDir.WriteReport(report)
|
|
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
|
RunSucceeded: true,
|
|
HasSkippedCorrections: hasSkippedCorrections,
|
|
}); err != nil {
|
|
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
|
return 1
|
|
}
|
|
}
|
|
|
|
return 0
|
|
}
|
|
|
|
func extractErrorPhase(err error) (phase string, message string) {
|
|
msg := err.Error()
|
|
if strings.Contains(msg, ": ") {
|
|
parts := strings.SplitN(msg, ": ", 2)
|
|
if len(parts) == 2 {
|
|
return parts[0], parts[1]
|
|
}
|
|
}
|
|
return "", msg
|
|
}
|
|
|
|
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
|
|
report := reporting.ProcessReport{
|
|
Phase: "default_pipeline",
|
|
Status: status,
|
|
Operation: "process",
|
|
TranscriptPath: inv.TranscriptPath,
|
|
GlossaryPath: inv.GlossaryPath,
|
|
OutputPath: inv.OutputPath,
|
|
Modules: append([]string(nil), inv.Config.Modules...),
|
|
StartedAt: startedAt,
|
|
CompletedAt: &completedAt,
|
|
ErrorPhase: errorPhase,
|
|
}
|
|
if runDir != nil {
|
|
diagnosticsDir := runDir.Path()
|
|
report.Diagnostics = &reporting.DiagnosticsMetadata{
|
|
DirectoryPath: diagnosticsDir,
|
|
SourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript.json"),
|
|
ParsedSourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript-parsed.json"),
|
|
NormalizedTranscriptPath: filepath.Join(diagnosticsDir, "normalized-transcript.json"),
|
|
NormalizationSummaryPath: filepath.Join(diagnosticsDir, "normalization-summary.json"),
|
|
ChunkingSummaryPath: filepath.Join(diagnosticsDir, "chunking-summary.json"),
|
|
InvocationMetadataPath: filepath.Join(diagnosticsDir, "invocation.json"),
|
|
RedactedEffectiveConfigPath: filepath.Join(diagnosticsDir, "effective-config.json"),
|
|
}
|
|
if status == "failed" {
|
|
report.Diagnostics.ErrorLogPath = filepath.Join(diagnosticsDir, "error.log")
|
|
}
|
|
}
|
|
if errorMessage != "" {
|
|
report.ErrorMessage = errorMessage
|
|
}
|
|
if normalizationSummary != nil {
|
|
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
|
|
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
|
|
report.NormalizationMerges = &normalizationSummary.MergesPerformed
|
|
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
|
|
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
|
|
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
|
|
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
|
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
|
}
|
|
if chunkingSummary != nil {
|
|
report.Chunking = &reporting.ChunkingSummary{
|
|
ChunkCount: chunkingSummary.ChunkCount,
|
|
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
|
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
|
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
|
TargetSections: chunkingSummary.TargetSections,
|
|
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
|
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
|
}
|
|
}
|
|
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
|
|
return report
|
|
}
|
|
|
|
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
|
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
|
for _, r := range runOutput.ModuleResults {
|
|
startedAt := r.StartedAt
|
|
completedAt := r.CompletedAt
|
|
moduleReports = append(moduleReports, reporting.ModuleReport{
|
|
ModuleKey: r.ModuleKey,
|
|
ModuleInstance: r.ModuleInstance,
|
|
ReplacementPolicy: string(r.ReplacementPolicy),
|
|
Status: r.Status,
|
|
ProposalCount: r.ProposalCount,
|
|
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
|
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
|
AppliedChanges: r.AppliedChanges,
|
|
SkippedChanges: r.SkippedChanges,
|
|
ErrorMessage: r.ErrorMessage,
|
|
StartedAt: &startedAt,
|
|
CompletedAt: &completedAt,
|
|
})
|
|
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
|
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
|
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
|
summary.FailedModuleInstance = r.ModuleInstance
|
|
}
|
|
}
|
|
|
|
return summary, moduleReports
|
|
}
|
|
|
|
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
|
if len(in) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]reporting.ValidatorDecisionReport, len(in))
|
|
for i, d := range in {
|
|
out[i] = reporting.ValidatorDecisionReport{
|
|
ValidatorName: d.ValidatorName,
|
|
ProposalIndex: d.ProposalIndex,
|
|
Approved: d.Approved,
|
|
ReasonCode: d.ReasonCode,
|
|
Message: d.Message,
|
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
|
if len(in) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]reporting.ValidatorRejectedReport, len(in))
|
|
for i, d := range in {
|
|
out[i] = reporting.ValidatorRejectedReport{
|
|
ValidatorName: d.ValidatorName,
|
|
ProposalIndex: d.ProposalIndex,
|
|
ModuleKey: d.ModuleKey,
|
|
ModuleInstance: d.ModuleInstance,
|
|
TargetSegmentID: d.TargetSegmentID,
|
|
OriginalText: d.OriginalText,
|
|
CorrectedText: d.CorrectedText,
|
|
ReasonCode: d.ReasonCode,
|
|
Message: d.Message,
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
type processFlags struct {
|
|
glossaryPath *string
|
|
outputPath *string
|
|
reportJSONPath *string
|
|
modules *string
|
|
llmAPIKey *string
|
|
validationLLMAPIKey *string
|
|
model *string
|
|
validationModel *string
|
|
baseURL *string
|
|
validationBaseURL *string
|
|
llmTimeoutSeconds *int
|
|
totalLLMConcurrency *int
|
|
proposalLLMConcurrency *int
|
|
llmConcurrency *int
|
|
validationLLMTimeoutSeconds *int
|
|
validationMaxPromptTokens *int
|
|
targetSections *int
|
|
maxRetries *int
|
|
validationMaxRetries *int
|
|
validationLLMConcurrency *int
|
|
maxSectionTokens *int
|
|
minSectionTokens *int
|
|
glossaryConfidenceThreshold *float64
|
|
grammarConfidenceThreshold *float64
|
|
homophonesConfidenceThreshold *float64
|
|
spokenWordConfidenceThreshold *float64
|
|
normalizeMaxSegmentGap *float64
|
|
normalizeEllipsisGap *float64
|
|
normalizeMaxSegmentDuration *float64
|
|
normalizeMaxSegmentTokens *int
|
|
workDir *string
|
|
workDirRetention *string
|
|
}
|
|
|
|
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
|
|
fs := flag.NewFlagSet("process", flag.ContinueOnError)
|
|
fs.SetOutput(stderr)
|
|
|
|
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
|
|
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
|
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
|
|
}
|
|
|
|
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
|
|
if cfg.ValidationLLM.MaxRetries != nil {
|
|
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
|
|
}
|
|
|
|
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
|
|
if cfg.ValidationLLMConcurrency != nil {
|
|
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
|
|
}
|
|
|
|
targetSectionsDefault := 0
|
|
if cfg.TargetSections != nil {
|
|
targetSectionsDefault = *cfg.TargetSections
|
|
}
|
|
|
|
pFlags := processFlags{
|
|
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
|
|
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
|
|
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
|
|
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
|
|
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
|
|
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
|
|
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
|
|
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
|
|
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
|
|
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
|
|
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
|
|
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
|
|
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
|
|
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
|
|
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
|
|
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
|
|
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
|
|
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
|
|
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
|
|
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
|
|
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
|
|
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
|
|
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
|
|
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
|
|
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
|
|
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
|
|
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
|
|
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
|
|
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
|
|
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
|
|
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
|
|
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
|
|
}
|
|
|
|
return fs, pFlags
|
|
}
|
|
|
|
func isHelpCommand(args []string) bool {
|
|
if len(args) == 0 {
|
|
return false
|
|
}
|
|
if len(args) == 1 {
|
|
switch args[0] {
|
|
case "help", "-h", "--help":
|
|
return true
|
|
}
|
|
}
|
|
if len(args) == 2 && args[0] == "help" {
|
|
switch args[1] {
|
|
case "process":
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func hasHelpFlag(args []string) bool {
|
|
for _, arg := range args {
|
|
if arg == "-h" || arg == "--help" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func writeRootUsage(w io.Writer) {
|
|
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Usage:")
|
|
fmt.Fprintln(w, " audita <command> [options]")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Commands:")
|
|
fmt.Fprintln(w, " process Process a transcript JSON file")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Example:")
|
|
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
|
}
|
|
|
|
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
|
|
fmt.Fprintln(w, "Process a transcript JSON file.")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Usage:")
|
|
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Flags:")
|
|
fs.VisitAll(func(f *flag.Flag) {
|
|
fmt.Fprintf(w, " --%s\n", f.Name)
|
|
})
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "Example:")
|
|
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
|
}
|