diff --git a/internal/cli/run.go b/internal/cli/run.go index f15c198..17d94b0 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -12,6 +12,19 @@ import ( const processNotImplementedMessage = "process command is not implemented yet" +type processInvocation struct { + TranscriptPath string + GlossaryPath string + OutputPath string + ReportJSONPath string + Config config.Config +} + +var processRunner = func(inv processInvocation) error { + _ = inv + return errors.New(processNotImplementedMessage) +} + // Run executes the Audita CLI with the provided arguments and streams. func Run(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { @@ -34,92 +47,29 @@ func Run(args []string, stdout, stderr io.Writer) int { } func runProcess(args []string, stdout, stderr io.Writer) int { - if isHelpCommand(args) || hasHelpFlag(args) { - writeProcessUsage(stdout) - return 0 - } - cfg, err := config.LoadFromEnv() if err != nil { fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", err) return 2 } - fs := flag.NewFlagSet("process", flag.ContinueOnError) - fs.SetOutput(stderr) + fs, pFlags := newProcessFlagSet(cfg, stderr) - glossaryPath := fs.String("glossary", "", "Path to glossary YAML file") - outputPath := fs.String("output", "", "Path to corrected transcript JSON output file") - reportJSONPath := fs.String("report-json", "", "Path to machine-readable report JSON output file") - modules := fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override") - llmAPIKey := fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key") - validationLLMAPIKey := fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key") - model := fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name") - validationModel := fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name") - baseURL := fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL") - validationBaseURL := fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL") - llmTimeoutSeconds := fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds") - validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds - if cfg.ValidationLLM.TimeoutSeconds != nil { - validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds + if isHelpCommand(args) || hasHelpFlag(args) { + writeProcessUsage(stdout, fs) + return 0 } - validationLLMTimeoutSeconds := fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds") - maxRetries := fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries") - validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries - if cfg.ValidationLLM.MaxRetries != nil { - validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries - } - validationMaxRetries := fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries") - validationMaxPromptTokens := fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens") - maxSectionTokens := fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Max section tokens") - minSectionTokens := fs.Int("min-section-tokens", cfg.MinSectionTokens, "Min section tokens") - targetSectionsDefault := 0 - if cfg.TargetSections != nil { - targetSectionsDefault = *cfg.TargetSections - } - targetSections := fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections") - glossaryConfidenceThreshold := fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold") - grammarConfidenceThreshold := fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold") - homophonesConfidenceThreshold := fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold") - spokenWordConfidenceThreshold := fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold") - normalizeMaxSegmentGap := fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap") - normalizeEllipsisGap := fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion") - normalizeMaxSegmentDuration := fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration") - normalizeMaxSegmentTokens := fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate") - workDir := fs.String("work-dir", cfg.WorkDir, "Per-run work directory") - workDirRetention := fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never") - _ = glossaryPath - _ = outputPath - _ = reportJSONPath - _ = llmAPIKey - _ = validationLLMAPIKey - _ = model - _ = validationModel - _ = baseURL - _ = validationBaseURL - _ = llmTimeoutSeconds - _ = validationLLMTimeoutSeconds - _ = maxRetries - _ = validationMaxRetries - _ = validationMaxPromptTokens - _ = maxSectionTokens - _ = minSectionTokens - _ = targetSections - _ = glossaryConfidenceThreshold - _ = grammarConfidenceThreshold - _ = homophonesConfidenceThreshold - _ = spokenWordConfidenceThreshold - _ = normalizeMaxSegmentGap - _ = normalizeEllipsisGap - _ = normalizeMaxSegmentDuration - _ = normalizeMaxSegmentTokens - _ = workDir - _ = workDirRetention + parseArgs := args + transcriptFromFront := "" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + transcriptFromFront = args[0] + parseArgs = args[1:] + } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(parseArgs); err != nil { if errors.Is(err, flag.ErrHelp) { - writeProcessUsage(stdout) + writeProcessUsage(stdout, fs) return 0 } return 2 @@ -129,55 +79,57 @@ func runProcess(args []string, stdout, stderr io.Writer) int { fs.Visit(func(f *flag.Flag) { switch f.Name { case "modules": - overrides.ModulesCSV = modules + overrides.ModulesCSV = pFlags.modules case "llm-api-key": - overrides.PrimaryLLMAPIKey = llmAPIKey + overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey case "validation-llm-api-key": - overrides.ValidationLLMAPIKey = validationLLMAPIKey + overrides.ValidationLLMAPIKey = pFlags.validationLLMAPIKey case "model": - overrides.PrimaryModel = model + overrides.PrimaryModel = pFlags.model case "validation-model": - overrides.ValidationModel = validationModel + overrides.ValidationModel = pFlags.validationModel case "base-url": - overrides.PrimaryBaseURL = baseURL + overrides.PrimaryBaseURL = pFlags.baseURL case "validation-base-url": - overrides.ValidationBaseURL = validationBaseURL + overrides.ValidationBaseURL = pFlags.validationBaseURL case "llm-timeout-seconds": - overrides.PrimaryLLMTimeoutSeconds = llmTimeoutSeconds + overrides.PrimaryLLMTimeoutSeconds = pFlags.llmTimeoutSeconds case "validation-llm-timeout-seconds": - overrides.ValidationLLMTimeoutSeconds = validationLLMTimeoutSeconds + overrides.ValidationLLMTimeoutSeconds = pFlags.validationLLMTimeoutSeconds case "max-retries": - overrides.MaxRetries = maxRetries + overrides.MaxRetries = pFlags.maxRetries case "validation-max-retries": - overrides.ValidationMaxRetries = validationMaxRetries + overrides.ValidationMaxRetries = pFlags.validationMaxRetries + case "validation-llm-concurrency": + overrides.ValidationLLMConcurrency = pFlags.validationLLMConcurrency case "validation-max-prompt-tokens": - overrides.ValidationMaxPromptTokens = validationMaxPromptTokens + overrides.ValidationMaxPromptTokens = pFlags.validationMaxPromptTokens case "max-section-tokens": - overrides.MaxSectionTokens = maxSectionTokens + overrides.MaxSectionTokens = pFlags.maxSectionTokens case "min-section-tokens": - overrides.MinSectionTokens = minSectionTokens + overrides.MinSectionTokens = pFlags.minSectionTokens case "target-sections": - overrides.TargetSections = targetSections + overrides.TargetSections = pFlags.targetSections case "glossary-confidence-threshold": - overrides.GlossaryConfidenceThreshold = glossaryConfidenceThreshold + overrides.GlossaryConfidenceThreshold = pFlags.glossaryConfidenceThreshold case "grammar-confidence-threshold": - overrides.GrammarConfidenceThreshold = grammarConfidenceThreshold + overrides.GrammarConfidenceThreshold = pFlags.grammarConfidenceThreshold case "homophones-confidence-threshold": - overrides.HomophonesConfidenceThreshold = homophonesConfidenceThreshold + overrides.HomophonesConfidenceThreshold = pFlags.homophonesConfidenceThreshold case "spoken-word-confidence-threshold": - overrides.SpokenWordConfidenceThreshold = spokenWordConfidenceThreshold + overrides.SpokenWordConfidenceThreshold = pFlags.spokenWordConfidenceThreshold case "normalize-max-segment-gap": - overrides.NormalizeMaxSegmentGap = normalizeMaxSegmentGap + overrides.NormalizeMaxSegmentGap = pFlags.normalizeMaxSegmentGap case "normalize-ellipsis-gap": - overrides.NormalizeEllipsisGap = normalizeEllipsisGap + overrides.NormalizeEllipsisGap = pFlags.normalizeEllipsisGap case "normalize-max-segment-duration": - overrides.NormalizeMaxSegmentDuration = normalizeMaxSegmentDuration + overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration case "normalize-max-segment-tokens": - overrides.NormalizeMaxSegmentTokens = normalizeMaxSegmentTokens + overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens case "work-dir": - overrides.WorkDir = workDir + overrides.WorkDir = pFlags.workDir case "work-dir-retention": - overrides.WorkDirRetention = workDirRetention + overrides.WorkDirRetention = pFlags.workDirRetention } }) @@ -187,15 +139,126 @@ func runProcess(args []string, stdout, stderr io.Writer) int { } remaining := fs.Args() - if len(remaining) != 1 { + positional := make([]string, 0, len(remaining)+1) + if transcriptFromFront != "" { + positional = append(positional, transcriptFromFront) + } + positional = append(positional, remaining...) + + if len(positional) != 1 { fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument") - fmt.Fprintln(stderr) - writeProcessUsage(stderr) + return 2 + } + if strings.TrimSpace(*pFlags.glossaryPath) == "" { + fmt.Fprintln(stderr, "audita process: --glossary is required") return 2 } - fmt.Fprintf(stderr, "audita process: %s\n", processNotImplementedMessage) - return 1 + inv := processInvocation{ + TranscriptPath: positional[0], + GlossaryPath: *pFlags.glossaryPath, + OutputPath: *pFlags.outputPath, + ReportJSONPath: *pFlags.reportJSONPath, + Config: cfg, + } + + if err := processRunner(inv); err != nil { + fmt.Fprintf(stderr, "audita process: %v\n", err) + return 1 + } + + return 0 +} + +type processFlags struct { + glossaryPath *string + outputPath *string + reportJSONPath *string + modules *string + llmAPIKey *string + validationLLMAPIKey *string + model *string + validationModel *string + baseURL *string + validationBaseURL *string + llmTimeoutSeconds *int + validationLLMTimeoutSeconds *int + validationMaxPromptTokens *int + targetSections *int + maxRetries *int + validationMaxRetries *int + validationLLMConcurrency *int + maxSectionTokens *int + minSectionTokens *int + glossaryConfidenceThreshold *float64 + grammarConfidenceThreshold *float64 + homophonesConfidenceThreshold *float64 + spokenWordConfidenceThreshold *float64 + normalizeMaxSegmentGap *float64 + normalizeEllipsisGap *float64 + normalizeMaxSegmentDuration *float64 + normalizeMaxSegmentTokens *int + workDir *string + workDirRetention *string +} + +func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) { + fs := flag.NewFlagSet("process", flag.ContinueOnError) + fs.SetOutput(stderr) + + validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds + if cfg.ValidationLLM.TimeoutSeconds != nil { + validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds + } + + validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries + if cfg.ValidationLLM.MaxRetries != nil { + validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries + } + + validationLLMConcurrencyDefault := cfg.PrimaryLLM.Concurrency + if cfg.ValidationLLM.Concurrency != nil { + validationLLMConcurrencyDefault = *cfg.ValidationLLM.Concurrency + } + + targetSectionsDefault := 0 + if cfg.TargetSections != nil { + targetSectionsDefault = *cfg.TargetSections + } + + pFlags := processFlags{ + glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"), + outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"), + reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"), + modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"), + llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"), + validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"), + model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"), + validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"), + baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"), + validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"), + llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"), + validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"), + validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"), + targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"), + maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"), + validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"), + validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Validation LLM concurrency"), + maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"), + minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"), + glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"), + grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"), + homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"), + spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"), + normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"), + normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"), + normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"), + normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"), + workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"), + workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"), + } + + return fs, pFlags } func isHelpCommand(args []string) bool { @@ -239,22 +302,16 @@ func writeRootUsage(w io.Writer) { fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json") } -func writeProcessUsage(w io.Writer) { +func writeProcessUsage(w io.Writer, fs *flag.FlagSet) { fmt.Fprintln(w, "Process a transcript JSON file.") fmt.Fprintln(w) fmt.Fprintln(w, "Usage:") fmt.Fprintln(w, " audita process [flags]") fmt.Fprintln(w) fmt.Fprintln(w, "Flags:") - fmt.Fprintln(w, " --glossary Path to glossary YAML file") - fmt.Fprintln(w, " --output Path to corrected transcript JSON output file") - fmt.Fprintln(w, " --report-json Path to machine-readable report JSON output file") - fmt.Fprintln(w, " --modules Comma-separated module sequence override") - fmt.Fprintln(w, " --llm-api-key Primary LLM API key") - fmt.Fprintln(w, " --model Primary LLM model") - fmt.Fprintln(w, " --base-url Primary OpenAI-compatible API base URL") - fmt.Fprintln(w, " --work-dir Per-run work directory") - fmt.Fprintln(w, " --work-dir-retention auto|always|never") + fs.VisitAll(func(f *flag.Flag) { + fmt.Fprintf(w, " --%s\n", f.Name) + }) fmt.Fprintln(w) fmt.Fprintln(w, "Example:") fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json") diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index eb8ea57..d9a43ef 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2,6 +2,7 @@ package cli import ( "bytes" + "errors" "strings" "testing" ) @@ -25,7 +26,7 @@ func TestRunRootHelp(t *testing.T) { } } -func TestRunProcessHelp(t *testing.T) { +func TestRunProcessHelpListsExpectedFlags(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer @@ -33,12 +34,43 @@ func TestRunProcessHelp(t *testing.T) { if exitCode != 0 { t.Fatalf("expected exit code 0, got %d", exitCode) } - if !strings.Contains(stdout.String(), "audita process [flags]") { - t.Fatalf("expected process usage in stdout, got %q", stdout.String()) - } - if !strings.Contains(stdout.String(), "--glossary") { - t.Fatalf("expected glossary flag in process help, got %q", stdout.String()) + + for _, expectedFlag := range []string{ + "--glossary", + "--output", + "--report-json", + "--modules", + "--llm-api-key", + "--validation-llm-api-key", + "--model", + "--validation-model", + "--base-url", + "--validation-base-url", + "--llm-timeout-seconds", + "--validation-llm-timeout-seconds", + "--validation-max-prompt-tokens", + "--target-sections", + "--max-retries", + "--validation-max-retries", + "--validation-llm-concurrency", + "--max-section-tokens", + "--min-section-tokens", + "--glossary-confidence-threshold", + "--grammar-confidence-threshold", + "--homophones-confidence-threshold", + "--spoken-word-confidence-threshold", + "--normalize-max-segment-gap", + "--normalize-ellipsis-gap", + "--normalize-max-segment-duration", + "--normalize-max-segment-tokens", + "--work-dir", + "--work-dir-retention", + } { + if !strings.Contains(stdout.String(), expectedFlag) { + t.Fatalf("expected process help to include %q, got %q", expectedFlag, stdout.String()) + } } + if stderr.Len() != 0 { t.Fatalf("expected empty stderr, got %q", stderr.String()) } @@ -60,13 +92,61 @@ func TestRunUnknownCommand(t *testing.T) { } } -func TestRunProcessNotImplemented(t *testing.T) { +func TestRunProcessMissingTranscriptPath(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + exitCode := Run([]string{"process", "--glossary", "glossary.yaml"}, &stdout, &stderr) + if exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "expected exactly 1 transcript JSON path argument") { + t.Fatalf("expected missing transcript error, got %q", stderr.String()) + } +} + +func TestRunProcessMissingGlossary(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", "transcript.json"}, &stdout, &stderr) if exitCode == 0 { - t.Fatalf("expected nonzero exit code for not-implemented process") + t.Fatalf("expected nonzero exit code") + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "--glossary is required") { + t.Fatalf("expected missing glossary error, got %q", stderr.String()) + } +} + +func TestRunProcessInvalidCLIConfig(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + exitCode := Run([]string{"process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir-retention", "invalid"}, &stdout, &stderr) + if exitCode != 2 { + t.Fatalf("expected exit code 2, got %d", exitCode) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "invalid CLI configuration") { + t.Fatalf("expected invalid config error, got %q", stderr.String()) + } +} + +func TestRunProcessNotImplemented(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + exitCode := Run([]string{"process", "transcript.json", "--glossary", "glossary.yaml"}, &stdout, &stderr) + if exitCode != 1 { + t.Fatalf("expected exit code 1 for not-implemented process, got %d", exitCode) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) @@ -75,3 +155,45 @@ func TestRunProcessNotImplemented(t *testing.T) { t.Fatalf("expected not-implemented message in stderr, got %q", stderr.String()) } } + +func TestRunProcessCLIOverridesEnvironment(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + t.Setenv("AUDITA_MODEL", "env-model") + t.Setenv("AUDITA_VALIDATION_LLM_CONCURRENCY", "2") + + var captured processInvocation + originalRunner := processRunner + processRunner = func(inv processInvocation) error { + captured = inv + return errors.New(processNotImplementedMessage) + } + t.Cleanup(func() { + processRunner = originalRunner + }) + + exitCode := Run([]string{ + "process", + "transcript.json", + "--glossary", + "glossary.yaml", + "--model", + "cli-model", + "--validation-llm-concurrency", + "5", + }, &stdout, &stderr) + + if exitCode != 1 { + t.Fatalf("expected exit code 1 for not implemented, got %d", exitCode) + } + if captured.Config.PrimaryLLM.Model != "cli-model" { + t.Fatalf("expected CLI model override, got %q", captured.Config.PrimaryLLM.Model) + } + if captured.Config.ValidationLLM.Concurrency == nil || *captured.Config.ValidationLLM.Concurrency != 5 { + t.Fatalf("expected CLI validation concurrency override, got %#v", captured.Config.ValidationLLM.Concurrency) + } + if captured.GlossaryPath != "glossary.yaml" { + t.Fatalf("unexpected glossary path: %q", captured.GlossaryPath) + } +} diff --git a/internal/core/config/config.go b/internal/core/config/config.go index 7de0483..d00d998 100644 --- a/internal/core/config/config.go +++ b/internal/core/config/config.go @@ -19,6 +19,7 @@ const ( DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1" DefaultPrimaryLLMTimeoutSeconds = 600 DefaultMaxRetries = 3 + DefaultLLMConcurrency = 1 DefaultValidationMaxPromptTokens = 2048 DefaultMaxSectionTokens = 8192 DefaultMinSectionTokens = 2048 @@ -51,6 +52,7 @@ type LLMConfig struct { BaseURL string TimeoutSeconds int MaxRetries int + Concurrency int } type ValidationLLMConfig struct { @@ -59,6 +61,7 @@ type ValidationLLMConfig struct { BaseURL string TimeoutSeconds *int MaxRetries *int + Concurrency *int } type ConfidenceThresholds struct { @@ -85,6 +88,7 @@ func Default() Config { BaseURL: DefaultPrimaryBaseURL, TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds, MaxRetries: DefaultMaxRetries, + Concurrency: DefaultLLMConcurrency, }, ValidationLLM: ValidationLLMConfig{}, ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens, @@ -142,6 +146,9 @@ func (c Config) EffectiveValidationLLMConfig() LLMConfig { if c.ValidationLLM.MaxRetries != nil { effective.MaxRetries = *c.ValidationLLM.MaxRetries } + if c.ValidationLLM.Concurrency != nil { + effective.Concurrency = *c.ValidationLLM.Concurrency + } return effective } diff --git a/internal/core/config/config_test.go b/internal/core/config/config_test.go index f9a5157..bc537b1 100644 --- a/internal/core/config/config_test.go +++ b/internal/core/config/config_test.go @@ -24,12 +24,18 @@ func TestDefaultConfigValues(t *testing.T) { if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries { t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries) } + if cfg.PrimaryLLM.Concurrency != DefaultLLMConcurrency { + t.Fatalf("unexpected default llm concurrency: %d", cfg.PrimaryLLM.Concurrency) + } if cfg.ValidationLLM.TimeoutSeconds != nil { t.Fatalf("expected validation timeout to be unset by default") } if cfg.ValidationLLM.MaxRetries != nil { t.Fatalf("expected validation max retries to be unset by default") } + if cfg.ValidationLLM.Concurrency != nil { + t.Fatalf("expected validation llm concurrency to be unset by default") + } if cfg.TargetSections != nil { t.Fatalf("expected target sections to be unset by default") } @@ -50,6 +56,8 @@ func TestLoadFromEnvOverridesAndFallback(t *testing.T) { "AUDITA_BASE_URL": "https://api.openai.com/v1", "AUDITA_LLM_TIMEOUT_SECONDS": "120", "AUDITA_MAX_RETRIES": "7", + "AUDITA_LLM_CONCURRENCY": "6", + "AUDITA_VALIDATION_LLM_CONCURRENCY": "2", "AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096", "AUDITA_MAX_SECTION_TOKENS": "9000", "AUDITA_MIN_SECTION_TOKENS": "3000", @@ -84,6 +92,12 @@ func TestLoadFromEnvOverridesAndFallback(t *testing.T) { if cfg.TargetSections == nil || *cfg.TargetSections != 5 { t.Fatalf("unexpected target sections: %#v", cfg.TargetSections) } + if cfg.PrimaryLLM.Concurrency != 6 { + t.Fatalf("unexpected primary llm concurrency: %d", cfg.PrimaryLLM.Concurrency) + } + if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 { + t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLM.Concurrency) + } if cfg.WorkDirRetention != WorkDirRetentionAlways { t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention) } @@ -137,6 +151,7 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) { func TestValidationFailures(t *testing.T) { cfg := Default() cfg.PrimaryLLM.TimeoutSeconds = -1 + cfg.PrimaryLLM.Concurrency = 0 cfg.ValidationMaxPromptTokens = 0 cfg.MaxSectionTokens = 100 cfg.MinSectionTokens = 200 @@ -151,6 +166,7 @@ func TestValidationFailures(t *testing.T) { message := err.Error() for _, expected := range []string{ "primary llm timeout seconds", + "primary llm concurrency", "validation max prompt tokens", "min section tokens", "grammar confidence threshold", @@ -169,9 +185,10 @@ func TestEffectiveValidationLLMInheritance(t *testing.T) { cfg.PrimaryLLM.BaseURL = "https://primary.example/v1" cfg.PrimaryLLM.TimeoutSeconds = 111 cfg.PrimaryLLM.MaxRetries = 2 + cfg.PrimaryLLM.Concurrency = 7 effective := cfg.EffectiveValidationLLMConfig() - if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 { + if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 { t.Fatalf("unexpected inherited config: %#v", effective) } @@ -182,9 +199,11 @@ func TestEffectiveValidationLLMInheritance(t *testing.T) { cfg.ValidationLLM.BaseURL = "https://validation.example/v1" cfg.ValidationLLM.TimeoutSeconds = &validationTimeout cfg.ValidationLLM.MaxRetries = &validationRetries + validationConcurrency := 4 + cfg.ValidationLLM.Concurrency = &validationConcurrency effective = cfg.EffectiveValidationLLMConfig() - if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 { + if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 { t.Fatalf("unexpected overridden validation config: %#v", effective) } } diff --git a/internal/core/config/env.go b/internal/core/config/env.go index 9a30ad3..0ef70c3 100644 --- a/internal/core/config/env.go +++ b/internal/core/config/env.go @@ -68,6 +68,13 @@ func loadFromLookup(lookup func(string) (string, bool)) (Config, error) { } cfg.PrimaryLLM.MaxRetries = value } + if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok { + value, err := parseInt(raw) + if err != nil { + return Config{}, fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err) + } + cfg.PrimaryLLM.Concurrency = value + } if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok { value, err := parseInt(raw) @@ -76,6 +83,13 @@ func loadFromLookup(lookup func(string) (string, bool)) (Config, error) { } cfg.ValidationLLM.MaxRetries = &value } + if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok { + value, err := parseInt(raw) + if err != nil { + return Config{}, fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err) + } + cfg.ValidationLLM.Concurrency = &value + } if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok { value, err := parseInt(raw) diff --git a/internal/core/config/flags.go b/internal/core/config/flags.go index 6fccdef..94d1a6e 100644 --- a/internal/core/config/flags.go +++ b/internal/core/config/flags.go @@ -14,6 +14,7 @@ type CLIOverrides struct { ValidationLLMTimeoutSeconds *int MaxRetries *int ValidationMaxRetries *int + ValidationLLMConcurrency *int ValidationMaxPromptTokens *int MaxSectionTokens *int MinSectionTokens *int @@ -71,6 +72,10 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error { value := *overrides.ValidationMaxRetries c.ValidationLLM.MaxRetries = &value } + if overrides.ValidationLLMConcurrency != nil { + value := *overrides.ValidationLLMConcurrency + c.ValidationLLM.Concurrency = &value + } if overrides.ValidationMaxPromptTokens != nil { c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens } diff --git a/internal/core/config/validation.go b/internal/core/config/validation.go index da7bdd2..5b76e96 100644 --- a/internal/core/config/validation.go +++ b/internal/core/config/validation.go @@ -24,6 +24,9 @@ func (c Config) Validate() error { if c.PrimaryLLM.MaxRetries < 0 { issues = append(issues, "max retries must be zero or greater") } + if c.PrimaryLLM.Concurrency <= 0 { + issues = append(issues, "primary llm concurrency must be greater than zero") + } if c.ValidationLLM.TimeoutSeconds != nil && *c.ValidationLLM.TimeoutSeconds <= 0 { issues = append(issues, "validation llm timeout seconds must be greater than zero") @@ -31,6 +34,9 @@ func (c Config) Validate() error { if c.ValidationLLM.MaxRetries != nil && *c.ValidationLLM.MaxRetries < 0 { issues = append(issues, "validation max retries must be zero or greater") } + if c.ValidationLLM.Concurrency != nil && *c.ValidationLLM.Concurrency <= 0 { + issues = append(issues, "validation llm concurrency must be greater than zero") + } if c.ValidationMaxPromptTokens <= 0 { issues = append(issues, "validation max prompt tokens must be greater than zero")