Implemented an --llm-concurrency flag in the Go application that enforces a global LLM concurrency cap

This commit is contained in:
2026-05-12 12:59:46 -05:00
parent cad172a758
commit af84249da0
11 changed files with 520 additions and 69 deletions

View File

@@ -127,10 +127,12 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
model := "cli-model"
workDir := "/cli/work"
modules := "grammar"
llmConcurrency := 5
overrides := CLIOverrides{
PrimaryModel: &model,
WorkDir: &workDir,
ModulesCSV: &modules,
PrimaryModel: &model,
WorkDir: &workDir,
ModulesCSV: &modules,
PrimaryLLMConcurrency: &llmConcurrency,
}
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
@@ -146,12 +148,17 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.PrimaryLLM.Concurrency != 5 {
t.Fatalf("expected CLI llm concurrency override, got %d", cfg.PrimaryLLM.Concurrency)
}
}
func TestValidationFailures(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.TimeoutSeconds = -1
cfg.PrimaryLLM.Concurrency = 0
validationConcurrency := 5
cfg.ValidationLLM.Concurrency = &validationConcurrency
cfg.ValidationMaxPromptTokens = 0
cfg.MaxSectionTokens = 100
cfg.MinSectionTokens = 200
@@ -167,6 +174,7 @@ func TestValidationFailures(t *testing.T) {
for _, expected := range []string{
"primary llm timeout seconds",
"primary llm concurrency",
"validation llm concurrency must be less than or equal to primary llm concurrency",
"validation max prompt tokens",
"min section tokens",
"grammar confidence threshold",
@@ -208,6 +216,37 @@ func TestEffectiveValidationLLMInheritance(t *testing.T) {
}
}
func TestValidationLLMConcurrencyCannotExceedPrimary(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.Concurrency = 2
validationConcurrency := 3
cfg.ValidationLLM.Concurrency = &validationConcurrency
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when validation llm concurrency exceeds primary")
}
validationConcurrency = 2
cfg.ValidationLLM.Concurrency = &validationConcurrency
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestCLIPrimaryLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
cfg := Default()
llmConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &llmConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.ValidationLLM.Concurrency != nil {
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLM.Concurrency)
}
if cfg.EffectiveValidationLLMConfig().Concurrency != 6 {
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConfig().Concurrency)
}
}
func TestRedactedConfig(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "secret-primary"

View File

@@ -11,6 +11,7 @@ type CLIOverrides struct {
PrimaryBaseURL *string
ValidationBaseURL *string
PrimaryLLMTimeoutSeconds *int
PrimaryLLMConcurrency *int
ValidationLLMTimeoutSeconds *int
MaxRetries *int
ValidationMaxRetries *int
@@ -61,6 +62,9 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.PrimaryLLMTimeoutSeconds != nil {
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
}
if overrides.PrimaryLLMConcurrency != nil {
c.PrimaryLLM.Concurrency = *overrides.PrimaryLLMConcurrency
}
if overrides.ValidationLLMTimeoutSeconds != nil {
value := *overrides.ValidationLLMTimeoutSeconds
c.ValidationLLM.TimeoutSeconds = &value

View File

@@ -37,6 +37,9 @@ func (c Config) Validate() error {
if c.ValidationLLM.Concurrency != nil && *c.ValidationLLM.Concurrency <= 0 {
issues = append(issues, "validation llm concurrency must be greater than zero")
}
if c.ValidationLLM.Concurrency != nil && *c.ValidationLLM.Concurrency > c.PrimaryLLM.Concurrency {
issues = append(issues, "validation llm concurrency must be less than or equal to primary llm concurrency")
}
if c.ValidationMaxPromptTokens <= 0 {
issues = append(issues, "validation max prompt tokens must be greater than zero")