Implemented an --llm-concurrency flag in the Go application that enforces a global LLM concurrency cap
This commit is contained in:
@@ -127,10 +127,12 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
model := "cli-model"
|
||||
workDir := "/cli/work"
|
||||
modules := "grammar"
|
||||
llmConcurrency := 5
|
||||
overrides := CLIOverrides{
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
PrimaryLLMConcurrency: &llmConcurrency,
|
||||
}
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
@@ -146,12 +148,17 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 5 {
|
||||
t.Fatalf("expected CLI llm concurrency override, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailures(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.TimeoutSeconds = -1
|
||||
cfg.PrimaryLLM.Concurrency = 0
|
||||
validationConcurrency := 5
|
||||
cfg.ValidationLLM.Concurrency = &validationConcurrency
|
||||
cfg.ValidationMaxPromptTokens = 0
|
||||
cfg.MaxSectionTokens = 100
|
||||
cfg.MinSectionTokens = 200
|
||||
@@ -167,6 +174,7 @@ func TestValidationFailures(t *testing.T) {
|
||||
for _, expected := range []string{
|
||||
"primary llm timeout seconds",
|
||||
"primary llm concurrency",
|
||||
"validation llm concurrency must be less than or equal to primary llm concurrency",
|
||||
"validation max prompt tokens",
|
||||
"min section tokens",
|
||||
"grammar confidence threshold",
|
||||
@@ -208,6 +216,37 @@ func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationLLMConcurrencyCannotExceedPrimary(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.Concurrency = 2
|
||||
validationConcurrency := 3
|
||||
cfg.ValidationLLM.Concurrency = &validationConcurrency
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when validation llm concurrency exceeds primary")
|
||||
}
|
||||
|
||||
validationConcurrency = 2
|
||||
cfg.ValidationLLM.Concurrency = &validationConcurrency
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLIPrimaryLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
|
||||
cfg := Default()
|
||||
llmConcurrency := 6
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &llmConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency != nil {
|
||||
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
if cfg.EffectiveValidationLLMConfig().Concurrency != 6 {
|
||||
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConfig().Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "secret-primary"
|
||||
|
||||
@@ -11,6 +11,7 @@ type CLIOverrides struct {
|
||||
PrimaryBaseURL *string
|
||||
ValidationBaseURL *string
|
||||
PrimaryLLMTimeoutSeconds *int
|
||||
PrimaryLLMConcurrency *int
|
||||
ValidationLLMTimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
ValidationMaxRetries *int
|
||||
@@ -61,6 +62,9 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
|
||||
if overrides.PrimaryLLMTimeoutSeconds != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
|
||||
}
|
||||
if overrides.PrimaryLLMConcurrency != nil {
|
||||
c.PrimaryLLM.Concurrency = *overrides.PrimaryLLMConcurrency
|
||||
}
|
||||
if overrides.ValidationLLMTimeoutSeconds != nil {
|
||||
value := *overrides.ValidationLLMTimeoutSeconds
|
||||
c.ValidationLLM.TimeoutSeconds = &value
|
||||
|
||||
@@ -37,6 +37,9 @@ func (c Config) Validate() error {
|
||||
if c.ValidationLLM.Concurrency != nil && *c.ValidationLLM.Concurrency <= 0 {
|
||||
issues = append(issues, "validation llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ValidationLLM.Concurrency != nil && *c.ValidationLLM.Concurrency > c.PrimaryLLM.Concurrency {
|
||||
issues = append(issues, "validation llm concurrency must be less than or equal to primary llm concurrency")
|
||||
}
|
||||
|
||||
if c.ValidationMaxPromptTokens <= 0 {
|
||||
issues = append(issues, "validation max prompt tokens must be greater than zero")
|
||||
|
||||
Reference in New Issue
Block a user