Add extract worker configuration controls
This commit is contained in:
@@ -43,7 +43,8 @@ type FileArtifactLaneProfile struct {
|
||||
}
|
||||
|
||||
type FileConcurrencyConfig struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
|
||||
}
|
||||
|
||||
type FileDiagnosticsConfig struct {
|
||||
@@ -316,6 +317,15 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
|
||||
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
|
||||
}
|
||||
if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil {
|
||||
workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Concurrency.StageWorkers = workers
|
||||
c.Concurrency.extractWorkersConfigured = configured
|
||||
}
|
||||
c.Concurrency.recomputeStageWorkerDefaults()
|
||||
if fileCfg.Diagnostics != nil {
|
||||
if fileCfg.Diagnostics.WorkDir != nil {
|
||||
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
|
||||
@@ -350,6 +360,26 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
|
||||
workers := make(map[string]int, len(values))
|
||||
configured := false
|
||||
for rawKey, value := range values {
|
||||
key := strings.TrimSpace(rawKey)
|
||||
if key == "" {
|
||||
return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty")
|
||||
}
|
||||
if key != "extract" {
|
||||
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey)
|
||||
}
|
||||
if _, exists := workers[key]; exists {
|
||||
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key)
|
||||
}
|
||||
workers[key] = value
|
||||
configured = true
|
||||
}
|
||||
return workers, configured, nil
|
||||
}
|
||||
|
||||
func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
|
||||
keys := make([]string, 0, len(values))
|
||||
rawByNormalized := make(map[string]string, len(values))
|
||||
|
||||
Reference in New Issue
Block a user