Add extract worker configuration controls

internal/core/config/config.go

@@ -24,7 +24,11 @@ type ScriptoriumConfig struct {
 }
 
 type ConcurrencyConfig struct {
-	TotalLLM int `json:"total_llm"`
+	TotalLLM     int            `json:"total_llm"`
+	StageWorkers map[string]int `json:"stage_workers"`
+
+	extractWorkersConfigured bool
+	defaultedExtractWorkers  int
 }
 
 type DiagnosticsConfig struct {
@@ -58,7 +62,9 @@ func Default() Config {
 	return Config{
 		Pipelines: map[string]pipeline.PipelineProfile{},
 		Concurrency: ConcurrencyConfig{
-			TotalLLM: 1,
+			TotalLLM:                1,
+			StageWorkers:            map[string]int{"extract": 1},
+			defaultedExtractWorkers: 1,
 		},
 		Diagnostics: DiagnosticsConfig{
 			WorkDir:   "/tmp/notarius",
@@ -101,6 +107,7 @@ func (c Config) workspaceDirectory() string {
 
 func cloneConfig(in Config) Config {
 	out := in
+	out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
 	out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
 	for key, profile := range in.Pipelines {
 		out.Pipelines[key] = clonePipelineProfile(profile)
@@ -108,6 +115,34 @@ func cloneConfig(in Config) Config {
 	return out
 }
 
+func cloneIntMap(in map[string]int) map[string]int {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make(map[string]int, len(in))
+	for key, value := range in {
+		out[key] = value
+	}
+	return out
+}
+
+func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
+	if c == nil {
+		return
+	}
+	if c.StageWorkers == nil {
+		c.StageWorkers = make(map[string]int)
+	}
+	if !c.extractWorkersConfigured {
+		if value, ok := c.StageWorkers["extract"]; ok && (c.defaultedExtractWorkers == 0 || value != c.defaultedExtractWorkers) {
+			c.extractWorkersConfigured = true
+			return
+		}
+		c.StageWorkers["extract"] = c.TotalLLM
+		c.defaultedExtractWorkers = c.TotalLLM
+	}
+}
+
 func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
 	out := in
 	out.Input = cloneModuleBinding(in.Input)