Add explicit LLM concurrency controls

internal/framework/llm/effective_config.go

@@ -21,13 +21,13 @@ type EffectiveConfig struct {
 
 // ResolvePrimaryConfig resolves the primary LLM settings into runtime form.
 func ResolvePrimaryConfig(cfg config.Config) EffectiveConfig {
-	return resolveFromLLMConfig(cfg.PrimaryLLM)
+	return resolveFromLLMConfig(cfg.PrimaryLLM, cfg.TotalLLMConcurrency)
 }
 
 // ResolveValidationConfig resolves validation LLM settings with inheritance
 // from primary values when validation overrides are unset.
 func ResolveValidationConfig(cfg config.Config) EffectiveConfig {
-	return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig())
+	return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency())
 }
 
 // ToInstructorClientConfig converts an effective runtime config into adapter
@@ -44,13 +44,13 @@ func (c EffectiveConfig) ToInstructorClientConfig(mode Mode, httpClient *http.Cl
 	}
 }
 
-func resolveFromLLMConfig(raw config.LLMConfig) EffectiveConfig {
+func resolveFromLLMConfig(raw config.LLMConfig, concurrency int) EffectiveConfig {
 	return EffectiveConfig{
 		APIKey:         strings.TrimSpace(raw.APIKey),
 		Model:          strings.TrimSpace(raw.Model),
 		BaseURL:        strings.TrimSpace(raw.BaseURL),
 		RequestTimeout: time.Duration(raw.TimeoutSeconds) * time.Second,
 		MaxRetries:     raw.MaxRetries,
-		Concurrency:    raw.Concurrency,
+		Concurrency:    concurrency,
 	}
 }

internal/framework/llm/effective_config_test.go

@@ -14,7 +14,8 @@ func TestResolvePrimaryConfig(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = " https://example.test/v1 "
 	cfg.PrimaryLLM.TimeoutSeconds = 42
 	cfg.PrimaryLLM.MaxRetries = 7
-	cfg.PrimaryLLM.Concurrency = 3
+	cfg.TotalLLMConcurrency = 3
+	cfg.ProposalLLMConcurrency = 3
 
 	effective := ResolvePrimaryConfig(cfg)
 	if effective.APIKey != "primary-key" {
@@ -44,7 +45,8 @@ func TestResolveValidationConfigInheritsPrimaryWhenUnset(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
 	cfg.PrimaryLLM.TimeoutSeconds = 90
 	cfg.PrimaryLLM.MaxRetries = 4
-	cfg.PrimaryLLM.Concurrency = 2
+	cfg.TotalLLMConcurrency = 2
+	cfg.ProposalLLMConcurrency = 2
 
 	effective := ResolveValidationConfig(cfg)
 	if effective.APIKey != "primary-key" ||
@@ -64,7 +66,8 @@ func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
 	cfg.PrimaryLLM.TimeoutSeconds = 90
 	cfg.PrimaryLLM.MaxRetries = 4
-	cfg.PrimaryLLM.Concurrency = 2
+	cfg.TotalLLMConcurrency = 2
+	cfg.ProposalLLMConcurrency = 2
 
 	timeout := 12
 	retries := 9
@@ -74,7 +77,7 @@ func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) {
 	cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
 	cfg.ValidationLLM.TimeoutSeconds = &timeout
 	cfg.ValidationLLM.MaxRetries = &retries
-	cfg.ValidationLLM.Concurrency = &concurrency
+	cfg.ValidationLLMConcurrency = &concurrency
 
 	effective := ResolveValidationConfig(cfg)
 	if effective.APIKey != "validation-key" ||

internal/framework/llm/scheduler.go

@@ -8,7 +8,16 @@ import (
 
 // Scheduler bounds concurrent backend LLM calls.
 type Scheduler struct {
-	permits chan struct{}
+	maxConcurrency int
+	mu             sync.Mutex
+	inFlight       int
+	queue          []*waiter
+}
+
+type waiter struct {
+	ready   chan struct{}
+	queued  bool
+	granted bool
 }
 
 // NewScheduler creates a scheduler with a fixed concurrency limit.
@@ -17,26 +26,86 @@ func NewScheduler(maxConcurrency int) (*Scheduler, error) {
 		return nil, fmt.Errorf("max concurrency must be greater than zero")
 	}
 	return &Scheduler{
-		permits: make(chan struct{}, maxConcurrency),
+		maxConcurrency: maxConcurrency,
 	}, nil
 }
 
 // Acquire blocks until a permit is available or the context is canceled.
 // The returned release function is safe to call multiple times.
 func (s *Scheduler) Acquire(ctx context.Context) (func(), error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	s.mu.Lock()
+	if s.inFlight < s.maxConcurrency && len(s.queue) == 0 {
+		s.inFlight++
+		s.mu.Unlock()
+		return s.releaseFunc(), nil
+	}
+	w := &waiter{
+		ready:  make(chan struct{}),
+		queued: true,
+	}
+	s.queue = append(s.queue, w)
+	s.mu.Unlock()
+
 	select {
-	case s.permits <- struct{}{}:
-		var once sync.Once
-		return func() {
-			once.Do(func() {
-				<-s.permits
-			})
-		}, nil
+	case <-w.ready:
+		return s.releaseFunc(), nil
 	case <-ctx.Done():
+		s.mu.Lock()
+		if w.queued {
+			s.removeQueuedWaiterLocked(w)
+			s.mu.Unlock()
+			return nil, ctx.Err()
+		}
+		if w.granted {
+			// Permit was granted concurrently with cancellation; release it to
+			// avoid leaks and unblock queued callers.
+			s.inFlight--
+			s.grantQueuedLocked()
+		}
+		s.mu.Unlock()
 		return nil, ctx.Err()
 	}
 }
 
+func (s *Scheduler) releaseFunc() func() {
+	var once sync.Once
+	return func() {
+		once.Do(func() {
+			s.mu.Lock()
+			if s.inFlight > 0 {
+				s.inFlight--
+				s.grantQueuedLocked()
+			}
+			s.mu.Unlock()
+		})
+	}
+}
+
+func (s *Scheduler) grantQueuedLocked() {
+	for s.inFlight < s.maxConcurrency && len(s.queue) > 0 {
+		w := s.queue[0]
+		s.queue = s.queue[1:]
+		w.queued = false
+		w.granted = true
+		s.inFlight++
+		close(w.ready)
+	}
+}
+
+func (s *Scheduler) removeQueuedWaiterLocked(target *waiter) {
+	for i, w := range s.queue {
+		if w == target {
+			w.queued = false
+			s.queue = append(s.queue[:i], s.queue[i+1:]...)
+			return
+		}
+	}
+}
+
 // Run acquires a permit, executes fn, and releases the permit.
 func (s *Scheduler) Run(ctx context.Context, fn func(context.Context) error) error {
 	release, err := s.Acquire(ctx)

internal/framework/llm/scheduler_test.go

@@ -3,12 +3,64 @@ package llm
 import (
 	"context"
 	"errors"
+	"reflect"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 )
 
+func TestSchedulerFIFOOrdering(t *testing.T) {
+	s, err := NewScheduler(1)
+	if err != nil {
+		t.Fatalf("NewScheduler: %v", err)
+	}
+
+	firstRelease, err := s.Acquire(context.Background())
+	if err != nil {
+		t.Fatalf("Acquire(first): %v", err)
+	}
+
+	gotOrder := make(chan int, 3)
+	waitChans := []chan struct{}{
+		make(chan struct{}),
+		make(chan struct{}),
+		make(chan struct{}),
+	}
+	var wg sync.WaitGroup
+	for i := 0; i < 3; i++ {
+		id := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			runErr := s.Run(context.Background(), func(context.Context) error {
+				gotOrder <- id
+				<-waitChans[id]
+				return nil
+			})
+			if runErr != nil {
+				t.Errorf("Run[%d] error: %v", id, runErr)
+			}
+		}()
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	firstRelease()
+
+	order := make([]int, 0, 3)
+	for i := 0; i < 3; i++ {
+		id := <-gotOrder
+		order = append(order, id)
+		close(waitChans[id])
+	}
+
+	wg.Wait()
+
+	if !reflect.DeepEqual(order, []int{0, 1, 2}) {
+		t.Fatalf("expected FIFO order [0 1 2], got %v", order)
+	}
+}
+
 func TestSchedulerEnforcesMaxConcurrency(t *testing.T) {
 	s, err := NewScheduler(2)
 	if err != nil {
@@ -79,7 +131,6 @@ func TestSchedulerReleasesPermitOnError(t *testing.T) {
 		t.Fatalf("expected %v, got %v", expectedErr, err)
 	}
 
-	// Must be able to run again after an error, proving permit release.
 	if err := s.Run(context.Background(), func(ctx context.Context) error {
 		_ = ctx
 		return nil
@@ -88,7 +139,7 @@ func TestSchedulerReleasesPermitOnError(t *testing.T) {
 	}
 }
 
-func TestSchedulerRespectsContextCancellation(t *testing.T) {
+func TestSchedulerContextCancellationWhileQueued(t *testing.T) {
 	s, err := NewScheduler(1)
 	if err != nil {
 		t.Fatalf("NewScheduler: %v", err)
@@ -111,3 +162,27 @@ func TestSchedulerRespectsContextCancellation(t *testing.T) {
 		t.Fatalf("expected deadline exceeded, got %v", err)
 	}
 }
+
+func TestSchedulerNoPermitLeakAfterQueuedCancellation(t *testing.T) {
+	s, err := NewScheduler(1)
+	if err != nil {
+		t.Fatalf("NewScheduler: %v", err)
+	}
+
+	firstRelease, err := s.Acquire(context.Background())
+	if err != nil {
+		t.Fatalf("Acquire(first): %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := s.Acquire(ctx); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context canceled, got %v", err)
+	}
+
+	firstRelease()
+
+	if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
+		t.Fatalf("expected scheduler to accept new work after cancellation, got %v", err)
+	}
+}