Add explicit LLM concurrency controls

internal/framework/llm/effective_config.go

@@ -21,13 +21,13 @@ type EffectiveConfig struct {
 
 // ResolvePrimaryConfig resolves the primary LLM settings into runtime form.
 func ResolvePrimaryConfig(cfg config.Config) EffectiveConfig {
-	return resolveFromLLMConfig(cfg.PrimaryLLM)
+	return resolveFromLLMConfig(cfg.PrimaryLLM, cfg.TotalLLMConcurrency)
 }
 
 // ResolveValidationConfig resolves validation LLM settings with inheritance
 // from primary values when validation overrides are unset.
 func ResolveValidationConfig(cfg config.Config) EffectiveConfig {
-	return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig())
+	return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency())
 }
 
 // ToInstructorClientConfig converts an effective runtime config into adapter
@@ -44,13 +44,13 @@ func (c EffectiveConfig) ToInstructorClientConfig(mode Mode, httpClient *http.Cl
 	}
 }
 
-func resolveFromLLMConfig(raw config.LLMConfig) EffectiveConfig {
+func resolveFromLLMConfig(raw config.LLMConfig, concurrency int) EffectiveConfig {
 	return EffectiveConfig{
 		APIKey:         strings.TrimSpace(raw.APIKey),
 		Model:          strings.TrimSpace(raw.Model),
 		BaseURL:        strings.TrimSpace(raw.BaseURL),
 		RequestTimeout: time.Duration(raw.TimeoutSeconds) * time.Second,
 		MaxRetries:     raw.MaxRetries,
-		Concurrency:    raw.Concurrency,
+		Concurrency:    concurrency,
 	}
 }

internal/framework/llm/effective_config_test.go

@@ -14,7 +14,8 @@ func TestResolvePrimaryConfig(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = " https://example.test/v1 "
 	cfg.PrimaryLLM.TimeoutSeconds = 42
 	cfg.PrimaryLLM.MaxRetries = 7
-	cfg.PrimaryLLM.Concurrency = 3
+	cfg.TotalLLMConcurrency = 3
+	cfg.ProposalLLMConcurrency = 3
 
 	effective := ResolvePrimaryConfig(cfg)
 	if effective.APIKey != "primary-key" {
@@ -44,7 +45,8 @@ func TestResolveValidationConfigInheritsPrimaryWhenUnset(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
 	cfg.PrimaryLLM.TimeoutSeconds = 90
 	cfg.PrimaryLLM.MaxRetries = 4
-	cfg.PrimaryLLM.Concurrency = 2
+	cfg.TotalLLMConcurrency = 2
+	cfg.ProposalLLMConcurrency = 2
 
 	effective := ResolveValidationConfig(cfg)
 	if effective.APIKey != "primary-key" ||
@@ -64,7 +66,8 @@ func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) {
 	cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
 	cfg.PrimaryLLM.TimeoutSeconds = 90
 	cfg.PrimaryLLM.MaxRetries = 4
-	cfg.PrimaryLLM.Concurrency = 2
+	cfg.TotalLLMConcurrency = 2
+	cfg.ProposalLLMConcurrency = 2
 
 	timeout := 12
 	retries := 9
@@ -74,7 +77,7 @@ func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) {
 	cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
 	cfg.ValidationLLM.TimeoutSeconds = &timeout
 	cfg.ValidationLLM.MaxRetries = &retries
-	cfg.ValidationLLM.Concurrency = &concurrency
+	cfg.ValidationLLMConcurrency = &concurrency
 
 	effective := ResolveValidationConfig(cfg)
 	if effective.APIKey != "validation-key" ||

internal/framework/llm/scheduler.go

@@ -8,7 +8,16 @@ import (
 
 // Scheduler bounds concurrent backend LLM calls.
 type Scheduler struct {
-	permits chan struct{}
+	maxConcurrency int
+	mu             sync.Mutex
+	inFlight       int
+	queue          []*waiter
+}
+
+type waiter struct {
+	ready   chan struct{}
+	queued  bool
+	granted bool
 }
 
 // NewScheduler creates a scheduler with a fixed concurrency limit.
@@ -17,26 +26,86 @@ func NewScheduler(maxConcurrency int) (*Scheduler, error) {
 		return nil, fmt.Errorf("max concurrency must be greater than zero")
 	}
 	return &Scheduler{
-		permits: make(chan struct{}, maxConcurrency),
+		maxConcurrency: maxConcurrency,
 	}, nil
 }
 
 // Acquire blocks until a permit is available or the context is canceled.
 // The returned release function is safe to call multiple times.
 func (s *Scheduler) Acquire(ctx context.Context) (func(), error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	s.mu.Lock()
+	if s.inFlight < s.maxConcurrency && len(s.queue) == 0 {
+		s.inFlight++
+		s.mu.Unlock()
+		return s.releaseFunc(), nil
+	}
+	w := &waiter{
+		ready:  make(chan struct{}),
+		queued: true,
+	}
+	s.queue = append(s.queue, w)
+	s.mu.Unlock()
+
 	select {
-	case s.permits <- struct{}{}:
-		var once sync.Once
-		return func() {
-			once.Do(func() {
-				<-s.permits
-			})
-		}, nil
+	case <-w.ready:
+		return s.releaseFunc(), nil
 	case <-ctx.Done():
+		s.mu.Lock()
+		if w.queued {
+			s.removeQueuedWaiterLocked(w)
+			s.mu.Unlock()
+			return nil, ctx.Err()
+		}
+		if w.granted {
+			// Permit was granted concurrently with cancellation; release it to
+			// avoid leaks and unblock queued callers.
+			s.inFlight--
+			s.grantQueuedLocked()
+		}
+		s.mu.Unlock()
 		return nil, ctx.Err()
 	}
 }
 
+func (s *Scheduler) releaseFunc() func() {
+	var once sync.Once
+	return func() {
+		once.Do(func() {
+			s.mu.Lock()
+			if s.inFlight > 0 {
+				s.inFlight--
+				s.grantQueuedLocked()
+			}
+			s.mu.Unlock()
+		})
+	}
+}
+
+func (s *Scheduler) grantQueuedLocked() {
+	for s.inFlight < s.maxConcurrency && len(s.queue) > 0 {
+		w := s.queue[0]
+		s.queue = s.queue[1:]
+		w.queued = false
+		w.granted = true
+		s.inFlight++
+		close(w.ready)
+	}
+}
+
+func (s *Scheduler) removeQueuedWaiterLocked(target *waiter) {
+	for i, w := range s.queue {
+		if w == target {
+			w.queued = false
+			s.queue = append(s.queue[:i], s.queue[i+1:]...)
+			return
+		}
+	}
+}
+
 // Run acquires a permit, executes fn, and releases the permit.
 func (s *Scheduler) Run(ctx context.Context, fn func(context.Context) error) error {
 	release, err := s.Acquire(ctx)

internal/framework/llm/scheduler_test.go

@@ -3,12 +3,64 @@ package llm
 import (
 	"context"
 	"errors"
+	"reflect"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 )
 
+func TestSchedulerFIFOOrdering(t *testing.T) {
+	s, err := NewScheduler(1)
+	if err != nil {
+		t.Fatalf("NewScheduler: %v", err)
+	}
+
+	firstRelease, err := s.Acquire(context.Background())
+	if err != nil {
+		t.Fatalf("Acquire(first): %v", err)
+	}
+
+	gotOrder := make(chan int, 3)
+	waitChans := []chan struct{}{
+		make(chan struct{}),
+		make(chan struct{}),
+		make(chan struct{}),
+	}
+	var wg sync.WaitGroup
+	for i := 0; i < 3; i++ {
+		id := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			runErr := s.Run(context.Background(), func(context.Context) error {
+				gotOrder <- id
+				<-waitChans[id]
+				return nil
+			})
+			if runErr != nil {
+				t.Errorf("Run[%d] error: %v", id, runErr)
+			}
+		}()
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	firstRelease()
+
+	order := make([]int, 0, 3)
+	for i := 0; i < 3; i++ {
+		id := <-gotOrder
+		order = append(order, id)
+		close(waitChans[id])
+	}
+
+	wg.Wait()
+
+	if !reflect.DeepEqual(order, []int{0, 1, 2}) {
+		t.Fatalf("expected FIFO order [0 1 2], got %v", order)
+	}
+}
+
 func TestSchedulerEnforcesMaxConcurrency(t *testing.T) {
 	s, err := NewScheduler(2)
 	if err != nil {
@@ -79,7 +131,6 @@ func TestSchedulerReleasesPermitOnError(t *testing.T) {
 		t.Fatalf("expected %v, got %v", expectedErr, err)
 	}
 
-	// Must be able to run again after an error, proving permit release.
 	if err := s.Run(context.Background(), func(ctx context.Context) error {
 		_ = ctx
 		return nil
@@ -88,7 +139,7 @@ func TestSchedulerReleasesPermitOnError(t *testing.T) {
 	}
 }
 
-func TestSchedulerRespectsContextCancellation(t *testing.T) {
+func TestSchedulerContextCancellationWhileQueued(t *testing.T) {
 	s, err := NewScheduler(1)
 	if err != nil {
 		t.Fatalf("NewScheduler: %v", err)
@@ -111,3 +162,27 @@ func TestSchedulerRespectsContextCancellation(t *testing.T) {
 		t.Fatalf("expected deadline exceeded, got %v", err)
 	}
 }
+
+func TestSchedulerNoPermitLeakAfterQueuedCancellation(t *testing.T) {
+	s, err := NewScheduler(1)
+	if err != nil {
+		t.Fatalf("NewScheduler: %v", err)
+	}
+
+	firstRelease, err := s.Acquire(context.Background())
+	if err != nil {
+		t.Fatalf("Acquire(first): %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := s.Acquire(ctx); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context canceled, got %v", err)
+	}
+
+	firstRelease()
+
+	if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
+		t.Fatalf("expected scheduler to accept new work after cancellation, got %v", err)
+	}
+}

internal/framework/proposal_generation/generate_test.go

@@ -7,10 +7,14 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"gitea.maximumdirect.net/eric/audita/internal/core/config"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
 )
 
@@ -47,6 +51,40 @@ func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) er
 	return fn(ctx)
 }
 
+type sleepingStructuredClient struct {
+	inFlight    int32
+	maxInFlight int32
+}
+
+func (c *sleepingStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
+	_ = req
+	current := atomic.AddInt32(&c.inFlight, 1)
+	for {
+		prior := atomic.LoadInt32(&c.maxInFlight)
+		if current <= prior || atomic.CompareAndSwapInt32(&c.maxInFlight, prior, current) {
+			break
+		}
+	}
+	select {
+	case <-time.After(20 * time.Millisecond):
+	case <-ctx.Done():
+		atomic.AddInt32(&c.inFlight, -1)
+		return contracts.StructuredCompletionResponse{}, ctx.Err()
+	}
+	atomic.AddInt32(&c.inFlight, -1)
+
+	target, ok := out.(*StructuredCorrectionSet)
+	if !ok {
+		return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
+	}
+	*target = StructuredCorrectionSet{
+		Corrections: []StructuredCorrectionProposal{
+			{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "y", Confidence: 0.9},
+		},
+	}
+	return contracts.StructuredCompletionResponse{}, nil
+}
+
 func defaultRequest(t *testing.T) Request {
 	t.Helper()
 	cfg := config.Default()
@@ -265,3 +303,35 @@ func TestGenerateCandidatesClientError(t *testing.T) {
 		t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
 	}
 }
+
+func TestGenerateCandidatesRespectsSchedulerConcurrency(t *testing.T) {
+	scheduler, err := llm.NewScheduler(2)
+	if err != nil {
+		t.Fatalf("NewScheduler: %v", err)
+	}
+	client := &sleepingStructuredClient{}
+	baseReq := defaultRequest(t)
+	baseReq.LLMClient = client
+	baseReq.Scheduler = scheduler
+
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			req := baseReq
+			req.StartIndex = idx
+			if _, runErr := GenerateCandidates(context.Background(), req); runErr != nil {
+				t.Errorf("GenerateCandidates[%d] error: %v", idx, runErr)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	if got := atomic.LoadInt32(&client.maxInFlight); got > 2 {
+		t.Fatalf("expected scheduler cap <= 2, got %d", got)
+	}
+	if got := atomic.LoadInt32(&client.maxInFlight); got < 2 {
+		t.Fatalf("expected observed concurrency of at least 2, got %d", got)
+	}
+}

internal/framework/runner/runner.go

@@ -296,8 +296,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
 	}
 
 	maxWorkers := 1
-	if input.Config != nil && input.Config.PrimaryLLM.Concurrency > 1 {
-		maxWorkers = input.Config.PrimaryLLM.Concurrency
+	if input.Config != nil && input.Config.EffectiveProposalLLMConcurrency() > 1 {
+		maxWorkers = input.Config.EffectiveProposalLLMConcurrency()
 	}
 	if maxWorkers > len(input.Sections) {
 		maxWorkers = len(input.Sections)

internal/framework/runner/runner_test.go

@@ -163,11 +163,12 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
 	}
 }
 
-func TestRunnerProposalSectionConcurrencyBoundedByPrimaryLLMConcurrency(t *testing.T) {
+func TestRunnerProposalSectionConcurrencyBoundedByProposalLLMConcurrency(t *testing.T) {
 	cfg := config.Default()
 	cfg.MaxSectionTokens = 3
 	cfg.MinSectionTokens = 0
-	cfg.PrimaryLLM.Concurrency = 2
+	cfg.TotalLLMConcurrency = 2
+	cfg.ProposalLLMConcurrency = 2
 
 	transcript := &schema.Transcript{Segments: []schema.Segment{
 		{ID: 1, Text: "one two"},
@@ -215,7 +216,8 @@ func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *tes
 	cfg := config.Default()
 	cfg.MaxSectionTokens = 3
 	cfg.MinSectionTokens = 0
-	cfg.PrimaryLLM.Concurrency = 3
+	cfg.TotalLLMConcurrency = 3
+	cfg.ProposalLLMConcurrency = 3
 
 	transcript := &schema.Transcript{Segments: []schema.Segment{
 		{ID: 1, Text: "alpha one"},
@@ -259,6 +261,12 @@ func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *tes
 			t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
 		}
 	}
+	wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
+	for i, seg := range out.FinalTranscript.Segments {
+		if seg.Text != wantFinal[i] {
+			t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
+		}
+	}
 }
 
 func TestRunnerSkippedRecorded(t *testing.T) {

internal/framework/validators/llm_validators_test.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"errors"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
 	"gitea.maximumdirect.net/eric/audita/internal/core/config"
@@ -18,6 +21,54 @@ type fakeStructuredLLMClient struct {
 	calls     []StructuredCompletionRequest
 }
 
+type sleepingValidationClient struct {
+	inFlight    int32
+	maxInFlight int32
+}
+
+type boundedScheduler struct {
+	permits chan struct{}
+}
+
+func newBoundedScheduler(max int) *boundedScheduler {
+	return &boundedScheduler{permits: make(chan struct{}, max)}
+}
+
+func (s *boundedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
+	select {
+	case s.permits <- struct{}{}:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+	defer func() { <-s.permits }()
+	return fn(ctx)
+}
+
+func (c *sleepingValidationClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
+	_ = req
+	current := atomic.AddInt32(&c.inFlight, 1)
+	for {
+		prior := atomic.LoadInt32(&c.maxInFlight)
+		if current <= prior || atomic.CompareAndSwapInt32(&c.maxInFlight, prior, current) {
+			break
+		}
+	}
+	select {
+	case <-time.After(20 * time.Millisecond):
+	case <-ctx.Done():
+		atomic.AddInt32(&c.inFlight, -1)
+		return StructuredCompletionResponse{}, ctx.Err()
+	}
+	atomic.AddInt32(&c.inFlight, -1)
+	target := out.(*LLMValidationResponse)
+	*target = LLMValidationResponse{
+		Validations: []LLMValidationDecision{
+			{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
+		},
+	}
+	return StructuredCompletionResponse{}, nil
+}
+
 func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
 	_ = ctx
 	f.calls = append(f.calls, req)
@@ -191,3 +242,34 @@ func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
 		t.Fatalf("expected unknown index error, got %v", err)
 	}
 }
+
+func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
+	scheduler := newBoundedScheduler(2)
+	client := &sleepingValidationClient{}
+	v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
+	if err != nil {
+		t.Fatalf("new validator error: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
+			req.LLMClient = client
+			req.Scheduler = scheduler
+			if _, runErr := v.Validate(context.Background(), req); runErr != nil {
+				t.Errorf("Validate error: %v", runErr)
+			}
+		}()
+	}
+	wg.Wait()
+
+	if got := atomic.LoadInt32(&client.maxInFlight); got > 2 {
+		t.Fatalf("expected scheduler cap <= 2, got %d", got)
+	}
+	if got := atomic.LoadInt32(&client.maxInFlight); got < 2 {
+		t.Fatalf("expected observed concurrency of at least 2, got %d", got)
+	}
+}