Record generation attempts and retry delays

internal/generation/coordinator.go

@@ -12,6 +12,7 @@ import (
 	"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
 	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
 	"gitea.maximumdirect.net/eric/promptkit/internal/llm"
+	"gitea.maximumdirect.net/eric/promptkit/internal/observation"
 )
 
 const (
@@ -80,15 +81,30 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
 	}
 
 	var attempts, retriesUsed uint64
+	recorder := observation.FromContext(ctx)
+	generation := req.Generation
+	if generation == 0 {
+		generation = 1
+	}
 	for {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		outcome, dispatched := c.execute(ctx, req.Target.BackendID, prepared)
+		ordinal := attempts + 1
+		outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
 		if dispatched {
 			attempts++
 		}
+		if recorder != nil {
+			recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
+				Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
+				Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
+				StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
+		}
 		if outcome.Err == nil {
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
 			return outcome.Response, nil
 		}
 		if !dispatched || !eligibleStatus(outcome) {
@@ -105,8 +121,14 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
 		}
 		retryOrdinal := retriesUsed + 1
 		delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
-		if err := c.wait(ctx, delay); err != nil {
-			return nil, err
+		waitStart := c.now()
+		waitErr := c.wait(ctx, delay)
+		if recorder != nil {
+			recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
+				Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
+		}
+		if waitErr != nil {
+			return nil, waitErr
 		}
 		if err := ctx.Err(); err != nil {
 			return nil, err
@@ -116,27 +138,113 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
 }
 
 func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	recorder := observation.FromContext(ctx)
+	generation := req.Generation
+	if generation == 0 {
+		generation = 1
+	}
+	start := c.now()
 	release, err := c.manager.Acquire(ctx, req.Target.BackendID)
+	capacityWait := c.capacityWait(req.Target.BackendID, start)
 	if err != nil {
+		if recorder != nil {
+			recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
+		}
 		return nil, err
 	}
 	defer release()
 	if err := ctx.Err(); err != nil {
+		release()
+		if recorder != nil {
+			recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
+		}
 		return nil, err
 	}
-	return c.injected.Generate(ctx, req)
+	dispatchStart := c.now()
+	response, err := c.injected.Generate(ctx, req)
+	duration := elapsed(dispatchStart, c.now())
+	if err == nil && response == nil {
+		err = llm.ErrMalformedResponse
+	}
+	release()
+	if recorder != nil {
+		data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
+			Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
+		if err == nil {
+			data.Usage = response.Usage
+			data.UsagePresence = response.UsagePresence
+		}
+		recorder.Attempt(data)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	return response, nil
 }
 
-func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool) {
+func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
+	start := c.now()
 	release, err := c.manager.Acquire(ctx, backendID)
+	capacityWait = c.capacityWait(backendID, start)
 	if err != nil {
-		return llm.AttemptOutcome{Err: err}, false
+		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
 	}
 	defer release()
 	if err := ctx.Err(); err != nil {
-		return llm.AttemptOutcome{Err: err}, false
+		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
 	}
-	return prepared.Execute(ctx), true
+	dispatchStart := c.now()
+	outcome = prepared.Execute(ctx)
+	return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
+}
+
+func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
+	if !c.manager.Limited(backendID) {
+		return 0
+	}
+	return elapsed(start, c.now())
+}
+
+func elapsed(start, end time.Time) time.Duration {
+	if d := end.Sub(start); d > 0 {
+		return d
+	}
+	return 0
+}
+
+func attemptOutcome(err error) domain.ExecutionOutcome {
+	if err == nil {
+		return domain.ExecutionOutcomeSuccess
+	}
+	return domain.ExecutionOutcomeError
+}
+
+func attemptFailure(err error) domain.ExecutionFailureKind {
+	if err == nil {
+		return domain.ExecutionFailureNone
+	}
+	if errors.Is(err, context.Canceled) {
+		return domain.ExecutionFailureCanceled
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return domain.ExecutionFailureDeadline
+	}
+	if errors.Is(err, llm.ErrUnexpectedStatus) {
+		return domain.ExecutionFailureProviderStatus
+	}
+	if errors.Is(err, llm.ErrTransport) {
+		return domain.ExecutionFailureTransport
+	}
+	if errors.Is(err, llm.ErrMalformedResponse) {
+		return domain.ExecutionFailureMalformedResponse
+	}
+	if errors.Is(err, llm.ErrInvalidRequest) {
+		return domain.ExecutionFailureInvalidRequest
+	}
+	return domain.ExecutionFailureGeneration
 }
 
 func eligibleStatus(outcome llm.AttemptOutcome) bool {