eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Carry evidence through built-in generation retries

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-13 16:56:56 +00:00
parent d91afc61a4
commit c5aed7a451
6 changed files with 372 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
internal/generation/coordinator.go
View File

@@ -72,14 +72,41 @@ func (e *ExhaustedError) Attempts() uint64 {
// Generate executes either one opaque client call or a frozen built-in HTTP
// sequence with one permit per actual dispatch.
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
response, err, _ := c.generate(ctx, req, false)
return response, err
}
// GenerateWithEvidence runs the built-in retry path and returns its factual
// attempt evidence alongside any response retained before cancellation.
func (c *Coordinator) GenerateWithEvidence(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error, GenerationEvidence) {
return c.generate(ctx, req, true)
}
func (c *Coordinator) generate(
ctx context.Context,
req domain.GenerateRequest,
captureEvidence bool,
) (*domain.GenerateResponse, error, GenerationEvidence) {
if c.injected != nil {
return c.generateInjected(ctx, req)
response, err := c.generateInjected(ctx, req)
return response, err, GenerationEvidence{}
}
prepared, err := c.builtIn.PrepareGeneration(req)
if err != nil {
return nil, err
return nil, err, GenerationEvidence{}
}
evidence := GenerationEvidence{}
var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence {
evidence.Usage = usage.Observed()
}
return response, err, evidence
}
var attempts, retriesUsed uint64
recorder := observation.FromContext(ctx)
generation := req.Generation
@@ -88,50 +115,82 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
}
for {
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
ordinal := attempts + 1
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
if dispatched {
attempts++
}
if captureEvidence {
attempt := AttemptEvidence{
Dispatched: dispatched,
HTTP: cloneHTTPEvidence(outcome.Evidence),
CapacityWait: capacityWait,
RequestDuration: generationDuration,
Failure: ClassifyFailure(outcome.Err),
ResponseDecoded: outcome.ResponseDecoded,
GenerationCompleted: outcome.GenerationCompleted,
}
if outcome.StructuredOutputRequested != nil {
requested := *outcome.StructuredOutputRequested
attempt.StructuredOutputRequested = &requested
}
if dispatched {
usage.Add(outcome.Usage, outcome.UsagePresence)
var attemptUsage observation.UsageAccumulator
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
attempt.Usage = attemptUsage.Observed()
}
evidence.Attempts = append(evidence.Attempts, attempt)
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
}
if outcome.Err == nil {
if err := ctx.Err(); err != nil {
return nil, err
if captureEvidence {
return finish(outcome.Response, err)
}
return finish(nil, err)
}
return outcome.Response, nil
return finish(outcome.Response, nil)
}
if !dispatched || !eligibleStatus(outcome) {
return nil, outcome.Err
if captureEvidence {
return finish(outcome.Response, outcome.Err)
}
return finish(nil, outcome.Err)
}
if retriesUsed == req.TransientRetries {
if retriesUsed == 0 {
return nil, outcome.Err
return finish(nil, outcome.Err)
}
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
}
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
retryOrdinal := retriesUsed + 1
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
waitStart := c.now()
waitErr := c.wait(ctx, delay)
waitDuration := elapsed(waitStart, c.now())
if captureEvidence {
evidence.RetryDelays = append(evidence.RetryDelays, waitDuration)
}
if recorder != nil {
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
Elapsed: waitDuration, Failure: ClassifyFailure(waitErr)})
}
if waitErr != nil {
return nil, waitErr
return finish(nil, waitErr)
}
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
retriesUsed++
}
@@ -148,7 +207,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
capacityWait := c.capacityWait(req.Target.BackendID, start)
if err != nil {
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return nil, err
}
@@ -156,7 +215,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
if err := ctx.Err(); err != nil {
release()
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return nil, err
}
@@ -169,7 +228,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
release()
if recorder != nil {
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
if err == nil {
data.Usage = response.Usage
data.UsagePresence = response.UsagePresence
@@ -185,7 +244,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
return response, nil
}
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
start := c.now()
release, err := c.manager.Acquire(ctx, backendID)
capacityWait = c.capacityWait(backendID, start)
@@ -197,8 +256,14 @@ func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *l
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
}
dispatchStart := c.now()
outcome = prepared.Execute(ctx)
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
if captureEvidence {
outcome = prepared.ExecuteWithEvidence(ctx)
dispatched = outcome.Evidence.DispatchInvoked
} else {
outcome = prepared.Execute(ctx)
dispatched = true
}
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now())
}
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
@@ -222,7 +287,8 @@ func attemptOutcome(err error) domain.ExecutionOutcome {
return domain.ExecutionOutcomeError
}
func attemptFailure(err error) domain.ExecutionFailureKind {
// ClassifyFailure maps an execution error to the shared content-free category.
func ClassifyFailure(err error) domain.ExecutionFailureKind {
if err == nil {
return domain.ExecutionFailureNone
}