Expose ordinary run execution observations
This commit is contained in:
177
observations.go
Normal file
177
observations.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// ExecutionObserver receives immutable metadata events synchronously. Callers
|
||||
// must synchronize shared state if separate runs use the same observer. A
|
||||
// callback must return for its run to progress; panics propagate to the caller.
|
||||
type ExecutionObserver func(ExecutionEvent)
|
||||
|
||||
// ExecutionEventKind identifies one point in an execution's lifecycle.
|
||||
type ExecutionEventKind string
|
||||
|
||||
const (
|
||||
ExecutionStarted ExecutionEventKind = "started"
|
||||
ExecutionAttempt ExecutionEventKind = "attempt"
|
||||
ExecutionRetryDelay ExecutionEventKind = "retry_delay"
|
||||
ExecutionFinished ExecutionEventKind = "finished"
|
||||
)
|
||||
|
||||
// ExecutionOutcome reports whether the operation or attempt succeeded.
|
||||
type ExecutionOutcome string
|
||||
|
||||
const (
|
||||
ExecutionOutcomeSuccess ExecutionOutcome = "success"
|
||||
ExecutionOutcomeInvalidOutput ExecutionOutcome = "invalid_output"
|
||||
ExecutionOutcomeError ExecutionOutcome = "error"
|
||||
)
|
||||
|
||||
// ExecutionFailureKind is a content-free classification of an error.
|
||||
type ExecutionFailureKind string
|
||||
|
||||
const (
|
||||
ExecutionFailureNone ExecutionFailureKind = ""
|
||||
ExecutionFailureCanceled ExecutionFailureKind = "canceled"
|
||||
ExecutionFailureDeadline ExecutionFailureKind = "deadline"
|
||||
ExecutionFailureTransport ExecutionFailureKind = "transport"
|
||||
ExecutionFailureProviderStatus ExecutionFailureKind = "provider_status"
|
||||
ExecutionFailureMalformedResponse ExecutionFailureKind = "malformed_response"
|
||||
ExecutionFailureCapacity ExecutionFailureKind = "capacity"
|
||||
ExecutionFailureInvalidConfig ExecutionFailureKind = "invalid_config"
|
||||
ExecutionFailureInvalidRequest ExecutionFailureKind = "invalid_request"
|
||||
ExecutionFailurePromptNotFound ExecutionFailureKind = "prompt_not_found"
|
||||
ExecutionFailureProfileNotFound ExecutionFailureKind = "profile_not_found"
|
||||
ExecutionFailurePromptLoad ExecutionFailureKind = "prompt_load"
|
||||
ExecutionFailureProfileLoad ExecutionFailureKind = "profile_load"
|
||||
ExecutionFailureArtifactLoad ExecutionFailureKind = "artifact_load"
|
||||
ExecutionFailurePromptRender ExecutionFailureKind = "prompt_render"
|
||||
ExecutionFailureValidation ExecutionFailureKind = "validation"
|
||||
ExecutionFailureGeneration ExecutionFailureKind = "generation"
|
||||
ExecutionFailureOther ExecutionFailureKind = "other"
|
||||
)
|
||||
|
||||
// ObservedTokenCount distinguishes an omitted count from a reported zero.
|
||||
// Complete means every dispatched attempt reported the field.
|
||||
type ObservedTokenCount struct {
|
||||
Value int
|
||||
Known bool
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// ObservedTokenUsage contains presence-aware usage for one attempt or an
|
||||
// aggregate. It does not prove complete provider billing.
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only, not provider text, request content,
|
||||
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
|
||||
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
|
||||
// zero; a terminal usage total must not be added to attempt usage.
|
||||
type ExecutionEvent struct {
|
||||
Kind ExecutionEventKind
|
||||
RunID string
|
||||
Time time.Time
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
BackendID string
|
||||
Model string
|
||||
Generation uint64
|
||||
Attempt uint64
|
||||
Dispatched bool
|
||||
CapacityWait time.Duration
|
||||
GenerationDuration time.Duration
|
||||
RetryDelay time.Duration
|
||||
Duration time.Duration
|
||||
Outcome ExecutionOutcome
|
||||
Failure ExecutionFailureKind
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
}
|
||||
|
||||
func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
||||
count := func(c domain.ObservedTokenCount) ObservedTokenCount {
|
||||
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
return ExecutionEvent{
|
||||
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
|
||||
PromptID: value.PromptID, PromptVersion: value.PromptVersion, ProfileID: value.ProfileID,
|
||||
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
||||
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
||||
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
|
||||
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
|
||||
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
|
||||
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
}
|
||||
}
|
||||
|
||||
func observedFinish(result *RunResult, err error) domain.ExecutionEvent {
|
||||
if err == nil {
|
||||
outcome := domain.ExecutionOutcomeSuccess
|
||||
if result.Validation.Status == ValidationFailed {
|
||||
outcome = domain.ExecutionOutcomeInvalidOutput
|
||||
}
|
||||
return domain.ExecutionEvent{Outcome: outcome}
|
||||
}
|
||||
value := domain.ExecutionEvent{Outcome: domain.ExecutionOutcomeError}
|
||||
var exhausted *RetryExhaustedError
|
||||
value.RetriesExhausted = errors.As(err, &exhausted)
|
||||
var provider *GenerationError
|
||||
if errors.As(err, &provider) {
|
||||
value.StatusCode = provider.StatusCode()
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
value.Failure = domain.ExecutionFailureCanceled
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
value.Failure = domain.ExecutionFailureDeadline
|
||||
case errors.Is(err, ErrTransport):
|
||||
value.Failure = domain.ExecutionFailureTransport
|
||||
case value.StatusCode != 0:
|
||||
value.Failure = domain.ExecutionFailureProviderStatus
|
||||
case errors.Is(err, ErrMalformedResponse):
|
||||
value.Failure = domain.ExecutionFailureMalformedResponse
|
||||
case errors.Is(err, ErrCapacityExceeded):
|
||||
value.Failure = domain.ExecutionFailureCapacity
|
||||
case errors.Is(err, ErrInvalidConfig):
|
||||
value.Failure = domain.ExecutionFailureInvalidConfig
|
||||
case errors.Is(err, ErrPromptNotFound):
|
||||
value.Failure = domain.ExecutionFailurePromptNotFound
|
||||
case errors.Is(err, ErrProfileNotFound):
|
||||
value.Failure = domain.ExecutionFailureProfileNotFound
|
||||
case errors.Is(err, ErrPromptLoad):
|
||||
value.Failure = domain.ExecutionFailurePromptLoad
|
||||
case errors.Is(err, ErrProfileLoad):
|
||||
value.Failure = domain.ExecutionFailureProfileLoad
|
||||
case errors.Is(err, ErrInvalidRequest):
|
||||
value.Failure = domain.ExecutionFailureInvalidRequest
|
||||
case errors.Is(err, ErrArtifactLoad):
|
||||
value.Failure = domain.ExecutionFailureArtifactLoad
|
||||
case errors.Is(err, ErrPromptRender):
|
||||
value.Failure = domain.ExecutionFailurePromptRender
|
||||
case errors.Is(err, ErrValidation):
|
||||
value.Failure = domain.ExecutionFailureValidation
|
||||
case errors.Is(err, ErrLLMGenerate):
|
||||
value.Failure = domain.ExecutionFailureGeneration
|
||||
default:
|
||||
value.Failure = domain.ExecutionFailureOther
|
||||
}
|
||||
return value
|
||||
}
|
||||
Reference in New Issue
Block a user