106 lines
4.1 KiB
Go
106 lines
4.1 KiB
Go
package report
|
|
|
|
import "gitea.maximumdirect.net/eric/promptkit"
|
|
|
|
// Execution is the terminal observation exposed by Scriptorium's opt-in adapters.
|
|
type Execution struct {
|
|
RunID string `json:"run_id,omitempty"`
|
|
PromptID string `json:"prompt_id,omitempty"`
|
|
PromptVersion string `json:"prompt_version,omitempty"`
|
|
ProfileID string `json:"profile_id,omitempty"`
|
|
BackendID string `json:"backend_id,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Outcome string `json:"outcome"`
|
|
Failure string `json:"failure"`
|
|
StatusCode int `json:"status_code,omitempty"`
|
|
RetriesExhausted bool `json:"retries_exhausted"`
|
|
Attempts uint64 `json:"attempts"`
|
|
TransientRetries uint64 `json:"transient_retries"`
|
|
RepairCalls uint64 `json:"repair_calls"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
|
GenerationDurationMS int64 `json:"generation_duration_ms"`
|
|
RetryDelayMS int64 `json:"retry_delay_ms"`
|
|
Usage Usage `json:"usage"`
|
|
}
|
|
|
|
type Usage struct {
|
|
PromptTokens TokenCount `json:"prompt_tokens"`
|
|
CompletionTokens TokenCount `json:"completion_tokens"`
|
|
TotalTokens TokenCount `json:"total_tokens"`
|
|
CachedTokens TokenCount `json:"cached_tokens"`
|
|
CacheWriteTokens TokenCount `json:"cache_write_tokens"`
|
|
}
|
|
|
|
type TokenCount struct {
|
|
Value int `json:"value"`
|
|
Known bool `json:"known"`
|
|
Complete bool `json:"complete"`
|
|
}
|
|
|
|
// Collector retains at most one terminal observation from a synchronous run.
|
|
type Collector struct {
|
|
report *Execution
|
|
}
|
|
|
|
func NewCollector() *Collector { return &Collector{} }
|
|
|
|
func (collector *Collector) Observe(event promptkit.ExecutionEvent) {
|
|
if event.Kind != promptkit.ExecutionFinished {
|
|
return
|
|
}
|
|
count := func(value promptkit.ObservedTokenCount) TokenCount {
|
|
return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
|
|
}
|
|
collector.report = &Execution{
|
|
RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion,
|
|
ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model,
|
|
Outcome: outcome(event.Outcome), Failure: failure(event.Failure),
|
|
StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
|
|
Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
|
DurationMS: event.Duration.Milliseconds(), CapacityWaitMS: event.CapacityWait.Milliseconds(),
|
|
GenerationDurationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(),
|
|
Usage: Usage{
|
|
PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens),
|
|
TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens),
|
|
CacheWriteTokens: count(event.Usage.CacheWriteTokens),
|
|
},
|
|
}
|
|
}
|
|
|
|
func (collector *Collector) Snapshot() *Execution {
|
|
if collector.report == nil {
|
|
return nil
|
|
}
|
|
value := *collector.report
|
|
return &value
|
|
}
|
|
|
|
func outcome(value promptkit.ExecutionOutcome) string {
|
|
switch value {
|
|
case promptkit.ExecutionOutcomeSuccess, promptkit.ExecutionOutcomeInvalidOutput, promptkit.ExecutionOutcomeError:
|
|
return string(value)
|
|
default:
|
|
return "other"
|
|
}
|
|
}
|
|
|
|
func failure(value promptkit.ExecutionFailureKind) string {
|
|
switch value {
|
|
case promptkit.ExecutionFailureNone:
|
|
return "none"
|
|
case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline,
|
|
promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus,
|
|
promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity,
|
|
promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest,
|
|
promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound,
|
|
promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad,
|
|
promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender,
|
|
promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration,
|
|
promptkit.ExecutionFailureOther:
|
|
return string(value)
|
|
default:
|
|
return "other"
|
|
}
|
|
}
|