Project terminal execution observations into shared reports

internal/report/report.go

@@ -0,0 +1,105 @@
+package report
+
+import "gitea.maximumdirect.net/eric/promptkit"
+
+// Execution is the terminal observation exposed by Scriptorium's opt-in adapters.
+type Execution struct {
+	RunID                string `json:"run_id,omitempty"`
+	PromptID             string `json:"prompt_id,omitempty"`
+	PromptVersion        string `json:"prompt_version,omitempty"`
+	ProfileID            string `json:"profile_id,omitempty"`
+	BackendID            string `json:"backend_id,omitempty"`
+	Model                string `json:"model,omitempty"`
+	Outcome              string `json:"outcome"`
+	Failure              string `json:"failure"`
+	StatusCode           int    `json:"status_code,omitempty"`
+	RetriesExhausted     bool   `json:"retries_exhausted"`
+	Attempts             uint64 `json:"attempts"`
+	TransientRetries     uint64 `json:"transient_retries"`
+	RepairCalls          uint64 `json:"repair_calls"`
+	DurationMS           int64  `json:"duration_ms"`
+	CapacityWaitMS       int64  `json:"capacity_wait_ms"`
+	GenerationDurationMS int64  `json:"generation_duration_ms"`
+	RetryDelayMS         int64  `json:"retry_delay_ms"`
+	Usage                Usage  `json:"usage"`
+}
+
+type Usage struct {
+	PromptTokens     TokenCount `json:"prompt_tokens"`
+	CompletionTokens TokenCount `json:"completion_tokens"`
+	TotalTokens      TokenCount `json:"total_tokens"`
+	CachedTokens     TokenCount `json:"cached_tokens"`
+	CacheWriteTokens TokenCount `json:"cache_write_tokens"`
+}
+
+type TokenCount struct {
+	Value    int  `json:"value"`
+	Known    bool `json:"known"`
+	Complete bool `json:"complete"`
+}
+
+// Collector retains at most one terminal observation from a synchronous run.
+type Collector struct {
+	report *Execution
+}
+
+func NewCollector() *Collector { return &Collector{} }
+
+func (collector *Collector) Observe(event promptkit.ExecutionEvent) {
+	if event.Kind != promptkit.ExecutionFinished {
+		return
+	}
+	count := func(value promptkit.ObservedTokenCount) TokenCount {
+		return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
+	}
+	collector.report = &Execution{
+		RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion,
+		ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model,
+		Outcome: outcome(event.Outcome), Failure: failure(event.Failure),
+		StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
+		Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
+		DurationMS: event.Duration.Milliseconds(), CapacityWaitMS: event.CapacityWait.Milliseconds(),
+		GenerationDurationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(),
+		Usage: Usage{
+			PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens),
+			TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens),
+			CacheWriteTokens: count(event.Usage.CacheWriteTokens),
+		},
+	}
+}
+
+func (collector *Collector) Snapshot() *Execution {
+	if collector.report == nil {
+		return nil
+	}
+	value := *collector.report
+	return &value
+}
+
+func outcome(value promptkit.ExecutionOutcome) string {
+	switch value {
+	case promptkit.ExecutionOutcomeSuccess, promptkit.ExecutionOutcomeInvalidOutput, promptkit.ExecutionOutcomeError:
+		return string(value)
+	default:
+		return "other"
+	}
+}
+
+func failure(value promptkit.ExecutionFailureKind) string {
+	switch value {
+	case promptkit.ExecutionFailureNone:
+		return "none"
+	case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline,
+		promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus,
+		promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity,
+		promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest,
+		promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound,
+		promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad,
+		promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender,
+		promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration,
+		promptkit.ExecutionFailureOther:
+		return string(value)
+	default:
+		return "other"
+	}
+}