promptkit/observations.go

package promptkit

import (
	"context"
	"errors"
	"time"

	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)

// ExecutionObserver receives immutable metadata events synchronously. Callers
// must synchronize shared state if separate runs use the same observer. A
// callback must return for its run to progress; panics propagate to the caller.
type ExecutionObserver func(ExecutionEvent)

// ExecutionEventKind identifies one point in an execution's lifecycle.
type ExecutionEventKind string

const (
	ExecutionStarted    ExecutionEventKind = "started"
	ExecutionAttempt    ExecutionEventKind = "attempt"
	ExecutionRetryDelay ExecutionEventKind = "retry_delay"
	ExecutionFinished   ExecutionEventKind = "finished"
)

// ExecutionOutcome reports whether the operation or attempt succeeded.
type ExecutionOutcome string

const (
	ExecutionOutcomeSuccess       ExecutionOutcome = "success"
	ExecutionOutcomeInvalidOutput ExecutionOutcome = "invalid_output"
	ExecutionOutcomeError         ExecutionOutcome = "error"
)

// ExecutionFailureKind is a content-free classification of an error.
type ExecutionFailureKind string

const (
	ExecutionFailureNone              ExecutionFailureKind = ""
	ExecutionFailureCanceled          ExecutionFailureKind = "canceled"
	ExecutionFailureDeadline          ExecutionFailureKind = "deadline"
	ExecutionFailureTransport         ExecutionFailureKind = "transport"
	ExecutionFailureProviderStatus    ExecutionFailureKind = "provider_status"
	ExecutionFailureMalformedResponse ExecutionFailureKind = "malformed_response"
	ExecutionFailureCapacity          ExecutionFailureKind = "capacity"
	ExecutionFailureInvalidConfig     ExecutionFailureKind = "invalid_config"
	ExecutionFailureInvalidRequest    ExecutionFailureKind = "invalid_request"
	ExecutionFailurePromptNotFound    ExecutionFailureKind = "prompt_not_found"
	ExecutionFailureProfileNotFound   ExecutionFailureKind = "profile_not_found"
	ExecutionFailurePromptLoad        ExecutionFailureKind = "prompt_load"
	ExecutionFailureProfileLoad       ExecutionFailureKind = "profile_load"
	ExecutionFailureArtifactLoad      ExecutionFailureKind = "artifact_load"
	ExecutionFailurePromptRender      ExecutionFailureKind = "prompt_render"
	ExecutionFailureValidation        ExecutionFailureKind = "validation"
	ExecutionFailureGeneration        ExecutionFailureKind = "generation"
	ExecutionFailureOther             ExecutionFailureKind = "other"
)

// ObservedTokenCount distinguishes an omitted count from a reported zero.
// Complete means every dispatched attempt reported the field.
type ObservedTokenCount struct {
	Value    int
	Known    bool
	Complete bool
}

// ObservedTokenUsage contains presence-aware usage for one attempt or an
// aggregate. It does not prove complete provider billing.
type ObservedTokenUsage struct {
	PromptTokens     ObservedTokenCount
	CompletionTokens ObservedTokenCount
	TotalTokens      ObservedTokenCount
	CachedTokens     ObservedTokenCount
	CacheWriteTokens ObservedTokenCount
}

// ExecutionEvent contains metadata only, not provider text, request content,
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
// zero; a terminal usage total must not be added to attempt usage.
type ExecutionEvent struct {
	Kind               ExecutionEventKind
	RunID              string
	Time               time.Time
	PromptID           string
	PromptVersion      string
	ProfileID          string
	BackendID          string
	Model              string
	Generation         uint64
	Attempt            uint64
	Dispatched         bool
	CapacityWait       time.Duration
	GenerationDuration time.Duration
	RetryDelay         time.Duration
	Duration           time.Duration
	Outcome            ExecutionOutcome
	Failure            ExecutionFailureKind
	StatusCode         int
	RetriesExhausted   bool
	Usage              ObservedTokenUsage
	Attempts           uint64
	TransientRetries   uint64
	RepairCalls        uint64
}

func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
	count := func(c domain.ObservedTokenCount) ObservedTokenCount {
		return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
	}
	return ExecutionEvent{
		Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
		PromptID: value.PromptID, PromptVersion: value.PromptVersion, ProfileID: value.ProfileID,
		BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
		Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
		GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
		Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
		StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
		Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
			TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
		Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
	}
}

func observedFinish(result *RunResult, err error) domain.ExecutionEvent {
	if err == nil {
		outcome := domain.ExecutionOutcomeSuccess
		if result.Validation.Status == ValidationFailed {
			outcome = domain.ExecutionOutcomeInvalidOutput
		}
		return domain.ExecutionEvent{Outcome: outcome}
	}
	value := domain.ExecutionEvent{Outcome: domain.ExecutionOutcomeError}
	var exhausted *RetryExhaustedError
	value.RetriesExhausted = errors.As(err, &exhausted)
	var provider *GenerationError
	if errors.As(err, &provider) {
		value.StatusCode = provider.StatusCode()
	}
	switch {
	case errors.Is(err, context.Canceled):
		value.Failure = domain.ExecutionFailureCanceled
	case errors.Is(err, context.DeadlineExceeded):
		value.Failure = domain.ExecutionFailureDeadline
	case errors.Is(err, ErrTransport):
		value.Failure = domain.ExecutionFailureTransport
	case value.StatusCode != 0:
		value.Failure = domain.ExecutionFailureProviderStatus
	case errors.Is(err, ErrMalformedResponse):
		value.Failure = domain.ExecutionFailureMalformedResponse
	case errors.Is(err, ErrCapacityExceeded):
		value.Failure = domain.ExecutionFailureCapacity
	case errors.Is(err, ErrInvalidConfig):
		value.Failure = domain.ExecutionFailureInvalidConfig
	case errors.Is(err, ErrPromptNotFound):
		value.Failure = domain.ExecutionFailurePromptNotFound
	case errors.Is(err, ErrProfileNotFound):
		value.Failure = domain.ExecutionFailureProfileNotFound
	case errors.Is(err, ErrPromptLoad):
		value.Failure = domain.ExecutionFailurePromptLoad
	case errors.Is(err, ErrProfileLoad):
		value.Failure = domain.ExecutionFailureProfileLoad
	case errors.Is(err, ErrInvalidRequest):
		value.Failure = domain.ExecutionFailureInvalidRequest
	case errors.Is(err, ErrArtifactLoad):
		value.Failure = domain.ExecutionFailureArtifactLoad
	case errors.Is(err, ErrPromptRender):
		value.Failure = domain.ExecutionFailurePromptRender
	case errors.Is(err, ErrValidation):
		value.Failure = domain.ExecutionFailureValidation
	case errors.Is(err, ErrLLMGenerate):
		value.Failure = domain.ExecutionFailureGeneration
	default:
		value.Failure = domain.ExecutionFailureOther
	}
	return value
}