promptkit/observations.go

package promptkit

import (
	"context"
	"errors"
	"time"

	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)

// ExecutionObserver receives immutable metadata events synchronously. Callers
// must synchronize shared state if separate runs use the same observer. A
// callback must return for its run to progress; cancellation cannot interrupt
// consumer code already in the callback. Do not block on work requiring the
// same run's admission slot. Panics propagate without a promise of a terminal
// event, and Promptkit keeps no asynchronous queue or durable history.
type ExecutionObserver func(ExecutionEvent)

// ExecutionEventKind identifies one point in an execution's lifecycle.
type ExecutionEventKind string

const (
	ExecutionStarted    ExecutionEventKind = "started"
	ExecutionAttempt    ExecutionEventKind = "attempt"
	ExecutionRetryDelay ExecutionEventKind = "retry_delay"
	ExecutionFinished   ExecutionEventKind = "finished"
)

// ExecutionOutcome reports whether the operation or attempt succeeded.
type ExecutionOutcome string

const (
	ExecutionOutcomeSuccess       ExecutionOutcome = "success"
	ExecutionOutcomeInvalidOutput ExecutionOutcome = "invalid_output"
	ExecutionOutcomeError         ExecutionOutcome = "error"
)

// ExecutionFailureKind is a content-free classification of an error.
type ExecutionFailureKind string

const (
	ExecutionFailureNone              ExecutionFailureKind = ""
	ExecutionFailureCanceled          ExecutionFailureKind = "canceled"
	ExecutionFailureDeadline          ExecutionFailureKind = "deadline"
	ExecutionFailureTransport         ExecutionFailureKind = "transport"
	ExecutionFailureProviderStatus    ExecutionFailureKind = "provider_status"
	ExecutionFailureMalformedResponse ExecutionFailureKind = "malformed_response"
	ExecutionFailureCapacity          ExecutionFailureKind = "capacity"
	ExecutionFailureInvalidConfig     ExecutionFailureKind = "invalid_config"
	ExecutionFailureInvalidRequest    ExecutionFailureKind = "invalid_request"
	ExecutionFailurePromptNotFound    ExecutionFailureKind = "prompt_not_found"
	ExecutionFailureProfileNotFound   ExecutionFailureKind = "profile_not_found"
	ExecutionFailurePromptLoad        ExecutionFailureKind = "prompt_load"
	ExecutionFailureProfileLoad       ExecutionFailureKind = "profile_load"
	ExecutionFailureArtifactLoad      ExecutionFailureKind = "artifact_load"
	ExecutionFailurePromptRender      ExecutionFailureKind = "prompt_render"
	ExecutionFailureValidation        ExecutionFailureKind = "validation"
	ExecutionFailureGeneration        ExecutionFailureKind = "generation"
	ExecutionFailureOther             ExecutionFailureKind = "other"
)

// ObservedTokenCount distinguishes an omitted count from a reported zero.
// On an attempt, Known and Complete are true together only for a reported
// non-null integer. An aggregate Value sums known contributions; Known means
// at least one attempt reported the field, and Complete means every dispatched
// attempt did. With no dispatched attempts, aggregate zero is known and
// complete. Unknown fields have Value zero.
type ObservedTokenCount struct {
	Value    int
	Known    bool
	Complete bool
}

// ObservedTokenUsage contains presence-aware usage for one attempt or an
// aggregate. It does not prove complete provider billing.
type ObservedTokenUsage struct {
	PromptTokens     ObservedTokenCount
	CompletionTokens ObservedTokenCount
	TotalTokens      ObservedTokenCount
	CachedTokens     ObservedTokenCount
	CacheWriteTokens ObservedTokenCount
}

// ExecutionEvent contains metadata only, not provider text, request content,
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
// zero; a terminal usage total must not be added to attempt usage.
//
// Generation is one for the initial generation and increases for repairs;
// Attempt is one-based within each generation. Dispatched means Promptkit
// invoked an HTTP attempt or injected Generate, not that a network packet was
// sent or billable work occurred. A canceled capacity acquisition emits a
// non-dispatched attempt with no usage. A retry-delay event describes the
// completed or interrupted wait after the indexed failed attempt.
//
// GenerationDuration excludes capacity waiting, retry delay, and callback
// work. Terminal phase durations sum measured phases; terminal Duration is
// wall time and may also include preparation, validation, and callbacks.
// Attempt success means generation succeeded, even if later validation rejects
// its output. Failure is classified metadata; RetriesExhausted is a separate
// flag and does not replace the underlying failure kind.
type ExecutionEvent struct {
	Kind     ExecutionEventKind
	RunID    string
	Time     time.Time
	PromptID string
	// PromptVersion is the selected definition's descriptive label. Each event
	// owns its pointer; nil means absent while a non-nil empty string is explicit.
	PromptVersion      *string
	ProfileID          string
	BackendID          string
	Model              string
	Generation         uint64
	Attempt            uint64
	Dispatched         bool
	CapacityWait       time.Duration
	GenerationDuration time.Duration
	RetryDelay         time.Duration
	Duration           time.Duration
	Outcome            ExecutionOutcome
	Failure            ExecutionFailureKind
	StatusCode         int
	RetriesExhausted   bool
	Usage              ObservedTokenUsage
	Attempts           uint64
	TransientRetries   uint64
	RepairCalls        uint64
}

func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
	count := func(c domain.ObservedTokenCount) ObservedTokenCount {
		return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
	}
	return ExecutionEvent{
		Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
		PromptID: value.PromptID, PromptVersion: copyStringPtr(value.PromptVersion), ProfileID: value.ProfileID,
		BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
		Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
		GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
		Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
		StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
		Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
			TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
		Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
	}
}

func observedFinish(result *RunResult, err error) domain.ExecutionEvent {
	if err == nil {
		outcome := domain.ExecutionOutcomeSuccess
		if result.Validation.Status == ValidationFailed {
			outcome = domain.ExecutionOutcomeInvalidOutput
		}
		return domain.ExecutionEvent{Outcome: outcome}
	}
	value := domain.ExecutionEvent{Outcome: domain.ExecutionOutcomeError}
	var exhausted *RetryExhaustedError
	value.RetriesExhausted = errors.As(err, &exhausted)
	var provider *GenerationError
	if errors.As(err, &provider) {
		value.StatusCode = provider.StatusCode()
	}
	switch {
	case errors.Is(err, context.Canceled):
		value.Failure = domain.ExecutionFailureCanceled
	case errors.Is(err, context.DeadlineExceeded):
		value.Failure = domain.ExecutionFailureDeadline
	case errors.Is(err, ErrTransport):
		value.Failure = domain.ExecutionFailureTransport
	case value.StatusCode != 0:
		value.Failure = domain.ExecutionFailureProviderStatus
	case errors.Is(err, ErrMalformedResponse):
		value.Failure = domain.ExecutionFailureMalformedResponse
	case errors.Is(err, ErrCapacityExceeded):
		value.Failure = domain.ExecutionFailureCapacity
	case errors.Is(err, ErrInvalidConfig):
		value.Failure = domain.ExecutionFailureInvalidConfig
	case errors.Is(err, ErrPromptNotFound):
		value.Failure = domain.ExecutionFailurePromptNotFound
	case errors.Is(err, ErrProfileNotFound):
		value.Failure = domain.ExecutionFailureProfileNotFound
	case errors.Is(err, ErrPromptLoad):
		value.Failure = domain.ExecutionFailurePromptLoad
	case errors.Is(err, ErrProfileLoad):
		value.Failure = domain.ExecutionFailureProfileLoad
	case errors.Is(err, ErrInvalidRequest):
		value.Failure = domain.ExecutionFailureInvalidRequest
	case errors.Is(err, ErrArtifactLoad):
		value.Failure = domain.ExecutionFailureArtifactLoad
	case errors.Is(err, ErrPromptRender):
		value.Failure = domain.ExecutionFailurePromptRender
	case errors.Is(err, ErrValidation):
		value.Failure = domain.ExecutionFailureValidation
	case errors.Is(err, ErrLLMGenerate):
		value.Failure = domain.ExecutionFailureGeneration
	default:
		value.Failure = domain.ExecutionFailureOther
	}
	return value
}