promptkit/smoke_diagnostics.go

package promptkit

import (
	"context"
	"fmt"
	"time"

	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
	"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)

// SmokeTestKind identifies one built-in diagnostic operation.
type SmokeTestKind string

const (
	// SmokeTestReachability performs an unauthenticated HTTP reachability probe.
	SmokeTestReachability SmokeTestKind = "reachability"
	// SmokeTestAuthentication checks the configured authentication behavior.
	SmokeTestAuthentication SmokeTestKind = "authentication"
	// SmokeTestTextInference checks text generation against a private case.
	SmokeTestTextInference SmokeTestKind = "text_inference"
	// SmokeTestStructuredOutput checks structured generation and local conformance.
	SmokeTestStructuredOutput SmokeTestKind = "structured_output"
)

// SmokeTestRequest selects one diagnostic and its execution target.
//
// APIKey is request-scoped and takes precedence over an environment credential.
// It is excluded from JSON and from String and GoString output. Execution values
// are copied before the internal engine uses them.
type SmokeTestRequest struct {
	// Kind selects one of the built-in smoke-test operations.
	Kind SmokeTestKind
	// BackendID selects a backend directly or supplies a fallback when inference
	// selection otherwise needs a profile. A selected profile determines its own
	// effective backend identity.
	BackendID string
	// PromptID selects the prompt's default profile when ProfileID is empty. A
	// namespace-scoped engine view qualifies a slashless caller ID. The prompt's
	// messages, inputs, templates, and output contract are not executed.
	PromptID string
	// ProfileID explicitly selects a profile; a namespace-scoped view qualifies
	// a slashless caller ID. It takes precedence over a prompt's default profile
	// and a backend's configured diagnostic default.
	ProfileID string
	// Execution overrides applicable target settings using the same semantics as
	// ordinary execution. HTTP probes do not require a model setting.
	Execution *ExecutionTargetOverride
	// APIKey supplies a direct request-scoped credential. It is never included
	// in JSON, String, or GoString output.
	APIKey string `json:"-"`
	// TransientRetries sets the inference retry count. Nil means zero retries for
	// this operation, independently of the engine's ordinary retry default.
	// HTTP-only probes remain single-attempt. Negative values are invalid.
	TransientRetries *int
}

// String returns a credential-safe summary of the request.
func (r SmokeTestRequest) String() string { return r.redactedString() }

// GoString returns a credential-safe summary of the request.
func (r SmokeTestRequest) GoString() string { return r.redactedString() }

func (r SmokeTestRequest) redactedString() string {
	retries := "nil"
	if r.TransientRetries != nil {
		retries = fmt.Sprintf("%d", *r.TransientRetries)
	}
	return fmt.Sprintf(
		"promptkit.SmokeTestRequest{Kind:%q BackendIDSet:%t PromptIDSet:%t ProfileIDSet:%t APIKeySet:%t TransientRetries:%s}",
		r.Kind,
		r.BackendID != "",
		r.PromptID != "",
		r.ProfileID != "",
		r.APIKey != "",
		retries,
	)
}

// SmokeTestReport contains factual diagnostic results, not generated content.
//
// A nil Go error from Engine.SmokeTest means the diagnostic concluded, not that
// the tested operation succeeded. Rejections, mismatches, unsupported
// operations, and known inapplicability can all be concluded reports with nil
// Go error. A fact pointer is true or false only when established; nil means it
// was not evaluated, is unknown, or is inapplicable. Reports own their mutable
// values. They intentionally have no stable JSON contract.
type SmokeTestReport struct {
	// Kind is the requested diagnostic.
	Kind SmokeTestKind
	// Supported reports whether this engine/client can perform the operation.
	Supported bool
	// Applicable is false only when known not to apply; nil means unknown.
	Applicable *bool
	// Completed indicates that the diagnostic concluded, regardless of findings.
	Completed bool
	// Reason is an additional factual code, not a severity or free-form message.
	Reason SmokeTestReason
	// Target identifies the selected target and its effective settings.
	Target SmokeTestTarget
	// Facts contains tri-state observations for the diagnostic.
	Facts SmokeTestFacts
	// Timing contains total and measured phase durations.
	Timing SmokeTestTiming
	// Attempts contains dispatched attempts and any interrupted acquisition.
	Attempts []SmokeTestAttempt
	// Usage is the presence-aware aggregate for dispatched attempts.
	Usage ObservedTokenUsage
	// Failure classifies the diagnostic execution outcome, when any.
	Failure ExecutionFailureKind
}

// SmokeTestTarget identifies the selected diagnostic target and its settings.
type SmokeTestTarget struct {
	// BackendID is the effective backend/capacity identity; it can be empty for
	// an endpoint-only profile.
	BackendID string
	// ProfileID is the selected profile, if any.
	ProfileID string
	// PromptID identifies the selected prompt; PromptVersion carries optional
	// descriptive metadata and never participates in selection. Nil means absent
	// while a non-nil empty string is explicit. The pointer belongs to this
	// returned target.
	PromptID      string
	PromptVersion *string
	// DefaultSourceBackendID identifies the backend whose configured default
	// selected the profile, when that selection path was used.
	DefaultSourceBackendID string
	// SelectionSource identifies the selection path (backend_default, profile,
	// prompt, or backend).
	SelectionSource string
	// Endpoint is the initially resolved diagnostic destination. It excludes URL
	// user information. For built-in inference it is the prepared completion URL,
	// not the configured base. It is empty when the HTTP destination is unknown,
	// including injected inference. The configured base remains available in
	// EffectiveSettings. Other endpoint components and effective settings may be
	// sensitive operator data.
	Endpoint string
	// EffectiveSettings is the selected target's resolved runtime configuration.
	EffectiveSettings ExecutionTarget
	// SettingsPresence records whether optional numeric settings were explicitly
	// present, including explicit zero values.
	SettingsPresence ExecutionTargetPresence
}

// SmokeTestFacts contains optional factual observations from a diagnostic.
// Nil means unknown, not evaluated, or inapplicable.
type SmokeTestFacts struct {
	// CredentialsSupplied reports whether a credential was attached to the last
	// observed request for that attempt's endpoint.
	CredentialsSupplied *bool
	// EndpointReachable reports whether an HTTP response was observed.
	EndpointReachable *bool
	// CredentialsAccepted reports only recognized authentication outcomes.
	CredentialsAccepted *bool
	// ResponseReceived reports whether a response was received from the selected
	// client; injected clients do not establish HTTP facts.
	ResponseReceived *bool
	// GenerationCompleted reports whether generation returned usable content.
	GenerationCompleted *bool
	// TextReturned reports whether non-empty text was returned.
	TextReturned *bool
	// ExpectedTextMatched reports whether the private text case matched.
	ExpectedTextMatched *bool
	// StructuredOutputRequested reports whether a native structured-output
	// request was made; it does not assert provider enforcement.
	StructuredOutputRequested *bool
	// ValidJSON reports whether generated content parsed as JSON.
	ValidJSON *bool
	// SchemaMatched reports whether parsed JSON conformed to the private schema.
	SchemaMatched *bool
}

// SmokeTestTiming records elapsed diagnostic and phase durations.
type SmokeTestTiming struct {
	// StartedAt and Duration bound total diagnostic orchestration.
	StartedAt time.Time
	Duration  time.Duration
	// CapacityWait, RequestDuration, RetryDelay, and EvaluationDuration are nil
	// when their phases did not occur. RequestDuration excludes capacity wait,
	// retry delay, and local evaluation.
	CapacityWait       *time.Duration
	RequestDuration    *time.Duration
	RetryDelay         *time.Duration
	EvaluationDuration *time.Duration
}

// SmokeTestAttempt records one dispatched attempt or an interrupted acquisition.
type SmokeTestAttempt struct {
	// Dispatched distinguishes an actual attempt from an interrupted capacity
	// acquisition.
	Dispatched bool
	// InitialEndpoint is the resolved request destination before redirects;
	// it is empty for injected inference whose HTTP destination is unknown.
	InitialEndpoint string
	// Endpoint is the last observed request destination for this attempt.
	Endpoint string
	// HTTPStatus is the last observed response status, if available.
	HTTPStatus *int
	// Facts, Timing, Failure, and Usage describe this attempt only.
	Facts   SmokeTestFacts
	Timing  SmokeTestTiming
	Failure ExecutionFailureKind
	Usage   ObservedTokenUsage
}

// SmokeTestReason identifies additional factual context for a diagnostic.
type SmokeTestReason string

const (
	// SmokeTestReasonNone indicates no additional reason.
	SmokeTestReasonNone SmokeTestReason = ""
	// SmokeTestReasonAuthenticationNotApplicable marks known inapplicability.
	SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
	// SmokeTestReasonProbeNotConfigured marks absent authentication metadata.
	SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
	// SmokeTestReasonHTTPDiagnosticsUnavailable marks an engine without a
	// diagnostic HTTP transport.
	SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
	// SmokeTestReasonRequiredCredentialMissing marks an inference credential
	// required locally but absent before dispatch.
	SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
	// SmokeTestReasonAuthenticationResponseUnrecognized marks an HTTP response
	// whose authentication meaning could not be established.
	SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
)

// SmokeTest runs one built-in diagnostic against the engine's captured
// configuration.
//
// On a namespace-scoped view, slashless caller-supplied prompt, profile, and
// backend IDs are qualified before selection; captured defaults and references
// remain unchanged. Selection prefers an explicit ProfileID, then the profile
// selected by PromptID, then a configured backend default profile when
// inference requires one. For backend-only probes, a selected profile supplies
// its backend metadata; otherwise BackendID selects a backend directly without
// resolving its default inference profile. A selected profile determines its
// own effective backend identity, even when it differs from BackendID.
// Application prompt messages, inputs, session templates, artifacts, and
// output schemas are never executed by this operation.
//
// A nil Go error means the diagnostic concluded, not that the tested operation
// succeeded. Concluded failures, mismatches, unsupported operations, and
// inapplicability are represented in the report with nil Go error. Invalid
// kinds, settings, or selections return an ordinary error before diagnostic
// work. Caller cancellation or deadline expiration returns available partial
// observations and an error recognizable with errors.Is; Completed remains
// false when cancellation interrupts the diagnostic. A request timeout while
// the caller context remains active is instead a concluded report.
//
// Built-in HTTP clients support reachability and configured authentication
// probes. An engine using an injected LLMClient can perform inference
// diagnostics, but HTTP diagnostics are reported unavailable and never fall
// back to another transport. Nil TransientRetries means zero inference retries;
// HTTP probes are single-attempt regardless of that setting. Diagnostics do
// not perform structural repair.
//
// Timing includes orchestration in Duration. CapacityWait, RequestDuration,
// RetryDelay, and EvaluationDuration measure only phases that occurred;
// RequestDuration excludes capacity waiting, retry delay, and local evaluation.
// Reports omit credentials, generated text, provider messages, raw HTTP bodies,
// and private fixture contents. Effective settings and identifiers may still
// contain sensitive operator data and require caller-controlled disclosure.
func (e *Engine) SmokeTest(ctx context.Context, req SmokeTestRequest) (*SmokeTestReport, error) {
	if e == nil || e.internal == nil {
		return nil, ErrInvalidRequest
	}
	domainRequest, err := toDomainSmokeTestRequest(req)
	if err != nil {
		return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
	}
	domainReport, err := e.internal.SmokeTest(ctx, domainRequest)
	if domainReport == nil {
		return nil, mapPublicError(err)
	}
	report, conversionErr := fromDomainSmokeTestReport(*domainReport)
	if conversionErr != nil {
		return nil, fmt.Errorf("%w: failed to copy smoke-test report", ErrInvalidRequest)
	}
	return report, mapPublicError(err)
}

func toDomainSmokeTestRequest(request SmokeTestRequest) (domain.SmokeTestRequest, error) {
	execution, err := toDomainExecutionTargetOverride(request.Execution)
	if err != nil {
		return domain.SmokeTestRequest{}, err
	}
	return domain.SmokeTestRequest{
		Kind:             domain.SmokeTestKind(request.Kind),
		BackendID:        request.BackendID,
		PromptID:         request.PromptID,
		ProfileID:        request.ProfileID,
		Execution:        execution,
		APIKey:           request.APIKey,
		TransientRetries: copyIntPtr(request.TransientRetries),
	}, nil
}

func fromDomainSmokeTestReport(report domain.SmokeTestReport) (*SmokeTestReport, error) {
	extraParams, err := jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
	if err != nil {
		return nil, err
	}
	result := &SmokeTestReport{
		Kind:       SmokeTestKind(report.Kind),
		Supported:  report.Supported,
		Applicable: copySmokeTestBoolPtr(report.Applicable),
		Completed:  report.Completed,
		Reason:     SmokeTestReason(report.Reason),
		Target: SmokeTestTarget{
			BackendID:              report.Target.BackendID,
			ProfileID:              report.Target.ProfileID,
			PromptID:               report.Target.PromptID,
			PromptVersion:          copyStringPtr(report.Target.PromptVersion),
			DefaultSourceBackendID: report.Target.DefaultSourceBackendID,
			SelectionSource:        report.Target.SelectionSource,
			Endpoint:               report.Target.Endpoint,
			EffectiveSettings:      fromDomainExecutionTargetWithOwnedParams(report.Target.EffectiveSettings, extraParams),
			SettingsPresence:       fromDomainExecutionTargetPresence(report.Target.SettingsPresence),
		},
		Facts:   fromDomainSmokeTestFacts(report.Facts),
		Timing:  fromDomainSmokeTestTiming(report.Timing),
		Usage:   fromDomainObservedTokenUsage(report.Usage),
		Failure: ExecutionFailureKind(report.Failure),
	}
	if report.Attempts != nil {
		result.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
		for index, attempt := range report.Attempts {
			result.Attempts[index] = SmokeTestAttempt{
				Dispatched:      attempt.Dispatched,
				InitialEndpoint: attempt.InitialEndpoint,
				Endpoint:        attempt.Endpoint,
				HTTPStatus:      copyIntPtr(attempt.HTTPStatus),
				Facts:           fromDomainSmokeTestFacts(attempt.Facts),
				Timing:          fromDomainSmokeTestTiming(attempt.Timing),
				Failure:         ExecutionFailureKind(attempt.Failure),
				Usage:           fromDomainObservedTokenUsage(attempt.Usage),
			}
		}
	}
	return result, nil
}

func fromDomainSmokeTestFacts(facts domain.SmokeTestFacts) SmokeTestFacts {
	return SmokeTestFacts{
		CredentialsSupplied:       copySmokeTestBoolPtr(facts.CredentialsSupplied),
		EndpointReachable:         copySmokeTestBoolPtr(facts.EndpointReachable),
		CredentialsAccepted:       copySmokeTestBoolPtr(facts.CredentialsAccepted),
		ResponseReceived:          copySmokeTestBoolPtr(facts.ResponseReceived),
		GenerationCompleted:       copySmokeTestBoolPtr(facts.GenerationCompleted),
		TextReturned:              copySmokeTestBoolPtr(facts.TextReturned),
		ExpectedTextMatched:       copySmokeTestBoolPtr(facts.ExpectedTextMatched),
		StructuredOutputRequested: copySmokeTestBoolPtr(facts.StructuredOutputRequested),
		ValidJSON:                 copySmokeTestBoolPtr(facts.ValidJSON),
		SchemaMatched:             copySmokeTestBoolPtr(facts.SchemaMatched),
	}
}

func copySmokeTestBoolPtr(value *bool) *bool {
	if value == nil {
		return nil
	}
	cloned := *value
	return &cloned
}

func fromDomainSmokeTestTiming(timing domain.SmokeTestTiming) SmokeTestTiming {
	return SmokeTestTiming{
		StartedAt:          timing.StartedAt,
		Duration:           timing.Duration,
		CapacityWait:       copyDurationPtr(timing.CapacityWait),
		RequestDuration:    copyDurationPtr(timing.RequestDuration),
		RetryDelay:         copyDurationPtr(timing.RetryDelay),
		EvaluationDuration: copyDurationPtr(timing.EvaluationDuration),
	}
}

func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedTokenUsage {
	return ObservedTokenUsage{
		PromptTokens:     fromDomainObservedTokenCount(usage.PromptTokens),
		CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
		TotalTokens:      fromDomainObservedTokenCount(usage.TotalTokens),
		CachedTokens:     fromDomainObservedTokenCount(usage.CachedTokens),
		CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),
	}
}

func fromDomainObservedTokenCount(count domain.ObservedTokenCount) ObservedTokenCount {
	return ObservedTokenCount{Value: count.Value, Known: count.Known, Complete: count.Complete}
}

func copyDurationPtr(value *time.Duration) *time.Duration {
	if value == nil {
		return nil
	}
	cloned := *value
	return &cloned
}