package promptkit import ( "context" "fmt" "time" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" ) // SmokeTestKind identifies one built-in diagnostic operation. type SmokeTestKind string const ( // SmokeTestReachability performs an unauthenticated HTTP reachability probe. SmokeTestReachability SmokeTestKind = "reachability" // SmokeTestAuthentication checks the configured authentication behavior. SmokeTestAuthentication SmokeTestKind = "authentication" // SmokeTestTextInference checks text generation against a private case. SmokeTestTextInference SmokeTestKind = "text_inference" // SmokeTestStructuredOutput checks structured generation and local conformance. SmokeTestStructuredOutput SmokeTestKind = "structured_output" ) // SmokeTestRequest selects one diagnostic and its execution target. // // APIKey is request-scoped and takes precedence over an environment credential. // It is excluded from JSON and from String and GoString output. Execution values // are copied before the internal engine uses them. type SmokeTestRequest struct { // Kind selects one of the built-in smoke-test operations. Kind SmokeTestKind // BackendID selects a backend directly or supplies a fallback when inference // selection otherwise needs a profile. A selected profile determines its own // effective backend identity. BackendID string // PromptID selects the prompt's default profile when ProfileID is empty. A // namespace-scoped engine view qualifies a slashless caller ID. The prompt's // messages, inputs, templates, and output contract are not executed. PromptID string // ProfileID explicitly selects a profile; a namespace-scoped view qualifies // a slashless caller ID. It takes precedence over a prompt's default profile // and a backend's configured diagnostic default. ProfileID string // Execution overrides applicable target settings using the same semantics as // ordinary execution. HTTP probes do not require a model setting. Execution *ExecutionTargetOverride // APIKey supplies a direct request-scoped credential. It is never included // in JSON, String, or GoString output. APIKey string `json:"-"` // TransientRetries sets the inference retry count. Nil means zero retries for // this operation, independently of the engine's ordinary retry default. // HTTP-only probes remain single-attempt. Negative values are invalid. TransientRetries *int } // String returns a credential-safe summary of the request. func (r SmokeTestRequest) String() string { return r.redactedString() } // GoString returns a credential-safe summary of the request. func (r SmokeTestRequest) GoString() string { return r.redactedString() } func (r SmokeTestRequest) redactedString() string { retries := "nil" if r.TransientRetries != nil { retries = fmt.Sprintf("%d", *r.TransientRetries) } return fmt.Sprintf( "promptkit.SmokeTestRequest{Kind:%q BackendIDSet:%t PromptIDSet:%t ProfileIDSet:%t APIKeySet:%t TransientRetries:%s}", r.Kind, r.BackendID != "", r.PromptID != "", r.ProfileID != "", r.APIKey != "", retries, ) } // SmokeTestReport contains factual diagnostic results, not generated content. // // A nil Go error from Engine.SmokeTest means the diagnostic concluded, not that // the tested operation succeeded. Rejections, mismatches, unsupported // operations, and known inapplicability can all be concluded reports with nil // Go error. A fact pointer is true or false only when established; nil means it // was not evaluated, is unknown, or is inapplicable. Reports own their mutable // values. They intentionally have no stable JSON contract. type SmokeTestReport struct { // Kind is the requested diagnostic. Kind SmokeTestKind // Supported reports whether this engine/client can perform the operation. Supported bool // Applicable is false only when known not to apply; nil means unknown. Applicable *bool // Completed indicates that the diagnostic concluded, regardless of findings. Completed bool // Reason is an additional factual code, not a severity or free-form message. Reason SmokeTestReason // Target identifies the selected target and its effective settings. Target SmokeTestTarget // Facts contains tri-state observations for the diagnostic. Facts SmokeTestFacts // Timing contains total and measured phase durations. Timing SmokeTestTiming // Attempts contains dispatched attempts and any interrupted acquisition. Attempts []SmokeTestAttempt // Usage is the presence-aware aggregate for dispatched attempts. Usage ObservedTokenUsage // Failure classifies the diagnostic execution outcome, when any. Failure ExecutionFailureKind } // SmokeTestTarget identifies the selected diagnostic target and its settings. type SmokeTestTarget struct { // BackendID is the effective backend/capacity identity; it can be empty for // an endpoint-only profile. BackendID string // ProfileID is the selected profile, if any. ProfileID string // PromptID identifies the selected prompt; PromptVersion carries optional // descriptive metadata and never participates in selection. Nil means absent // while a non-nil empty string is explicit. The pointer belongs to this // returned target. PromptID string PromptVersion *string // DefaultSourceBackendID identifies the backend whose configured default // selected the profile, when that selection path was used. DefaultSourceBackendID string // SelectionSource identifies the selection path (backend_default, profile, // prompt, or backend). SelectionSource string // Endpoint is the initially resolved diagnostic destination. It excludes URL // user information. For built-in inference it is the prepared completion URL, // not the configured base. It is empty when the HTTP destination is unknown, // including injected inference. The configured base remains available in // EffectiveSettings. Other endpoint components and effective settings may be // sensitive operator data. Endpoint string // EffectiveSettings is the selected target's resolved runtime configuration. EffectiveSettings ExecutionTarget // SettingsPresence records whether optional numeric settings were explicitly // present, including explicit zero values. SettingsPresence ExecutionTargetPresence } // SmokeTestFacts contains optional factual observations from a diagnostic. // Nil means unknown, not evaluated, or inapplicable. type SmokeTestFacts struct { // CredentialsSupplied reports whether a credential was attached to the last // observed request for that attempt's endpoint. CredentialsSupplied *bool // EndpointReachable reports whether an HTTP response was observed. EndpointReachable *bool // CredentialsAccepted reports only recognized authentication outcomes. CredentialsAccepted *bool // ResponseReceived reports whether a response was received from the selected // client; injected clients do not establish HTTP facts. ResponseReceived *bool // GenerationCompleted reports whether generation returned usable content. GenerationCompleted *bool // TextReturned reports whether non-empty text was returned. TextReturned *bool // ExpectedTextMatched reports whether the private text case matched. ExpectedTextMatched *bool // StructuredOutputRequested reports whether a native structured-output // request was made; it does not assert provider enforcement. StructuredOutputRequested *bool // ValidJSON reports whether generated content parsed as JSON. ValidJSON *bool // SchemaMatched reports whether parsed JSON conformed to the private schema. SchemaMatched *bool } // SmokeTestTiming records elapsed diagnostic and phase durations. type SmokeTestTiming struct { // StartedAt and Duration bound total diagnostic orchestration. StartedAt time.Time Duration time.Duration // CapacityWait, RequestDuration, RetryDelay, and EvaluationDuration are nil // when their phases did not occur. RequestDuration excludes capacity wait, // retry delay, and local evaluation. CapacityWait *time.Duration RequestDuration *time.Duration RetryDelay *time.Duration EvaluationDuration *time.Duration } // SmokeTestAttempt records one dispatched attempt or an interrupted acquisition. type SmokeTestAttempt struct { // Dispatched distinguishes an actual attempt from an interrupted capacity // acquisition. Dispatched bool // InitialEndpoint is the resolved request destination before redirects; // it is empty for injected inference whose HTTP destination is unknown. InitialEndpoint string // Endpoint is the last observed request destination for this attempt. Endpoint string // HTTPStatus is the last observed response status, if available. HTTPStatus *int // Facts, Timing, Failure, and Usage describe this attempt only. Facts SmokeTestFacts Timing SmokeTestTiming Failure ExecutionFailureKind Usage ObservedTokenUsage } // SmokeTestReason identifies additional factual context for a diagnostic. type SmokeTestReason string const ( // SmokeTestReasonNone indicates no additional reason. SmokeTestReasonNone SmokeTestReason = "" // SmokeTestReasonAuthenticationNotApplicable marks known inapplicability. SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable" // SmokeTestReasonProbeNotConfigured marks absent authentication metadata. SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured" // SmokeTestReasonHTTPDiagnosticsUnavailable marks an engine without a // diagnostic HTTP transport. SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable" // SmokeTestReasonRequiredCredentialMissing marks an inference credential // required locally but absent before dispatch. SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing" // SmokeTestReasonAuthenticationResponseUnrecognized marks an HTTP response // whose authentication meaning could not be established. SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized" ) // SmokeTest runs one built-in diagnostic against the engine's captured // configuration. // // On a namespace-scoped view, slashless caller-supplied prompt, profile, and // backend IDs are qualified before selection; captured defaults and references // remain unchanged. Selection prefers an explicit ProfileID, then the profile // selected by PromptID, then a configured backend default profile when // inference requires one. For backend-only probes, a selected profile supplies // its backend metadata; otherwise BackendID selects a backend directly without // resolving its default inference profile. A selected profile determines its // own effective backend identity, even when it differs from BackendID. // Application prompt messages, inputs, session templates, artifacts, and // output schemas are never executed by this operation. // // A nil Go error means the diagnostic concluded, not that the tested operation // succeeded. Concluded failures, mismatches, unsupported operations, and // inapplicability are represented in the report with nil Go error. Invalid // kinds, settings, or selections return an ordinary error before diagnostic // work. Caller cancellation or deadline expiration returns available partial // observations and an error recognizable with errors.Is; Completed remains // false when cancellation interrupts the diagnostic. A request timeout while // the caller context remains active is instead a concluded report. // // Built-in HTTP clients support reachability and configured authentication // probes. An engine using an injected LLMClient can perform inference // diagnostics, but HTTP diagnostics are reported unavailable and never fall // back to another transport. Nil TransientRetries means zero inference retries; // HTTP probes are single-attempt regardless of that setting. Diagnostics do // not perform structural repair. // // Timing includes orchestration in Duration. CapacityWait, RequestDuration, // RetryDelay, and EvaluationDuration measure only phases that occurred; // RequestDuration excludes capacity waiting, retry delay, and local evaluation. // Reports omit credentials, generated text, provider messages, raw HTTP bodies, // and private fixture contents. Effective settings and identifiers may still // contain sensitive operator data and require caller-controlled disclosure. func (e *Engine) SmokeTest(ctx context.Context, req SmokeTestRequest) (*SmokeTestReport, error) { if e == nil || e.internal == nil { return nil, ErrInvalidRequest } domainRequest, err := toDomainSmokeTestRequest(req) if err != nil { return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) } domainReport, err := e.internal.SmokeTest(ctx, domainRequest) if domainReport == nil { return nil, mapPublicError(err) } report, conversionErr := fromDomainSmokeTestReport(*domainReport) if conversionErr != nil { return nil, fmt.Errorf("%w: failed to copy smoke-test report", ErrInvalidRequest) } return report, mapPublicError(err) } func toDomainSmokeTestRequest(request SmokeTestRequest) (domain.SmokeTestRequest, error) { execution, err := toDomainExecutionTargetOverride(request.Execution) if err != nil { return domain.SmokeTestRequest{}, err } return domain.SmokeTestRequest{ Kind: domain.SmokeTestKind(request.Kind), BackendID: request.BackendID, PromptID: request.PromptID, ProfileID: request.ProfileID, Execution: execution, APIKey: request.APIKey, TransientRetries: copyIntPtr(request.TransientRetries), }, nil } func fromDomainSmokeTestReport(report domain.SmokeTestReport) (*SmokeTestReport, error) { extraParams, err := jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams) if err != nil { return nil, err } result := &SmokeTestReport{ Kind: SmokeTestKind(report.Kind), Supported: report.Supported, Applicable: copySmokeTestBoolPtr(report.Applicable), Completed: report.Completed, Reason: SmokeTestReason(report.Reason), Target: SmokeTestTarget{ BackendID: report.Target.BackendID, ProfileID: report.Target.ProfileID, PromptID: report.Target.PromptID, PromptVersion: copyStringPtr(report.Target.PromptVersion), DefaultSourceBackendID: report.Target.DefaultSourceBackendID, SelectionSource: report.Target.SelectionSource, Endpoint: report.Target.Endpoint, EffectiveSettings: fromDomainExecutionTargetWithOwnedParams(report.Target.EffectiveSettings, extraParams), SettingsPresence: fromDomainExecutionTargetPresence(report.Target.SettingsPresence), }, Facts: fromDomainSmokeTestFacts(report.Facts), Timing: fromDomainSmokeTestTiming(report.Timing), Usage: fromDomainObservedTokenUsage(report.Usage), Failure: ExecutionFailureKind(report.Failure), } if report.Attempts != nil { result.Attempts = make([]SmokeTestAttempt, len(report.Attempts)) for index, attempt := range report.Attempts { result.Attempts[index] = SmokeTestAttempt{ Dispatched: attempt.Dispatched, InitialEndpoint: attempt.InitialEndpoint, Endpoint: attempt.Endpoint, HTTPStatus: copyIntPtr(attempt.HTTPStatus), Facts: fromDomainSmokeTestFacts(attempt.Facts), Timing: fromDomainSmokeTestTiming(attempt.Timing), Failure: ExecutionFailureKind(attempt.Failure), Usage: fromDomainObservedTokenUsage(attempt.Usage), } } } return result, nil } func fromDomainSmokeTestFacts(facts domain.SmokeTestFacts) SmokeTestFacts { return SmokeTestFacts{ CredentialsSupplied: copySmokeTestBoolPtr(facts.CredentialsSupplied), EndpointReachable: copySmokeTestBoolPtr(facts.EndpointReachable), CredentialsAccepted: copySmokeTestBoolPtr(facts.CredentialsAccepted), ResponseReceived: copySmokeTestBoolPtr(facts.ResponseReceived), GenerationCompleted: copySmokeTestBoolPtr(facts.GenerationCompleted), TextReturned: copySmokeTestBoolPtr(facts.TextReturned), ExpectedTextMatched: copySmokeTestBoolPtr(facts.ExpectedTextMatched), StructuredOutputRequested: copySmokeTestBoolPtr(facts.StructuredOutputRequested), ValidJSON: copySmokeTestBoolPtr(facts.ValidJSON), SchemaMatched: copySmokeTestBoolPtr(facts.SchemaMatched), } } func copySmokeTestBoolPtr(value *bool) *bool { if value == nil { return nil } cloned := *value return &cloned } func fromDomainSmokeTestTiming(timing domain.SmokeTestTiming) SmokeTestTiming { return SmokeTestTiming{ StartedAt: timing.StartedAt, Duration: timing.Duration, CapacityWait: copyDurationPtr(timing.CapacityWait), RequestDuration: copyDurationPtr(timing.RequestDuration), RetryDelay: copyDurationPtr(timing.RetryDelay), EvaluationDuration: copyDurationPtr(timing.EvaluationDuration), } } func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedTokenUsage { return ObservedTokenUsage{ PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens), CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens), TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens), CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens), CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens), } } func fromDomainObservedTokenCount(count domain.ObservedTokenCount) ObservedTokenCount { return ObservedTokenCount{Value: count.Value, Known: count.Known, Complete: count.Complete} } func copyDurationPtr(value *time.Duration) *time.Duration { if value == nil { return nil } cloned := *value return &cloned }