Expose the public smoke-test API
This commit is contained in:
391
smoke_diagnostics.go
Normal file
391
smoke_diagnostics.go
Normal file
@@ -0,0 +1,391 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// SmokeTestKind identifies one built-in diagnostic operation.
|
||||
type SmokeTestKind string
|
||||
|
||||
const (
|
||||
// SmokeTestReachability performs an unauthenticated HTTP reachability probe.
|
||||
SmokeTestReachability SmokeTestKind = "reachability"
|
||||
// SmokeTestAuthentication checks the configured authentication behavior.
|
||||
SmokeTestAuthentication SmokeTestKind = "authentication"
|
||||
// SmokeTestTextInference checks text generation against a private case.
|
||||
SmokeTestTextInference SmokeTestKind = "text_inference"
|
||||
// SmokeTestStructuredOutput checks structured generation and local conformance.
|
||||
SmokeTestStructuredOutput SmokeTestKind = "structured_output"
|
||||
)
|
||||
|
||||
// SmokeTestRequest selects one diagnostic and its execution target.
|
||||
//
|
||||
// APIKey is request-scoped and takes precedence over an environment credential.
|
||||
// It is excluded from JSON and from String and GoString output. Execution values
|
||||
// are copied before the internal engine uses them.
|
||||
type SmokeTestRequest struct {
|
||||
// Kind selects one of the built-in smoke-test operations.
|
||||
Kind SmokeTestKind
|
||||
// BackendID selects a backend directly or supplies a fallback when inference
|
||||
// selection otherwise needs a profile. A selected profile determines its own
|
||||
// effective backend identity.
|
||||
BackendID string
|
||||
// PromptID selects the prompt's default profile when ProfileID is empty. The
|
||||
// prompt's messages, inputs, templates, and output contract are not executed.
|
||||
PromptID string
|
||||
// PromptVersion selects a version of PromptID using ordinary prompt-version
|
||||
// selection semantics.
|
||||
PromptVersion string
|
||||
// ProfileID explicitly selects a profile and takes precedence over a prompt's
|
||||
// default profile and a backend's configured diagnostic default.
|
||||
ProfileID string
|
||||
// Execution overrides applicable target settings using the same semantics as
|
||||
// ordinary execution. HTTP probes do not require a model setting.
|
||||
Execution *ExecutionTargetOverride
|
||||
// APIKey supplies a direct request-scoped credential. It is never included
|
||||
// in JSON, String, or GoString output.
|
||||
APIKey string `json:"-"`
|
||||
// TransientRetries sets the inference retry count. Nil means zero retries for
|
||||
// this operation, independently of the engine's ordinary retry default.
|
||||
// HTTP-only probes remain single-attempt. Negative values are invalid.
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// String returns a credential-safe summary of the request.
|
||||
func (r SmokeTestRequest) String() string { return r.redactedString() }
|
||||
|
||||
// GoString returns a credential-safe summary of the request.
|
||||
func (r SmokeTestRequest) GoString() string { return r.redactedString() }
|
||||
|
||||
func (r SmokeTestRequest) redactedString() string {
|
||||
retries := "nil"
|
||||
if r.TransientRetries != nil {
|
||||
retries = fmt.Sprintf("%d", *r.TransientRetries)
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"promptkit.SmokeTestRequest{Kind:%q BackendIDSet:%t PromptIDSet:%t PromptVersionSet:%t ProfileIDSet:%t APIKeySet:%t TransientRetries:%s}",
|
||||
r.Kind,
|
||||
r.BackendID != "",
|
||||
r.PromptID != "",
|
||||
r.PromptVersion != "",
|
||||
r.ProfileID != "",
|
||||
r.APIKey != "",
|
||||
retries,
|
||||
)
|
||||
}
|
||||
|
||||
// SmokeTestReport contains factual diagnostic results, not generated content.
|
||||
//
|
||||
// A nil Go error from Engine.SmokeTest means the diagnostic concluded, not that
|
||||
// the tested operation succeeded. Rejections, mismatches, unsupported
|
||||
// operations, and known inapplicability can all be concluded reports with nil
|
||||
// Go error. A fact pointer is true or false only when established; nil means it
|
||||
// was not evaluated, is unknown, or is inapplicable. Reports own their mutable
|
||||
// values. They intentionally have no stable JSON contract.
|
||||
type SmokeTestReport struct {
|
||||
// Kind is the requested diagnostic.
|
||||
Kind SmokeTestKind
|
||||
// Supported reports whether this engine/client can perform the operation.
|
||||
Supported bool
|
||||
// Applicable is false only when known not to apply; nil means unknown.
|
||||
Applicable *bool
|
||||
// Completed indicates that the diagnostic concluded, regardless of findings.
|
||||
Completed bool
|
||||
// Reason is an additional factual code, not a severity or free-form message.
|
||||
Reason SmokeTestReason
|
||||
// Target identifies the selected target and its effective settings.
|
||||
Target SmokeTestTarget
|
||||
// Facts contains tri-state observations for the diagnostic.
|
||||
Facts SmokeTestFacts
|
||||
// Timing contains total and measured phase durations.
|
||||
Timing SmokeTestTiming
|
||||
// Attempts contains dispatched attempts and any interrupted acquisition.
|
||||
Attempts []SmokeTestAttempt
|
||||
// Usage is the presence-aware aggregate for dispatched attempts.
|
||||
Usage ObservedTokenUsage
|
||||
// Failure classifies the diagnostic execution outcome, when any.
|
||||
Failure ExecutionFailureKind
|
||||
}
|
||||
|
||||
// SmokeTestTarget identifies the selected diagnostic target and its settings.
|
||||
type SmokeTestTarget struct {
|
||||
// BackendID is the effective backend/capacity identity; it can be empty for
|
||||
// an endpoint-only profile.
|
||||
BackendID string
|
||||
// ProfileID is the selected profile, if any.
|
||||
ProfileID string
|
||||
// PromptID and PromptVersion identify the prompt used for selection, if any.
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
// DefaultSourceBackendID identifies the backend whose configured default
|
||||
// selected the profile, when that selection path was used.
|
||||
DefaultSourceBackendID string
|
||||
// SelectionSource identifies the selection path (backend_default, profile,
|
||||
// prompt, or backend).
|
||||
SelectionSource string
|
||||
// Endpoint is the initially resolved diagnostic destination. It excludes URL
|
||||
// user information; other endpoint components and effective settings may be
|
||||
// sensitive operator data.
|
||||
Endpoint string
|
||||
// EffectiveSettings is the selected target's resolved runtime configuration.
|
||||
EffectiveSettings ExecutionTarget
|
||||
// SettingsPresence records whether optional numeric settings were explicitly
|
||||
// present, including explicit zero values.
|
||||
SettingsPresence ExecutionTargetPresence
|
||||
}
|
||||
|
||||
// SmokeTestFacts contains optional factual observations from a diagnostic.
|
||||
// Nil means unknown, not evaluated, or inapplicable.
|
||||
type SmokeTestFacts struct {
|
||||
// CredentialsSupplied reports whether a credential was attached to the last
|
||||
// observed request for that attempt's endpoint.
|
||||
CredentialsSupplied *bool
|
||||
// EndpointReachable reports whether an HTTP response was observed.
|
||||
EndpointReachable *bool
|
||||
// CredentialsAccepted reports only recognized authentication outcomes.
|
||||
CredentialsAccepted *bool
|
||||
// ResponseReceived reports whether a response was received from the selected
|
||||
// client; injected clients do not establish HTTP facts.
|
||||
ResponseReceived *bool
|
||||
// GenerationCompleted reports whether generation returned usable content.
|
||||
GenerationCompleted *bool
|
||||
// TextReturned reports whether non-empty text was returned.
|
||||
TextReturned *bool
|
||||
// ExpectedTextMatched reports whether the private text case matched.
|
||||
ExpectedTextMatched *bool
|
||||
// StructuredOutputRequested reports whether a native structured-output
|
||||
// request was made; it does not assert provider enforcement.
|
||||
StructuredOutputRequested *bool
|
||||
// ValidJSON reports whether generated content parsed as JSON.
|
||||
ValidJSON *bool
|
||||
// SchemaMatched reports whether parsed JSON conformed to the private schema.
|
||||
SchemaMatched *bool
|
||||
}
|
||||
|
||||
// SmokeTestTiming records elapsed diagnostic and phase durations.
|
||||
type SmokeTestTiming struct {
|
||||
// StartedAt and Duration bound total diagnostic orchestration.
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
// CapacityWait, RequestDuration, RetryDelay, and EvaluationDuration are nil
|
||||
// when their phases did not occur. RequestDuration excludes capacity wait,
|
||||
// retry delay, and local evaluation.
|
||||
CapacityWait *time.Duration
|
||||
RequestDuration *time.Duration
|
||||
RetryDelay *time.Duration
|
||||
EvaluationDuration *time.Duration
|
||||
}
|
||||
|
||||
// SmokeTestAttempt records one dispatched attempt or an interrupted acquisition.
|
||||
type SmokeTestAttempt struct {
|
||||
// Dispatched distinguishes an actual attempt from an interrupted capacity
|
||||
// acquisition.
|
||||
Dispatched bool
|
||||
// InitialEndpoint is the destination before redirects.
|
||||
InitialEndpoint string
|
||||
// Endpoint is the last observed request destination for this attempt.
|
||||
Endpoint string
|
||||
// HTTPStatus is the last observed response status, if available.
|
||||
HTTPStatus *int
|
||||
// Facts, Timing, Failure, and Usage describe this attempt only.
|
||||
Facts SmokeTestFacts
|
||||
Timing SmokeTestTiming
|
||||
Failure ExecutionFailureKind
|
||||
Usage ObservedTokenUsage
|
||||
}
|
||||
|
||||
// SmokeTestReason identifies additional factual context for a diagnostic.
|
||||
type SmokeTestReason string
|
||||
|
||||
const (
|
||||
// SmokeTestReasonNone indicates no additional reason.
|
||||
SmokeTestReasonNone SmokeTestReason = ""
|
||||
// SmokeTestReasonAuthenticationNotApplicable marks known inapplicability.
|
||||
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||
// SmokeTestReasonProbeNotConfigured marks absent authentication metadata.
|
||||
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||
// SmokeTestReasonHTTPDiagnosticsUnavailable marks an engine without a
|
||||
// diagnostic HTTP transport.
|
||||
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||
// SmokeTestReasonRequiredCredentialMissing marks an inference credential
|
||||
// required locally but absent before dispatch.
|
||||
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||
// SmokeTestReasonAuthenticationResponseUnrecognized marks an HTTP response
|
||||
// whose authentication meaning could not be established.
|
||||
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||
)
|
||||
|
||||
// SmokeTest runs one built-in diagnostic against the engine's captured
|
||||
// configuration.
|
||||
//
|
||||
// Selection prefers an explicit ProfileID, then the profile selected by
|
||||
// PromptID and PromptVersion, then a configured backend default profile when
|
||||
// inference requires one. For backend-only probes, a selected profile supplies
|
||||
// its backend metadata; otherwise BackendID selects a backend directly without
|
||||
// resolving its default inference profile. A selected profile determines its
|
||||
// own effective backend identity, even when it differs from BackendID.
|
||||
// Application prompt messages, inputs, session templates, artifacts, and
|
||||
// output schemas are never executed by this operation.
|
||||
//
|
||||
// A nil Go error means the diagnostic concluded, not that the tested operation
|
||||
// succeeded. Concluded failures, mismatches, unsupported operations, and
|
||||
// inapplicability are represented in the report with nil Go error. Invalid
|
||||
// kinds, settings, or selections return an ordinary error before diagnostic
|
||||
// work. Caller cancellation or deadline expiration returns available partial
|
||||
// observations and an error recognizable with errors.Is; Completed remains
|
||||
// false when cancellation interrupts the diagnostic. A request timeout while
|
||||
// the caller context remains active is instead a concluded report.
|
||||
//
|
||||
// Built-in HTTP clients support reachability and configured authentication
|
||||
// probes. An engine using an injected LLMClient can perform inference
|
||||
// diagnostics, but HTTP diagnostics are reported unavailable and never fall
|
||||
// back to another transport. Nil TransientRetries means zero inference retries;
|
||||
// HTTP probes are single-attempt regardless of that setting. Diagnostics do
|
||||
// not perform structural repair.
|
||||
//
|
||||
// Timing includes orchestration in Duration. CapacityWait, RequestDuration,
|
||||
// RetryDelay, and EvaluationDuration measure only phases that occurred;
|
||||
// RequestDuration excludes capacity waiting, retry delay, and local evaluation.
|
||||
// Reports omit credentials, generated text, provider messages, raw HTTP bodies,
|
||||
// and private fixture contents. Effective settings and identifiers may still
|
||||
// contain sensitive operator data and require caller-controlled disclosure.
|
||||
func (e *Engine) SmokeTest(ctx context.Context, req SmokeTestRequest) (*SmokeTestReport, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, ErrInvalidRequest
|
||||
}
|
||||
domainRequest, err := toDomainSmokeTestRequest(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
domainReport, err := e.internal.SmokeTest(ctx, domainRequest)
|
||||
if domainReport == nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
report, conversionErr := fromDomainSmokeTestReport(*domainReport)
|
||||
if conversionErr != nil {
|
||||
return nil, fmt.Errorf("%w: failed to copy smoke-test report", ErrInvalidRequest)
|
||||
}
|
||||
return report, mapPublicError(err)
|
||||
}
|
||||
|
||||
func toDomainSmokeTestRequest(request SmokeTestRequest) (domain.SmokeTestRequest, error) {
|
||||
execution, err := toDomainExecutionTargetOverride(request.Execution)
|
||||
if err != nil {
|
||||
return domain.SmokeTestRequest{}, err
|
||||
}
|
||||
return domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestKind(request.Kind),
|
||||
BackendID: request.BackendID,
|
||||
PromptID: request.PromptID,
|
||||
PromptVersion: request.PromptVersion,
|
||||
ProfileID: request.ProfileID,
|
||||
Execution: execution,
|
||||
APIKey: request.APIKey,
|
||||
TransientRetries: copyIntPtr(request.TransientRetries),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fromDomainSmokeTestReport(report domain.SmokeTestReport) (*SmokeTestReport, error) {
|
||||
owned, err := domain.CloneSmokeTestReport(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := &SmokeTestReport{
|
||||
Kind: SmokeTestKind(owned.Kind),
|
||||
Supported: owned.Supported,
|
||||
Applicable: copySmokeTestBoolPtr(owned.Applicable),
|
||||
Completed: owned.Completed,
|
||||
Reason: SmokeTestReason(owned.Reason),
|
||||
Target: SmokeTestTarget{
|
||||
BackendID: owned.Target.BackendID,
|
||||
ProfileID: owned.Target.ProfileID,
|
||||
PromptID: owned.Target.PromptID,
|
||||
PromptVersion: owned.Target.PromptVersion,
|
||||
DefaultSourceBackendID: owned.Target.DefaultSourceBackendID,
|
||||
SelectionSource: owned.Target.SelectionSource,
|
||||
Endpoint: owned.Target.Endpoint,
|
||||
EffectiveSettings: fromDomainExecutionTarget(owned.Target.EffectiveSettings),
|
||||
SettingsPresence: fromDomainExecutionTargetPresence(owned.Target.SettingsPresence),
|
||||
},
|
||||
Facts: fromDomainSmokeTestFacts(owned.Facts),
|
||||
Timing: fromDomainSmokeTestTiming(owned.Timing),
|
||||
Usage: fromDomainObservedTokenUsage(owned.Usage),
|
||||
Failure: ExecutionFailureKind(owned.Failure),
|
||||
}
|
||||
if owned.Attempts != nil {
|
||||
result.Attempts = make([]SmokeTestAttempt, len(owned.Attempts))
|
||||
for index, attempt := range owned.Attempts {
|
||||
result.Attempts[index] = SmokeTestAttempt{
|
||||
Dispatched: attempt.Dispatched,
|
||||
InitialEndpoint: attempt.InitialEndpoint,
|
||||
Endpoint: attempt.Endpoint,
|
||||
HTTPStatus: copyIntPtr(attempt.HTTPStatus),
|
||||
Facts: fromDomainSmokeTestFacts(attempt.Facts),
|
||||
Timing: fromDomainSmokeTestTiming(attempt.Timing),
|
||||
Failure: ExecutionFailureKind(attempt.Failure),
|
||||
Usage: fromDomainObservedTokenUsage(attempt.Usage),
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func fromDomainSmokeTestFacts(facts domain.SmokeTestFacts) SmokeTestFacts {
|
||||
return SmokeTestFacts{
|
||||
CredentialsSupplied: copySmokeTestBoolPtr(facts.CredentialsSupplied),
|
||||
EndpointReachable: copySmokeTestBoolPtr(facts.EndpointReachable),
|
||||
CredentialsAccepted: copySmokeTestBoolPtr(facts.CredentialsAccepted),
|
||||
ResponseReceived: copySmokeTestBoolPtr(facts.ResponseReceived),
|
||||
GenerationCompleted: copySmokeTestBoolPtr(facts.GenerationCompleted),
|
||||
TextReturned: copySmokeTestBoolPtr(facts.TextReturned),
|
||||
ExpectedTextMatched: copySmokeTestBoolPtr(facts.ExpectedTextMatched),
|
||||
StructuredOutputRequested: copySmokeTestBoolPtr(facts.StructuredOutputRequested),
|
||||
ValidJSON: copySmokeTestBoolPtr(facts.ValidJSON),
|
||||
SchemaMatched: copySmokeTestBoolPtr(facts.SchemaMatched),
|
||||
}
|
||||
}
|
||||
|
||||
func copySmokeTestBoolPtr(value *bool) *bool {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func fromDomainSmokeTestTiming(timing domain.SmokeTestTiming) SmokeTestTiming {
|
||||
return SmokeTestTiming{
|
||||
StartedAt: timing.StartedAt,
|
||||
Duration: timing.Duration,
|
||||
CapacityWait: copyDurationPtr(timing.CapacityWait),
|
||||
RequestDuration: copyDurationPtr(timing.RequestDuration),
|
||||
RetryDelay: copyDurationPtr(timing.RetryDelay),
|
||||
EvaluationDuration: copyDurationPtr(timing.EvaluationDuration),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedTokenUsage {
|
||||
return ObservedTokenUsage{
|
||||
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
|
||||
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
|
||||
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
|
||||
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
|
||||
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainObservedTokenCount(count domain.ObservedTokenCount) ObservedTokenCount {
|
||||
return ObservedTokenCount{Value: count.Value, Known: count.Known, Complete: count.Complete}
|
||||
}
|
||||
|
||||
func copyDurationPtr(value *time.Duration) *time.Duration {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
Reference in New Issue
Block a user