Resolve authentication diagnostics by provider
This commit is contained in:
@@ -253,7 +253,7 @@ backends:
|
||||
endpoint: https://openrouter.ai/api/v1/key
|
||||
```
|
||||
|
||||
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||
An omitted or null `smoke_tests` block removes its declared metadata.
|
||||
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||
`default_profile` means there is no inference default. Omitted/null
|
||||
@@ -262,14 +262,17 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||
means fallback, while an explicit blank value is invalid.
|
||||
|
||||
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||
non-null object's optional `kind` is trimmed and case-normalized; accepted
|
||||
Omitted/null `authentication` leaves the provider's diagnostic default in
|
||||
effect; `openrouter` defaults to `openrouter_key` and the generic provider has
|
||||
no default. A non-null object's optional `kind` is trimmed and case-normalized; accepted
|
||||
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
|
||||
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||
Without a selected adapter, or for `vllm_models`, authentication diagnostics
|
||||
currently report an unsupported selection.
|
||||
currently report an unsupported selection. An endpoint-only authentication
|
||||
block overrides the destination of a provider default; `kind: none` suppresses
|
||||
that default.
|
||||
These values are configuration only: engine construction does not contact a
|
||||
backend.
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ report facts, and injected-client behavior.
|
||||
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||
is selected from backend metadata, not inferred from the host. The
|
||||
is selected from backend metadata and provider defaults, not inferred from the host. The
|
||||
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||
the [backend format reference](../formats.md#backend-definitions)). It may send
|
||||
|
||||
@@ -176,6 +176,8 @@ Validation: configuration capture, backend inspection, bundle, and preparation t
|
||||
|
||||
## Stage 5 — Generic Authentication Dispatch And OpenRouter Defaults
|
||||
|
||||
**Completed.**
|
||||
|
||||
Current provider branches occur in `internal/usecase/smoke_authentication.go` and
|
||||
`smoke_selection.go`, including `ProbeOpenRouterKey` and endpoint derivation.
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -26,8 +27,11 @@ func OpenRouterKeyEndpoint(base string) (string, error) {
|
||||
return url.JoinPath(base, "key")
|
||||
}
|
||||
|
||||
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
|
||||
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||
// ProbeAuthentication executes the selected authentication adapter.
|
||||
func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapter, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||
if !SupportedAuthenticationAdapter(adapter) {
|
||||
return AuthenticationEvidence{}, fmt.Errorf("unsupported authentication adapter %q", adapter)
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return AuthenticationEvidence{}, err
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestProbeOpenRouterKey(t *testing.T) {
|
||||
authorization = req.Header.Get("Authorization")
|
||||
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
|
||||
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
|
||||
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
|
||||
}
|
||||
@@ -127,7 +127,7 @@ func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
|
||||
})}})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(ctx, "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
|
||||
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package llm
|
||||
|
||||
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||
|
||||
// EffectiveProvider resolves an omitted provider to the generic conventions.
|
||||
func EffectiveProvider(provider string) string {
|
||||
if provider == "" {
|
||||
@@ -18,3 +20,28 @@ func SupportedProvider(provider string) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// AuthenticationAdapter selects declared diagnostic metadata or provider defaults.
|
||||
func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig) string {
|
||||
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil {
|
||||
return *smoke.Authentication.Kind
|
||||
}
|
||||
if provider == "openrouter" {
|
||||
return "openrouter_key"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// AuthenticationEndpoint resolves a diagnostic destination without changing configuration.
|
||||
func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTestConfig) (string, error) {
|
||||
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Endpoint != nil {
|
||||
return *smoke.Authentication.Endpoint, nil
|
||||
}
|
||||
if adapter == "openrouter_key" {
|
||||
return OpenRouterKeyEndpoint(base)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// SupportedAuthenticationAdapter identifies implemented authentication probes.
|
||||
func SupportedAuthenticationAdapter(adapter string) bool { return adapter == "openrouter_key" }
|
||||
|
||||
30
internal/llm/provider_test.go
Normal file
30
internal/llm/provider_test.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestAuthenticationSelection(t *testing.T) {
|
||||
none, explicit, endpoint := "none", "openrouter_key", "https://diagnostics.example/probe"
|
||||
for _, tc := range []struct {
|
||||
name, provider, adapter, destination string
|
||||
metadata *domain.BackendSmokeTestConfig
|
||||
}{
|
||||
{name: "provider default", provider: "openrouter", adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "cleared block", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "endpoint only", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "openrouter_key", destination: endpoint},
|
||||
{name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
|
||||
{name: "generic unsupported", destination: "https://example.test/v1"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kind := AuthenticationAdapter(tc.provider, tc.metadata)
|
||||
destination, err := AuthenticationEndpoint(kind, "https://example.test/v1", tc.metadata)
|
||||
if err != nil || kind != tc.adapter || destination != tc.destination {
|
||||
t.Fatalf("selection=%q destination=%q error=%v", kind, destination, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
type smokeAuthenticationClient interface {
|
||||
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||
ProbeAuthentication(context.Context, string, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||
}
|
||||
|
||||
type authenticationProbe struct{ client smokeAuthenticationClient }
|
||||
@@ -30,7 +30,8 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
|
||||
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
||||
return outcome, err
|
||||
}
|
||||
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
|
||||
result, requestErr := p.client.ProbeAuthentication(ctx, adapter, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||
outcome.evidence = result.HTTP
|
||||
outcome.credentialsAccepted = result.Accepted
|
||||
if result.Overflow {
|
||||
@@ -55,22 +56,18 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
|
||||
}
|
||||
|
||||
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
||||
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil {
|
||||
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
|
||||
if adapter == "" {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
|
||||
}
|
||||
authentication := backend.SmokeTests.Authentication
|
||||
if authentication.Kind == nil {
|
||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
|
||||
}
|
||||
switch *authentication.Kind {
|
||||
case "none":
|
||||
if adapter == "none" {
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
|
||||
case "openrouter_key":
|
||||
if p.client == nil {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||
}
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
|
||||
default:
|
||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
||||
}
|
||||
if !llm.SupportedAuthenticationAdapter(adapter) {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
|
||||
}
|
||||
if p.client == nil {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||
}
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
|
||||
}
|
||||
|
||||
@@ -110,6 +110,8 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
|
||||
} else {
|
||||
settings := defaults.ExecutionTargetDefault()
|
||||
settings.BackendID = selection.backend.ID
|
||||
settings.Provider = selection.backend.Provider
|
||||
settings.ZeroCost = selection.backend.ZeroCost
|
||||
settings.Endpoint = selection.backend.Endpoint
|
||||
settings.APIKeyEnv = selection.backend.APIKeyEnv
|
||||
if len(selection.backend.ExtraParams) > 0 {
|
||||
@@ -132,7 +134,7 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
|
||||
if promptInspection != nil {
|
||||
selection.target.PromptID = promptInspection.PromptID
|
||||
}
|
||||
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests)
|
||||
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
@@ -165,22 +167,15 @@ func isInferenceSmokeTest(kind domain.SmokeTestKind) bool {
|
||||
return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput
|
||||
}
|
||||
|
||||
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) {
|
||||
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, backend domain.Backend) (string, error) {
|
||||
smokeTests := backend.SmokeTests
|
||||
switch kind {
|
||||
case domain.SmokeTestReachability:
|
||||
if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil {
|
||||
return *smokeTests.Reachability.Endpoint, nil
|
||||
}
|
||||
case domain.SmokeTestAuthentication:
|
||||
if smokeTests != nil && smokeTests.Authentication != nil {
|
||||
authentication := smokeTests.Authentication
|
||||
if authentication.Endpoint != nil {
|
||||
return *authentication.Endpoint, nil
|
||||
}
|
||||
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
|
||||
return llm.OpenRouterKeyEndpoint(baseEndpoint)
|
||||
}
|
||||
}
|
||||
return llm.AuthenticationEndpoint(llm.AuthenticationAdapter(backend.Provider, smokeTests), baseEndpoint, smokeTests)
|
||||
}
|
||||
return baseEndpoint, nil
|
||||
}
|
||||
|
||||
@@ -5,13 +5,40 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestSmokeAuthenticationUsesProviderDefault(t *testing.T) {
|
||||
var path string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path = r.URL.Path
|
||||
fmt.Fprint(w, `{"data":{"label":"member","is_free_tier":false,"usage":0}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("diagnostic", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "renamed", "provider": "openrouter", "endpoint": server.URL + "/v1"}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectBackend(context.Background(), "renamed")
|
||||
if err != nil || inspection.SmokeTests != nil {
|
||||
t.Fatalf("declared metadata: %+v, %v", inspection, err)
|
||||
}
|
||||
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "renamed"})
|
||||
if err != nil || !report.Supported || !report.Completed || path != "/v1/key" || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted {
|
||||
t.Fatalf("report=%+v path=%q err=%v", report, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
type gatedSmokeClient struct {
|
||||
started chan promptkit.GenerateRequest
|
||||
release <-chan struct{}
|
||||
|
||||
Reference in New Issue
Block a user