diff --git a/docs/formats.md b/docs/formats.md index b2b9d80..8fc1d5d 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -253,7 +253,7 @@ backends: endpoint: https://openrouter.ai/api/v1/key ``` -An omitted or null `smoke_tests` block disables these metadata defaults. +An omitted or null `smoke_tests` block removes its declared metadata. `default_profile` is preserved as an exact, nonblank profile ID; its existence is checked only if an inference diagnostic selects that default. Omitted/null `default_profile` means there is no inference default. Omitted/null @@ -262,14 +262,17 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a host; unlike the backend base endpoint, its path and query are allowed. Null means fallback, while an explicit blank value is invalid. -Omitted/null `authentication` means no authentication adapter is declared. A -non-null object's optional `kind` is trimmed and case-normalized; accepted +Omitted/null `authentication` leaves the provider's diagnostic default in +effect; `openrouter` defaults to `openrouter_key` and the generic provider has +no default. A non-null object's optional `kind` is trimmed and case-normalized; accepted values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint uses the adapter's backend-relative destination (`openrouter_key` appends `/key` to the effective backend base); an explicit endpoint follows the same full-URL rules as reachability. `none` declares authentication inapplicable. Without a selected adapter, or for `vllm_models`, authentication diagnostics -currently report an unsupported selection. +currently report an unsupported selection. An endpoint-only authentication +block overrides the destination of a provider default; `kind: none` suppresses +that default. These values are configuration only: engine construction does not contact a backend. diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index 9ca2864..2565384 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -58,7 +58,7 @@ report facts, and injected-client behavior. HTTP-only diagnostics use the configured built-in HTTP client. Reachability is one unauthenticated `GET` to the selected diagnostic URL; an observed response counts as reachable even when its status is non-2xx. Authentication behavior -is selected from backend metadata, not inferred from the host. The +is selected from backend metadata and provider defaults, not inferred from the host. The `openrouter_key` adapter makes one `GET` to `/key` relative to the effective backend base, unless configuration supplies a full diagnostic endpoint (see the [backend format reference](../formats.md#backend-definitions)). It may send diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 04eec4a..e0263ae 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -176,6 +176,8 @@ Validation: configuration capture, backend inspection, bundle, and preparation t ## Stage 5 — Generic Authentication Dispatch And OpenRouter Defaults +**Completed.** + Current provider branches occur in `internal/usecase/smoke_authentication.go` and `smoke_selection.go`, including `ProbeOpenRouterKey` and endpoint derivation. diff --git a/internal/llm/openrouter_authentication.go b/internal/llm/openrouter_authentication.go index b6ee30a..98dcdf9 100644 --- a/internal/llm/openrouter_authentication.go +++ b/internal/llm/openrouter_authentication.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "net/url" @@ -26,8 +27,11 @@ func OpenRouterKeyEndpoint(base string) (string, error) { return url.JoinPath(base, "key") } -// ProbeOpenRouterKey executes and interprets an OpenRouter key request. -func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) { +// ProbeAuthentication executes the selected authentication adapter. +func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapter, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) { + if !SupportedAuthenticationAdapter(adapter) { + return AuthenticationEvidence{}, fmt.Errorf("unsupported authentication adapter %q", adapter) + } parsed, err := url.Parse(endpoint) if err != nil { return AuthenticationEvidence{}, err diff --git a/internal/llm/openrouter_authentication_test.go b/internal/llm/openrouter_authentication_test.go index 8ecebd2..a65dbe9 100644 --- a/internal/llm/openrouter_authentication_test.go +++ b/internal/llm/openrouter_authentication_test.go @@ -38,7 +38,7 @@ func TestProbeOpenRouterKey(t *testing.T) { authorization = req.Header.Get("Authorization") return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil })}}) - got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key}) + got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key}) if err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) { client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil })}}) - got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{}) + got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{}) if !errors.Is(err, want) || !got.HTTP.ResponseReceived { t.Fatalf("evidence = %+v, error = %v", got, err) } @@ -105,7 +105,7 @@ func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) { client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil })}}) - got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{}) + got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{}) if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) { t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err) } @@ -127,7 +127,7 @@ func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) { })}}) ctx, cancel := context.WithCancel(context.Background()) cancel() - got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{}) + got, err := client.ProbeAuthentication(ctx, "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{}) if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived { t.Fatalf("evidence = %+v, error = %v", got, err) } diff --git a/internal/llm/provider.go b/internal/llm/provider.go index 3eeaf88..5f7f0bf 100644 --- a/internal/llm/provider.go +++ b/internal/llm/provider.go @@ -1,5 +1,7 @@ package llm +import "go.maximumdirect.net/promptkit/internal/domain" + // EffectiveProvider resolves an omitted provider to the generic conventions. func EffectiveProvider(provider string) string { if provider == "" { @@ -18,3 +20,28 @@ func SupportedProvider(provider string) bool { return false } } + +// AuthenticationAdapter selects declared diagnostic metadata or provider defaults. +func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig) string { + if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil { + return *smoke.Authentication.Kind + } + if provider == "openrouter" { + return "openrouter_key" + } + return "" +} + +// AuthenticationEndpoint resolves a diagnostic destination without changing configuration. +func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTestConfig) (string, error) { + if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Endpoint != nil { + return *smoke.Authentication.Endpoint, nil + } + if adapter == "openrouter_key" { + return OpenRouterKeyEndpoint(base) + } + return base, nil +} + +// SupportedAuthenticationAdapter identifies implemented authentication probes. +func SupportedAuthenticationAdapter(adapter string) bool { return adapter == "openrouter_key" } diff --git a/internal/llm/provider_test.go b/internal/llm/provider_test.go new file mode 100644 index 0000000..ee53c5f --- /dev/null +++ b/internal/llm/provider_test.go @@ -0,0 +1,30 @@ +package llm + +import ( + "testing" + + "go.maximumdirect.net/promptkit/internal/domain" +) + +func TestAuthenticationSelection(t *testing.T) { + none, explicit, endpoint := "none", "openrouter_key", "https://diagnostics.example/probe" + for _, tc := range []struct { + name, provider, adapter, destination string + metadata *domain.BackendSmokeTestConfig + }{ + {name: "provider default", provider: "openrouter", adapter: "openrouter_key", destination: "https://example.test/v1/key"}, + {name: "cleared block", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{}, adapter: "openrouter_key", destination: "https://example.test/v1/key"}, + {name: "endpoint only", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "openrouter_key", destination: endpoint}, + {name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"}, + {name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"}, + {name: "generic unsupported", destination: "https://example.test/v1"}, + } { + t.Run(tc.name, func(t *testing.T) { + kind := AuthenticationAdapter(tc.provider, tc.metadata) + destination, err := AuthenticationEndpoint(kind, "https://example.test/v1", tc.metadata) + if err != nil || kind != tc.adapter || destination != tc.destination { + t.Fatalf("selection=%q destination=%q error=%v", kind, destination, err) + } + }) + } +} diff --git a/internal/usecase/smoke_authentication.go b/internal/usecase/smoke_authentication.go index cb81507..0d18603 100644 --- a/internal/usecase/smoke_authentication.go +++ b/internal/usecase/smoke_authentication.go @@ -11,7 +11,7 @@ import ( ) type smokeAuthenticationClient interface { - ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error) + ProbeAuthentication(context.Context, string, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error) } type authenticationProbe struct{ client smokeAuthenticationClient } @@ -30,7 +30,8 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) { return outcome, err } - result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings) + adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests) + result, requestErr := p.client.ProbeAuthentication(ctx, adapter, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings) outcome.evidence = result.HTTP outcome.credentialsAccepted = result.Accepted if result.Overflow { @@ -55,22 +56,18 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta } func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) { - if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil { + adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests) + if adapter == "" { return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil } - authentication := backend.SmokeTests.Authentication - if authentication.Kind == nil { - return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest) - } - switch *authentication.Kind { - case "none": + if adapter == "none" { return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil - case "openrouter_key": - if p.client == nil { - return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil - } - return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil - default: - return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind) } + if !llm.SupportedAuthenticationAdapter(adapter) { + return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil + } + if p.client == nil { + return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil + } + return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil } diff --git a/internal/usecase/smoke_selection.go b/internal/usecase/smoke_selection.go index f557bc0..fbbfb12 100644 --- a/internal/usecase/smoke_selection.go +++ b/internal/usecase/smoke_selection.go @@ -110,6 +110,8 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes } else { settings := defaults.ExecutionTargetDefault() settings.BackendID = selection.backend.ID + settings.Provider = selection.backend.Provider + settings.ZeroCost = selection.backend.ZeroCost settings.Endpoint = selection.backend.Endpoint settings.APIKeyEnv = selection.backend.APIKeyEnv if len(selection.backend.ExtraParams) > 0 { @@ -132,7 +134,7 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes if promptInspection != nil { selection.target.PromptID = promptInspection.PromptID } - selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests) + selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend) if err != nil { return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err) } @@ -165,22 +167,15 @@ func isInferenceSmokeTest(kind domain.SmokeTestKind) bool { return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput } -func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) { +func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, backend domain.Backend) (string, error) { + smokeTests := backend.SmokeTests switch kind { case domain.SmokeTestReachability: if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil { return *smokeTests.Reachability.Endpoint, nil } case domain.SmokeTestAuthentication: - if smokeTests != nil && smokeTests.Authentication != nil { - authentication := smokeTests.Authentication - if authentication.Endpoint != nil { - return *authentication.Endpoint, nil - } - if authentication.Kind != nil && *authentication.Kind == "openrouter_key" { - return llm.OpenRouterKeyEndpoint(baseEndpoint) - } - } + return llm.AuthenticationEndpoint(llm.AuthenticationAdapter(backend.Provider, smokeTests), baseEndpoint, smokeTests) } return baseEndpoint, nil } diff --git a/smoke_test_contract_test.go b/smoke_test_contract_test.go index fe7de9f..54f0823 100644 --- a/smoke_test_contract_test.go +++ b/smoke_test_contract_test.go @@ -5,13 +5,40 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "net/http/httptest" "strings" "testing" "time" + "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) +func TestSmokeAuthenticationUsesProviderDefault(t *testing.T) { + var path string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + fmt.Fprint(w, `{"data":{"label":"member","is_free_tier":false,"usage":0}}`) + })) + defer server.Close() + input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("diagnostic", map[string]any{ + "backends": []any{map[string]any{"id": "renamed", "provider": "openrouter", "endpoint": server.URL + "/v1"}}, + })} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false)) + if err != nil { + t.Fatal(err) + } + inspection, err := engine.InspectBackend(context.Background(), "renamed") + if err != nil || inspection.SmokeTests != nil { + t.Fatalf("declared metadata: %+v, %v", inspection, err) + } + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "renamed"}) + if err != nil || !report.Supported || !report.Completed || path != "/v1/key" || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted { + t.Fatalf("report=%+v path=%q err=%v", report, path, err) + } +} + type gatedSmokeClient struct { started chan promptkit.GenerateRequest release <-chan struct{}