Add OpenRouter authentication diagnostics
This commit is contained in:
@@ -540,6 +540,14 @@ rejected, anonymous rejection, malformed/unrecognized 200, other status, and
|
|||||||
transport failure. Add applicability/support tests without duplicating HTTP
|
transport failure. Add applicability/support tests without duplicating HTTP
|
||||||
client tests. Synthetic credentials must not appear in report-facing values.
|
client tests. Synthetic credentials must not appear in report-facing values.
|
||||||
|
|
||||||
|
**Completed: 2026-10-13.** Added metadata-driven authentication applicability
|
||||||
|
and an OpenRouter key probe with optional direct/environment credentials. It
|
||||||
|
recognizes the documented minimal 200 response, records 401 rejection, leaves
|
||||||
|
other outcomes undetermined as specified, and distinguishes interrupted body
|
||||||
|
reads from malformed JSON. Validation passed: `go test -race
|
||||||
|
./internal/usecase ./internal/llm`, `go vet ./...`, `go test ./...`, and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
## Stage 13: Build And Evaluate Private Inference Cases
|
## Stage 13: Build And Evaluate Private Inference Cases
|
||||||
|
|
||||||
Scope: private fixtures and validation collaborators in `internal/usecase`.
|
Scope: private fixtures and validation collaborators in `internal/usecase`.
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTPExecutionEvidence contains only facts observable at the configured
|
// HTTPExecutionEvidence contains only facts observable at the configured
|
||||||
@@ -99,6 +101,27 @@ func (c *OpenAICompatibleClient) GetWithEvidence(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
endpoint string,
|
endpoint string,
|
||||||
timeoutSeconds int,
|
timeoutSeconds int,
|
||||||
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
|
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWithAuthenticationEvidence performs a plain GET with credentials resolved
|
||||||
|
// from the selected target. Missing credentials are allowed for diagnostic
|
||||||
|
// authentication probes even when generation requires a key.
|
||||||
|
func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
endpoint string,
|
||||||
|
timeoutSeconds int,
|
||||||
|
target domain.ExecutionTarget,
|
||||||
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
|
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *OpenAICompatibleClient) getWithEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
endpoint string,
|
||||||
|
timeoutSeconds int,
|
||||||
|
apiKey string,
|
||||||
) (*http.Response, HTTPExecutionEvidence, error) {
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
requestContext := ctx
|
requestContext := ctx
|
||||||
var cancel context.CancelFunc
|
var cancel context.CancelFunc
|
||||||
@@ -113,6 +136,9 @@ func (c *OpenAICompatibleClient) GetWithEvidence(
|
|||||||
}
|
}
|
||||||
return nil, HTTPExecutionEvidence{}, err
|
return nil, HTTPExecutionEvidence{}, err
|
||||||
}
|
}
|
||||||
|
if apiKey != "" {
|
||||||
|
request.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
}
|
||||||
|
|
||||||
response, err, evidence := doWithEvidence(c.httpClient, request)
|
response, err, evidence := doWithEvidence(c.httpClient, request)
|
||||||
if response == nil || response.Body == nil {
|
if response == nil || response.Body == nil {
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
||||||
@@ -54,6 +56,69 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
target domain.ExecutionTarget
|
||||||
|
environment string
|
||||||
|
wantAuth string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "direct key takes precedence",
|
||||||
|
target: domain.ExecutionTarget{
|
||||||
|
APIKey: " direct-key ",
|
||||||
|
APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY",
|
||||||
|
},
|
||||||
|
environment: "environment-key",
|
||||||
|
wantAuth: "Bearer direct-key",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "environment key is used",
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY"},
|
||||||
|
environment: " environment-key ",
|
||||||
|
wantAuth: "Bearer environment-key",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing required key is optional here",
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", APIKeyRequired: true},
|
||||||
|
environment: " ",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
||||||
|
var authorization string
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
authorization = request.Header.Get("Authorization")
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusUnauthorized,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: io.NopCloser(strings.NewReader("")),
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, evidence, err := client.GetWithAuthenticationEvidence(
|
||||||
|
context.Background(), "https://example.invalid/key", 0, tc.target,
|
||||||
|
)
|
||||||
|
if err != nil || response == nil {
|
||||||
|
t.Fatalf("response=%v error=%v", response, err)
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
if authorization != tc.wantAuth || evidence.CredentialsSupplied != (tc.wantAuth != "") {
|
||||||
|
t.Fatalf("authorization=%q evidence=%+v", authorization, evidence)
|
||||||
|
}
|
||||||
|
if evidence.Endpoint != "https://example.invalid/key" || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("HTTP evidence = %+v", evidence)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) {
|
func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) {
|
||||||
transportErr := errors.New("transport unavailable")
|
transportErr := errors.New("transport unavailable")
|
||||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
|||||||
@@ -164,11 +164,8 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
apiKey := resolveAPIKey(req.Target)
|
||||||
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||||
if apiKey == "" && envName != "" {
|
|
||||||
apiKey = strings.TrimSpace(os.Getenv(envName))
|
|
||||||
}
|
|
||||||
if apiKey == "" && req.Target.APIKeyRequired {
|
if apiKey == "" && req.Target.APIKeyRequired {
|
||||||
if envName != "" {
|
if envName != "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
@@ -186,6 +183,16 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveAPIKey(target domain.ExecutionTarget) string {
|
||||||
|
apiKey := strings.TrimSpace(target.APIKey)
|
||||||
|
if apiKey == "" {
|
||||||
|
if envName := strings.TrimSpace(target.APIKeyEnv); envName != "" {
|
||||||
|
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return apiKey
|
||||||
|
}
|
||||||
|
|
||||||
// Execute dispatches one attempt. The returned response is non-nil only when
|
// Execute dispatches one attempt. The returned response is non-nil only when
|
||||||
// the successful envelope contains a usable first choice.
|
// the successful envelope contains a usable first choice.
|
||||||
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
||||||
|
|||||||
187
internal/usecase/smoke_authentication.go
Normal file
187
internal/usecase/smoke_authentication.go
Normal file
@@ -0,0 +1,187 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type authenticationProbe struct {
|
||||||
|
client smokeHTTPGetter
|
||||||
|
}
|
||||||
|
|
||||||
|
type authenticationProbeOutcome struct {
|
||||||
|
supported bool
|
||||||
|
applicable *bool
|
||||||
|
reason domain.SmokeTestReason
|
||||||
|
evidence llm.HTTPExecutionEvidence
|
||||||
|
credentialsAccepted *bool
|
||||||
|
failure domain.ExecutionFailureKind
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p authenticationProbe) run(
|
||||||
|
ctx context.Context,
|
||||||
|
backend domain.Backend,
|
||||||
|
target domain.SmokeTestTarget,
|
||||||
|
) (authenticationProbeOutcome, error) {
|
||||||
|
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil {
|
||||||
|
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
|
||||||
|
}
|
||||||
|
authentication := backend.SmokeTests.Authentication
|
||||||
|
if authentication.Kind == nil {
|
||||||
|
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
switch *authentication.Kind {
|
||||||
|
case "none":
|
||||||
|
return authenticationProbeOutcome{
|
||||||
|
supported: true,
|
||||||
|
applicable: boolPointer(false),
|
||||||
|
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
|
||||||
|
}, nil
|
||||||
|
case "openrouter_key":
|
||||||
|
return p.runOpenRouterKey(ctx, target)
|
||||||
|
default:
|
||||||
|
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p authenticationProbe) runOpenRouterKey(
|
||||||
|
ctx context.Context,
|
||||||
|
target domain.SmokeTestTarget,
|
||||||
|
) (authenticationProbeOutcome, error) {
|
||||||
|
applicable := boolPointer(true)
|
||||||
|
if p.client == nil {
|
||||||
|
return authenticationProbeOutcome{
|
||||||
|
applicable: applicable,
|
||||||
|
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint, err := endpointWithoutUserInfo(target.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err)
|
||||||
|
}
|
||||||
|
response, evidence, requestErr := p.client.GetWithAuthenticationEvidence(
|
||||||
|
ctx,
|
||||||
|
endpoint,
|
||||||
|
target.EffectiveSettings.TimeoutSeconds,
|
||||||
|
target.EffectiveSettings,
|
||||||
|
)
|
||||||
|
if response != nil && response.Body != nil {
|
||||||
|
defer response.Body.Close()
|
||||||
|
}
|
||||||
|
outcome := authenticationProbeOutcome{
|
||||||
|
supported: true,
|
||||||
|
applicable: applicable,
|
||||||
|
evidence: evidence,
|
||||||
|
}
|
||||||
|
if evidence.ResponseReceived || response != nil {
|
||||||
|
status := evidence.HTTPStatus
|
||||||
|
if status == nil && response != nil {
|
||||||
|
status = &response.StatusCode
|
||||||
|
}
|
||||||
|
if status == nil {
|
||||||
|
outcome.failure = domain.ExecutionFailureMalformedResponse
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
switch *status {
|
||||||
|
case http.StatusUnauthorized:
|
||||||
|
outcome.credentialsAccepted = boolPointer(false)
|
||||||
|
case http.StatusOK:
|
||||||
|
if response == nil || response.Body == nil {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
||||||
|
outcome.failure = domain.ExecutionFailureMalformedResponse
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body)
|
||||||
|
if bodyErr != nil {
|
||||||
|
outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, bodyErr))
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
if recognized {
|
||||||
|
outcome.credentialsAccepted = boolPointer(true)
|
||||||
|
} else {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
||||||
|
outcome.failure = domain.ExecutionFailureMalformedResponse
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
|
||||||
|
}
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if evidence.DispatchInvoked {
|
||||||
|
if requestErr != nil {
|
||||||
|
outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, requestErr))
|
||||||
|
} else {
|
||||||
|
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
|
||||||
|
}
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
if requestErr != nil {
|
||||||
|
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
|
||||||
|
}
|
||||||
|
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) {
|
||||||
|
contents, err := io.ReadAll(body)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var envelope map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
dataJSON, ok := envelope["data"]
|
||||||
|
if !ok || !isJSONObject(dataJSON) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var data map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
label, labelOK := data["label"]
|
||||||
|
freeTier, freeTierOK := data["is_free_tier"]
|
||||||
|
usage, usageOK := data["usage"]
|
||||||
|
if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var number any
|
||||||
|
usageDecoder := json.NewDecoder(bytes.NewReader(usage))
|
||||||
|
usageDecoder.UseNumber()
|
||||||
|
if usageDecoder.Decode(&number) != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
_, ok = number.(json.Number)
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONObject(value json.RawMessage) bool {
|
||||||
|
value = bytes.TrimSpace(value)
|
||||||
|
return len(value) > 0 && value[0] == '{'
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONString(value json.RawMessage) bool {
|
||||||
|
value = bytes.TrimSpace(value)
|
||||||
|
if len(value) == 0 || value[0] != '"' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var parsed string
|
||||||
|
return json.Unmarshal(value, &parsed) == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONBoolean(value json.RawMessage) bool {
|
||||||
|
value = bytes.TrimSpace(value)
|
||||||
|
return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false"))
|
||||||
|
}
|
||||||
253
internal/usecase/smoke_authentication_test.go
Normal file
253
internal/usecase/smoke_authentication_test.go
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
||||||
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
t.Fatal("authentication applicability unexpectedly dispatched")
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
backend domain.Backend
|
||||||
|
client bool
|
||||||
|
supported bool
|
||||||
|
applicable *bool
|
||||||
|
reason domain.SmokeTestReason
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "not configured",
|
||||||
|
backend: domain.Backend{},
|
||||||
|
reason: domain.SmokeTestReasonProbeNotConfigured,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "none works with injected client",
|
||||||
|
backend: backendWithAuthenticationKind("none"),
|
||||||
|
supported: true,
|
||||||
|
applicable: boolPointer(false),
|
||||||
|
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "HTTP adapter unavailable",
|
||||||
|
backend: backendWithAuthenticationKind("openrouter_key"),
|
||||||
|
applicable: boolPointer(true),
|
||||||
|
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
probe := authenticationProbe{}
|
||||||
|
if tc.client {
|
||||||
|
probe.client = client
|
||||||
|
}
|
||||||
|
outcome, err := probe.run(context.Background(), tc.backend, domain.SmokeTestTarget{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.supported != tc.supported || outcome.reason != tc.reason || outcome.evidence.DispatchInvoked {
|
||||||
|
t.Fatalf("outcome = %+v", outcome)
|
||||||
|
}
|
||||||
|
if !sameBool(outcome.applicable, tc.applicable) {
|
||||||
|
t.Fatalf("applicability = %v, want %v", outcome.applicable, tc.applicable)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) {
|
||||||
|
validResponse := `{"data":{"label":"member","is_free_tier":false,"usage":0,"limit":12,"extra":{"ignored":true}},"other":"ignored"}`
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
key string
|
||||||
|
wantAccepted *bool
|
||||||
|
wantFailure domain.ExecutionFailureKind
|
||||||
|
wantReason domain.SmokeTestReason
|
||||||
|
wantCredentials bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "recognized success",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: validResponse,
|
||||||
|
key: "synthetic-secret",
|
||||||
|
wantAccepted: boolPointer(true),
|
||||||
|
wantCredentials: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "anonymous unauthorized",
|
||||||
|
status: http.StatusUnauthorized,
|
||||||
|
wantAccepted: boolPointer(false),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "credential unauthorized",
|
||||||
|
status: http.StatusUnauthorized,
|
||||||
|
key: "synthetic-secret",
|
||||||
|
wantAccepted: boolPointer(false),
|
||||||
|
wantCredentials: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unrecognized success",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: `{"data":{"label":"member","is_free_tier":false,"usage":"0"}}`,
|
||||||
|
key: "synthetic-secret",
|
||||||
|
wantFailure: domain.ExecutionFailureMalformedResponse,
|
||||||
|
wantReason: domain.SmokeTestReasonAuthenticationResponseUnrecognized,
|
||||||
|
wantCredentials: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "other status",
|
||||||
|
status: http.StatusNotFound,
|
||||||
|
key: "synthetic-secret",
|
||||||
|
wantFailure: domain.ExecutionFailureProviderStatus,
|
||||||
|
wantCredentials: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var authorization string
|
||||||
|
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.Method != http.MethodGet {
|
||||||
|
t.Errorf("method = %q, want GET", request.Method)
|
||||||
|
}
|
||||||
|
authorization = request.Header.Get("Authorization")
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: tc.status,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: ioNopCloser(tc.body),
|
||||||
|
}, nil
|
||||||
|
})
|
||||||
|
backend := backendWithAuthenticationKind("openrouter_key")
|
||||||
|
settings := domain.ExecutionTarget{APIKey: tc.key, APIKeyRequired: true}
|
||||||
|
target := domain.SmokeTestTarget{
|
||||||
|
Endpoint: "https://endpoint-user:endpoint-pass@example.invalid/v1/key",
|
||||||
|
EffectiveSettings: settings,
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backend, target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.credentialsAccepted == nil && tc.wantAccepted != nil ||
|
||||||
|
outcome.credentialsAccepted != nil && (tc.wantAccepted == nil || *outcome.credentialsAccepted != *tc.wantAccepted) {
|
||||||
|
t.Fatalf("accepted = %v, want %v", outcome.credentialsAccepted, tc.wantAccepted)
|
||||||
|
}
|
||||||
|
if outcome.failure != tc.wantFailure || outcome.reason != tc.wantReason {
|
||||||
|
t.Fatalf("failure/reason = %v/%q", outcome.failure, outcome.reason)
|
||||||
|
}
|
||||||
|
if !outcome.supported || outcome.applicable == nil || !*outcome.applicable ||
|
||||||
|
!outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != tc.status {
|
||||||
|
t.Fatalf("response evidence = %+v", outcome)
|
||||||
|
}
|
||||||
|
if outcome.evidence.CredentialsSupplied != tc.wantCredentials ||
|
||||||
|
(authorization != "") != tc.wantCredentials {
|
||||||
|
t.Fatalf("credential evidence=%+v authorization=%q", outcome.evidence, authorization)
|
||||||
|
}
|
||||||
|
if strings.Contains(fmt.Sprintf("%+v", outcome), "synthetic-secret") ||
|
||||||
|
strings.Contains(outcome.evidence.Endpoint, "endpoint-pass") {
|
||||||
|
t.Fatalf("sensitive value escaped into outcome: %+v", outcome)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
|
||||||
|
for _, body := range []string{
|
||||||
|
`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`,
|
||||||
|
`{"data":[]}`,
|
||||||
|
`{"data":{"label":"member","is_free_tier":false,"usage":null}}`,
|
||||||
|
`{"data":{"label":"member","is_free_tier":0,"usage":1}}`,
|
||||||
|
`{"data":{"is_free_tier":false,"usage":1}}`,
|
||||||
|
`{"data":{"label":null,"is_free_tier":false,"usage":1}}`,
|
||||||
|
} {
|
||||||
|
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized {
|
||||||
|
t.Errorf("recognized invalid account response %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized {
|
||||||
|
t.Fatal("valid minimal identifying response was rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
|
||||||
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: interruptedReadCloser{err: context.DeadlineExceeded},
|
||||||
|
}, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(
|
||||||
|
context.Background(),
|
||||||
|
backendWithAuthenticationKind("openrouter_key"),
|
||||||
|
domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureDeadline ||
|
||||||
|
!outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != http.StatusOK {
|
||||||
|
t.Fatalf("interrupted response outcome = %+v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthenticationProbeReportsTransportFailureAndAllowsMissingRequiredKey(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_AUTH_REQUIRED_KEY", "")
|
||||||
|
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.Header.Get("Authorization") != "" {
|
||||||
|
t.Errorf("Authorization = %q, want absent", request.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
return nil, errors.New("opaque transport failure")
|
||||||
|
})
|
||||||
|
target := domain.SmokeTestTarget{
|
||||||
|
Endpoint: "https://example.invalid/key",
|
||||||
|
EffectiveSettings: domain.ExecutionTarget{
|
||||||
|
APIKeyEnv: "PROMPTKIT_AUTH_REQUIRED_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureTransport ||
|
||||||
|
!outcome.evidence.DispatchInvoked || !outcome.evidence.RequestObserved || outcome.evidence.CredentialsSupplied {
|
||||||
|
t.Fatalf("transport outcome = %+v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func backendWithAuthenticationKind(kind string) domain.Backend {
|
||||||
|
return domain.Backend{SmokeTests: &domain.BackendSmokeTestConfig{
|
||||||
|
Authentication: &domain.AuthenticationProbeConfig{Kind: &kind},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameBool(got, want *bool) bool {
|
||||||
|
if got == nil || want == nil {
|
||||||
|
return got == nil && want == nil
|
||||||
|
}
|
||||||
|
return *got == *want
|
||||||
|
}
|
||||||
|
|
||||||
|
type stringReadCloser struct{ *strings.Reader }
|
||||||
|
|
||||||
|
func ioNopCloser(value string) stringReadCloser {
|
||||||
|
return stringReadCloser{Reader: strings.NewReader(value)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (stringReadCloser) Close() error { return nil }
|
||||||
|
|
||||||
|
type interruptedReadCloser struct{ err error }
|
||||||
|
|
||||||
|
func (r interruptedReadCloser) Read([]byte) (int, error) { return 0, r.err }
|
||||||
|
func (interruptedReadCloser) Close() error { return nil }
|
||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
type smokeHTTPGetter interface {
|
type smokeHTTPGetter interface {
|
||||||
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
|
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
|
||||||
|
GetWithAuthenticationEvidence(context.Context, string, int, domain.ExecutionTarget) (*http.Response, llm.HTTPExecutionEvidence, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type reachabilityProbe struct {
|
type reachabilityProbe struct {
|
||||||
|
|||||||
Reference in New Issue
Block a user