Add OpenRouter authentication diagnostics

This commit is contained in:
2026-10-13 17:23:47 +00:00
parent 2e81c9e61b
commit edaaecc982
7 changed files with 551 additions and 4 deletions

View File

@@ -540,6 +540,14 @@ rejected, anonymous rejection, malformed/unrecognized 200, other status, and
transport failure. Add applicability/support tests without duplicating HTTP
client tests. Synthetic credentials must not appear in report-facing values.
**Completed: 2026-10-13.** Added metadata-driven authentication applicability
and an OpenRouter key probe with optional direct/environment credentials. It
recognizes the documented minimal 200 response, records 401 rejection, leaves
other outcomes undetermined as specified, and distinguishes interrupted body
reads from malformed JSON. Validation passed: `go test -race
./internal/usecase ./internal/llm`, `go vet ./...`, `go test ./...`, and
`git diff --check`.
## Stage 13: Build And Evaluate Private Inference Cases
Scope: private fixtures and validation collaborators in `internal/usecase`.

View File

@@ -6,6 +6,8 @@ import (
"net/http"
"sync"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// HTTPExecutionEvidence contains only facts observable at the configured
@@ -99,6 +101,27 @@ func (c *OpenAICompatibleClient) GetWithEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
) (*http.Response, HTTPExecutionEvidence, error) {
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "")
}
// GetWithAuthenticationEvidence performs a plain GET with credentials resolved
// from the selected target. Missing credentials are allowed for diagnostic
// authentication probes even when generation requires a key.
func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
target domain.ExecutionTarget,
) (*http.Response, HTTPExecutionEvidence, error) {
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target))
}
func (c *OpenAICompatibleClient) getWithEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
apiKey string,
) (*http.Response, HTTPExecutionEvidence, error) {
requestContext := ctx
var cancel context.CancelFunc
@@ -113,6 +136,9 @@ func (c *OpenAICompatibleClient) GetWithEvidence(
}
return nil, HTTPExecutionEvidence{}, err
}
if apiKey != "" {
request.Header.Set("Authorization", "Bearer "+apiKey)
}
response, err, evidence := doWithEvidence(c.httpClient, request)
if response == nil || response.Body == nil {

View File

@@ -12,6 +12,8 @@ import (
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
@@ -54,6 +56,69 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
}
}
func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T) {
tests := []struct {
name string
target domain.ExecutionTarget
environment string
wantAuth string
}{
{
name: "direct key takes precedence",
target: domain.ExecutionTarget{
APIKey: " direct-key ",
APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY",
},
environment: "environment-key",
wantAuth: "Bearer direct-key",
},
{
name: "environment key is used",
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY"},
environment: " environment-key ",
wantAuth: "Bearer environment-key",
},
{
name: "missing required key is optional here",
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", APIKeyRequired: true},
environment: " ",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
var authorization string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
authorization = request.Header.Get("Authorization")
return &http.Response{
StatusCode: http.StatusUnauthorized,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("")),
}, nil
})},
})
if err != nil {
t.Fatal(err)
}
response, evidence, err := client.GetWithAuthenticationEvidence(
context.Background(), "https://example.invalid/key", 0, tc.target,
)
if err != nil || response == nil {
t.Fatalf("response=%v error=%v", response, err)
}
response.Body.Close()
if authorization != tc.wantAuth || evidence.CredentialsSupplied != (tc.wantAuth != "") {
t.Fatalf("authorization=%q evidence=%+v", authorization, evidence)
}
if evidence.Endpoint != "https://example.invalid/key" || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
t.Fatalf("HTTP evidence = %+v", evidence)
}
})
}
}
func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) {
transportErr := errors.New("transport unavailable")
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {

View File

@@ -164,11 +164,8 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
}
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
}
apiKey := strings.TrimSpace(req.Target.APIKey)
apiKey := resolveAPIKey(req.Target)
envName := strings.TrimSpace(req.Target.APIKeyEnv)
if apiKey == "" && envName != "" {
apiKey = strings.TrimSpace(os.Getenv(envName))
}
if apiKey == "" && req.Target.APIKeyRequired {
if envName != "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
@@ -186,6 +183,16 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
}, nil
}
func resolveAPIKey(target domain.ExecutionTarget) string {
apiKey := strings.TrimSpace(target.APIKey)
if apiKey == "" {
if envName := strings.TrimSpace(target.APIKeyEnv); envName != "" {
apiKey = strings.TrimSpace(os.Getenv(envName))
}
}
return apiKey
}
// Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {

View File

@@ -0,0 +1,187 @@
package usecase
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
type authenticationProbe struct {
client smokeHTTPGetter
}
type authenticationProbeOutcome struct {
supported bool
applicable *bool
reason domain.SmokeTestReason
evidence llm.HTTPExecutionEvidence
credentialsAccepted *bool
failure domain.ExecutionFailureKind
}
func (p authenticationProbe) run(
ctx context.Context,
backend domain.Backend,
target domain.SmokeTestTarget,
) (authenticationProbeOutcome, error) {
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
}
authentication := backend.SmokeTests.Authentication
if authentication.Kind == nil {
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
}
switch *authentication.Kind {
case "none":
return authenticationProbeOutcome{
supported: true,
applicable: boolPointer(false),
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
}, nil
case "openrouter_key":
return p.runOpenRouterKey(ctx, target)
default:
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
}
}
func (p authenticationProbe) runOpenRouterKey(
ctx context.Context,
target domain.SmokeTestTarget,
) (authenticationProbeOutcome, error) {
applicable := boolPointer(true)
if p.client == nil {
return authenticationProbeOutcome{
applicable: applicable,
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
}, nil
}
endpoint, err := endpointWithoutUserInfo(target.Endpoint)
if err != nil {
return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err)
}
response, evidence, requestErr := p.client.GetWithAuthenticationEvidence(
ctx,
endpoint,
target.EffectiveSettings.TimeoutSeconds,
target.EffectiveSettings,
)
if response != nil && response.Body != nil {
defer response.Body.Close()
}
outcome := authenticationProbeOutcome{
supported: true,
applicable: applicable,
evidence: evidence,
}
if evidence.ResponseReceived || response != nil {
status := evidence.HTTPStatus
if status == nil && response != nil {
status = &response.StatusCode
}
if status == nil {
outcome.failure = domain.ExecutionFailureMalformedResponse
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
return outcome, nil
}
switch *status {
case http.StatusUnauthorized:
outcome.credentialsAccepted = boolPointer(false)
case http.StatusOK:
if response == nil || response.Body == nil {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
outcome.failure = domain.ExecutionFailureMalformedResponse
return outcome, nil
}
recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body)
if bodyErr != nil {
outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, bodyErr))
return outcome, nil
}
if recognized {
outcome.credentialsAccepted = boolPointer(true)
} else {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
outcome.failure = domain.ExecutionFailureMalformedResponse
}
default:
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
}
return outcome, nil
}
if evidence.DispatchInvoked {
if requestErr != nil {
outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, requestErr))
} else {
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
}
return outcome, nil
}
if requestErr != nil {
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
}
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
return outcome, nil
}
func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) {
contents, err := io.ReadAll(body)
if err != nil {
return false, err
}
var envelope map[string]json.RawMessage
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
return false, nil
}
dataJSON, ok := envelope["data"]
if !ok || !isJSONObject(dataJSON) {
return false, nil
}
var data map[string]json.RawMessage
if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil {
return false, nil
}
label, labelOK := data["label"]
freeTier, freeTierOK := data["is_free_tier"]
usage, usageOK := data["usage"]
if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK {
return false, nil
}
var number any
usageDecoder := json.NewDecoder(bytes.NewReader(usage))
usageDecoder.UseNumber()
if usageDecoder.Decode(&number) != nil {
return false, nil
}
_, ok = number.(json.Number)
return ok, nil
}
func isJSONObject(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
return len(value) > 0 && value[0] == '{'
}
func isJSONString(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
if len(value) == 0 || value[0] != '"' {
return false
}
var parsed string
return json.Unmarshal(value, &parsed) == nil
}
func isJSONBoolean(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false"))
}

View File

@@ -0,0 +1,253 @@
package usecase
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
t.Fatal("authentication applicability unexpectedly dispatched")
return nil, nil
})
for _, tc := range []struct {
name string
backend domain.Backend
client bool
supported bool
applicable *bool
reason domain.SmokeTestReason
}{
{
name: "not configured",
backend: domain.Backend{},
reason: domain.SmokeTestReasonProbeNotConfigured,
},
{
name: "none works with injected client",
backend: backendWithAuthenticationKind("none"),
supported: true,
applicable: boolPointer(false),
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
},
{
name: "HTTP adapter unavailable",
backend: backendWithAuthenticationKind("openrouter_key"),
applicable: boolPointer(true),
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
},
} {
t.Run(tc.name, func(t *testing.T) {
probe := authenticationProbe{}
if tc.client {
probe.client = client
}
outcome, err := probe.run(context.Background(), tc.backend, domain.SmokeTestTarget{})
if err != nil {
t.Fatal(err)
}
if outcome.supported != tc.supported || outcome.reason != tc.reason || outcome.evidence.DispatchInvoked {
t.Fatalf("outcome = %+v", outcome)
}
if !sameBool(outcome.applicable, tc.applicable) {
t.Fatalf("applicability = %v, want %v", outcome.applicable, tc.applicable)
}
})
}
}
func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) {
validResponse := `{"data":{"label":"member","is_free_tier":false,"usage":0,"limit":12,"extra":{"ignored":true}},"other":"ignored"}`
for _, tc := range []struct {
name string
status int
body string
key string
wantAccepted *bool
wantFailure domain.ExecutionFailureKind
wantReason domain.SmokeTestReason
wantCredentials bool
}{
{
name: "recognized success",
status: http.StatusOK,
body: validResponse,
key: "synthetic-secret",
wantAccepted: boolPointer(true),
wantCredentials: true,
},
{
name: "anonymous unauthorized",
status: http.StatusUnauthorized,
wantAccepted: boolPointer(false),
},
{
name: "credential unauthorized",
status: http.StatusUnauthorized,
key: "synthetic-secret",
wantAccepted: boolPointer(false),
wantCredentials: true,
},
{
name: "unrecognized success",
status: http.StatusOK,
body: `{"data":{"label":"member","is_free_tier":false,"usage":"0"}}`,
key: "synthetic-secret",
wantFailure: domain.ExecutionFailureMalformedResponse,
wantReason: domain.SmokeTestReasonAuthenticationResponseUnrecognized,
wantCredentials: true,
},
{
name: "other status",
status: http.StatusNotFound,
key: "synthetic-secret",
wantFailure: domain.ExecutionFailureProviderStatus,
wantCredentials: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
var authorization string
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
if request.Method != http.MethodGet {
t.Errorf("method = %q, want GET", request.Method)
}
authorization = request.Header.Get("Authorization")
return &http.Response{
StatusCode: tc.status,
Header: make(http.Header),
Body: ioNopCloser(tc.body),
}, nil
})
backend := backendWithAuthenticationKind("openrouter_key")
settings := domain.ExecutionTarget{APIKey: tc.key, APIKeyRequired: true}
target := domain.SmokeTestTarget{
Endpoint: "https://endpoint-user:endpoint-pass@example.invalid/v1/key",
EffectiveSettings: settings,
}
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backend, target)
if err != nil {
t.Fatal(err)
}
if outcome.credentialsAccepted == nil && tc.wantAccepted != nil ||
outcome.credentialsAccepted != nil && (tc.wantAccepted == nil || *outcome.credentialsAccepted != *tc.wantAccepted) {
t.Fatalf("accepted = %v, want %v", outcome.credentialsAccepted, tc.wantAccepted)
}
if outcome.failure != tc.wantFailure || outcome.reason != tc.wantReason {
t.Fatalf("failure/reason = %v/%q", outcome.failure, outcome.reason)
}
if !outcome.supported || outcome.applicable == nil || !*outcome.applicable ||
!outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != tc.status {
t.Fatalf("response evidence = %+v", outcome)
}
if outcome.evidence.CredentialsSupplied != tc.wantCredentials ||
(authorization != "") != tc.wantCredentials {
t.Fatalf("credential evidence=%+v authorization=%q", outcome.evidence, authorization)
}
if strings.Contains(fmt.Sprintf("%+v", outcome), "synthetic-secret") ||
strings.Contains(outcome.evidence.Endpoint, "endpoint-pass") {
t.Fatalf("sensitive value escaped into outcome: %+v", outcome)
}
})
}
}
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
for _, body := range []string{
`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`,
`{"data":[]}`,
`{"data":{"label":"member","is_free_tier":false,"usage":null}}`,
`{"data":{"label":"member","is_free_tier":0,"usage":1}}`,
`{"data":{"is_free_tier":false,"usage":1}}`,
`{"data":{"label":null,"is_free_tier":false,"usage":1}}`,
} {
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized {
t.Errorf("recognized invalid account response %s", body)
}
}
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized {
t.Fatal("valid minimal identifying response was rejected")
}
}
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: interruptedReadCloser{err: context.DeadlineExceeded},
}, nil
})
outcome, err := (authenticationProbe{client: client}).run(
context.Background(),
backendWithAuthenticationKind("openrouter_key"),
domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"},
)
if err != nil {
t.Fatal(err)
}
if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureDeadline ||
!outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != http.StatusOK {
t.Fatalf("interrupted response outcome = %+v", outcome)
}
}
func TestAuthenticationProbeReportsTransportFailureAndAllowsMissingRequiredKey(t *testing.T) {
t.Setenv("PROMPTKIT_AUTH_REQUIRED_KEY", "")
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
if request.Header.Get("Authorization") != "" {
t.Errorf("Authorization = %q, want absent", request.Header.Get("Authorization"))
}
return nil, errors.New("opaque transport failure")
})
target := domain.SmokeTestTarget{
Endpoint: "https://example.invalid/key",
EffectiveSettings: domain.ExecutionTarget{
APIKeyEnv: "PROMPTKIT_AUTH_REQUIRED_KEY",
APIKeyRequired: true,
},
}
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), target)
if err != nil {
t.Fatal(err)
}
if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureTransport ||
!outcome.evidence.DispatchInvoked || !outcome.evidence.RequestObserved || outcome.evidence.CredentialsSupplied {
t.Fatalf("transport outcome = %+v", outcome)
}
}
func backendWithAuthenticationKind(kind string) domain.Backend {
return domain.Backend{SmokeTests: &domain.BackendSmokeTestConfig{
Authentication: &domain.AuthenticationProbeConfig{Kind: &kind},
}}
}
func sameBool(got, want *bool) bool {
if got == nil || want == nil {
return got == nil && want == nil
}
return *got == *want
}
type stringReadCloser struct{ *strings.Reader }
func ioNopCloser(value string) stringReadCloser {
return stringReadCloser{Reader: strings.NewReader(value)}
}
func (stringReadCloser) Close() error { return nil }
type interruptedReadCloser struct{ err error }
func (r interruptedReadCloser) Read([]byte) (int, error) { return 0, r.err }
func (interruptedReadCloser) Close() error { return nil }

View File

@@ -13,6 +13,7 @@ import (
type smokeHTTPGetter interface {
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
GetWithAuthenticationEvidence(context.Context, string, int, domain.ExecutionTarget) (*http.Response, llm.HTTPExecutionEvidence, error)
}
type reachabilityProbe struct {