diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 344ca9c..76efa11 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -540,6 +540,14 @@ rejected, anonymous rejection, malformed/unrecognized 200, other status, and transport failure. Add applicability/support tests without duplicating HTTP client tests. Synthetic credentials must not appear in report-facing values. +**Completed: 2026-10-13.** Added metadata-driven authentication applicability +and an OpenRouter key probe with optional direct/environment credentials. It +recognizes the documented minimal 200 response, records 401 rejection, leaves +other outcomes undetermined as specified, and distinguishes interrupted body +reads from malformed JSON. Validation passed: `go test -race +./internal/usecase ./internal/llm`, `go vet ./...`, `go test ./...`, and +`git diff --check`. + ## Stage 13: Build And Evaluate Private Inference Cases Scope: private fixtures and validation collaborators in `internal/usecase`. diff --git a/internal/llm/http_evidence.go b/internal/llm/http_evidence.go index 205a820..b2d7b46 100644 --- a/internal/llm/http_evidence.go +++ b/internal/llm/http_evidence.go @@ -6,6 +6,8 @@ import ( "net/http" "sync" "time" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) // HTTPExecutionEvidence contains only facts observable at the configured @@ -99,6 +101,27 @@ func (c *OpenAICompatibleClient) GetWithEvidence( ctx context.Context, endpoint string, timeoutSeconds int, +) (*http.Response, HTTPExecutionEvidence, error) { + return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "") +} + +// GetWithAuthenticationEvidence performs a plain GET with credentials resolved +// from the selected target. Missing credentials are allowed for diagnostic +// authentication probes even when generation requires a key. +func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence( + ctx context.Context, + endpoint string, + timeoutSeconds int, + target domain.ExecutionTarget, +) (*http.Response, HTTPExecutionEvidence, error) { + return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target)) +} + +func (c *OpenAICompatibleClient) getWithEvidence( + ctx context.Context, + endpoint string, + timeoutSeconds int, + apiKey string, ) (*http.Response, HTTPExecutionEvidence, error) { requestContext := ctx var cancel context.CancelFunc @@ -113,6 +136,9 @@ func (c *OpenAICompatibleClient) GetWithEvidence( } return nil, HTTPExecutionEvidence{}, err } + if apiKey != "" { + request.Header.Set("Authorization", "Bearer "+apiKey) + } response, err, evidence := doWithEvidence(c.httpClient, request) if response == nil || response.Body == nil { diff --git a/internal/llm/http_evidence_test.go b/internal/llm/http_evidence_test.go index 7423006..26cb7c8 100644 --- a/internal/llm/http_evidence_test.go +++ b/internal/llm/http_evidence_test.go @@ -12,6 +12,8 @@ import ( "sync" "testing" "time" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) { @@ -54,6 +56,69 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) { } } +func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T) { + tests := []struct { + name string + target domain.ExecutionTarget + environment string + wantAuth string + }{ + { + name: "direct key takes precedence", + target: domain.ExecutionTarget{ + APIKey: " direct-key ", + APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", + }, + environment: "environment-key", + wantAuth: "Bearer direct-key", + }, + { + name: "environment key is used", + target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY"}, + environment: " environment-key ", + wantAuth: "Bearer environment-key", + }, + { + name: "missing required key is optional here", + target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", APIKeyRequired: true}, + environment: " ", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment) + var authorization string + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + authorization = request.Header.Get("Authorization") + return &http.Response{ + StatusCode: http.StatusUnauthorized, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("")), + }, nil + })}, + }) + if err != nil { + t.Fatal(err) + } + + response, evidence, err := client.GetWithAuthenticationEvidence( + context.Background(), "https://example.invalid/key", 0, tc.target, + ) + if err != nil || response == nil { + t.Fatalf("response=%v error=%v", response, err) + } + response.Body.Close() + if authorization != tc.wantAuth || evidence.CredentialsSupplied != (tc.wantAuth != "") { + t.Fatalf("authorization=%q evidence=%+v", authorization, evidence) + } + if evidence.Endpoint != "https://example.invalid/key" || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized { + t.Fatalf("HTTP evidence = %+v", evidence) + } + }) + } +} + func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) { transportErr := errors.New("transport unavailable") client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index 14e38a6..f82f8dc 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -164,11 +164,8 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) ( } return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) } - apiKey := strings.TrimSpace(req.Target.APIKey) + apiKey := resolveAPIKey(req.Target) envName := strings.TrimSpace(req.Target.APIKeyEnv) - if apiKey == "" && envName != "" { - apiKey = strings.TrimSpace(os.Getenv(envName)) - } if apiKey == "" && req.Target.APIKeyRequired { if envName != "" { return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName) @@ -186,6 +183,16 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) ( }, nil } +func resolveAPIKey(target domain.ExecutionTarget) string { + apiKey := strings.TrimSpace(target.APIKey) + if apiKey == "" { + if envName := strings.TrimSpace(target.APIKeyEnv); envName != "" { + apiKey = strings.TrimSpace(os.Getenv(envName)) + } + } + return apiKey +} + // Execute dispatches one attempt. The returned response is non-nil only when // the successful envelope contains a usable first choice. func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { diff --git a/internal/usecase/smoke_authentication.go b/internal/usecase/smoke_authentication.go new file mode 100644 index 0000000..6920923 --- /dev/null +++ b/internal/usecase/smoke_authentication.go @@ -0,0 +1,187 @@ +package usecase + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/generation" + "gitea.maximumdirect.net/eric/promptkit/internal/llm" +) + +type authenticationProbe struct { + client smokeHTTPGetter +} + +type authenticationProbeOutcome struct { + supported bool + applicable *bool + reason domain.SmokeTestReason + evidence llm.HTTPExecutionEvidence + credentialsAccepted *bool + failure domain.ExecutionFailureKind +} + +func (p authenticationProbe) run( + ctx context.Context, + backend domain.Backend, + target domain.SmokeTestTarget, +) (authenticationProbeOutcome, error) { + if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil { + return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil + } + authentication := backend.SmokeTests.Authentication + if authentication.Kind == nil { + return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest) + } + switch *authentication.Kind { + case "none": + return authenticationProbeOutcome{ + supported: true, + applicable: boolPointer(false), + reason: domain.SmokeTestReasonAuthenticationNotApplicable, + }, nil + case "openrouter_key": + return p.runOpenRouterKey(ctx, target) + default: + return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind) + } +} + +func (p authenticationProbe) runOpenRouterKey( + ctx context.Context, + target domain.SmokeTestTarget, +) (authenticationProbeOutcome, error) { + applicable := boolPointer(true) + if p.client == nil { + return authenticationProbeOutcome{ + applicable: applicable, + reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable, + }, nil + } + + endpoint, err := endpointWithoutUserInfo(target.Endpoint) + if err != nil { + return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err) + } + response, evidence, requestErr := p.client.GetWithAuthenticationEvidence( + ctx, + endpoint, + target.EffectiveSettings.TimeoutSeconds, + target.EffectiveSettings, + ) + if response != nil && response.Body != nil { + defer response.Body.Close() + } + outcome := authenticationProbeOutcome{ + supported: true, + applicable: applicable, + evidence: evidence, + } + if evidence.ResponseReceived || response != nil { + status := evidence.HTTPStatus + if status == nil && response != nil { + status = &response.StatusCode + } + if status == nil { + outcome.failure = domain.ExecutionFailureMalformedResponse + outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized + return outcome, nil + } + switch *status { + case http.StatusUnauthorized: + outcome.credentialsAccepted = boolPointer(false) + case http.StatusOK: + if response == nil || response.Body == nil { + outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized + outcome.failure = domain.ExecutionFailureMalformedResponse + return outcome, nil + } + recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body) + if bodyErr != nil { + outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, bodyErr)) + return outcome, nil + } + if recognized { + outcome.credentialsAccepted = boolPointer(true) + } else { + outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized + outcome.failure = domain.ExecutionFailureMalformedResponse + } + default: + outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus) + } + return outcome, nil + } + + if evidence.DispatchInvoked { + if requestErr != nil { + outcome.failure = generation.ClassifyFailure(fmt.Errorf("%w: %w", llm.ErrTransport, requestErr)) + } else { + outcome.failure = generation.ClassifyFailure(llm.ErrTransport) + } + return outcome, nil + } + if requestErr != nil { + return outcome, fmt.Errorf("execute authentication request: %w", requestErr) + } + outcome.failure = generation.ClassifyFailure(llm.ErrTransport) + return outcome, nil +} + +func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) { + contents, err := io.ReadAll(body) + if err != nil { + return false, err + } + var envelope map[string]json.RawMessage + if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil { + return false, nil + } + + dataJSON, ok := envelope["data"] + if !ok || !isJSONObject(dataJSON) { + return false, nil + } + var data map[string]json.RawMessage + if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil { + return false, nil + } + label, labelOK := data["label"] + freeTier, freeTierOK := data["is_free_tier"] + usage, usageOK := data["usage"] + if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK { + return false, nil + } + var number any + usageDecoder := json.NewDecoder(bytes.NewReader(usage)) + usageDecoder.UseNumber() + if usageDecoder.Decode(&number) != nil { + return false, nil + } + _, ok = number.(json.Number) + return ok, nil +} + +func isJSONObject(value json.RawMessage) bool { + value = bytes.TrimSpace(value) + return len(value) > 0 && value[0] == '{' +} + +func isJSONString(value json.RawMessage) bool { + value = bytes.TrimSpace(value) + if len(value) == 0 || value[0] != '"' { + return false + } + var parsed string + return json.Unmarshal(value, &parsed) == nil +} + +func isJSONBoolean(value json.RawMessage) bool { + value = bytes.TrimSpace(value) + return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false")) +} diff --git a/internal/usecase/smoke_authentication_test.go b/internal/usecase/smoke_authentication_test.go new file mode 100644 index 0000000..fed0a8b --- /dev/null +++ b/internal/usecase/smoke_authentication_test.go @@ -0,0 +1,253 @@ +package usecase + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) { + client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { + t.Fatal("authentication applicability unexpectedly dispatched") + return nil, nil + }) + + for _, tc := range []struct { + name string + backend domain.Backend + client bool + supported bool + applicable *bool + reason domain.SmokeTestReason + }{ + { + name: "not configured", + backend: domain.Backend{}, + reason: domain.SmokeTestReasonProbeNotConfigured, + }, + { + name: "none works with injected client", + backend: backendWithAuthenticationKind("none"), + supported: true, + applicable: boolPointer(false), + reason: domain.SmokeTestReasonAuthenticationNotApplicable, + }, + { + name: "HTTP adapter unavailable", + backend: backendWithAuthenticationKind("openrouter_key"), + applicable: boolPointer(true), + reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable, + }, + } { + t.Run(tc.name, func(t *testing.T) { + probe := authenticationProbe{} + if tc.client { + probe.client = client + } + outcome, err := probe.run(context.Background(), tc.backend, domain.SmokeTestTarget{}) + if err != nil { + t.Fatal(err) + } + if outcome.supported != tc.supported || outcome.reason != tc.reason || outcome.evidence.DispatchInvoked { + t.Fatalf("outcome = %+v", outcome) + } + if !sameBool(outcome.applicable, tc.applicable) { + t.Fatalf("applicability = %v, want %v", outcome.applicable, tc.applicable) + } + }) + } +} + +func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) { + validResponse := `{"data":{"label":"member","is_free_tier":false,"usage":0,"limit":12,"extra":{"ignored":true}},"other":"ignored"}` + for _, tc := range []struct { + name string + status int + body string + key string + wantAccepted *bool + wantFailure domain.ExecutionFailureKind + wantReason domain.SmokeTestReason + wantCredentials bool + }{ + { + name: "recognized success", + status: http.StatusOK, + body: validResponse, + key: "synthetic-secret", + wantAccepted: boolPointer(true), + wantCredentials: true, + }, + { + name: "anonymous unauthorized", + status: http.StatusUnauthorized, + wantAccepted: boolPointer(false), + }, + { + name: "credential unauthorized", + status: http.StatusUnauthorized, + key: "synthetic-secret", + wantAccepted: boolPointer(false), + wantCredentials: true, + }, + { + name: "unrecognized success", + status: http.StatusOK, + body: `{"data":{"label":"member","is_free_tier":false,"usage":"0"}}`, + key: "synthetic-secret", + wantFailure: domain.ExecutionFailureMalformedResponse, + wantReason: domain.SmokeTestReasonAuthenticationResponseUnrecognized, + wantCredentials: true, + }, + { + name: "other status", + status: http.StatusNotFound, + key: "synthetic-secret", + wantFailure: domain.ExecutionFailureProviderStatus, + wantCredentials: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var authorization string + client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodGet { + t.Errorf("method = %q, want GET", request.Method) + } + authorization = request.Header.Get("Authorization") + return &http.Response{ + StatusCode: tc.status, + Header: make(http.Header), + Body: ioNopCloser(tc.body), + }, nil + }) + backend := backendWithAuthenticationKind("openrouter_key") + settings := domain.ExecutionTarget{APIKey: tc.key, APIKeyRequired: true} + target := domain.SmokeTestTarget{ + Endpoint: "https://endpoint-user:endpoint-pass@example.invalid/v1/key", + EffectiveSettings: settings, + } + + outcome, err := (authenticationProbe{client: client}).run(context.Background(), backend, target) + if err != nil { + t.Fatal(err) + } + if outcome.credentialsAccepted == nil && tc.wantAccepted != nil || + outcome.credentialsAccepted != nil && (tc.wantAccepted == nil || *outcome.credentialsAccepted != *tc.wantAccepted) { + t.Fatalf("accepted = %v, want %v", outcome.credentialsAccepted, tc.wantAccepted) + } + if outcome.failure != tc.wantFailure || outcome.reason != tc.wantReason { + t.Fatalf("failure/reason = %v/%q", outcome.failure, outcome.reason) + } + if !outcome.supported || outcome.applicable == nil || !*outcome.applicable || + !outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != tc.status { + t.Fatalf("response evidence = %+v", outcome) + } + if outcome.evidence.CredentialsSupplied != tc.wantCredentials || + (authorization != "") != tc.wantCredentials { + t.Fatalf("credential evidence=%+v authorization=%q", outcome.evidence, authorization) + } + if strings.Contains(fmt.Sprintf("%+v", outcome), "synthetic-secret") || + strings.Contains(outcome.evidence.Endpoint, "endpoint-pass") { + t.Fatalf("sensitive value escaped into outcome: %+v", outcome) + } + }) + } +} + +func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) { + for _, body := range []string{ + `{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`, + `{"data":[]}`, + `{"data":{"label":"member","is_free_tier":false,"usage":null}}`, + `{"data":{"label":"member","is_free_tier":0,"usage":1}}`, + `{"data":{"is_free_tier":false,"usage":1}}`, + `{"data":{"label":null,"is_free_tier":false,"usage":1}}`, + } { + if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized { + t.Errorf("recognized invalid account response %s", body) + } + } + if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized { + t.Fatal("valid minimal identifying response was rejected") + } +} + +func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) { + client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: interruptedReadCloser{err: context.DeadlineExceeded}, + }, nil + }) + + outcome, err := (authenticationProbe{client: client}).run( + context.Background(), + backendWithAuthenticationKind("openrouter_key"), + domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"}, + ) + if err != nil { + t.Fatal(err) + } + if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureDeadline || + !outcome.evidence.ResponseReceived || outcome.evidence.HTTPStatus == nil || *outcome.evidence.HTTPStatus != http.StatusOK { + t.Fatalf("interrupted response outcome = %+v", outcome) + } +} + +func TestAuthenticationProbeReportsTransportFailureAndAllowsMissingRequiredKey(t *testing.T) { + t.Setenv("PROMPTKIT_AUTH_REQUIRED_KEY", "") + client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) { + if request.Header.Get("Authorization") != "" { + t.Errorf("Authorization = %q, want absent", request.Header.Get("Authorization")) + } + return nil, errors.New("opaque transport failure") + }) + target := domain.SmokeTestTarget{ + Endpoint: "https://example.invalid/key", + EffectiveSettings: domain.ExecutionTarget{ + APIKeyEnv: "PROMPTKIT_AUTH_REQUIRED_KEY", + APIKeyRequired: true, + }, + } + + outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), target) + if err != nil { + t.Fatal(err) + } + if outcome.credentialsAccepted != nil || outcome.failure != domain.ExecutionFailureTransport || + !outcome.evidence.DispatchInvoked || !outcome.evidence.RequestObserved || outcome.evidence.CredentialsSupplied { + t.Fatalf("transport outcome = %+v", outcome) + } +} + +func backendWithAuthenticationKind(kind string) domain.Backend { + return domain.Backend{SmokeTests: &domain.BackendSmokeTestConfig{ + Authentication: &domain.AuthenticationProbeConfig{Kind: &kind}, + }} +} + +func sameBool(got, want *bool) bool { + if got == nil || want == nil { + return got == nil && want == nil + } + return *got == *want +} + +type stringReadCloser struct{ *strings.Reader } + +func ioNopCloser(value string) stringReadCloser { + return stringReadCloser{Reader: strings.NewReader(value)} +} + +func (stringReadCloser) Close() error { return nil } + +type interruptedReadCloser struct{ err error } + +func (r interruptedReadCloser) Read([]byte) (int, error) { return 0, r.err } +func (interruptedReadCloser) Close() error { return nil } diff --git a/internal/usecase/smoke_reachability.go b/internal/usecase/smoke_reachability.go index 9774a40..ffddf70 100644 --- a/internal/usecase/smoke_reachability.go +++ b/internal/usecase/smoke_reachability.go @@ -13,6 +13,7 @@ import ( type smokeHTTPGetter interface { GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error) + GetWithAuthenticationEvidence(context.Context, string, int, domain.ExecutionTarget) (*http.Response, llm.HTTPExecutionEvidence, error) } type reachabilityProbe struct {