Files
weatherreporter/internal/adapters/promptkit/adapter_test.go

627 lines
23 KiB
Go

package promptkitadapter
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
promptkit "gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
type fakeClient struct {
mu sync.Mutex
response *promptkit.GenerateResponse
err error
calls int
requests []promptkit.GenerateRequest
block bool
started chan struct{}
}
type recordingReader struct {
ref promptkit.ArtifactRef
}
func (reader *recordingReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
reader.ref = ref
return &promptkit.Artifact{
Name: "data_package",
ContentType: "application/yaml",
Body: []byte(ref.Body),
URI: ref.URI,
Hash: "input-hash",
}, nil
}
func (client *fakeClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
client.calls++
client.requests = append(client.requests, request)
block := client.block
started := client.started
response := client.response
err := client.err
client.mu.Unlock()
if started != nil {
started <- struct{}{}
}
if block {
<-ctx.Done()
return nil, ctx.Err()
}
return response, err
}
func TestExecuteSupportsConcurrentCalls(t *testing.T) {
client := &fakeClient{response: validResponse(), block: true, started: make(chan struct{}, 2)}
adapter := newTestAdapter(t, client)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
executionErrors := make(chan error, 2)
for range 2 {
go func() {
_, err := adapter.Execute(ctx, testExecuteRequest(), nil)
executionErrors <- err
}()
}
for range 2 {
select {
case <-client.started:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for concurrent Promptkit calls")
}
}
cancel()
for range 2 {
if err := <-executionErrors; promptexec.CategoryOf(err) != promptexec.Canceled {
t.Fatalf("Execute() error/category = %v/%q", err, promptexec.CategoryOf(err))
}
}
if client.callCount() != 2 {
t.Fatalf("provider calls = %d, want 2", client.callCount())
}
}
func (client *fakeClient) callCount() int {
client.mu.Lock()
defer client.mu.Unlock()
return client.calls
}
func (client *fakeClient) request() promptkit.GenerateRequest {
client.mu.Lock()
defer client.mu.Unlock()
return client.requests[0]
}
func TestInspectPromptAndProfile(t *testing.T) {
adapter := newTestAdapter(t, &fakeClient{})
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "2.0.0")
if err != nil {
t.Fatalf("InspectPrompt() error = %v", err)
}
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != "weather-balanced" {
t.Fatalf("inspection = %#v", inspection)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" {
t.Fatalf("inputs = %#v", inspection.Inputs)
}
if inspection.Output.Format != "json" || inspection.Output.ValidationMode != "json_schema" || inspection.Output.SchemaPath != "daily.generated_text.schema.json" {
t.Fatalf("output = %#v", inspection.Output)
}
profile, err := adapter.InspectProfile(context.Background(), "test-profile")
if err != nil {
t.Fatalf("InspectProfile() error = %v", err)
}
if profile.ProfileID != "test-profile" || profile.BackendID != "" || profile.ModelName != "test-model" || profile.CredentialRequired {
t.Fatalf("profile = %#v", profile)
}
if strings.Contains(fmt.Sprintf("%#v", profile), "https://profile.example") {
t.Fatalf("profile leaks endpoint: %#v", profile)
}
builtin, err := adapter.InspectProfile(context.Background(), "gemini-flash-latest")
if err != nil {
t.Fatalf("InspectProfile(builtin) error = %v", err)
}
if builtin.ProfileID != "gemini-flash-latest" || builtin.ModelName == "" {
t.Fatalf("builtin profile = %#v", builtin)
}
}
func TestEmbeddedProfilesAreAvailableToProductionAndTestAdapters(t *testing.T) {
adapter, err := New(Config{})
if err != nil {
t.Fatalf("New() error = %v", err)
}
for _, want := range []struct {
id string
backend string
model string
}{
{"weather-light", "openrouter", "deepseek/deepseek-v4-flash"},
{"weather-balanced", "openrouter", "~google/gemini-flash-latest"},
{"weather-deep", "openrouter", "~anthropic/claude-sonnet-latest"},
} {
t.Run(want.id, func(t *testing.T) {
assertProfile(t, adapter, want.id, want.backend, want.model)
})
}
testAdapter, err := newAdapterForTest(Config{}, &fakeClient{})
if err != nil {
t.Fatalf("newAdapterForTest() error = %v", err)
}
assertProfile(t, testAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash")
}
func TestConfiguredProfilesOverrideEmbeddedFallbacks(t *testing.T) {
file := writeProfileFile(t, `id: weather-light
endpoint: https://local-file.example/v1
model: file-light
`)
fileAdapter, err := New(Config{ProfileFile: file})
if err != nil {
t.Fatalf("New(profile file) error = %v", err)
}
assertProfile(t, fileAdapter, "weather-light", "", "file-light")
directory := testProfileDirectory(t, `id: weather-light
backend: local
model: directory-light
`)
directoryAdapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"})
if err != nil {
t.Fatalf("New(profile directory) error = %v", err)
}
assertProfile(t, directoryAdapter, "weather-light", promptkit.BackendLocal, "directory-light")
}
func TestMaintainedWeatherLightLocalProfileExampleInspectsOffline(t *testing.T) {
adapter, err := New(Config{ProfileFile: filepath.Join("..", "..", "..", "examples", "weather-light-local-profile.yml")})
if err != nil {
t.Fatalf("New() error = %v", err)
}
assertProfile(t, adapter, "weather-light", "", "weather-local")
}
func TestProfileResolutionFallsThroughOnlyWhenTheConfiguredIDIsAbsent(t *testing.T) {
absentAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: other-profile
backend: openrouter
model: other-model
`)})
if err != nil {
t.Fatalf("New(absent profile) error = %v", err)
}
assertProfile(t, absentAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash")
malformedAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: weather-light
backend: openrouter
`)})
if err != nil {
t.Fatalf("New(malformed profile) error = %v", err)
}
if _, err := malformedAdapter.InspectProfile(context.Background(), "weather-light"); err == nil {
t.Fatal("InspectProfile() error = nil, want malformed configured profile error")
}
}
func TestProfileResolutionPreservesBuiltInAndExplicitPrecedence(t *testing.T) {
adapter, err := New(Config{})
if err != nil {
t.Fatalf("New() error = %v", err)
}
builtin, err := adapter.InspectProfile(context.Background(), "gemini-flash-latest")
if err != nil {
t.Fatalf("InspectProfile(builtin) error = %v", err)
}
if builtin.ProfileID != "gemini-flash-latest" || builtin.BackendID != "openrouter" || builtin.ModelName == "" {
t.Fatalf("builtin profile = %#v", builtin)
}
explicit, err := newAdapter(Config{}, promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
Endpoint: "https://explicit.example/v1",
Model: "explicit-light",
}))
if err != nil {
t.Fatalf("newAdapter(explicit profile) error = %v", err)
}
assertProfile(t, explicit, "weather-light", "", "explicit-light")
}
func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)
request := testExecuteRequest()
callbackCalls := 0
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
callbackCalls++
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" {
t.Fatalf("preparation = %#v", preparation)
}
if debug != nil {
t.Fatalf("debug = %#v, want nil", debug)
}
if client.callCount() != 0 {
t.Fatal("provider called before preparation callback")
}
return nil
})
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if callbackCalls != 1 || client.callCount() != 1 {
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
}
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content {
t.Fatalf("result = %#v", result)
}
if result.Debug != nil {
t.Fatalf("debug = %#v, want nil", result.Debug)
}
providerRequest := client.request()
if providerRequest.Target.Model != "test-model" || providerRequest.Target.Endpoint != "https://profile.example/v1" {
t.Fatalf("provider target = %#v", providerRequest.Target)
}
if len(providerRequest.Prompt.Messages) == 0 || !strings.Contains(providerRequest.Prompt.Messages[2].Content, string(request.DataPackage)) {
t.Fatalf("rendered messages do not contain exact data package: %#v", providerRequest.Prompt.Messages)
}
}
func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-openrouter-key")
client := &fakeClient{response: hourlyValidResponse()}
adapter, err := newAdapter(Config{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("newAdapter() error = %v", err)
}
request := promptexec.ExecuteRequest{
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
}
var preparation promptexec.Preparation
prepared := false
result, err := adapter.Execute(context.Background(), request, func(value promptexec.Preparation, _ *promptexec.PreparationDebug) error {
if client.callCount() != 0 {
t.Fatal("provider was called before preparation completed")
}
preparation = value
prepared = true
return nil
})
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if !prepared || preparation.ProfileID != "weather-light" || preparation.BackendID != "openrouter" || preparation.ModelName != "deepseek/deepseek-v4-flash" {
t.Fatalf("preparation = %#v", preparation)
}
if result == nil || result.ProfileID != "weather-light" || result.BackendID != "openrouter" || result.ModelName != "deepseek/deepseek-v4-flash" || result.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution = %#v", result)
}
if client.callCount() != 1 || client.request().Target.Model != "deepseek/deepseek-v4-flash" {
t.Fatalf("provider calls/request = %d/%#v", client.callCount(), client.request())
}
}
func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
client := &fakeClient{response: validResponse()}
reader := &recordingReader{}
adapter := newTestAdapterWithOptions(t, client, promptkit.WithArtifactReader(reader))
request := testExecuteRequest()
if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
t.Fatalf("Execute() error = %v", err)
}
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) {
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
}
}
func TestExecuteCapturesSensitiveDebugOnlyWhenRequested(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)
request := testExecuteRequest()
request.CaptureDebug = true
var preparationDebug *promptexec.PreparationDebug
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
preparationDebug = debug
if strings.Contains(fmt.Sprintf("%#v", preparation), "https://profile.example") || strings.Contains(fmt.Sprintf("%#v", preparation), string(request.DataPackage)) {
t.Fatalf("safe preparation leaks sensitive content: %#v", preparation)
}
return nil
})
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if preparationDebug == nil || preparationDebug.Endpoint != "https://profile.example/v1" || len(preparationDebug.RenderedMessages) == 0 || len(preparationDebug.StructuredSchema) == 0 || len(preparationDebug.ParametersJSON) == 0 {
t.Fatalf("preparation debug = %#v", preparationDebug)
}
if result.Debug == nil || string(result.Debug.RawOutput) != client.response.Content {
t.Fatalf("execution debug = %#v", result.Debug)
}
}
func TestMarshalDebugParametersOmitsProviderExtras(t *testing.T) {
const marker = "private-debug-marker"
parameters := string(marshalDebugParameters(promptkit.ExecutionTarget{
Temperature: 0.2,
MaxTokens: 400,
TopP: 0.9,
TimeoutSeconds: 30,
ServiceTier: "flex",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"access-key": marker,
"signature": marker,
},
}))
if strings.Contains(parameters, marker) || strings.Contains(parameters, "extra_params") {
t.Fatalf("debug parameters leaked provider extras: %s", parameters)
}
for _, want := range []string{`"temperature":0.2`, `"max_tokens":400`, `"top_p":0.9`, `"timeout_seconds":30`, `"service_tier":"flex"`, `"reasoning_effort":"high"`} {
if !strings.Contains(parameters, want) {
t.Fatalf("debug parameters missing safe value %q: %s", want, parameters)
}
}
}
func TestExecuteCallbackFailurePreventsGeneration(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)
callbackError := errors.New("save preparation")
result, err := adapter.Execute(context.Background(), testExecuteRequest(), func(promptexec.Preparation, *promptexec.PreparationDebug) error {
return callbackError
})
if result != nil || !errors.Is(err, callbackError) || client.callCount() != 0 {
t.Fatalf("result/error/provider calls = %#v/%v/%d", result, err, client.callCount())
}
}
func TestExecuteReturnsCompletedValidationRejection(t *testing.T) {
client := &fakeClient{response: &promptkit.GenerateResponse{Content: `{"summary":42}`, Usage: promptkit.TokenUsage{TotalTokens: 5}}}
adapter := newTestAdapter(t, client)
result, err := adapter.Execute(context.Background(), testExecuteRequest(), nil)
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if result == nil || result.Validation.Status != promptexec.ValidationFailed || len(result.Validation.Diagnostics) == 0 || string(result.RawOutput) != client.response.Content {
t.Fatalf("result = %#v", result)
}
}
func TestExecuteDropsOversizedGeneratedOutput(t *testing.T) {
client := &fakeClient{response: &promptkit.GenerateResponse{Content: strings.Repeat("x", generatedtext.MaxGeneratedTextBytes+1)}}
adapter := newTestAdapter(t, client)
request := testExecuteRequest()
request.CaptureDebug = true
result, err := adapter.Execute(context.Background(), request, nil)
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if result == nil || result.Validation.Status != promptexec.ValidationFailed || len(result.RawOutput) != 0 || result.Debug == nil || len(result.Debug.RawOutput) != 0 {
t.Fatalf("execution = %#v", result)
}
if len(result.Validation.Diagnostics) != 1 || result.Validation.Diagnostics[0] != "generated output exceeds the configured size limit" {
t.Fatalf("diagnostics = %#v", result.Validation.Diagnostics)
}
}
func TestExecuteClassifiesOperationalFailures(t *testing.T) {
tests := []struct {
name string
client *fakeClient
context func() (context.Context, context.CancelFunc)
category promptexec.ErrorCategory
}{
{
name: "generation",
client: &fakeClient{err: errors.New("provider response body")},
context: func() (context.Context, context.CancelFunc) {
return context.WithCancel(context.Background())
},
category: promptexec.Generation,
},
{
name: "canceled",
client: &fakeClient{block: true},
context: func() (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
return ctx, func() {}
},
category: promptexec.Canceled,
},
{
name: "deadline",
client: &fakeClient{block: true},
context: func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), time.Nanosecond)
},
category: promptexec.DeadlineExceeded,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
adapter := newTestAdapter(t, test.client)
ctx, cancel := test.context()
defer cancel()
result, err := adapter.Execute(ctx, testExecuteRequest(), nil)
if result != nil || err == nil || promptexec.CategoryOf(err) != test.category {
t.Fatalf("result/error/category = %#v/%v/%q, want %q", result, err, promptexec.CategoryOf(err), test.category)
}
if strings.Contains(err.Error(), "provider response body") {
t.Fatalf("error leaks provider detail: %v", err)
}
})
}
}
func TestClassifyPromptkitErrors(t *testing.T) {
tests := []struct {
err error
category promptexec.ErrorCategory
}{
{promptkit.ErrInvalidConfig, promptexec.InvalidConfiguration},
{promptkit.ErrInvalidRequest, promptexec.InvalidRequest},
{promptkit.ErrPromptNotFound, promptexec.PromptNotFound},
{promptkit.ErrPromptLoad, promptexec.PromptLoad},
{promptkit.ErrProfileNotFound, promptexec.ProfileNotFound},
{promptkit.ErrProfileLoad, promptexec.ProfileLoad},
{promptkit.ErrAPIKeyEnvMissing, promptexec.MissingCredential},
{promptkit.ErrArtifactLoad, promptexec.ArtifactLoad},
{promptkit.ErrPromptRender, promptexec.PromptRender},
{promptkit.ErrLLMGenerate, promptexec.Generation},
{promptkit.ErrValidation, promptexec.OperationalValidation},
{&promptkit.CapacityError{BackendID: "local"}, promptexec.Capacity},
}
for _, test := range tests {
t.Run(string(test.category), func(t *testing.T) {
got := classifyError(test.err)
if promptexec.CategoryOf(got) != test.category {
t.Fatalf("category = %q, want %q", promptexec.CategoryOf(got), test.category)
}
})
}
}
func TestNewValidatesConfiguration(t *testing.T) {
if _, err := New(Config{ProfileDirectory: "profiles", ProfileFile: "profile.yml"}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("profile source error = %v", err)
}
if _, err := New(Config{LocalConcurrencyLimit: 1}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("local concurrency error = %v", err)
}
if _, err := New(Config{LocalEndpoint: "not a URL"}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("local endpoint error = %v", err)
}
}
func TestLocalBackendAndMissingCredentialBehavior(t *testing.T) {
t.Setenv("WEATHERREPORTER_TEST_MISSING_KEY", "")
profiles := testProfileDirectory(t, `id: local-profile
backend: local
model: local-model
`)
adapter, err := newAdapterForTest(Config{
ProfileDirectory: profiles,
LocalEndpoint: "https://local.example/v1",
LocalConcurrencyLimit: 1,
}, &fakeClient{})
if err != nil {
t.Fatalf("newAdapterForTest(local) error = %v", err)
}
profile, err := adapter.InspectProfile(context.Background(), "local-profile")
if err != nil || profile.BackendID != promptkit.BackendLocal || profile.ModelName != "local-model" {
t.Fatalf("local profile/error = %#v/%v", profile, err)
}
if got := classifyError(&promptkit.CapacityError{BackendID: promptkit.BackendLocal}); promptexec.CategoryOf(got) != promptexec.Capacity {
t.Fatalf("capacity classification = %v", got)
}
credentialProfiles := testProfileDirectory(t, `id: credential-profile
endpoint: https://profile.example/v1
model: test-model
api_key_env: WEATHERREPORTER_TEST_MISSING_KEY
`)
client := &fakeClient{response: validResponse()}
credentialAdapter, err := newAdapterForTest(Config{ProfileDirectory: credentialProfiles}, client)
if err != nil {
t.Fatalf("newAdapterForTest(credential) error = %v", err)
}
credentialProfile, err := credentialAdapter.InspectProfile(context.Background(), "credential-profile")
if err != nil || credentialProfile.CredentialRequired || credentialProfile.APIKeyEnv != "WEATHERREPORTER_TEST_MISSING_KEY" {
t.Fatalf("credential profile/error = %#v/%v", credentialProfile, err)
}
request := testExecuteRequest()
request.ProfileID = "credential-profile"
result, err := credentialAdapter.Execute(context.Background(), request, nil)
if result != nil || promptexec.CategoryOf(err) != promptexec.MissingCredential || client.callCount() != 0 {
t.Fatalf("credential result/category/calls = %#v/%q/%d", result, promptexec.CategoryOf(err), client.callCount())
}
}
func newTestAdapter(t *testing.T, client promptkit.LLMClient) *Adapter {
return newTestAdapterWithOptions(t, client)
}
func assertProfile(t *testing.T, adapter *Adapter, id string, backend string, model string) {
t.Helper()
profile, err := adapter.InspectProfile(context.Background(), id)
if err != nil {
t.Fatalf("InspectProfile(%q) error = %v", id, err)
}
if profile.ProfileID != id || profile.BackendID != backend || profile.ModelName != model {
t.Fatalf("profile = %#v, want %q with backend/model %q/%q", profile, id, backend, model)
}
}
func newTestAdapterWithOptions(t *testing.T, client promptkit.LLMClient, options ...promptkit.Option) *Adapter {
t.Helper()
profiles := testProfileDirectory(t, `id: test-profile
endpoint: https://profile.example/v1
model: test-model
temperature: 0.2
max_tokens: 300
top_p: 1
timeout_seconds: 30
`)
options = append(options, promptkit.WithLLMClient(client))
adapter, err := newAdapter(Config{ProfileDirectory: profiles, Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("newAdapter() error = %v", err)
}
return adapter
}
func testProfileDirectory(t *testing.T, profile string) string {
t.Helper()
profiles := t.TempDir()
if err := os.WriteFile(filepath.Join(profiles, "profile.yml"), []byte(profile), 0o600); err != nil {
t.Fatalf("write profile: %v", err)
}
return profiles
}
func writeProfileFile(t *testing.T, profile string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "profile.yml")
if err := os.WriteFile(path, []byte(profile), 0o600); err != nil {
t.Fatalf("write profile: %v", err)
}
return path
}
func testExecuteRequest() promptexec.ExecuteRequest {
return promptexec.ExecuteRequest{
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
}
}
func validResponse() *promptkit.GenerateResponse {
return &promptkit.GenerateResponse{
Content: `{"summary":"A quiet day is expected.","forecast_discussion":["High pressure keeps conditions settled."],"precipitation_timing":""}`,
Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20},
}
}
func hourlyValidResponse() *promptkit.GenerateResponse {
return &promptkit.GenerateResponse{
Content: `{"summary":"A quiet hour is expected.","forecast_discussion":"Conditions remain settled.","precipitation_timing":""}`,
Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20},
}
}