package promptkitadapter import ( "context" "errors" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "strings" "sync" "testing" "testing/fstest" "time" promptkit "gitea.maximumdirect.net/eric/promptkit" "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) type fakeClient struct { mu sync.Mutex response *promptkit.GenerateResponse err error outcomes []generationOutcome next int calls int requests []promptkit.GenerateRequest block bool started chan struct{} } type generationOutcome struct { response *promptkit.GenerateResponse err error } type recordingReader struct { ref promptkit.ArtifactRef } func (reader *recordingReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) { reader.ref = ref return &promptkit.Artifact{ Name: "data_package", ContentType: "application/yaml", Body: []byte(ref.Body), URI: ref.URI, Hash: "input-hash", }, nil } func (client *fakeClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.mu.Lock() client.calls++ client.requests = append(client.requests, request) block := client.block started := client.started response := client.response err := client.err if client.next < len(client.outcomes) { outcome := client.outcomes[client.next] client.next++ response, err = outcome.response, outcome.err } client.mu.Unlock() if started != nil { started <- struct{}{} } if block { <-ctx.Done() return nil, ctx.Err() } return response, err } func TestExecuteSupportsConcurrentCalls(t *testing.T) { client := &fakeClient{response: validResponse(), block: true, started: make(chan struct{}, 2)} adapter := newTestAdapter(t, client) ctx, cancel := context.WithCancel(context.Background()) defer cancel() executionErrors := make(chan error, 2) for range 2 { go func() { _, err := adapter.Execute(ctx, testExecuteRequest(), nil) executionErrors <- err }() } for range 2 { select { case <-client.started: case <-time.After(5 * time.Second): t.Fatal("timed out waiting for concurrent Promptkit calls") } } cancel() for range 2 { if err := <-executionErrors; promptexec.CategoryOf(err) != promptexec.Canceled { t.Fatalf("Execute() error/category = %v/%q", err, promptexec.CategoryOf(err)) } } if client.callCount() != 2 { t.Fatalf("provider calls = %d, want 2", client.callCount()) } } func (client *fakeClient) callCount() int { client.mu.Lock() defer client.mu.Unlock() return client.calls } func (client *fakeClient) request() promptkit.GenerateRequest { client.mu.Lock() defer client.mu.Unlock() return client.requests[0] } func (client *fakeClient) allRequests() []promptkit.GenerateRequest { client.mu.Lock() defer client.mu.Unlock() return append([]promptkit.GenerateRequest(nil), client.requests...) } func TestInspectPromptAndProfile(t *testing.T) { adapter := newTestAdapter(t, &fakeClient{}) inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "2.0.0") if err != nil { t.Fatalf("InspectPrompt() error = %v", err) } if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != "weather-balanced" { t.Fatalf("inspection = %#v", inspection) } if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" { t.Fatalf("inputs = %#v", inspection.Inputs) } if inspection.Output.Format != "json" || inspection.Output.ValidationMode != "json_schema" || inspection.Output.SchemaPath != "daily.generated_text.schema.json" { t.Fatalf("output = %#v", inspection.Output) } profile, err := adapter.InspectProfile(context.Background(), "test-profile") if err != nil { t.Fatalf("InspectProfile() error = %v", err) } if profile.ProfileID != "test-profile" || profile.BackendID != "" || profile.ModelName != "test-model" || profile.CredentialRequired { t.Fatalf("profile = %#v", profile) } if strings.Contains(fmt.Sprintf("%#v", profile), "https://profile.example") { t.Fatalf("profile leaks endpoint: %#v", profile) } builtin, err := adapter.InspectProfile(context.Background(), "gemini-flash-latest") if err != nil { t.Fatalf("InspectProfile(builtin) error = %v", err) } if builtin.ProfileID != "gemini-flash-latest" || builtin.ModelName == "" { t.Fatalf("builtin profile = %#v", builtin) } } func TestEmbeddedProfilesAreAvailableToProductionAndTestAdapters(t *testing.T) { adapter, err := New(Config{}) if err != nil { t.Fatalf("New() error = %v", err) } for _, want := range []struct { id string backend string model string }{ {"weather-light", "openrouter", "deepseek/deepseek-v4-flash"}, {"weather-balanced", "openrouter", "~google/gemini-flash-latest"}, {"weather-deep", "openrouter", "~anthropic/claude-sonnet-latest"}, } { t.Run(want.id, func(t *testing.T) { assertProfile(t, adapter, want.id, want.backend, want.model) }) } testAdapter, err := newAdapterForTest(Config{}, &fakeClient{}) if err != nil { t.Fatalf("newAdapterForTest() error = %v", err) } assertProfile(t, testAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash") } func TestConfiguredProfilesOverrideEmbeddedFallbacks(t *testing.T) { file := writeProfileFile(t, `id: weather-light endpoint: https://local-file.example/v1 model: file-light `) fileAdapter, err := New(Config{ProfileFile: file}) if err != nil { t.Fatalf("New(profile file) error = %v", err) } assertProfile(t, fileAdapter, "weather-light", "", "file-light") directory := testProfileDirectory(t, map[string]string{"profile.yml": `id: weather-light backend: local model: directory-light `}) directoryAdapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"}) if err != nil { t.Fatalf("New(profile directory) error = %v", err) } assertProfile(t, directoryAdapter, "weather-light", promptkit.BackendLocal, "directory-light") derived := writeProfileFile(t, `id: weather-light base_profile: gemini-flash-latest `) derivedAdapter, err := New(Config{ProfileFile: derived}) if err != nil { t.Fatalf("New(derived profile) error = %v", err) } assertProfile(t, derivedAdapter, "weather-light", "openrouter", "~google/gemini-flash-latest") } func TestConfiguredBaseProfileOverridesEmbeddedProfileTarget(t *testing.T) { directory := testProfileDirectory(t, map[string]string{ "deepseek.yml": `id: deepseek-4-flash backend: local model: shadowed-deepseek `, }) adapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"}) if err != nil { t.Fatalf("New() error = %v", err) } assertProfile(t, adapter, "weather-light", promptkit.BackendLocal, "shadowed-deepseek") } func TestMaintainedWeatherLightLocalProfileExampleInspectsOffline(t *testing.T) { adapter, err := New(Config{ProfileFile: filepath.Join("..", "..", "..", "examples", "weather-light-local-profile.yml")}) if err != nil { t.Fatalf("New() error = %v", err) } assertProfile(t, adapter, "weather-light", "", "weather-local") } func TestProfileResolutionFallsThroughOnlyWhenTheConfiguredIDIsAbsent(t *testing.T) { absentAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, map[string]string{"profile.yml": `id: other-profile backend: openrouter model: other-model `})}) if err != nil { t.Fatalf("New(absent profile) error = %v", err) } assertProfile(t, absentAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash") malformedAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, map[string]string{"profile.yml": `id: weather-light backend: openrouter `})}) if err != nil { t.Fatalf("New(malformed profile) error = %v", err) } if _, err := malformedAdapter.InspectProfile(context.Background(), "weather-light"); err == nil { t.Fatal("InspectProfile() error = nil, want malformed configured profile error") } } func TestProfileResolutionReturnsConfiguredInheritanceFailures(t *testing.T) { tests := []struct { name string profile string profiles map[string]string }{ { name: "missing base", profile: "missing-base", profiles: map[string]string{"missing.yml": `id: missing-base base_profile: unavailable `}, }, { name: "cyclic bases", profile: "first", profiles: map[string]string{ "first.yml": `id: first base_profile: second `, "second.yml": `id: second base_profile: first `, }, }, { name: "malformed base", profile: "child", profiles: map[string]string{ "child.yml": `id: child base_profile: malformed `, "malformed.yml": `id: malformed base_profile: [not-a-profile] `, }, }, { name: "incomplete target", profile: "incomplete", profiles: map[string]string{"incomplete.yml": `id: incomplete backend: openrouter `}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { adapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, test.profiles)}) if err != nil { t.Fatalf("New() error = %v", err) } if _, err := adapter.InspectProfile(context.Background(), test.profile); err == nil { t.Fatal("InspectProfile() error = nil, want configured inheritance error") } }) } } func TestRakestrawhomeBuiltInProfileInspectsOffline(t *testing.T) { adapter, err := New(Config{}) if err != nil { t.Fatalf("New() error = %v", err) } profile, err := adapter.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b") if err != nil { t.Fatalf("InspectProfile() error = %v", err) } if profile.ProfileID != "rakestrawhome-gemma-4-31b" || profile.BackendID != "rakestrawhome" || profile.ModelName == "" { t.Fatalf("profile = %#v", profile) } } func TestProfileResolutionPreservesBuiltInAndExplicitPrecedence(t *testing.T) { adapter, err := New(Config{}) if err != nil { t.Fatalf("New() error = %v", err) } builtin, err := adapter.InspectProfile(context.Background(), "gemini-flash-latest") if err != nil { t.Fatalf("InspectProfile(builtin) error = %v", err) } if builtin.ProfileID != "gemini-flash-latest" || builtin.BackendID != "openrouter" || builtin.ModelName == "" { t.Fatalf("builtin profile = %#v", builtin) } explicit, err := newAdapter(Config{}, promptkit.WithProfiles(promptkit.Profile{ ID: "weather-light", Endpoint: "https://explicit.example/v1", Model: "explicit-light", })) if err != nil { t.Fatalf("newAdapter(explicit profile) error = %v", err) } assertProfile(t, explicit, "weather-light", "", "explicit-light") } func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) { client := &fakeClient{response: validResponse()} adapter := newTestAdapter(t, client) request := testExecuteRequest() callbackCalls := 0 result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { callbackCalls++ if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" { t.Fatalf("preparation = %#v", preparation) } if debug != nil { t.Fatalf("debug = %#v, want nil", debug) } if client.callCount() != 0 { t.Fatal("provider called before preparation callback") } return nil }) if err != nil { t.Fatalf("Execute() error = %v", err) } if callbackCalls != 1 || client.callCount() != 1 { t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount()) } if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content { t.Fatalf("result = %#v", result) } if result.Debug != nil { t.Fatalf("debug = %#v, want nil", result.Debug) } providerRequest := client.request() if providerRequest.Target.Model != "test-model" || providerRequest.Target.Endpoint != "https://profile.example/v1" { t.Fatalf("provider target = %#v", providerRequest.Target) } if len(providerRequest.Prompt.Messages) == 0 || !strings.Contains(providerRequest.Prompt.Messages[2].Content, string(request.DataPackage)) { t.Fatalf("rendered messages do not contain exact data package: %#v", providerRequest.Prompt.Messages) } } func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "test-openrouter-key") client := &fakeClient{response: hourlyValidResponse()} adapter, err := newAdapter(Config{}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("newAdapter() error = %v", err) } request := promptexec.ExecuteRequest{ PromptID: "weather.hourly_generated_text", PromptVersion: "2.0.0", ProfileID: "weather-light", DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"), } var preparation promptexec.Preparation prepared := false result, err := adapter.Execute(context.Background(), request, func(value promptexec.Preparation, _ *promptexec.PreparationDebug) error { if client.callCount() != 0 { t.Fatal("provider was called before preparation completed") } preparation = value prepared = true return nil }) if err != nil { t.Fatalf("Execute() error = %v", err) } if !prepared || preparation.ProfileID != "weather-light" || preparation.BackendID != "openrouter" || preparation.ModelName != "deepseek/deepseek-v4-flash" { t.Fatalf("preparation = %#v", preparation) } if result == nil || result.ProfileID != "weather-light" || result.BackendID != "openrouter" || result.ModelName != "deepseek/deepseek-v4-flash" || result.Validation.Status != promptexec.ValidationPassed { t.Fatalf("execution = %#v", result) } if client.callCount() != 1 || client.request().Target.Model != "deepseek/deepseek-v4-flash" { t.Fatalf("provider calls/request = %d/%#v", client.callCount(), client.request()) } } func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) { client := &fakeClient{response: validResponse()} reader := &recordingReader{} adapter := newTestAdapterWithOptions(t, client, promptkit.WithArtifactReader(reader)) request := testExecuteRequest() if _, err := adapter.Execute(context.Background(), request, nil); err != nil { t.Fatalf("Execute() error = %v", err) } if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) { t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref) } } func TestExecuteCapturesSensitiveDebugOnlyWhenRequested(t *testing.T) { client := &fakeClient{response: validResponse()} adapter := newTestAdapter(t, client) request := testExecuteRequest() request.CaptureDebug = true var preparationDebug *promptexec.PreparationDebug result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { preparationDebug = debug if strings.Contains(fmt.Sprintf("%#v", preparation), "https://profile.example") || strings.Contains(fmt.Sprintf("%#v", preparation), string(request.DataPackage)) { t.Fatalf("safe preparation leaks sensitive content: %#v", preparation) } return nil }) if err != nil { t.Fatalf("Execute() error = %v", err) } if preparationDebug == nil || preparationDebug.Endpoint != "https://profile.example/v1" || len(preparationDebug.RenderedMessages) == 0 || len(preparationDebug.StructuredSchema) == 0 || len(preparationDebug.ParametersJSON) == 0 { t.Fatalf("preparation debug = %#v", preparationDebug) } if result.Debug == nil || string(result.Debug.RawOutput) != client.response.Content { t.Fatalf("execution debug = %#v", result.Debug) } } func TestMarshalDebugParametersOmitsProviderExtras(t *testing.T) { const marker = "private-debug-marker" parameters := string(marshalDebugParameters(promptkit.ExecutionTarget{ Temperature: 0.2, MaxTokens: 400, TopP: 0.9, TimeoutSeconds: 30, ServiceTier: "flex", ReasoningEffort: "high", ExtraParams: map[string]any{ "access-key": marker, "signature": marker, }, })) if strings.Contains(parameters, marker) || strings.Contains(parameters, "extra_params") { t.Fatalf("debug parameters leaked provider extras: %s", parameters) } for _, want := range []string{`"temperature":0.2`, `"max_tokens":400`, `"top_p":0.9`, `"timeout_seconds":30`, `"service_tier":"flex"`, `"reasoning_effort":"high"`} { if !strings.Contains(parameters, want) { t.Fatalf("debug parameters missing safe value %q: %s", want, parameters) } } } func TestExecuteCallbackFailurePreventsGeneration(t *testing.T) { client := &fakeClient{response: validResponse()} adapter := newTestAdapter(t, client) callbackError := errors.New("save preparation") result, err := adapter.Execute(context.Background(), testExecuteRequest(), func(promptexec.Preparation, *promptexec.PreparationDebug) error { return callbackError }) if result != nil || !errors.Is(err, callbackError) || client.callCount() != 0 { t.Fatalf("result/error/provider calls = %#v/%v/%d", result, err, client.callCount()) } } func TestExecuteReturnsCompletedValidationRejection(t *testing.T) { client := &fakeClient{response: &promptkit.GenerateResponse{Content: `{"summary":42}`, Usage: promptkit.TokenUsage{TotalTokens: 5}}} adapter := newTestAdapter(t, client) result, err := adapter.Execute(context.Background(), testExecuteRequest(), nil) if err != nil { t.Fatalf("Execute() error = %v", err) } if result == nil || result.Validation.Status != promptexec.ValidationFailed || len(result.Validation.Diagnostics) == 0 || string(result.RawOutput) != client.response.Content { t.Fatalf("result = %#v", result) } } func TestExecuteMapsCorrectiveGenerationResults(t *testing.T) { valid := `{"summary":"valid"}` invalid := `{"summary":42}` tests := []struct { name string outcomes []generationOutcome wantStatus promptexec.ValidationStatus wantRepairs int wantCalls int wantRaw string wantUsage promptexec.TokenUsage }{ { name: "first pass valid", outcomes: []generationOutcome{{response: generationResponse(valid, 2, 3, 5)}}, wantStatus: promptexec.ValidationPassed, wantRepairs: 0, wantCalls: 1, wantRaw: valid, wantUsage: promptexec.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5}, }, { name: "empty output repaired", outcomes: []generationOutcome{{response: generationResponse("", 2, 3, 5)}, {response: generationResponse(valid, 7, 11, 18)}}, wantStatus: promptexec.ValidationPassed, wantRepairs: 1, wantCalls: 2, wantRaw: valid, wantUsage: promptexec.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}, }, { name: "invalid output repaired", outcomes: []generationOutcome{{response: generationResponse(invalid, 2, 3, 5)}, {response: generationResponse(valid, 7, 11, 18)}}, wantStatus: promptexec.ValidationPassed, wantRepairs: 1, wantCalls: 2, wantRaw: valid, wantUsage: promptexec.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}, }, { name: "repair budget exhausted", outcomes: []generationOutcome{{response: generationResponse(invalid, 2, 3, 5)}, {response: generationResponse(invalid, 7, 11, 18)}}, wantStatus: promptexec.ValidationFailed, wantRepairs: 1, wantCalls: 2, wantRaw: invalid, wantUsage: promptexec.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { client := &fakeClient{outcomes: test.outcomes} adapter := newRepairAdapter(t, client, "https://repair.example/v1") var preparation promptexec.Preparation result, err := adapter.Execute(context.Background(), repairExecuteRequest(), func(value promptexec.Preparation, _ *promptexec.PreparationDebug) error { preparation = value return nil }) if err != nil { t.Fatalf("Execute() error = %v", err) } if preparation.Output.RepairAttempts != 1 || result == nil || result.Validation.Status != test.wantStatus || result.Validation.RepairAttempts != test.wantRepairs || string(result.RawOutput) != test.wantRaw || result.Usage != test.wantUsage { t.Fatalf("preparation/result = %#v/%#v", preparation, result) } requests := client.allRequests() if len(requests) != test.wantCalls { t.Fatalf("provider requests = %d, want %d", len(requests), test.wantCalls) } if test.wantCalls == 2 && !reflect.DeepEqual(requests[0].Target, requests[1].Target) { t.Fatalf("corrective target = %#v, want same prepared identity as %#v", requests[1].Target, requests[0].Target) } if result.ProfileID != preparation.ProfileID || result.BackendID != preparation.BackendID || result.ModelName != preparation.ModelName || result.PromptID != preparation.PromptID || result.PromptVersion != preparation.PromptVersion || result.PromptHash != preparation.PromptHash { t.Fatalf("prepared/result identity = %#v/%#v", preparation, result) } }) } } func TestExecuteMapsCorrectiveGenerationError(t *testing.T) { const providerBody = `{"error":{"code":"repair-code","type":"repair-type","message":"repair-message"}}` calls := 0 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { calls++ if calls == 1 { writer.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf(writer, `{"choices":[{"message":{"content":%q}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, `{"summary":42}`) return } writer.Header().Set("Content-Type", "application/json") writer.WriteHeader(http.StatusUnprocessableEntity) _, _ = writer.Write([]byte(providerBody)) })) defer server.Close() adapter := newRepairAdapter(t, nil, server.URL) result, err := adapter.Execute(context.Background(), repairExecuteRequest(), nil) if result != nil || err == nil || calls != 2 { t.Fatalf("result/error/calls = %#v/%v/%d", result, err, calls) } var generationError *promptexec.GenerationError if !errors.As(err, &generationError) || generationError.StatusCode() != http.StatusUnprocessableEntity || generationError.ProviderCode() != "repair-code" || generationError.ProviderType() != "repair-type" || generationError.ProviderMessage() != "repair-message" { t.Fatalf("generation error = %#v", err) } if strings.Contains(err.Error(), "repair-message") || !errors.Is(err, promptkit.ErrLLMGenerate) { t.Fatalf("generation error = %v", err) } } func TestExecuteDropsOversizedGeneratedOutput(t *testing.T) { client := &fakeClient{response: &promptkit.GenerateResponse{Content: strings.Repeat("x", generatedtext.MaxGeneratedTextBytes+1)}} adapter := newTestAdapter(t, client) request := testExecuteRequest() request.CaptureDebug = true result, err := adapter.Execute(context.Background(), request, nil) if err != nil { t.Fatalf("Execute() error = %v", err) } if result == nil || result.Validation.Status != promptexec.ValidationFailed || len(result.RawOutput) != 0 || result.Debug == nil || len(result.Debug.RawOutput) != 0 { t.Fatalf("execution = %#v", result) } if len(result.Validation.Diagnostics) != 1 || result.Validation.Diagnostics[0] != "generated output exceeds the configured size limit" { t.Fatalf("diagnostics = %#v", result.Validation.Diagnostics) } } func TestExecuteClassifiesOperationalFailures(t *testing.T) { tests := []struct { name string client *fakeClient context func() (context.Context, context.CancelFunc) category promptexec.ErrorCategory }{ { name: "generation", client: &fakeClient{err: errors.New("provider response body")}, context: func() (context.Context, context.CancelFunc) { return context.WithCancel(context.Background()) }, category: promptexec.Generation, }, { name: "canceled", client: &fakeClient{block: true}, context: func() (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) cancel() return ctx, func() {} }, category: promptexec.Canceled, }, { name: "deadline", client: &fakeClient{block: true}, context: func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Nanosecond) }, category: promptexec.DeadlineExceeded, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { adapter := newTestAdapter(t, test.client) ctx, cancel := test.context() defer cancel() result, err := adapter.Execute(ctx, testExecuteRequest(), nil) if result != nil || err == nil || promptexec.CategoryOf(err) != test.category { t.Fatalf("result/error/category = %#v/%v/%q, want %q", result, err, promptexec.CategoryOf(err), test.category) } if strings.Contains(err.Error(), "provider response body") { t.Fatalf("error leaks provider detail: %v", err) } }) } } func TestClassifyPromptkitErrors(t *testing.T) { tests := []struct { err error category promptexec.ErrorCategory }{ {promptkit.ErrInvalidConfig, promptexec.InvalidConfiguration}, {promptkit.ErrInvalidRequest, promptexec.InvalidRequest}, {promptkit.ErrPromptNotFound, promptexec.PromptNotFound}, {promptkit.ErrPromptLoad, promptexec.PromptLoad}, {promptkit.ErrProfileNotFound, promptexec.ProfileNotFound}, {promptkit.ErrProfileLoad, promptexec.ProfileLoad}, {promptkit.ErrAPIKeyEnvMissing, promptexec.MissingCredential}, {promptkit.ErrArtifactLoad, promptexec.ArtifactLoad}, {promptkit.ErrPromptRender, promptexec.PromptRender}, {promptkit.ErrLLMGenerate, promptexec.Generation}, {promptkit.ErrValidation, promptexec.OperationalValidation}, {&promptkit.CapacityError{BackendID: "local"}, promptexec.Capacity}, } for _, test := range tests { t.Run(string(test.category), func(t *testing.T) { got := classifyError(test.err) if promptexec.CategoryOf(got) != test.category { t.Fatalf("category = %q, want %q", promptexec.CategoryOf(got), test.category) } }) } } func TestNewValidatesConfiguration(t *testing.T) { if _, err := New(Config{ProfileDirectory: "profiles", ProfileFile: "profile.yml"}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("profile source error = %v", err) } if _, err := New(Config{LocalConcurrencyLimit: 1}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("local concurrency error = %v", err) } if _, err := New(Config{LocalEndpoint: "not a URL"}); promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("local endpoint error = %v", err) } } func TestLocalBackendAndOptionalCredentialSourceBehavior(t *testing.T) { t.Setenv("WEATHERREPORTER_TEST_MISSING_KEY", "") profiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: local-profile backend: local model: local-model `}) adapter, err := newAdapterForTest(Config{ ProfileDirectory: profiles, LocalEndpoint: "https://local.example/v1", LocalConcurrencyLimit: 1, }, &fakeClient{}) if err != nil { t.Fatalf("newAdapterForTest(local) error = %v", err) } profile, err := adapter.InspectProfile(context.Background(), "local-profile") if err != nil || profile.BackendID != promptkit.BackendLocal || profile.ModelName != "local-model" { t.Fatalf("local profile/error = %#v/%v", profile, err) } if got := classifyError(&promptkit.CapacityError{BackendID: promptkit.BackendLocal}); promptexec.CategoryOf(got) != promptexec.Capacity { t.Fatalf("capacity classification = %v", got) } credentialProfiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: credential-profile endpoint: https://profile.example/v1 model: test-model api_key_env: WEATHERREPORTER_TEST_MISSING_KEY `}) client := &fakeClient{response: validResponse()} credentialAdapter, err := newAdapterForTest(Config{ProfileDirectory: credentialProfiles}, client) if err != nil { t.Fatalf("newAdapterForTest(credential) error = %v", err) } credentialProfile, err := credentialAdapter.InspectProfile(context.Background(), "credential-profile") if err != nil || credentialProfile.CredentialRequired || credentialProfile.APIKeyEnv != "WEATHERREPORTER_TEST_MISSING_KEY" { t.Fatalf("credential profile/error = %#v/%v", credentialProfile, err) } request := testExecuteRequest() request.ProfileID = "credential-profile" result, err := credentialAdapter.Execute(context.Background(), request, nil) if err != nil || result == nil || client.callCount() != 1 { t.Fatalf("credential result/error/calls = %#v/%v/%d", result, err, client.callCount()) } } func newTestAdapter(t *testing.T, client promptkit.LLMClient) *Adapter { return newTestAdapterWithOptions(t, client) } func assertProfile(t *testing.T, adapter *Adapter, id string, backend string, model string) { t.Helper() profile, err := adapter.InspectProfile(context.Background(), id) if err != nil { t.Fatalf("InspectProfile(%q) error = %v", id, err) } if profile.ProfileID != id || profile.BackendID != backend || profile.ModelName != model { t.Fatalf("profile = %#v, want %q with backend/model %q/%q", profile, id, backend, model) } } func newTestAdapterWithOptions(t *testing.T, client promptkit.LLMClient, options ...promptkit.Option) *Adapter { t.Helper() profiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: test-profile endpoint: https://profile.example/v1 model: test-model temperature: 0.2 max_tokens: 300 top_p: 1 timeout_seconds: 30 `}) options = append(options, promptkit.WithLLMClient(client)) adapter, err := newAdapter(Config{ProfileDirectory: profiles, Timeout: time.Second}, options...) if err != nil { t.Fatalf("newAdapter() error = %v", err) } return adapter } func testProfileDirectory(t *testing.T, profiles map[string]string) string { t.Helper() directory := t.TempDir() for name, profile := range profiles { if err := os.WriteFile(filepath.Join(directory, name), []byte(profile), 0o600); err != nil { t.Fatalf("write profile: %v", err) } } return directory } func writeProfileFile(t *testing.T, profile string) string { t.Helper() path := filepath.Join(t.TempDir(), "profile.yml") if err := os.WriteFile(path, []byte(profile), 0o600); err != nil { t.Fatalf("write profile: %v", err) } return path } func testExecuteRequest() promptexec.ExecuteRequest { return promptexec.ExecuteRequest{ PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", ProfileID: "test-profile", DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"), } } func validResponse() *promptkit.GenerateResponse { return &promptkit.GenerateResponse{ Content: `{"summary":"A quiet day is expected.","forecast_discussion":["High pressure keeps conditions settled."],"precipitation_timing":""}`, Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20}, } } func generationResponse(content string, promptTokens int, completionTokens int, totalTokens int) *promptkit.GenerateResponse { return &promptkit.GenerateResponse{ Content: content, Usage: promptkit.TokenUsage{ PromptTokens: promptTokens, CompletionTokens: completionTokens, TotalTokens: totalTokens, }, } } func newRepairAdapter(t *testing.T, client promptkit.LLMClient, endpoint string) *Adapter { t.Helper() profiles := testProfileDirectory(t, map[string]string{"profile.yml": "id: repair-profile\nendpoint: " + endpoint + "\nmodel: repair-model\n"}) options := []promptkit.Option{ promptkit.WithPromptFS(fstest.MapFS{ "repair.yml": &fstest.MapFile{Data: []byte(`id: weather.repair version: "1.0.0" default_profile: repair-profile inputs: - name: data_package required: true content_type: application/yaml messages: - role: user content: "{{input \"data_package\"}}" output: format: json validation_mode: json_schema schema_path: repair.schema.json repair_attempts: 1 `)}}, "."), promptkit.WithSchemaFS(fstest.MapFS{ "repair.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"],"additionalProperties":false}`)}, }, "."), } if client != nil { options = append(options, promptkit.WithLLMClient(client)) } adapter, err := newAdapter(Config{ProfileDirectory: profiles}, options...) if err != nil { t.Fatalf("newAdapter() error = %v", err) } return adapter } func repairExecuteRequest() promptexec.ExecuteRequest { return promptexec.ExecuteRequest{ PromptID: "weather.repair", PromptVersion: "1.0.0", ProfileID: "repair-profile", DataPackage: []byte("report: repair\n"), } } func hourlyValidResponse() *promptkit.GenerateResponse { return &promptkit.GenerateResponse{ Content: `{"summary":"A quiet hour is expected.","forecast_discussion":"Conditions remain settled.","precipitation_timing":""}`, Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20}, } }