package llm import ( "context" "encoding/json" "errors" "io" "net/http" "os" "path/filepath" "reflect" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/promptkit" ) func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out struct { OK bool `json:"ok"` } resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ StageName: "test-stage", PromptID: "adapter.direct-session", PromptVersion: "v1", ProfileID: "explicit-profile", SessionID: " session-123 ", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"), }, Vars: map[string]any{"custom": "value"}, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if !out.OK { t.Fatalf("decoded output OK = false, want true") } if resp.Provider != "promptkit" || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" { t.Fatalf("response metadata = %#v", resp) } if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { t.Fatalf("usage = %#v, want mapped token counts", resp) } if resp.Debug == nil || resp.Debug.Prompt == nil { t.Fatalf("debug prompt = nil, want prepared prompt material") } if resp.Debug.Prompt.PromptID != "adapter.direct-session" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" || resp.Debug.Prompt.SelectedBackendID != "test-backend" || resp.Debug.Prompt.SessionID != "session-123" { t.Fatalf("debug prompt metadata = %#v, want prompt/profile/backend/session", resp.Debug.Prompt) } if resp.Debug.Prompt.EffectiveModelParams["backend_id"] != "test-backend" || resp.Debug.Prompt.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" { t.Fatalf("debug effective model params = %#v, want backend and reasoning", resp.Debug.Prompt.EffectiveModelParams) } if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) { t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages) } if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response) } if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 { t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage) } if resp.Debug.Response.EffectiveModelParams["backend_id"] != "test-backend" || resp.Debug.Response.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" { t.Fatalf("debug response effective model params = %#v, want backend and reasoning", resp.Debug.Response.EffectiveModelParams) } debugJSON, err := json.Marshal(resp.Debug) if err != nil { t.Fatalf("marshal debug material: %v", err) } if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") { t.Fatalf("debug material contains secret material: %s", debugJSON) } gotReq := fake.lastRequest() if gotReq.Prompt.SessionID != "session-123" { t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID) } if gotReq.Target.Model != "explicit-model" { t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model) } if gotReq.Target.BackendID != "test-backend" { t.Fatalf("backend id = %q, want test-backend", gotReq.Target.BackendID) } if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) || !strings.Contains(gotReq.Prompt.Messages[0].Content, "value") { t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages) } if gotReq.StructuredOutput == nil { t.Fatalf("structured output = nil, want JSON schema") } manifests := client.LLMProfileManifests() if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Provider != "promptkit" || manifests[0].Model != "explicit-model" || manifests[0].BackendID != "test-backend" || manifests[0].ReasoningEffort != "profile-reasoning" { t.Fatalf("profile manifests = %#v", manifests) } } func TestPromptKitClientRetainsSessionPromptVariable(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out map[string]any _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: " canonical-session ", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, Vars: map[string]any{ "custom": "value", "session_id": "caller-session", }, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } gotReq := fake.lastRequest() if gotReq.Prompt.SessionID != "canonical-session" { t.Fatalf("session id = %q, want canonical-session", gotReq.Prompt.SessionID) } if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, "Session: canonical-session") || strings.Contains(gotReq.Prompt.Messages[0].Content, "caller-session") { t.Fatalf("rendered messages = %#v, want canonical session compatibility variable", gotReq.Prompt.Messages) } } func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.direct-session", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, Vars: map[string]any{"custom": "value"}, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if got := fake.lastRequest().Prompt.SessionID; got != "" { t.Fatalf("session id = %q, want empty", got) } if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Prompt.SessionID != "" { t.Fatalf("debug prompt = %#v, want no effective session", resp.Debug) } } func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) { tests := []struct { name string override func() *string mutateAfterCreate bool want string }{ { name: "inherit", want: "profile-reasoning", }, { name: "replace", override: func() *string { value := "focused"; return &value }, want: "focused", }, { name: "clear", override: func() *string { value := ""; return &value }, want: "", }, { name: "defensive copy", override: func() *string { value := "original"; return &value }, mutateAfterCreate: true, want: "original", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} var override *string if tt.override != nil { override = tt.override() } client := newTestPromptKitClientWithReasoning(t, fake, override) if tt.mutateAfterCreate { *override = "mutated" } var out map[string]any _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.direct-session", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, Vars: map[string]any{"custom": "value"}, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want { t.Fatalf("reasoning effort = %q, want %q", got, tt.want) } }) } } func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) { t.Run("assets", func(t *testing.T) { registry := NewAssetRegistry() for _, root := range []string{"one", "two"} { if err := registry.RegisterPromptFS(fstest.MapFS{ root + "/prompt.yaml": {Data: []byte("id: duplicate")}, }, root); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) } } if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil || !strings.Contains(err.Error(), "duplicate asset path") { t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err) } }) t.Run("engine", func(t *testing.T) { if _, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")}, }); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") { t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err) } }) } func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { profilePath := filepath.Join(t.TempDir(), "profiles.yml") writeProfile := func(model string) { t.Helper() content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\n" if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { t.Fatal(err) } } fingerprintFor := func() CheckpointFingerprint { t.Helper() client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), ProfileFile: profilePath, }) if err != nil { t.Fatal(err) } values, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName { t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values) } return values[0] } writeProfile("model-one") first := fingerprintFor() writeProfile("model-two") second := fingerprintFor() if first == second { t.Fatalf("profile-source fingerprint = %#v for both profile models", first) } if strings.Contains(first.Value, profilePath) || strings.Contains(first.Value, "model-one") { t.Fatalf("profile-source fingerprint exposes source details: %#v", first) } client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)}) if err != nil { t.Fatal(err) } copy, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } copy[0].Value = "mutated" fresh, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } if fresh[0].Value == "mutated" { t.Fatal("LLMCheckpointFingerprints exposed mutable backing storage") } const wantBuiltinFingerprint = "sha256:5218b1dec48f5fdd46836826e0b25906c33efbdf943c5f18085b8d82467e0276" if fresh[0].Value != wantBuiltinFingerprint { t.Fatalf("built-in profile fingerprint = %q, want %q", fresh[0].Value, wantBuiltinFingerprint) } } func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out); err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if got := fake.lastRequest().Target.Model; got != "default-model" { t.Fatalf("model = %q, want prompt default profile model", got) } manifests := client.LLMProfileManifests() if len(manifests) != 1 || manifests[0].BackendID != "" || manifests[0].ReasoningEffort != "profile-reasoning" { t.Fatalf("endpoint-only profile manifests = %#v, want omitted backend and effective reasoning", manifests) } } func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) { recorder := NewLLMProfileRecorder() for _, profile := range []artifacts.LLMProfileManifest{ {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, {ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "}, {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, } { recorder.Record(profile) } want := []artifacts.LLMProfileManifest{ {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, } if got := recorder.Manifests(); !reflect.DeepEqual(got, want) { t.Fatalf("profile manifests = %#v, want %#v", got, want) } } func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") { t.Fatalf("CompleteStructured() error = %v, want validation failure", err) } if got := string(resp.Content); got != `{"bad":true}` { t.Fatalf("response content = %q, want raw failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` { t.Fatalf("debug response = %#v, want raw failed output", resp.Debug) } if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 { t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt) } } func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out []any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") { t.Fatalf("CompleteStructured() error = %v, want decode failure", err) } if got := string(resp.Content); got != `{"ok":true}` { t.Fatalf("response content = %q, want raw decode-failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug) } } func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil { t.Fatalf("CompleteStructured() error = nil, want provider error") } if errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("provider error = %v, must not be classified as invalid structured output", err) } if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) { t.Fatalf("error = %q, want operation context", err.Error()) } if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") { t.Fatalf("error = %q, want redacted bearer token", err.Error()) } if resp.Debug != nil { t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug) } } func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) { queueCapacity := 0 fake := &fakePromptKitLLM{ err: errors.New("provider failed with Bearer secret-token"), block: make(chan struct{}), } client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), EngineOptions: []promptkit.Option{ promptkit.WithBackend(promptkit.Backend{ ID: "limited-backend", Endpoint: "http://127.0.0.1:1/v1", ConcurrencyLimit: 1, QueueCapacity: &queueCapacity, }), promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "limited-profile", BackendID: "limited-backend", Model: "limited-model", })), promptkit.WithLLMClient(fake), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) } defer func() { select { case <-fake.block: default: close(fake.block) } }() request := contracts.StructuredCompletionRequest{ PromptID: "adapter.direct-session", ProfileID: "limited-profile", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, Vars: map[string]any{"custom": "value"}, } firstResult := make(chan error, 1) go func() { var out map[string]any _, callErr := client.CompleteStructured(context.Background(), request, &out) firstResult <- callErr }() waitForAtomicAtLeast(t, &fake.calls, 1) var out map[string]any response, capacityErr := client.CompleteStructured(context.Background(), request, &out) if len(response.Content) != 0 { t.Fatalf("capacity response = %#v, want empty", response) } if !errors.Is(capacityErr, contracts.ErrLLMCapacityExceeded) { t.Fatalf("capacity error = %v, want ErrLLMCapacityExceeded", capacityErr) } if errors.Is(capacityErr, contracts.ErrInvalidStructuredOutput) { t.Fatalf("capacity error = %v, must not be invalid structured output", capacityErr) } if errors.Is(capacityErr, promptkit.ErrCapacityExceeded) { t.Fatalf("capacity error exposes PromptKit sentinel: %v", capacityErr) } if !strings.Contains(capacityErr.Error(), `run PromptKit prompt "adapter.direct-session"`) || !strings.Contains(capacityErr.Error(), "backend capacity exceeded") { t.Fatalf("capacity error = %q, want prompt context and upstream diagnostic", capacityErr) } if calls := atomic.LoadInt32(&fake.calls); calls != 1 { t.Fatalf("provider calls after capacity rejection = %d, want 1", calls) } close(fake.block) firstErr := <-firstResult if firstErr == nil || strings.Contains(firstErr.Error(), "secret-token") || !strings.Contains(firstErr.Error(), "Bearer [REDACTED]") { t.Fatalf("admitted provider error = %v, want redacted diagnostic", firstErr) } if calls := atomic.LoadInt32(&fake.calls); calls != 1 { t.Fatalf("provider calls after release = %d, want no adapter retry", calls) } } func TestPromptKitClientContextCancellationIsRespected(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out map[string]any _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ PromptID: "adapter.test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want context canceled", err) } } func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) { var remaining time.Duration transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { deadline, ok := req.Context().Deadline() if !ok { t.Fatal("outbound request context has no deadline") } remaining = time.Until(deadline) return &http.Response{ StatusCode: http.StatusOK, Status: "200 OK", Header: make(http.Header), Body: io.NopCloser(strings.NewReader( `{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`, )), Request: req, }, nil }) const configuredTimeout = 2 * time.Second client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), Timeout: configuredTimeout, HTTPClient: &http.Client{Transport: transport}, EngineOptions: []promptkit.Option{ promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model", })), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v", err) } var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out); err != nil { t.Fatalf("CompleteStructured() error = %v", err) } if remaining < configuredTimeout-time.Second || remaining > configuredTimeout { t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout) } } func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true}) var out map[string]any _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want invalid structured output", err) } } func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) { fake := &fakePromptKitLLM{ content: `{"ok":true}`, block: make(chan struct{}), } client := newTestPromptKitClient(t, fake) scheduler, err := NewScheduler(1) if err != nil { t.Fatalf("NewScheduler() error = %v, want nil", err) } scheduled := NewScheduledClient(client, scheduler) var wg sync.WaitGroup for i := 0; i < 3; i++ { wg.Add(1) go func() { defer wg.Done() var out map[string]any _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if callErr != nil { t.Errorf("CompleteStructured() error = %v, want nil", callErr) } }() } waitForAtomicAtLeast(t, &fake.calls, 1) time.Sleep(20 * time.Millisecond) if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 { t.Fatalf("max in-flight calls = %d, want <= 1", got) } close(fake.block) wg.Wait() } func TestPromptKitClientValidatesRequest(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") { t.Fatalf("missing prompt id error = %v, want prompt_id validation", err) } if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") { t.Fatalf("nil output error = %v, want output validation", err) } } func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient { return newTestPromptKitClientWithReasoning(t, fake, nil) } func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient { t.Helper() registry := newTestPromptKitAssets(t) client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: registry, ReasoningEffort: reasoningEffort, EngineOptions: []promptkit.Option{ promptkit.WithBackend(promptkit.Backend{ ID: "test-backend", Endpoint: "http://127.0.0.1:1/v1", }), promptkit.WithProfiles( promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "default-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "default-model", ReasoningEffort: "profile-reasoning", }), promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "explicit-profile", BackendID: "test-backend", Model: "explicit-model", ReasoningEffort: "profile-reasoning", }), ), promptkit.WithLLMClient(fake), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) } return client } func newTestPromptKitAssets(t *testing.T) *AssetRegistry { t.Helper() registry := NewAssetRegistry() if err := registry.RegisterPromptFS(fstest.MapFS{ "adapter.test.yaml": {Data: []byte(`id: adapter.test version: "v1" default_profile: default-profile session_id: "{{ .session_id }}" inputs: - name: transcript required: true content_type: application/json messages: - role: user content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}" output: format: json validation_mode: json_schema schema_path: adapter.schema.json repair_attempts: 0 `)}, "adapter.direct-session.yaml": {Data: []byte(`id: adapter.direct-session version: "v1" default_profile: default-profile inputs: - name: transcript required: true content_type: application/json messages: - role: user content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}" output: format: json validation_mode: json_schema schema_path: adapter.schema.json repair_attempts: 0 `)}, }, "."); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) } if err := registry.RegisterSchemaFS(fstest.MapFS{ "adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)}, }, "."); err != nil { t.Fatalf("RegisterSchemaFS() error = %v", err) } return registry } type fakePromptKitLLM struct { content string allowEmpty bool err error block chan struct{} mu sync.Mutex last promptkit.GenerateRequest calls int32 inFlight int32 maxInFlight int32 } func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { f.mu.Lock() f.last = req f.mu.Unlock() atomic.AddInt32(&f.calls, 1) current := atomic.AddInt32(&f.inFlight, 1) for { seen := atomic.LoadInt32(&f.maxInFlight) if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) { break } } defer atomic.AddInt32(&f.inFlight, -1) if f.block != nil { select { case <-f.block: case <-ctx.Done(): return nil, ctx.Err() } } if f.err != nil { return nil, f.err } content := f.content if content == "" && !f.allowEmpty { content = `{"ok":true}` } if !f.allowEmpty && !json.Valid([]byte(content)) { return nil, errors.New("test fake must return JSON content") } return &promptkit.GenerateResponse{ Content: content, Usage: promptkit.TokenUsage{ PromptTokens: 11, CompletionTokens: 7, TotalTokens: 18, CachedTokens: 5, CacheWriteTokens: 3, }, }, nil } type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest { f.mu.Lock() defer f.mu.Unlock() return f.last }