958 lines
34 KiB
Go
958 lines
34 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/fstest"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
|
client := newTestPromptKitClient(t, fake)
|
|
|
|
var out struct {
|
|
OK bool `json:"ok"`
|
|
}
|
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
StageName: "test-stage",
|
|
PromptID: "adapter.direct-session",
|
|
PromptVersion: "v1",
|
|
ProfileID: "explicit-profile",
|
|
SessionID: " session-123 ",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"),
|
|
},
|
|
Vars: map[string]any{"custom": "value"},
|
|
}, &out)
|
|
if err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
if !out.OK {
|
|
t.Fatalf("decoded output OK = false, want true")
|
|
}
|
|
if resp.Provider != "promptkit" || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
|
|
t.Fatalf("response metadata = %#v", resp)
|
|
}
|
|
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
|
t.Fatalf("usage = %#v, want mapped token counts", resp)
|
|
}
|
|
if resp.Debug == nil || resp.Debug.Prompt == nil {
|
|
t.Fatalf("debug prompt = nil, want prepared prompt material")
|
|
}
|
|
if resp.Debug.Prompt.PromptID != "adapter.direct-session" ||
|
|
resp.Debug.Prompt.SelectedProfileID != "explicit-profile" ||
|
|
resp.Debug.Prompt.SelectedBackendID != "test-backend" ||
|
|
resp.Debug.Prompt.SessionID != "session-123" {
|
|
t.Fatalf("debug prompt metadata = %#v, want prompt/profile/backend/session", resp.Debug.Prompt)
|
|
}
|
|
if resp.Debug.Prompt.EffectiveModelParams["backend_id"] != "test-backend" ||
|
|
resp.Debug.Prompt.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
|
|
t.Fatalf("debug effective model params = %#v, want backend and reasoning", resp.Debug.Prompt.EffectiveModelParams)
|
|
}
|
|
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
|
|
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
|
|
}
|
|
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
|
|
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
|
|
}
|
|
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
|
|
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
|
|
}
|
|
if resp.Debug.Response.EffectiveModelParams["backend_id"] != "test-backend" ||
|
|
resp.Debug.Response.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
|
|
t.Fatalf("debug response effective model params = %#v, want backend and reasoning", resp.Debug.Response.EffectiveModelParams)
|
|
}
|
|
debugJSON, err := json.Marshal(resp.Debug)
|
|
if err != nil {
|
|
t.Fatalf("marshal debug material: %v", err)
|
|
}
|
|
if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") {
|
|
t.Fatalf("debug material contains secret material: %s", debugJSON)
|
|
}
|
|
gotReq := fake.lastRequest()
|
|
if gotReq.Prompt.SessionID != "session-123" {
|
|
t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
|
|
}
|
|
if gotReq.Target.Model != "explicit-model" {
|
|
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
|
|
}
|
|
if gotReq.Target.BackendID != "test-backend" {
|
|
t.Fatalf("backend id = %q, want test-backend", gotReq.Target.BackendID)
|
|
}
|
|
if len(gotReq.Prompt.Messages) != 1 ||
|
|
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
|
|
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
|
|
t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages)
|
|
}
|
|
if gotReq.StructuredOutput == nil {
|
|
t.Fatalf("structured output = nil, want JSON schema")
|
|
}
|
|
manifests := client.LLMProfileManifests()
|
|
if len(manifests) != 1 ||
|
|
manifests[0].ID != "explicit-profile" ||
|
|
manifests[0].Provider != "promptkit" ||
|
|
manifests[0].Model != "explicit-model" ||
|
|
manifests[0].BackendID != "test-backend" ||
|
|
manifests[0].ReasoningEffort != "profile-reasoning" {
|
|
t.Fatalf("profile manifests = %#v", manifests)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientRetainsSessionPromptVariable(t *testing.T) {
|
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
|
client := newTestPromptKitClient(t, fake)
|
|
|
|
var out map[string]any
|
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: " canonical-session ",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
Vars: map[string]any{
|
|
"custom": "value",
|
|
"session_id": "caller-session",
|
|
},
|
|
}, &out)
|
|
if err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
gotReq := fake.lastRequest()
|
|
if gotReq.Prompt.SessionID != "canonical-session" {
|
|
t.Fatalf("session id = %q, want canonical-session", gotReq.Prompt.SessionID)
|
|
}
|
|
if len(gotReq.Prompt.Messages) != 1 ||
|
|
!strings.Contains(gotReq.Prompt.Messages[0].Content, "Session: canonical-session") ||
|
|
strings.Contains(gotReq.Prompt.Messages[0].Content, "caller-session") {
|
|
t.Fatalf("rendered messages = %#v, want canonical session compatibility variable", gotReq.Prompt.Messages)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) {
|
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
|
client := newTestPromptKitClient(t, fake)
|
|
|
|
var out map[string]any
|
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.direct-session",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
Vars: map[string]any{"custom": "value"},
|
|
}, &out)
|
|
if err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
if got := fake.lastRequest().Prompt.SessionID; got != "" {
|
|
t.Fatalf("session id = %q, want empty", got)
|
|
}
|
|
if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Prompt.SessionID != "" {
|
|
t.Fatalf("debug prompt = %#v, want no effective session", resp.Debug)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
override func() *string
|
|
mutateAfterCreate bool
|
|
want string
|
|
}{
|
|
{
|
|
name: "inherit",
|
|
want: "profile-reasoning",
|
|
},
|
|
{
|
|
name: "replace",
|
|
override: func() *string { value := "focused"; return &value },
|
|
want: "focused",
|
|
},
|
|
{
|
|
name: "clear",
|
|
override: func() *string { value := ""; return &value },
|
|
want: "",
|
|
},
|
|
{
|
|
name: "defensive copy",
|
|
override: func() *string { value := "original"; return &value },
|
|
mutateAfterCreate: true,
|
|
want: "original",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
|
var override *string
|
|
if tt.override != nil {
|
|
override = tt.override()
|
|
}
|
|
client := newTestPromptKitClientWithReasoning(t, fake, override)
|
|
if tt.mutateAfterCreate {
|
|
*override = "mutated"
|
|
}
|
|
|
|
var out map[string]any
|
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.direct-session",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
Vars: map[string]any{"custom": "value"},
|
|
}, &out)
|
|
if err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want {
|
|
t.Fatalf("reasoning effort = %q, want %q", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
|
t.Run("assets", func(t *testing.T) {
|
|
registry := NewAssetRegistry()
|
|
for _, root := range []string{"one", "two"} {
|
|
if err := registry.RegisterPromptFS(fstest.MapFS{
|
|
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
|
|
}, root); err != nil {
|
|
t.Fatalf("RegisterPromptFS() error = %v", err)
|
|
}
|
|
}
|
|
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
|
|
!strings.Contains(err.Error(), "duplicate asset path") {
|
|
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
|
|
}
|
|
})
|
|
|
|
t.Run("engine", func(t *testing.T) {
|
|
if _, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
|
|
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
|
|
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
|
writeProfile := func(model string) {
|
|
t.Helper()
|
|
content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\n"
|
|
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
fingerprintFor := func() CheckpointFingerprint {
|
|
t.Helper()
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
ProfileFile: profilePath,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
values, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName {
|
|
t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values)
|
|
}
|
|
return values[0]
|
|
}
|
|
|
|
writeProfile("model-one")
|
|
first := fingerprintFor()
|
|
writeProfile("model-two")
|
|
second := fingerprintFor()
|
|
if first == second {
|
|
t.Fatalf("profile-source fingerprint = %#v for both profile models", first)
|
|
}
|
|
if strings.Contains(first.Value, profilePath) || strings.Contains(first.Value, "model-one") {
|
|
t.Fatalf("profile-source fingerprint exposes source details: %#v", first)
|
|
}
|
|
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
copy, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
copy[0].Value = "mutated"
|
|
fresh, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fresh[0].Value == "mutated" {
|
|
t.Fatal("LLMCheckpointFingerprints exposed mutable backing storage")
|
|
}
|
|
const wantBuiltinFingerprint = "sha256:5218b1dec48f5fdd46836826e0b25906c33efbdf943c5f18085b8d82467e0276"
|
|
if fresh[0].Value != wantBuiltinFingerprint {
|
|
t.Fatalf("built-in profile fingerprint = %q, want %q", fresh[0].Value, wantBuiltinFingerprint)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
|
const (
|
|
firstEndpoint = "http://localhost:8000/v1"
|
|
secondEndpoint = "https://inference.example.test/v1"
|
|
)
|
|
fingerprintsFor := func(endpoint string, concurrencyLimit int) []CheckpointFingerprint {
|
|
t.Helper()
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
LocalBackend: &PromptKitLocalBackendConfig{
|
|
Endpoint: endpoint,
|
|
ConcurrencyLimit: concurrencyLimit,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
values, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return values
|
|
}
|
|
|
|
baseline := fingerprintsFor(firstEndpoint, 0)
|
|
if len(baseline) != 2 ||
|
|
baseline[0].Name != promptKitProfileFingerprintName ||
|
|
baseline[1].Name != promptKitLocalBackendFingerprintName {
|
|
t.Fatalf("checkpoint fingerprints = %#v, want profile source then local backend target", baseline)
|
|
}
|
|
endpointChanged := fingerprintsFor(secondEndpoint, 0)
|
|
if baseline[0] != endpointChanged[0] || baseline[1] == endpointChanged[1] {
|
|
t.Fatalf("endpoint fingerprints = %#v and %#v, want only local target to change", baseline, endpointChanged)
|
|
}
|
|
concurrencyChanged := fingerprintsFor(firstEndpoint, 4)
|
|
if !reflect.DeepEqual(baseline, concurrencyChanged) {
|
|
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
|
|
}
|
|
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
|
|
for _, fingerprint := range values {
|
|
if strings.Contains(fingerprint.Value, firstEndpoint) ||
|
|
strings.Contains(fingerprint.Value, secondEndpoint) {
|
|
t.Fatalf("checkpoint fingerprint exposes endpoint: %#v", fingerprint)
|
|
}
|
|
}
|
|
}
|
|
|
|
localBackend := &PromptKitLocalBackendConfig{
|
|
Endpoint: " " + firstEndpoint + " ",
|
|
ConcurrencyLimit: 0,
|
|
}
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
LocalBackend: localBackend,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
localBackend.Endpoint = secondEndpoint
|
|
copy, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(copy, baseline) {
|
|
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
|
|
}
|
|
copy[0].Value = "mutated-profile"
|
|
copy[1].Value = "mutated-target"
|
|
fresh, err := client.LLMCheckpointFingerprints()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(fresh, baseline) {
|
|
t.Fatalf("fingerprints after returned-slice mutation = %#v, want %#v", fresh, baseline)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
|
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
|
client := newTestPromptKitClient(t, fake)
|
|
|
|
var out map[string]any
|
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out); err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
if got := fake.lastRequest().Target.Model; got != "default-model" {
|
|
t.Fatalf("model = %q, want prompt default profile model", got)
|
|
}
|
|
manifests := client.LLMProfileManifests()
|
|
if len(manifests) != 1 || manifests[0].BackendID != "" || manifests[0].ReasoningEffort != "profile-reasoning" {
|
|
t.Fatalf("endpoint-only profile manifests = %#v, want omitted backend and effective reasoning", manifests)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) {
|
|
var providerCalls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{
|
|
"choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}],
|
|
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
|
|
}`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
|
if err := os.WriteFile(profilePath, []byte(`id: local-profile
|
|
backend: local
|
|
model: local-model
|
|
`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
localBackend := &PromptKitLocalBackendConfig{
|
|
Endpoint: " " + server.URL + "/v1 ",
|
|
ConcurrencyLimit: 2,
|
|
}
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
ProfileFile: profilePath,
|
|
LocalBackend: localBackend,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
|
}
|
|
localBackend.Endpoint = "http://127.0.0.1:1/v1"
|
|
|
|
request := contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.direct-session",
|
|
ProfileID: "local-profile",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
Vars: map[string]any{"custom": "value"},
|
|
}
|
|
var out map[string]any
|
|
response, err := client.CompleteStructured(context.Background(), request, &out)
|
|
if err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
|
}
|
|
if providerCalls.Load() != 1 {
|
|
t.Fatalf("provider calls = %d, want 1", providerCalls.Load())
|
|
}
|
|
if response.ProfileID != "local-profile" || response.Model != "local-model" {
|
|
t.Fatalf("response metadata = %#v", response)
|
|
}
|
|
if response.Debug == nil || response.Debug.Prompt == nil ||
|
|
response.Debug.Prompt.SelectedBackendID != promptkit.BackendLocal {
|
|
t.Fatalf("response debug prompt = %#v, want local backend", response.Debug)
|
|
}
|
|
manifests := client.LLMProfileManifests()
|
|
if len(manifests) != 1 || manifests[0].BackendID != promptkit.BackendLocal {
|
|
t.Fatalf("profile manifests = %#v, want local backend", manifests)
|
|
}
|
|
|
|
clientWithoutRegistration, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
ProfileFile: profilePath,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewPromptKitClient() without local registration error = %v, want nil", err)
|
|
}
|
|
_, err = clientWithoutRegistration.CompleteStructured(context.Background(), request, &out)
|
|
if err == nil ||
|
|
!strings.Contains(err.Error(), "prepare PromptKit prompt") ||
|
|
!strings.Contains(err.Error(), promptkit.BackendLocal) {
|
|
t.Fatalf("CompleteStructured() without registration error = %v, want preparation failure with local backend context", err)
|
|
}
|
|
if providerCalls.Load() != 1 {
|
|
t.Fatalf("provider calls after missing-registration failure = %d, want 1", providerCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
|
|
recorder := NewLLMProfileRecorder()
|
|
for _, profile := range []artifacts.LLMProfileManifest{
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
|
|
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
|
|
} {
|
|
recorder.Record(profile)
|
|
}
|
|
want := []artifacts.LLMProfileManifest{
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
|
|
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
|
|
}
|
|
if got := recorder.Manifests(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("profile manifests = %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
|
|
|
|
var out map[string]any
|
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
|
|
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
|
}
|
|
if got := string(resp.Content); got != `{"bad":true}` {
|
|
t.Fatalf("response content = %q, want raw failed output", got)
|
|
}
|
|
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` {
|
|
t.Fatalf("debug response = %#v, want raw failed output", resp.Debug)
|
|
}
|
|
if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 {
|
|
t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
|
|
|
var out []any
|
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") {
|
|
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
|
|
}
|
|
if got := string(resp.Content); got != `{"ok":true}` {
|
|
t.Fatalf("response content = %q, want raw decode-failed output", got)
|
|
}
|
|
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
|
|
t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")})
|
|
|
|
var out map[string]any
|
|
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if err == nil {
|
|
t.Fatalf("CompleteStructured() error = nil, want provider error")
|
|
}
|
|
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
|
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
|
|
}
|
|
if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) {
|
|
t.Fatalf("error = %q, want operation context", err.Error())
|
|
}
|
|
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
|
|
t.Fatalf("error = %q, want redacted bearer token", err.Error())
|
|
}
|
|
if resp.Debug != nil {
|
|
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
|
|
queueCapacity := 0
|
|
fake := &fakePromptKitLLM{
|
|
err: errors.New("provider failed with Bearer secret-token"),
|
|
block: make(chan struct{}),
|
|
}
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
EngineOptions: []promptkit.Option{
|
|
promptkit.WithBackend(promptkit.Backend{
|
|
ID: "limited-backend",
|
|
Endpoint: "http://127.0.0.1:1/v1",
|
|
ConcurrencyLimit: 1,
|
|
QueueCapacity: &queueCapacity,
|
|
}),
|
|
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "limited-profile",
|
|
BackendID: "limited-backend",
|
|
Model: "limited-model",
|
|
})),
|
|
promptkit.WithLLMClient(fake),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
|
}
|
|
defer func() {
|
|
select {
|
|
case <-fake.block:
|
|
default:
|
|
close(fake.block)
|
|
}
|
|
}()
|
|
|
|
request := contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.direct-session",
|
|
ProfileID: "limited-profile",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
Vars: map[string]any{"custom": "value"},
|
|
}
|
|
firstResult := make(chan error, 1)
|
|
go func() {
|
|
var out map[string]any
|
|
_, callErr := client.CompleteStructured(context.Background(), request, &out)
|
|
firstResult <- callErr
|
|
}()
|
|
waitForAtomicAtLeast(t, &fake.calls, 1)
|
|
|
|
var out map[string]any
|
|
response, capacityErr := client.CompleteStructured(context.Background(), request, &out)
|
|
if len(response.Content) != 0 {
|
|
t.Fatalf("capacity response = %#v, want empty", response)
|
|
}
|
|
if !errors.Is(capacityErr, contracts.ErrLLMCapacityExceeded) {
|
|
t.Fatalf("capacity error = %v, want ErrLLMCapacityExceeded", capacityErr)
|
|
}
|
|
if errors.Is(capacityErr, contracts.ErrInvalidStructuredOutput) {
|
|
t.Fatalf("capacity error = %v, must not be invalid structured output", capacityErr)
|
|
}
|
|
if errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
|
|
t.Fatalf("capacity error exposes PromptKit sentinel: %v", capacityErr)
|
|
}
|
|
if !strings.Contains(capacityErr.Error(), `run PromptKit prompt "adapter.direct-session"`) ||
|
|
!strings.Contains(capacityErr.Error(), "backend capacity exceeded") {
|
|
t.Fatalf("capacity error = %q, want prompt context and upstream diagnostic", capacityErr)
|
|
}
|
|
if calls := atomic.LoadInt32(&fake.calls); calls != 1 {
|
|
t.Fatalf("provider calls after capacity rejection = %d, want 1", calls)
|
|
}
|
|
|
|
close(fake.block)
|
|
firstErr := <-firstResult
|
|
if firstErr == nil || strings.Contains(firstErr.Error(), "secret-token") ||
|
|
!strings.Contains(firstErr.Error(), "Bearer [REDACTED]") {
|
|
t.Fatalf("admitted provider error = %v, want redacted diagnostic", firstErr)
|
|
}
|
|
if calls := atomic.LoadInt32(&fake.calls); calls != 1 {
|
|
t.Fatalf("provider calls after release = %d, want no adapter retry", calls)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
|
|
|
var out map[string]any
|
|
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
|
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
|
|
var remaining time.Duration
|
|
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
deadline, ok := req.Context().Deadline()
|
|
if !ok {
|
|
t.Fatal("outbound request context has no deadline")
|
|
}
|
|
remaining = time.Until(deadline)
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Status: "200 OK",
|
|
Header: make(http.Header),
|
|
Body: io.NopCloser(strings.NewReader(
|
|
`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`,
|
|
)),
|
|
Request: req,
|
|
}, nil
|
|
})
|
|
const configuredTimeout = 2 * time.Second
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: newTestPromptKitAssets(t),
|
|
Timeout: configuredTimeout,
|
|
HTTPClient: &http.Client{Transport: transport},
|
|
EngineOptions: []promptkit.Option{
|
|
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "default-profile",
|
|
Endpoint: "http://promptkit.test/v1",
|
|
Model: "default-model",
|
|
})),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
|
}
|
|
var out map[string]any
|
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out); err != nil {
|
|
t.Fatalf("CompleteStructured() error = %v", err)
|
|
}
|
|
if remaining < configuredTimeout-time.Second || remaining > configuredTimeout {
|
|
t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})
|
|
|
|
var out map[string]any
|
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if !errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
|
t.Fatalf("CompleteStructured() error = %v, want invalid structured output", err)
|
|
}
|
|
}
|
|
|
|
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
|
fake := &fakePromptKitLLM{
|
|
content: `{"ok":true}`,
|
|
block: make(chan struct{}),
|
|
}
|
|
client := newTestPromptKitClient(t, fake)
|
|
scheduler, err := NewScheduler(1)
|
|
if err != nil {
|
|
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
|
}
|
|
scheduled := NewScheduledClient(client, scheduler)
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 3; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
var out map[string]any
|
|
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
PromptID: "adapter.test",
|
|
SessionID: "session-123",
|
|
Inputs: contracts.LLMInputSet{
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
},
|
|
}, &out)
|
|
if callErr != nil {
|
|
t.Errorf("CompleteStructured() error = %v, want nil", callErr)
|
|
}
|
|
}()
|
|
}
|
|
waitForAtomicAtLeast(t, &fake.calls, 1)
|
|
time.Sleep(20 * time.Millisecond)
|
|
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
|
t.Fatalf("max in-flight calls = %d, want <= 1", got)
|
|
}
|
|
close(fake.block)
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
|
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
|
var out map[string]any
|
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
|
|
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
|
|
}
|
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") {
|
|
t.Fatalf("nil output error = %v, want output validation", err)
|
|
}
|
|
}
|
|
|
|
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
|
|
return newTestPromptKitClientWithReasoning(t, fake, nil)
|
|
}
|
|
|
|
func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient {
|
|
t.Helper()
|
|
registry := newTestPromptKitAssets(t)
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
Assets: registry,
|
|
ReasoningEffort: reasoningEffort,
|
|
EngineOptions: []promptkit.Option{
|
|
promptkit.WithBackend(promptkit.Backend{
|
|
ID: "test-backend",
|
|
Endpoint: "http://127.0.0.1:1/v1",
|
|
}),
|
|
promptkit.WithProfiles(
|
|
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "default-profile",
|
|
Endpoint: "http://127.0.0.1:1/v1",
|
|
Model: "default-model",
|
|
ReasoningEffort: "profile-reasoning",
|
|
}),
|
|
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "explicit-profile",
|
|
BackendID: "test-backend",
|
|
Model: "explicit-model",
|
|
ReasoningEffort: "profile-reasoning",
|
|
}),
|
|
),
|
|
promptkit.WithLLMClient(fake),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
|
}
|
|
return client
|
|
}
|
|
|
|
func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
|
|
t.Helper()
|
|
registry := NewAssetRegistry()
|
|
if err := registry.RegisterPromptFS(fstest.MapFS{
|
|
"adapter.test.yaml": {Data: []byte(`id: adapter.test
|
|
version: "v1"
|
|
default_profile: default-profile
|
|
session_id: "{{ .session_id }}"
|
|
inputs:
|
|
- name: transcript
|
|
required: true
|
|
content_type: application/json
|
|
messages:
|
|
- role: user
|
|
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}"
|
|
output:
|
|
format: json
|
|
validation_mode: json_schema
|
|
schema_path: adapter.schema.json
|
|
repair_attempts: 0
|
|
`)},
|
|
"adapter.direct-session.yaml": {Data: []byte(`id: adapter.direct-session
|
|
version: "v1"
|
|
default_profile: default-profile
|
|
inputs:
|
|
- name: transcript
|
|
required: true
|
|
content_type: application/json
|
|
messages:
|
|
- role: user
|
|
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
|
|
output:
|
|
format: json
|
|
validation_mode: json_schema
|
|
schema_path: adapter.schema.json
|
|
repair_attempts: 0
|
|
`)},
|
|
}, "."); err != nil {
|
|
t.Fatalf("RegisterPromptFS() error = %v", err)
|
|
}
|
|
if err := registry.RegisterSchemaFS(fstest.MapFS{
|
|
"adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)},
|
|
}, "."); err != nil {
|
|
t.Fatalf("RegisterSchemaFS() error = %v", err)
|
|
}
|
|
return registry
|
|
}
|
|
|
|
type fakePromptKitLLM struct {
|
|
content string
|
|
allowEmpty bool
|
|
err error
|
|
block chan struct{}
|
|
mu sync.Mutex
|
|
last promptkit.GenerateRequest
|
|
calls int32
|
|
inFlight int32
|
|
maxInFlight int32
|
|
}
|
|
|
|
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
f.mu.Lock()
|
|
f.last = req
|
|
f.mu.Unlock()
|
|
atomic.AddInt32(&f.calls, 1)
|
|
current := atomic.AddInt32(&f.inFlight, 1)
|
|
for {
|
|
seen := atomic.LoadInt32(&f.maxInFlight)
|
|
if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) {
|
|
break
|
|
}
|
|
}
|
|
defer atomic.AddInt32(&f.inFlight, -1)
|
|
if f.block != nil {
|
|
select {
|
|
case <-f.block:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
content := f.content
|
|
if content == "" && !f.allowEmpty {
|
|
content = `{"ok":true}`
|
|
}
|
|
if !f.allowEmpty && !json.Valid([]byte(content)) {
|
|
return nil, errors.New("test fake must return JSON content")
|
|
}
|
|
return &promptkit.GenerateResponse{
|
|
Content: content,
|
|
Usage: promptkit.TokenUsage{
|
|
PromptTokens: 11,
|
|
CompletionTokens: 7,
|
|
TotalTokens: 18,
|
|
CachedTokens: 5,
|
|
CacheWriteTokens: 3,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
return f(req)
|
|
}
|
|
|
|
func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.last
|
|
}
|