package llm import ( "context" "encoding/json" "errors" "io" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "time" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/promptkit" ) func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out struct { OK bool `json:"ok"` } resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ StageName: "test-stage", PromptID: "adapter.test", PromptVersion: "v1", ProfileID: "explicit-profile", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"), }, Vars: map[string]any{"custom": "value"}, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if !out.OK { t.Fatalf("decoded output OK = false, want true") } if resp.Provider != "promptkit" || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" { t.Fatalf("response metadata = %#v", resp) } if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { t.Fatalf("usage = %#v, want mapped token counts", resp) } if resp.Debug == nil || resp.Debug.Prompt == nil { t.Fatalf("debug prompt = nil, want prepared prompt material") } if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" { t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt) } if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) { t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages) } if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response) } if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 { t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage) } debugJSON, err := json.Marshal(resp.Debug) if err != nil { t.Fatalf("marshal debug material: %v", err) } if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") { t.Fatalf("debug material contains secret material: %s", debugJSON) } gotReq := fake.lastRequest() if gotReq.Prompt.SessionID != "session-123" { t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID) } if gotReq.Target.Model != "explicit-model" { t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model) } if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) || !strings.Contains(gotReq.Prompt.Messages[0].Content, "value") { t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages) } if gotReq.StructuredOutput == nil { t.Fatalf("structured output = nil, want JSON schema") } manifests := client.LLMProfileManifests() if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Provider != "promptkit" || manifests[0].Model != "explicit-model" { t.Fatalf("profile manifests = %#v", manifests) } } func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) { t.Run("assets", func(t *testing.T) { registry := NewAssetRegistry() for _, root := range []string{"one", "two"} { if err := registry.RegisterPromptFS(fstest.MapFS{ root + "/prompt.yaml": {Data: []byte("id: duplicate")}, }, root); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) } } if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil || !strings.Contains(err.Error(), "duplicate asset path") { t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err) } }) t.Run("engine", func(t *testing.T) { if _, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")}, }); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") { t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err) } }) } func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { profilePath := filepath.Join(t.TempDir(), "profiles.yml") writeProfile := func(model string) { t.Helper() content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\n" if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { t.Fatal(err) } } fingerprintFor := func() CheckpointFingerprint { t.Helper() client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), ProfileFile: profilePath, }) if err != nil { t.Fatal(err) } values, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName { t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values) } return values[0] } writeProfile("model-one") first := fingerprintFor() writeProfile("model-two") second := fingerprintFor() if first == second { t.Fatalf("profile-source fingerprint = %#v for both profile models", first) } if strings.Contains(first.Value, profilePath) || strings.Contains(first.Value, "model-one") { t.Fatalf("profile-source fingerprint exposes source details: %#v", first) } client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)}) if err != nil { t.Fatal(err) } copy, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } copy[0].Value = "mutated" fresh, err := client.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } if fresh[0].Value == "mutated" { t.Fatal("LLMCheckpointFingerprints exposed mutable backing storage") } const wantBuiltinFingerprint = "sha256:4b48cd1ef010b587cd8d56bc13b73a0cdd84da1414a8de9f9a2adc695dfcf0a7" if fresh[0].Value != wantBuiltinFingerprint { t.Fatalf("built-in profile fingerprint = %q, want %q", fresh[0].Value, wantBuiltinFingerprint) } } func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out); err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if got := fake.lastRequest().Target.Model; got != "default-model" { t.Fatalf("model = %q, want prompt default profile model", got) } } func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") { t.Fatalf("CompleteStructured() error = %v, want validation failure", err) } if got := string(resp.Content); got != `{"bad":true}` { t.Fatalf("response content = %q, want raw failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` { t.Fatalf("debug response = %#v, want raw failed output", resp.Debug) } if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 { t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt) } } func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out []any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") { t.Fatalf("CompleteStructured() error = %v, want decode failure", err) } if got := string(resp.Content); got != `{"ok":true}` { t.Fatalf("response content = %q, want raw decode-failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug) } } func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil { t.Fatalf("CompleteStructured() error = nil, want provider error") } if errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("provider error = %v, must not be classified as invalid structured output", err) } if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) { t.Fatalf("error = %q, want operation context", err.Error()) } if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") { t.Fatalf("error = %q, want redacted bearer token", err.Error()) } if resp.Debug != nil { t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug) } } func TestPromptKitClientContextCancellationIsRespected(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out map[string]any _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ PromptID: "adapter.test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want context canceled", err) } } func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) { var remaining time.Duration transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { deadline, ok := req.Context().Deadline() if !ok { t.Fatal("outbound request context has no deadline") } remaining = time.Until(deadline) return &http.Response{ StatusCode: http.StatusOK, Status: "200 OK", Header: make(http.Header), Body: io.NopCloser(strings.NewReader( `{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`, )), Request: req, }, nil }) const configuredTimeout = 2 * time.Second client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), Timeout: configuredTimeout, HTTPClient: &http.Client{Transport: transport}, EngineOptions: []promptkit.Option{ promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model", })), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v", err) } var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out); err != nil { t.Fatalf("CompleteStructured() error = %v", err) } if remaining < configuredTimeout-time.Second || remaining > configuredTimeout { t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout) } } func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true}) var out map[string]any _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want invalid structured output", err) } } func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) { fake := &fakePromptKitLLM{ content: `{"ok":true}`, block: make(chan struct{}), } client := newTestPromptKitClient(t, fake) scheduler, err := NewScheduler(1) if err != nil { t.Fatalf("NewScheduler() error = %v, want nil", err) } scheduled := NewScheduledClient(client, scheduler) var wg sync.WaitGroup for i := 0; i < 3; i++ { wg.Add(1) go func() { defer wg.Done() var out map[string]any _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if callErr != nil { t.Errorf("CompleteStructured() error = %v, want nil", callErr) } }() } waitForAtomicAtLeast(t, &fake.calls, 1) time.Sleep(20 * time.Millisecond) if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 { t.Fatalf("max in-flight calls = %d, want <= 1", got) } close(fake.block) wg.Wait() } func TestPromptKitClientValidatesRequest(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`}) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") { t.Fatalf("missing prompt id error = %v, want prompt_id validation", err) } if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") { t.Fatalf("nil output error = %v, want output validation", err) } } func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient { t.Helper() registry := newTestPromptKitAssets(t) client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: registry, EngineOptions: []promptkit.Option{ promptkit.WithProfiles( promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "default-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "default-model", }), promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "explicit-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "explicit-model", }), ), promptkit.WithLLMClient(fake), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) } return client } func newTestPromptKitAssets(t *testing.T) *AssetRegistry { t.Helper() registry := NewAssetRegistry() if err := registry.RegisterPromptFS(fstest.MapFS{ "adapter.test.yaml": {Data: []byte(`id: adapter.test version: "v1" default_profile: default-profile session_id: "{{ .session_id }}" inputs: - name: transcript required: true content_type: application/json messages: - role: user content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}" output: format: json validation_mode: json_schema schema_path: adapter.schema.json repair_attempts: 0 `)}, }, "."); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) } if err := registry.RegisterSchemaFS(fstest.MapFS{ "adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)}, }, "."); err != nil { t.Fatalf("RegisterSchemaFS() error = %v", err) } return registry } type fakePromptKitLLM struct { content string allowEmpty bool err error block chan struct{} mu sync.Mutex last promptkit.GenerateRequest calls int32 inFlight int32 maxInFlight int32 } func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { f.mu.Lock() f.last = req f.mu.Unlock() atomic.AddInt32(&f.calls, 1) current := atomic.AddInt32(&f.inFlight, 1) for { seen := atomic.LoadInt32(&f.maxInFlight) if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) { break } } defer atomic.AddInt32(&f.inFlight, -1) if f.block != nil { select { case <-f.block: case <-ctx.Done(): return nil, ctx.Err() } } if f.err != nil { return nil, f.err } content := f.content if content == "" && !f.allowEmpty { content = `{"ok":true}` } if !f.allowEmpty && !json.Valid([]byte(content)) { return nil, errors.New("test fake must return JSON content") } return &promptkit.GenerateResponse{ Content: content, Usage: promptkit.TokenUsage{ PromptTokens: 11, CompletionTokens: 7, TotalTokens: 18, CachedTokens: 5, CacheWriteTokens: 3, }, }, nil } type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest { f.mu.Lock() defer f.mu.Unlock() return f.last }