package llm import ( "context" "encoding/json" "errors" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "time" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/scriptorium" ) func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { fake := &fakeScriptoriumLLM{content: `{"ok":true}`} client := newTestScriptoriumClient(t, fake) var out struct { OK bool `json:"ok"` } resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ StageName: "test-stage", PromptID: "adapter.test", PromptVersion: "v1", ProfileID: "explicit-profile", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"), }, Vars: map[string]any{"custom": "value"}, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if !out.OK { t.Fatalf("decoded output OK = false, want true") } if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" { t.Fatalf("response metadata = %#v", resp) } if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { t.Fatalf("usage = %#v, want mapped token counts", resp) } if resp.Debug == nil || resp.Debug.Prompt == nil { t.Fatalf("debug prompt = nil, want prepared prompt material") } if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" { t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt) } if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) { t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages) } if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response) } debugJSON, err := json.Marshal(resp.Debug) if err != nil { t.Fatalf("marshal debug material: %v", err) } if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") { t.Fatalf("debug material contains secret material: %s", debugJSON) } gotReq := fake.lastRequest() if gotReq.Prompt.SessionID != "session-123" { t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID) } if gotReq.Target.Model != "explicit-model" { t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model) } if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) { t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages) } if gotReq.StructuredOutput == nil { t.Fatalf("structured output = nil, want JSON schema") } manifests := client.LLMProfileManifests() if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" { t.Fatalf("profile manifests = %#v", manifests) } } func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { fake := &fakeScriptoriumLLM{content: `{"ok":true}`} client := newTestScriptoriumClient(t, fake) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out); err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if got := fake.lastRequest().Target.Model; got != "default-model" { t.Fatalf("model = %q, want prompt default profile model", got) } } func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) { client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") { t.Fatalf("CompleteStructured() error = %v, want validation failure", err) } if got := string(resp.Content); got != `{"bad":true}` { t.Fatalf("response content = %q, want raw failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` { t.Fatalf("debug response = %#v, want raw failed output", resp.Debug) } if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 { t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt) } } func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) { client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`}) var out []any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") { t.Fatalf("CompleteStructured() error = %v, want decode failure", err) } if got := string(resp.Content); got != `{"ok":true}` { t.Fatalf("response content = %q, want raw decode-failed output", got) } if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` { t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug) } } func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) { client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")}) var out map[string]any resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil { t.Fatalf("CompleteStructured() error = nil, want provider error") } if errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("provider error = %v, must not be classified as invalid structured output", err) } if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) { t.Fatalf("error = %q, want operation context", err.Error()) } if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") { t.Fatalf("error = %q, want redacted bearer token", err.Error()) } if resp.Debug != nil { t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug) } } func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`}) var out map[string]any _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ PromptID: "adapter.test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want context canceled", err) } } func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) { client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true}) var out map[string]any _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if !errors.Is(err, contracts.ErrInvalidStructuredOutput) { t.Fatalf("CompleteStructured() error = %v, want invalid structured output", err) } } func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) { fake := &fakeScriptoriumLLM{ content: `{"ok":true}`, block: make(chan struct{}), } client := newTestScriptoriumClient(t, fake) scheduler, err := NewScheduler(1) if err != nil { t.Fatalf("NewScheduler() error = %v, want nil", err) } scheduled := NewScheduledClient(client, scheduler) var wg sync.WaitGroup for i := 0; i < 3; i++ { wg.Add(1) go func() { defer wg.Done() var out map[string]any _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", SessionID: "session-123", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if callErr != nil { t.Errorf("CompleteStructured() error = %v, want nil", callErr) } }() } waitForAtomicAtLeast(t, &fake.calls, 1) time.Sleep(20 * time.Millisecond) if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 { t.Fatalf("max in-flight calls = %d, want <= 1", got) } close(fake.block) wg.Wait() } func TestScriptoriumClientValidatesRequest(t *testing.T) { client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`}) var out map[string]any if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") { t.Fatalf("missing prompt id error = %v, want prompt_id validation", err) } if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") { t.Fatalf("nil output error = %v, want output validation", err) } } func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient { t.Helper() registry := NewAssetRegistry() if err := registry.RegisterPromptFS(fstest.MapFS{ "adapter.test.yaml": {Data: []byte(`id: adapter.test version: "v1" default_profile: default-profile session_id: "{{ .session_id }}" inputs: - name: transcript required: true content_type: application/json messages: - role: user content: "Transcript: {{ input \"transcript\" }}" output: format: json validation_mode: json_schema schema_path: adapter.schema.json repair_attempts: 0 `)}, }, "."); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) } if err := registry.RegisterSchemaFS(fstest.MapFS{ "adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)}, }, "."); err != nil { t.Fatalf("RegisterSchemaFS() error = %v", err) } client, err := NewScriptoriumClient(ScriptoriumClientConfig{ Assets: registry, EngineOptions: []scriptorium.Option{ scriptorium.WithProfiles( scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ ID: "default-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "default-model", }), scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ ID: "explicit-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "explicit-model", }), ), scriptorium.WithLLMClient(fake), }, }) if err != nil { t.Fatalf("NewScriptoriumClient() error = %v, want nil", err) } return client } type fakeScriptoriumLLM struct { content string allowEmpty bool err error block chan struct{} mu sync.Mutex last scriptorium.GenerateRequest calls int32 inFlight int32 maxInFlight int32 } func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { f.mu.Lock() f.last = req f.mu.Unlock() atomic.AddInt32(&f.calls, 1) current := atomic.AddInt32(&f.inFlight, 1) for { seen := atomic.LoadInt32(&f.maxInFlight) if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) { break } } defer atomic.AddInt32(&f.inFlight, -1) if f.block != nil { select { case <-f.block: case <-ctx.Done(): return nil, ctx.Err() } } if f.err != nil { return nil, f.err } content := f.content if content == "" && !f.allowEmpty { content = `{"ok":true}` } if !f.allowEmpty && !json.Valid([]byte(content)) { return nil, errors.New("test fake must return JSON content") } return &scriptorium.GenerateResponse{ Content: content, Usage: scriptorium.TokenUsage{ PromptTokens: 11, CompletionTokens: 7, TotalTokens: 18, }, }, nil } func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest { f.mu.Lock() defer f.mu.Unlock() return f.last }