From f6224dcbee5fcbc3f243e78e2bd6c26b05cbb266 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 5 Jul 2026 18:21:37 +0000 Subject: [PATCH] Add Scriptorium-backed LLM runtime --- docs/internal/llm.md | 73 +++-- docs/internal/overview.md | 5 +- internal/cli/catalog.go | 22 +- internal/cli/run_test.go | 15 +- internal/framework/contracts/contracts.go | 5 + internal/framework/llm/scheduled_client.go | 12 + internal/framework/llm/scriptorium_client.go | 267 ++++++++++++++++ .../framework/llm/scriptorium_client_test.go | 299 ++++++++++++++++++ internal/framework/pipeline/runner.go | 50 ++- internal/framework/pipeline/runner_test.go | 31 ++ .../scriptorium/prompts/dnd.scenes.yaml | 1 + .../scriptorium/prompts/dnd.spells.yaml | 1 + 12 files changed, 742 insertions(+), 39 deletions(-) create mode 100644 internal/framework/llm/scriptorium_client.go create mode 100644 internal/framework/llm/scriptorium_client_test.go diff --git a/docs/internal/llm.md b/docs/internal/llm.md index fd0b49b..642983a 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -1,9 +1,10 @@ # LLM Runtime The implemented LLM runtime lives in `internal/framework/llm`. It provides -transport-neutral structured completion contracts, an OpenAI-compatible HTTP -adapter, concurrency scheduling, schema registry helpers, retry behavior, and -secret redaction. +transport-neutral structured completion contracts, a Scriptorium-backed +production client, an OpenAI-compatible HTTP adapter retained for legacy tests +and helpers, concurrency scheduling, prompt/schema asset registration, schema +registry helpers, and secret redaction. ## Contract @@ -13,26 +14,48 @@ Modules depend on `contracts.StructuredLLMClient`: CompleteStructured(ctx, request, out) (response, error) ``` -The request contains messages, optional model override, response schema name, -and response schema JSON. The caller supplies a pointer target for decoded -structured output. +The request contains prompt ID/version, profile ID, session ID, prompt input +materials, variables, and legacy rendered-message/schema fields used by modules +that have not yet moved to prompt-asset execution. The caller supplies a pointer +target for decoded structured output. -Modules that call the LLM own their prompts and schemas. Provider adapters -should not contain domain-specific prompt logic. +Modules that call the LLM own their prompts, schemas, prompt IDs, validators, +and domain-specific interpretation. Provider adapters should not contain +domain-specific prompt logic. ## Production Client Construction `internal/cli` builds the production LLM client from the effective config: -1. find the effective LLM profile; -2. build `OpenAICompatibleClientConfig`; -3. create an OpenAI-compatible client; -4. create a scheduler from profile or global concurrency; -5. wrap the client with `NewScheduledClient`; -6. return non-secret LLM profile manifest metadata. +1. collect production Scriptorium prompt and schema assets from module packages; +2. create a Scriptorium-backed structured client using effective Scriptorium + profile source settings; +3. create a scheduler from global LLM concurrency; +4. wrap the client with `NewScheduledClient`; +5. let the runtime report non-secret profile manifest metadata after calls. -The current run command requires exactly one distinct effective LLM profile for -the resolved pipeline. +The runtime records the actual selected Scriptorium profile, provider, and model +used during execution. Manifest population does not rely on a precomputed +profile ID before pipeline execution. + +## Scriptorium Adapter + +`ScriptoriumClient` implements `contracts.StructuredLLMClient` by converting +Notarius prompt requests into Scriptorium `RunRequest` values. It: + +- validates the caller output target and prompt ID; +- converts `LLMInputMaterial` values into inline Scriptorium artifacts; +- passes `session_id` through Scriptorium variables when present; +- sends explicit profile IDs only when the request supplies one; +- lets Scriptorium render prompts, call the configured provider, and validate + structured output; +- unmarshals successful JSON into the caller-provided target; +- maps token usage and selected profile/model metadata into the Notarius + response and manifest profile recorder. + +Generated-output validation failures are returned as Notarius errors. Provider +and runtime errors are wrapped with prompt context and bearer tokens are +redacted from error strings. ## OpenAI-Compatible Adapter @@ -73,8 +96,8 @@ The adapter retries: - malformed assistant JSON; - structured-output decode failures. -Non-retryable `4xx` responses are returned without retry. Request timeout comes -from the effective LLM profile. Context cancellation is respected. +Non-retryable `4xx` responses are returned without retry. Context cancellation +is respected. ## Scheduler @@ -87,9 +110,8 @@ inside the scheduler. Effective concurrency is: -1. `llm_profiles..max_concurrency`, when greater than zero; -2. `concurrency.total_llm`, when greater than zero; -3. `1`. +1. `concurrency.total_llm`, when greater than zero; +2. `1`. ## Schema Registry @@ -104,13 +126,14 @@ helpers for caller-owned schemas: `DiagnosticsMap` omits raw schema content and includes metadata such as key, ID, version, name, and SHA-256. -The D&D spell extractor owns and loads its own embedded response schema. +Production modules own and register their Scriptorium prompt and schema assets. +Framework packages may collect those files but must not contain D&D-specific +prompt content. ## Secret Redaction -Provider errors are passed through `ErrorWithSecretsRedacted` with the API key -and bearer-token value. Config diagnostics use redacted effective config -payloads. +Provider errors are redacted before surfacing through the Scriptorium-backed +client. Config diagnostics use redacted effective config payloads. Do not add raw provider request bodies, response bodies, API keys, or prompt payloads to diagnostics by default. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 5cca600..b1539db 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -41,8 +41,9 @@ production modules. - `internal/framework/pipeline`: module registries, module specs, profile resolution, capability checks, run orchestration, warnings, validation, and manifest population. -- `internal/framework/llm`: OpenAI-compatible structured-output client, - scheduler, schema registry, retries, and secret redaction. +- `internal/framework/llm`: Scriptorium-backed structured-output client, + prompt/schema asset registry, scheduler, schema registry, retries, and secret + redaction. - `internal/framework/prompt`: embedded prompt registry and template rendering. - `internal/framework/validate`: validator decision helpers and cardinality enforcement. diff --git a/internal/cli/catalog.go b/internal/cli/catalog.go index ae75d7e..e1f9c12 100644 --- a/internal/cli/catalog.go +++ b/internal/cli/catalog.go @@ -3,7 +3,6 @@ package cli import ( "context" "fmt" - "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" @@ -144,6 +143,23 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI if err := ctx.Err(); err != nil { return nil, nil, err } - trimmedID := strings.TrimSpace(profileID) - return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client for profile %q: not implemented yet", trimmedID) + assets, err := productionPromptAssets() + if err != nil { + return nil, nil, err + } + recorder := llm.NewLLMProfileRecorder() + client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{ + ProfileDir: cfg.Scriptorium.ProfileDir, + ProfileFile: cfg.Scriptorium.ProfileFile, + Assets: assets, + Recorder: recorder, + }) + if err != nil { + return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err) + } + scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM) + if err != nil { + return nil, nil, fmt.Errorf("create LLM scheduler: %w", err) + } + return llm.NewScheduledClient(client, scheduler), nil, nil } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 4836a24..e953fad 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -445,15 +445,18 @@ func TestRunInvalidFlagsExitTwo(t *testing.T) { } } -func TestProductionLLMClientFactoryReportsPendingScriptoriumRuntime(t *testing.T) { +func TestProductionLLMClientFactoryBuildsScriptoriumRuntime(t *testing.T) { cfg := config.Default() - _, _, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3") - if err == nil { - t.Fatal("productionLLMClientFactory() error = nil, want error") + client, profiles, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3") + if err != nil { + t.Fatalf("productionLLMClientFactory() error = %v, want nil", err) } - if !strings.Contains(err.Error(), "Scriptorium-backed LLM client") || !strings.Contains(err.Error(), "not implemented yet") { - t.Fatalf("error = %q, want pending Scriptorium runtime context", err.Error()) + if client == nil { + t.Fatal("productionLLMClientFactory() client = nil, want client") + } + if profiles != nil { + t.Fatalf("productionLLMClientFactory() profiles = %#v, want runtime-reported profiles", profiles) } } diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 142b65e..d4802f5 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -31,6 +31,7 @@ type StructuredCompletionResponse struct { Content json.RawMessage `json:"content"` Provider string `json:"provider,omitempty"` Model string `json:"model,omitempty"` + ProfileID string `json:"profile_id,omitempty"` PromptTokens int `json:"prompt_tokens,omitempty"` CompletionTokens int `json:"completion_tokens,omitempty"` TotalTokens int `json:"total_tokens,omitempty"` @@ -40,6 +41,10 @@ type StructuredLLMClient interface { CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) } +type LLMProfileManifestProvider interface { + LLMProfileManifests() []artifacts.LLMProfileManifest +} + type LLMInputMaterial struct { Name string `json:"name"` MediaType string `json:"media_type,omitempty"` diff --git a/internal/framework/llm/scheduled_client.go b/internal/framework/llm/scheduled_client.go index 88d923a..379f64c 100644 --- a/internal/framework/llm/scheduled_client.go +++ b/internal/framework/llm/scheduled_client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) @@ -41,3 +42,14 @@ func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts. } return response, nil } + +func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest { + if c == nil || c.client == nil { + return nil + } + provider, ok := c.client.(contracts.LLMProfileManifestProvider) + if !ok { + return nil + } + return provider.LLMProfileManifests() +} diff --git a/internal/framework/llm/scriptorium_client.go b/internal/framework/llm/scriptorium_client.go new file mode 100644 index 0000000..ce8f6dd --- /dev/null +++ b/internal/framework/llm/scriptorium_client.go @@ -0,0 +1,267 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "regexp" + "sort" + "strings" + "sync" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/scriptorium" +) + +const scriptoriumProviderName = "openai-compatible" + +type ScriptoriumClientConfig struct { + ProfileDir string + ProfileFile string + Assets *AssetRegistry + Timeout time.Duration + HTTPClient *http.Client + EngineOptions []scriptorium.Option + Recorder *LLMProfileRecorder +} + +type ScriptoriumClient struct { + engine *scriptorium.Engine + recorder *LLMProfileRecorder +} + +type LLMProfileRecorder struct { + mu sync.Mutex + profiles map[string]artifacts.LLMProfileManifest +} + +var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil) +var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil) + +func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) { + if cfg.Assets == nil { + return nil, fmt.Errorf("scriptorium client assets must not be nil") + } + if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" { + return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive") + } + options, err := cfg.Assets.ScriptoriumOptions() + if err != nil { + return nil, err + } + if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" { + options = append(options, scriptorium.WithProfileFile(profileFile)) + } + options = append(options, cfg.EngineOptions...) + + engine, err := scriptorium.NewEngine(scriptorium.Config{ + ProfileDir: strings.TrimSpace(cfg.ProfileDir), + Timeout: cfg.Timeout, + HTTPClient: cfg.HTTPClient, + }, options...) + if err != nil { + return nil, fmt.Errorf("create Scriptorium engine: %w", err) + } + recorder := cfg.Recorder + if recorder == nil { + recorder = NewLLMProfileRecorder() + } + return &ScriptoriumClient{ + engine: engine, + recorder: recorder, + }, nil +} + +func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + if c == nil { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil") + } + if c.engine == nil { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil") + } + if err := validateOutputTarget(out); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + promptID := strings.TrimSpace(req.PromptID) + if promptID == "" { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty") + } + + runReq := scriptorium.RunRequest{ + PromptID: promptID, + PromptVersion: strings.TrimSpace(req.PromptVersion), + ProfileID: strings.TrimSpace(req.ProfileID), + Inputs: scriptoriumInputs(req.Inputs), + Vars: scriptoriumVars(req), + Metadata: scriptoriumMetadata(req), + } + result, err := c.engine.Run(ctx, runReq) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return contracts.StructuredCompletionResponse{}, ctxErr + } + return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err)) + } + if result == nil { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID) + } + if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; ")) + } + + content := result.Artifact.Body + if len(content) == 0 { + content = []byte(result.RawOutput) + } + if len(strings.TrimSpace(string(content))) == 0 { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty structured output", promptID) + } + if err := json.Unmarshal(content, out); err != nil { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w", promptID, err) + } + + profile := artifacts.LLMProfileManifest{ + ID: strings.TrimSpace(result.SelectedProfileID), + Provider: scriptoriumProviderName, + Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model), + } + if c.recorder != nil { + c.recorder.Record(profile) + } + return contracts.StructuredCompletionResponse{ + Content: append(json.RawMessage(nil), content...), + Provider: profile.Provider, + Model: profile.Model, + ProfileID: profile.ID, + PromptTokens: result.Usage.PromptTokens, + CompletionTokens: result.Usage.CompletionTokens, + TotalTokens: result.Usage.TotalTokens, + }, nil +} + +func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest { + if c == nil || c.recorder == nil { + return nil + } + return c.recorder.Manifests() +} + +func NewLLMProfileRecorder() *LLMProfileRecorder { + return &LLMProfileRecorder{profiles: map[string]artifacts.LLMProfileManifest{}} +} + +func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) { + if r == nil { + return + } + profile.ID = strings.TrimSpace(profile.ID) + profile.Provider = strings.TrimSpace(profile.Provider) + profile.Model = strings.TrimSpace(profile.Model) + key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + r.mu.Lock() + defer r.mu.Unlock() + if r.profiles == nil { + r.profiles = map[string]artifacts.LLMProfileManifest{} + } + r.profiles[key] = profile +} + +func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest { + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + if len(r.profiles) == 0 { + return nil + } + keys := make([]string, 0, len(r.profiles)) + for key := range r.profiles { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]artifacts.LLMProfileManifest, 0, len(keys)) + for _, key := range keys { + out = append(out, r.profiles[key]) + } + return out +} + +func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef { + if len(inputs) == 0 { + return nil + } + out := make(map[string]scriptorium.ArtifactRef, len(inputs)) + for key, material := range inputs { + name := strings.TrimSpace(key) + if name == "" { + name = strings.TrimSpace(material.Name) + } + if name == "" { + continue + } + body := string(material.Content) + if body == "" { + body = " " + } + if origin := strings.TrimSpace(material.OriginURI); origin != "" { + out[name] = scriptorium.InlineWithURI(origin, body) + } else { + out[name] = scriptorium.Inline(body) + } + } + return out +} + +func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string { + vars := make(map[string]string, len(req.Vars)+1) + for key, value := range req.Vars { + name := strings.TrimSpace(key) + if name == "" || value == nil { + continue + } + vars[name] = fmt.Sprint(value) + } + if sessionID := strings.TrimSpace(req.SessionID); sessionID != "" { + vars["session_id"] = sessionID + } + if len(vars) == 0 { + return nil + } + return vars +} + +func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string { + metadata := map[string]string{} + if stageName := strings.TrimSpace(req.StageName); stageName != "" { + metadata["stage_name"] = stageName + } + if len(metadata) == 0 { + return nil + } + return metadata +} + +var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`) + +func redactScriptoriumError(err error) error { + if err == nil { + return nil + } + return redactedProviderError{err: err} +} + +type redactedProviderError struct { + err error +} + +func (e redactedProviderError) Error() string { + return bearerTokenPattern.ReplaceAllString(e.err.Error(), "Bearer "+secretReplacement) +} + +func (e redactedProviderError) Unwrap() error { + return e.err +} diff --git a/internal/framework/llm/scriptorium_client_test.go b/internal/framework/llm/scriptorium_client_test.go new file mode 100644 index 0000000..1ed2bb8 --- /dev/null +++ b/internal/framework/llm/scriptorium_client_test.go @@ -0,0 +1,299 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/fstest" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { + fake := &fakeScriptoriumLLM{content: `{"ok":true}`} + client := newTestScriptoriumClient(t, fake) + + var out struct { + OK bool `json:"ok"` + } + resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + StageName: "test-stage", + PromptID: "adapter.test", + PromptVersion: "v1", + ProfileID: "explicit-profile", + SessionID: "session-123", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"), + }, + Vars: map[string]any{"custom": "value"}, + }, &out) + if err != nil { + t.Fatalf("CompleteStructured() error = %v, want nil", err) + } + if !out.OK { + t.Fatalf("decoded output OK = false, want true") + } + if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" { + t.Fatalf("response metadata = %#v", resp) + } + if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { + t.Fatalf("usage = %#v, want mapped token counts", resp) + } + gotReq := fake.lastRequest() + if gotReq.Prompt.SessionID != "session-123" { + t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID) + } + if gotReq.Target.Model != "explicit-model" { + t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model) + } + if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) { + t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages) + } + if gotReq.StructuredOutput == nil { + t.Fatalf("structured output = nil, want JSON schema") + } + manifests := client.LLMProfileManifests() + if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" { + t.Fatalf("profile manifests = %#v", manifests) + } +} + +func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { + fake := &fakeScriptoriumLLM{content: `{"ok":true}`} + client := newTestScriptoriumClient(t, fake) + + var out map[string]any + if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "session-123", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out); err != nil { + t.Fatalf("CompleteStructured() error = %v, want nil", err) + } + if got := fake.lastRequest().Target.Model; got != "default-model" { + t.Fatalf("model = %q, want prompt default profile model", got) + } +} + +func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) { + client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`}) + + var out map[string]any + _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "session-123", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err == nil || !strings.Contains(err.Error(), "validation failed") { + t.Fatalf("CompleteStructured() error = %v, want validation failure", err) + } +} + +func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) { + client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")}) + + var out map[string]any + _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "session-123", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err == nil { + t.Fatalf("CompleteStructured() error = nil, want provider error") + } + if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) { + t.Fatalf("error = %q, want operation context", err.Error()) + } + if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") { + t.Fatalf("error = %q, want redacted bearer token", err.Error()) + } +} + +func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`}) + + var out map[string]any + _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("CompleteStructured() error = %v, want context canceled", err) + } +} + +func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) { + fake := &fakeScriptoriumLLM{ + content: `{"ok":true}`, + block: make(chan struct{}), + } + client := newTestScriptoriumClient(t, fake) + scheduler, err := NewScheduler(1) + if err != nil { + t.Fatalf("NewScheduler() error = %v, want nil", err) + } + scheduled := NewScheduledClient(client, scheduler) + + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + var out map[string]any + _, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + SessionID: "session-123", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if callErr != nil { + t.Errorf("CompleteStructured() error = %v, want nil", callErr) + } + }() + } + waitForAtomicAtLeast(t, &fake.calls, 1) + time.Sleep(20 * time.Millisecond) + if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 { + t.Fatalf("max in-flight calls = %d, want <= 1", got) + } + close(fake.block) + wg.Wait() +} + +func TestScriptoriumClientValidatesRequest(t *testing.T) { + client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`}) + var out map[string]any + if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") { + t.Fatalf("missing prompt id error = %v, want prompt_id validation", err) + } + if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") { + t.Fatalf("nil output error = %v, want output validation", err) + } +} + +func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient { + t.Helper() + registry := NewAssetRegistry() + if err := registry.RegisterPromptFS(fstest.MapFS{ + "adapter.test.yaml": {Data: []byte(`id: adapter.test +version: "v1" +default_profile: default-profile +session_id: "{{ .session_id }}" +inputs: + - name: transcript + required: true + content_type: application/json +messages: + - role: user + content: "Transcript: {{ input \"transcript\" }}" +output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 +`)}, + }, "."); err != nil { + t.Fatalf("RegisterPromptFS() error = %v", err) + } + if err := registry.RegisterSchemaFS(fstest.MapFS{ + "adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)}, + }, "."); err != nil { + t.Fatalf("RegisterSchemaFS() error = %v", err) + } + client, err := NewScriptoriumClient(ScriptoriumClientConfig{ + Assets: registry, + EngineOptions: []scriptorium.Option{ + scriptorium.WithProfiles( + scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "default-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "default-model", + }), + scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "explicit-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "explicit-model", + }), + ), + scriptorium.WithLLMClient(fake), + }, + }) + if err != nil { + t.Fatalf("NewScriptoriumClient() error = %v, want nil", err) + } + return client +} + +type fakeScriptoriumLLM struct { + content string + err error + block chan struct{} + mu sync.Mutex + last scriptorium.GenerateRequest + calls int32 + inFlight int32 + maxInFlight int32 +} + +func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { + f.mu.Lock() + f.last = req + f.mu.Unlock() + atomic.AddInt32(&f.calls, 1) + current := atomic.AddInt32(&f.inFlight, 1) + for { + seen := atomic.LoadInt32(&f.maxInFlight) + if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) { + break + } + } + defer atomic.AddInt32(&f.inFlight, -1) + if f.block != nil { + select { + case <-f.block: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if f.err != nil { + return nil, f.err + } + content := f.content + if content == "" { + content = `{"ok":true}` + } + if !json.Valid([]byte(content)) { + return nil, errors.New("test fake must return JSON content") + } + return &scriptorium.GenerateResponse{ + Content: content, + Usage: scriptorium.TokenUsage{ + PromptTokens: 11, + CompletionTokens: 7, + TotalTokens: 18, + }, + }, nil +} + +func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest { + f.mu.Lock() + defer f.mu.Unlock() + return f.last +} diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index b430391..88f5195 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -8,6 +8,7 @@ import ( "mime" "path" "path/filepath" + "sort" "strings" "time" @@ -57,8 +58,7 @@ type RunOutput struct { OutputFiles []contracts.OutputFile `json:"-"` } -func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { - var output RunOutput +func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) { if r == nil { return output, fmt.Errorf("runner must not be nil") } @@ -69,8 +69,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { return output, err } - output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...) output.Manifest = manifestFromPipeline(input) + defer func() { + output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient)) + }() + output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...) adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module) if err != nil { @@ -524,6 +527,47 @@ func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMPr return append([]artifacts.LLMProfileManifest(nil), profiles...) } +func llmProfileManifests(client contracts.StructuredLLMClient) []artifacts.LLMProfileManifest { + provider, ok := client.(contracts.LLMProfileManifestProvider) + if !ok { + return nil + } + return provider.LLMProfileManifests() +} + +func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest { + merged := make(map[string]artifacts.LLMProfileManifest) + for _, source := range sources { + for _, profile := range source { + id := strings.TrimSpace(profile.ID) + provider := strings.TrimSpace(profile.Provider) + model := strings.TrimSpace(profile.Model) + key := id + "\x00" + provider + "\x00" + model + if _, exists := merged[key]; exists { + continue + } + merged[key] = artifacts.LLMProfileManifest{ + ID: id, + Provider: provider, + Model: model, + } + } + } + if len(merged) == 0 { + return nil + } + keys := make([]string, 0, len(merged)) + for key := range merged { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]artifacts.LLMProfileManifest, 0, len(keys)) + for _, key := range keys { + out = append(out, merged[key]) + } + return out +} + func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMaterial { return contracts.NewLLMInputMaterial( "source", diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 64b099b..bcc7d4d 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -1212,6 +1212,28 @@ func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) { } } +func TestRunManifestIncludesProfilesReportedByLLMClient(t *testing.T) { + output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{ + Pipeline: resolvedPipeline(), + LLMClient: manifestReportingLLMClient{profiles: []artifacts.LLMProfileManifest{ + {ID: "profile-b", Provider: "openai-compatible", Model: "model-b"}, + {ID: "profile-a", Provider: "openai-compatible", Model: "model-a"}, + {ID: "profile-b", Provider: "openai-compatible", Model: "model-b"}, + }}, + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + want := []artifacts.LLMProfileManifest{ + {ID: "profile-a", Provider: "openai-compatible", Model: "model-a"}, + {ID: "profile-b", Provider: "openai-compatible", Model: "model-b"}, + } + if !reflect.DeepEqual(output.Manifest.LLMProfiles, want) { + t.Fatalf("LLMProfiles = %#v, want %#v", output.Manifest.LLMProfiles, want) + } +} + func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) { output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) if err != nil { @@ -1693,6 +1715,15 @@ func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contract return contracts.StructuredCompletionResponse{}, nil } +type manifestReportingLLMClient struct { + fakeLLMClient + profiles []artifacts.LLMProfileManifest +} + +func (client manifestReportingLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest { + return append([]artifacts.LLMProfileManifest(nil), client.profiles...) +} + func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision { decisions := make([]contracts.ValidationDecision, 0, len(candidates)) for _, candidate := range candidates { diff --git a/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml index 08f7ef5..b28a467 100644 --- a/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml +++ b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml @@ -1,5 +1,6 @@ id: dnd.scenes version: "v1" +default_profile: mistral-small-3 inputs: - name: transcript required: true diff --git a/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml index ae1fe44..445b141 100644 --- a/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml +++ b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml @@ -1,5 +1,6 @@ id: dnd.spells version: "v1" +default_profile: mistral-small-3 inputs: - name: transcript required: true