Add Scriptorium-backed LLM runtime

2026-07-05 18:21:37 +00:00
parent de6689bc1d
commit f6224dcbee
12 changed files with 742 additions and 39 deletions
--- a/internal/framework/contracts/contracts.go
+++ b/internal/framework/contracts/contracts.go
@@ -31,6 +31,7 @@ type StructuredCompletionResponse struct {
 	Content          json.RawMessage `json:"content"`
 	Provider         string          `json:"provider,omitempty"`
 	Model            string          `json:"model,omitempty"`
+	ProfileID        string          `json:"profile_id,omitempty"`
 	PromptTokens     int             `json:"prompt_tokens,omitempty"`
 	CompletionTokens int             `json:"completion_tokens,omitempty"`
 	TotalTokens      int             `json:"total_tokens,omitempty"`
@@ -40,6 +41,10 @@ type StructuredLLMClient interface {
 	CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error)
 }

+type LLMProfileManifestProvider interface {
+	LLMProfileManifests() []artifacts.LLMProfileManifest
+}
+
 type LLMInputMaterial struct {
 	Name      string `json:"name"`
 	MediaType string `json:"media_type,omitempty"`
--- a/internal/framework/llm/scheduled_client.go
+++ b/internal/framework/llm/scheduled_client.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"

+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 )

@@ -41,3 +42,14 @@ func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.
 	}
 	return response, nil
 }
+
+func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
+	if c == nil || c.client == nil {
+		return nil
+	}
+	provider, ok := c.client.(contracts.LLMProfileManifestProvider)
+	if !ok {
+		return nil
+	}
+	return provider.LLMProfileManifests()
+}
--- a/internal/framework/llm/scriptorium_client.go
+++ b/internal/framework/llm/scriptorium_client.go
@@ -0,0 +1,267 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/scriptorium"
+)
+
+const scriptoriumProviderName = "openai-compatible"
+
+type ScriptoriumClientConfig struct {
+	ProfileDir    string
+	ProfileFile   string
+	Assets        *AssetRegistry
+	Timeout       time.Duration
+	HTTPClient    *http.Client
+	EngineOptions []scriptorium.Option
+	Recorder      *LLMProfileRecorder
+}
+
+type ScriptoriumClient struct {
+	engine   *scriptorium.Engine
+	recorder *LLMProfileRecorder
+}
+
+type LLMProfileRecorder struct {
+	mu       sync.Mutex
+	profiles map[string]artifacts.LLMProfileManifest
+}
+
+var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
+var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
+
+func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
+	if cfg.Assets == nil {
+		return nil, fmt.Errorf("scriptorium client assets must not be nil")
+	}
+	if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
+		return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
+	}
+	options, err := cfg.Assets.ScriptoriumOptions()
+	if err != nil {
+		return nil, err
+	}
+	if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
+		options = append(options, scriptorium.WithProfileFile(profileFile))
+	}
+	options = append(options, cfg.EngineOptions...)
+
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		ProfileDir: strings.TrimSpace(cfg.ProfileDir),
+		Timeout:    cfg.Timeout,
+		HTTPClient: cfg.HTTPClient,
+	}, options...)
+	if err != nil {
+		return nil, fmt.Errorf("create Scriptorium engine: %w", err)
+	}
+	recorder := cfg.Recorder
+	if recorder == nil {
+		recorder = NewLLMProfileRecorder()
+	}
+	return &ScriptoriumClient{
+		engine:   engine,
+		recorder: recorder,
+	}, nil
+}
+
+func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
+	if c == nil {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
+	}
+	if c.engine == nil {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
+	}
+	if err := validateOutputTarget(out); err != nil {
+		return contracts.StructuredCompletionResponse{}, err
+	}
+	promptID := strings.TrimSpace(req.PromptID)
+	if promptID == "" {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
+	}
+
+	runReq := scriptorium.RunRequest{
+		PromptID:      promptID,
+		PromptVersion: strings.TrimSpace(req.PromptVersion),
+		ProfileID:     strings.TrimSpace(req.ProfileID),
+		Inputs:        scriptoriumInputs(req.Inputs),
+		Vars:          scriptoriumVars(req),
+		Metadata:      scriptoriumMetadata(req),
+	}
+	result, err := c.engine.Run(ctx, runReq)
+	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return contracts.StructuredCompletionResponse{}, ctxErr
+		}
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
+	}
+	if result == nil {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
+	}
+	if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
+	}
+
+	content := result.Artifact.Body
+	if len(content) == 0 {
+		content = []byte(result.RawOutput)
+	}
+	if len(strings.TrimSpace(string(content))) == 0 {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty structured output", promptID)
+	}
+	if err := json.Unmarshal(content, out); err != nil {
+		return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w", promptID, err)
+	}
+
+	profile := artifacts.LLMProfileManifest{
+		ID:       strings.TrimSpace(result.SelectedProfileID),
+		Provider: scriptoriumProviderName,
+		Model:    firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
+	}
+	if c.recorder != nil {
+		c.recorder.Record(profile)
+	}
+	return contracts.StructuredCompletionResponse{
+		Content:          append(json.RawMessage(nil), content...),
+		Provider:         profile.Provider,
+		Model:            profile.Model,
+		ProfileID:        profile.ID,
+		PromptTokens:     result.Usage.PromptTokens,
+		CompletionTokens: result.Usage.CompletionTokens,
+		TotalTokens:      result.Usage.TotalTokens,
+	}, nil
+}
+
+func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
+	if c == nil || c.recorder == nil {
+		return nil
+	}
+	return c.recorder.Manifests()
+}
+
+func NewLLMProfileRecorder() *LLMProfileRecorder {
+	return &LLMProfileRecorder{profiles: map[string]artifacts.LLMProfileManifest{}}
+}
+
+func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) {
+	if r == nil {
+		return
+	}
+	profile.ID = strings.TrimSpace(profile.ID)
+	profile.Provider = strings.TrimSpace(profile.Provider)
+	profile.Model = strings.TrimSpace(profile.Model)
+	key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.profiles == nil {
+		r.profiles = map[string]artifacts.LLMProfileManifest{}
+	}
+	r.profiles[key] = profile
+}
+
+func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
+	if r == nil {
+		return nil
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if len(r.profiles) == 0 {
+		return nil
+	}
+	keys := make([]string, 0, len(r.profiles))
+	for key := range r.profiles {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	out := make([]artifacts.LLMProfileManifest, 0, len(keys))
+	for _, key := range keys {
+		out = append(out, r.profiles[key])
+	}
+	return out
+}
+
+func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
+	if len(inputs) == 0 {
+		return nil
+	}
+	out := make(map[string]scriptorium.ArtifactRef, len(inputs))
+	for key, material := range inputs {
+		name := strings.TrimSpace(key)
+		if name == "" {
+			name = strings.TrimSpace(material.Name)
+		}
+		if name == "" {
+			continue
+		}
+		body := string(material.Content)
+		if body == "" {
+			body = " "
+		}
+		if origin := strings.TrimSpace(material.OriginURI); origin != "" {
+			out[name] = scriptorium.InlineWithURI(origin, body)
+		} else {
+			out[name] = scriptorium.Inline(body)
+		}
+	}
+	return out
+}
+
+func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
+	vars := make(map[string]string, len(req.Vars)+1)
+	for key, value := range req.Vars {
+		name := strings.TrimSpace(key)
+		if name == "" || value == nil {
+			continue
+		}
+		vars[name] = fmt.Sprint(value)
+	}
+	if sessionID := strings.TrimSpace(req.SessionID); sessionID != "" {
+		vars["session_id"] = sessionID
+	}
+	if len(vars) == 0 {
+		return nil
+	}
+	return vars
+}
+
+func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
+	metadata := map[string]string{}
+	if stageName := strings.TrimSpace(req.StageName); stageName != "" {
+		metadata["stage_name"] = stageName
+	}
+	if len(metadata) == 0 {
+		return nil
+	}
+	return metadata
+}
+
+var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
+
+func redactScriptoriumError(err error) error {
+	if err == nil {
+		return nil
+	}
+	return redactedProviderError{err: err}
+}
+
+type redactedProviderError struct {
+	err error
+}
+
+func (e redactedProviderError) Error() string {
+	return bearerTokenPattern.ReplaceAllString(e.err.Error(), "Bearer "+secretReplacement)
+}
+
+func (e redactedProviderError) Unwrap() error {
+	return e.err
+}
--- a/internal/framework/llm/scriptorium_client_test.go
+++ b/internal/framework/llm/scriptorium_client_test.go
@@ -0,0 +1,299 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"testing/fstest"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/scriptorium"
+)
+
+func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
+	fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
+	client := newTestScriptoriumClient(t, fake)
+
+	var out struct {
+		OK bool `json:"ok"`
+	}
+	resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+		StageName:     "test-stage",
+		PromptID:      "adapter.test",
+		PromptVersion: "v1",
+		ProfileID:     "explicit-profile",
+		SessionID:     "session-123",
+		Inputs: contracts.LLMInputSet{
+			"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"),
+		},
+		Vars: map[string]any{"custom": "value"},
+	}, &out)
+	if err != nil {
+		t.Fatalf("CompleteStructured() error = %v, want nil", err)
+	}
+	if !out.OK {
+		t.Fatalf("decoded output OK = false, want true")
+	}
+	if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
+		t.Fatalf("response metadata = %#v", resp)
+	}
+	if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
+		t.Fatalf("usage = %#v, want mapped token counts", resp)
+	}
+	gotReq := fake.lastRequest()
+	if gotReq.Prompt.SessionID != "session-123" {
+		t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
+	}
+	if gotReq.Target.Model != "explicit-model" {
+		t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
+	}
+	if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
+		t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
+	}
+	if gotReq.StructuredOutput == nil {
+		t.Fatalf("structured output = nil, want JSON schema")
+	}
+	manifests := client.LLMProfileManifests()
+	if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
+		t.Fatalf("profile manifests = %#v", manifests)
+	}
+}
+
+func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
+	fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
+	client := newTestScriptoriumClient(t, fake)
+
+	var out map[string]any
+	if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+		PromptID:  "adapter.test",
+		SessionID: "session-123",
+		Inputs: contracts.LLMInputSet{
+			"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+		},
+	}, &out); err != nil {
+		t.Fatalf("CompleteStructured() error = %v, want nil", err)
+	}
+	if got := fake.lastRequest().Target.Model; got != "default-model" {
+		t.Fatalf("model = %q, want prompt default profile model", got)
+	}
+}
+
+func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
+	client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
+
+	var out map[string]any
+	_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+		PromptID:  "adapter.test",
+		SessionID: "session-123",
+		Inputs: contracts.LLMInputSet{
+			"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+		},
+	}, &out)
+	if err == nil || !strings.Contains(err.Error(), "validation failed") {
+		t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
+	}
+}
+
+func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
+	client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
+
+	var out map[string]any
+	_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+		PromptID:  "adapter.test",
+		SessionID: "session-123",
+		Inputs: contracts.LLMInputSet{
+			"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+		},
+	}, &out)
+	if err == nil {
+		t.Fatalf("CompleteStructured() error = nil, want provider error")
+	}
+	if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
+		t.Fatalf("error = %q, want operation context", err.Error())
+	}
+	if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
+		t.Fatalf("error = %q, want redacted bearer token", err.Error())
+	}
+}
+
+func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
+
+	var out map[string]any
+	_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
+		PromptID: "adapter.test",
+		Inputs: contracts.LLMInputSet{
+			"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+		},
+	}, &out)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
+	}
+}
+
+func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
+	fake := &fakeScriptoriumLLM{
+		content: `{"ok":true}`,
+		block:   make(chan struct{}),
+	}
+	client := newTestScriptoriumClient(t, fake)
+	scheduler, err := NewScheduler(1)
+	if err != nil {
+		t.Fatalf("NewScheduler() error = %v, want nil", err)
+	}
+	scheduled := NewScheduledClient(client, scheduler)
+
+	var wg sync.WaitGroup
+	for i := 0; i < 3; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var out map[string]any
+			_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
+				PromptID:  "adapter.test",
+				SessionID: "session-123",
+				Inputs: contracts.LLMInputSet{
+					"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
+				},
+			}, &out)
+			if callErr != nil {
+				t.Errorf("CompleteStructured() error = %v, want nil", callErr)
+			}
+		}()
+	}
+	waitForAtomicAtLeast(t, &fake.calls, 1)
+	time.Sleep(20 * time.Millisecond)
+	if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
+		t.Fatalf("max in-flight calls = %d, want <= 1", got)
+	}
+	close(fake.block)
+	wg.Wait()
+}
+
+func TestScriptoriumClientValidatesRequest(t *testing.T) {
+	client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
+	var out map[string]any
+	if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
+		t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
+	}
+	if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") {
+		t.Fatalf("nil output error = %v, want output validation", err)
+	}
+}
+
+func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
+	t.Helper()
+	registry := NewAssetRegistry()
+	if err := registry.RegisterPromptFS(fstest.MapFS{
+		"adapter.test.yaml": {Data: []byte(`id: adapter.test
+version: "v1"
+default_profile: default-profile
+session_id: "{{ .session_id }}"
+inputs:
+  - name: transcript
+    required: true
+    content_type: application/json
+messages:
+  - role: user
+    content: "Transcript: {{ input \"transcript\" }}"
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: adapter.schema.json
+  repair_attempts: 0
+`)},
+	}, "."); err != nil {
+		t.Fatalf("RegisterPromptFS() error = %v", err)
+	}
+	if err := registry.RegisterSchemaFS(fstest.MapFS{
+		"adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)},
+	}, "."); err != nil {
+		t.Fatalf("RegisterSchemaFS() error = %v", err)
+	}
+	client, err := NewScriptoriumClient(ScriptoriumClientConfig{
+		Assets: registry,
+		EngineOptions: []scriptorium.Option{
+			scriptorium.WithProfiles(
+				scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
+					ID:       "default-profile",
+					Endpoint: "http://127.0.0.1:1/v1",
+					Model:    "default-model",
+				}),
+				scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
+					ID:       "explicit-profile",
+					Endpoint: "http://127.0.0.1:1/v1",
+					Model:    "explicit-model",
+				}),
+			),
+			scriptorium.WithLLMClient(fake),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
+	}
+	return client
+}
+
+type fakeScriptoriumLLM struct {
+	content     string
+	err         error
+	block       chan struct{}
+	mu          sync.Mutex
+	last        scriptorium.GenerateRequest
+	calls       int32
+	inFlight    int32
+	maxInFlight int32
+}
+
+func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
+	f.mu.Lock()
+	f.last = req
+	f.mu.Unlock()
+	atomic.AddInt32(&f.calls, 1)
+	current := atomic.AddInt32(&f.inFlight, 1)
+	for {
+		seen := atomic.LoadInt32(&f.maxInFlight)
+		if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) {
+			break
+		}
+	}
+	defer atomic.AddInt32(&f.inFlight, -1)
+	if f.block != nil {
+		select {
+		case <-f.block:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	if f.err != nil {
+		return nil, f.err
+	}
+	content := f.content
+	if content == "" {
+		content = `{"ok":true}`
+	}
+	if !json.Valid([]byte(content)) {
+		return nil, errors.New("test fake must return JSON content")
+	}
+	return &scriptorium.GenerateResponse{
+		Content: content,
+		Usage: scriptorium.TokenUsage{
+			PromptTokens:     11,
+			CompletionTokens: 7,
+			TotalTokens:      18,
+		},
+	}, nil
+}
+
+func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.last
+}
--- a/internal/framework/pipeline/runner.go
+++ b/internal/framework/pipeline/runner.go
@@ -8,6 +8,7 @@ import (
 	"mime"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"

@@ -57,8 +58,7 @@ type RunOutput struct {
 	OutputFiles []contracts.OutputFile       `json:"-"`
 }

-func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
-	var output RunOutput
+func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
 	if r == nil {
 		return output, fmt.Errorf("runner must not be nil")
 	}
@@ -69,8 +69,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
 		return output, err
 	}

-	output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
 	output.Manifest = manifestFromPipeline(input)
+	defer func() {
+		output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
+	}()
+	output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)

 	adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
 	if err != nil {
@@ -524,6 +527,47 @@ func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMPr
 	return append([]artifacts.LLMProfileManifest(nil), profiles...)
 }

+func llmProfileManifests(client contracts.StructuredLLMClient) []artifacts.LLMProfileManifest {
+	provider, ok := client.(contracts.LLMProfileManifestProvider)
+	if !ok {
+		return nil
+	}
+	return provider.LLMProfileManifests()
+}
+
+func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
+	merged := make(map[string]artifacts.LLMProfileManifest)
+	for _, source := range sources {
+		for _, profile := range source {
+			id := strings.TrimSpace(profile.ID)
+			provider := strings.TrimSpace(profile.Provider)
+			model := strings.TrimSpace(profile.Model)
+			key := id + "\x00" + provider + "\x00" + model
+			if _, exists := merged[key]; exists {
+				continue
+			}
+			merged[key] = artifacts.LLMProfileManifest{
+				ID:       id,
+				Provider: provider,
+				Model:    model,
+			}
+		}
+	}
+	if len(merged) == 0 {
+		return nil
+	}
+	keys := make([]string, 0, len(merged))
+	for key := range merged {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	out := make([]artifacts.LLMProfileManifest, 0, len(keys))
+	for _, key := range keys {
+		out = append(out, merged[key])
+	}
+	return out
+}
+
 func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMaterial {
 	return contracts.NewLLMInputMaterial(
 		"source",
--- a/internal/framework/pipeline/runner_test.go
+++ b/internal/framework/pipeline/runner_test.go
@@ -1212,6 +1212,28 @@ func TestRunManifestIncludesRunTimingAndLLMProfiles(t *testing.T) {
 	}
 }

+func TestRunManifestIncludesProfilesReportedByLLMClient(t *testing.T) {
+	output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{
+		Pipeline: resolvedPipeline(),
+		LLMClient: manifestReportingLLMClient{profiles: []artifacts.LLMProfileManifest{
+			{ID: "profile-b", Provider: "openai-compatible", Model: "model-b"},
+			{ID: "profile-a", Provider: "openai-compatible", Model: "model-a"},
+			{ID: "profile-b", Provider: "openai-compatible", Model: "model-b"},
+		}},
+	})
+	if err != nil {
+		t.Fatalf("Run() error = %v, want nil", err)
+	}
+
+	want := []artifacts.LLMProfileManifest{
+		{ID: "profile-a", Provider: "openai-compatible", Model: "model-a"},
+		{ID: "profile-b", Provider: "openai-compatible", Model: "model-b"},
+	}
+	if !reflect.DeepEqual(output.Manifest.LLMProfiles, want) {
+		t.Fatalf("LLMProfiles = %#v, want %#v", output.Manifest.LLMProfiles, want)
+	}
+}
+
 func TestRunManifestGeneratesRunIDAndTimestamps(t *testing.T) {
 	output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
 	if err != nil {
@@ -1693,6 +1715,15 @@ func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contract
 	return contracts.StructuredCompletionResponse{}, nil
 }

+type manifestReportingLLMClient struct {
+	fakeLLMClient
+	profiles []artifacts.LLMProfileManifest
+}
+
+func (client manifestReportingLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
+	return append([]artifacts.LLMProfileManifest(nil), client.profiles...)
+}
+
 func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
 	decisions := make([]contracts.ValidationDecision, 0, len(candidates))
 	for _, candidate := range candidates {