From 44056d7b8f128b534fbfe8b678d9f5bbaec814e1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 6 May 2026 15:12:58 +0000 Subject: [PATCH] Add Runner.Prepare and refactor Runner.Run to reuse pre-LLM prepare flow --- internal/domain/domain.go | 1 + internal/usecase/runner.go | 149 ++++++++------ internal/usecase/runner_test.go | 331 ++++++++++++++++++++++++++++++++ 3 files changed, 420 insertions(+), 61 deletions(-) diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 4b0d4a5..03c7e24 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -83,6 +83,7 @@ type PreparedRun struct { PromptHash string `json:"prompt_hash,omitempty"` SelectedProfileID string `json:"selected_profile_id"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + OutputContract OutputContract `json:"output_contract"` InputHashes map[string]string `json:"input_hashes,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash"` Messages []RenderedMessage `json:"messages"` diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index dfcc9a1..cd1ef5e 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -74,10 +74,6 @@ func NewRunnerWithRepairer( } func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { - if strings.TrimSpace(req.PromptID) == "" { - return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) - } - runID, err := newRunID() if err != nil { return nil, fmt.Errorf("failed to create run id: %w", err) @@ -85,6 +81,85 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes start := time.Now().UTC() + prepared, err := r.Prepare(ctx, req) + if err != nil { + return nil, err + } + + genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ + Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, + Target: prepared.EffectiveModelParams, + }) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) + } + + outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) + validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrValidation, err) + } + + if r.shouldAttemptRepair(prepared.OutputContract, validationResult) { + attemptsUsed := 0 + for attemptsUsed < prepared.OutputContract.RepairAttempts && validationResult.Status == domain.ValidationFailed { + attemptsUsed++ + + repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{ + PreviousOutput: genResp.Content, + ValidationErrors: validationResult.Errors, + Target: prepared.EffectiveModelParams, + Attempt: attemptsUsed, + MaxAttempts: prepared.OutputContract.RepairAttempts, + Mode: prepared.OutputContract.ValidationMode, + }) + if repairErr != nil { + return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr) + } + if repairResp == nil { + return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation) + } + + genResp = repairResp + outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) + + validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrValidation, err) + } + } + } + + end := time.Now().UTC() + + return &domain.RunResult{ + RunID: runID, + Artifact: outputArtifact, + RawOutput: genResp.Content, + Validation: validationResult, + PromptID: prepared.PromptID, + PromptVersion: prepared.PromptVersion, + PromptHash: prepared.PromptHash, + RenderedPromptHash: prepared.RenderedPromptHash, + SelectedProfileID: prepared.SelectedProfileID, + ModelName: prepared.EffectiveModelParams.Model, + Endpoint: prepared.EffectiveModelParams.Endpoint, + EffectiveModelParams: prepared.EffectiveModelParams, + InputHashes: prepared.InputHashes, + Usage: genResp.Usage, + StartTime: start, + EndTime: end, + Duration: end.Sub(start), + }, nil +} + +func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) { + if strings.TrimSpace(req.PromptID) == "" { + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) + } + + start := time.Now().UTC() + def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) @@ -93,6 +168,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes if err != nil { return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err) } + selectedProfileID := strings.TrimSpace(req.ProfileID) if selectedProfileID == "" { selectedProfileID = strings.TrimSpace(def.DefaultProfile) @@ -100,10 +176,12 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes if selectedProfileID == "" { return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest) } + execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) } + effectiveModel := resolveExecutionTarget(execProfile, req.Execution) if strings.TrimSpace(effectiveModel.Endpoint) == "" { return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) @@ -114,6 +192,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) } + effectiveContract := resolveOutputContract(def, req.Validation) resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs)) @@ -135,72 +214,20 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes return nil, fmt.Errorf("%w: %w", ErrPromptRender, err) } - renderedPromptHash := hashRenderedPrompt(*renderedPrompt) - - genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: *renderedPrompt, - Target: effectiveModel, - }) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) - } - - outputArtifact := buildOutputArtifact(genResp.Content, effectiveContract.Format) - validationResult, err := r.validateOutput(ctx, &outputArtifact, effectiveContract, 0) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, err) - } - - if r.shouldAttemptRepair(effectiveContract, validationResult) { - attemptsUsed := 0 - for attemptsUsed < effectiveContract.RepairAttempts && validationResult.Status == domain.ValidationFailed { - attemptsUsed++ - - repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{ - PreviousOutput: genResp.Content, - ValidationErrors: validationResult.Errors, - Target: effectiveModel, - Attempt: attemptsUsed, - MaxAttempts: effectiveContract.RepairAttempts, - Mode: effectiveContract.ValidationMode, - }) - if repairErr != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr) - } - if repairResp == nil { - return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation) - } - - genResp = repairResp - outputArtifact = buildOutputArtifact(genResp.Content, effectiveContract.Format) - - validationResult, err = r.validateOutput(ctx, &outputArtifact, effectiveContract, attemptsUsed) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, err) - } - } - } - end := time.Now().UTC() - - return &domain.RunResult{ - RunID: runID, - Artifact: outputArtifact, - RawOutput: genResp.Content, - Validation: validationResult, + return &domain.PreparedRun{ PromptID: def.ID, PromptVersion: def.Version, PromptHash: promptDefinitionHash, - RenderedPromptHash: renderedPromptHash, SelectedProfileID: selectedProfileID, - ModelName: effectiveModel.Model, - Endpoint: effectiveModel.Endpoint, EffectiveModelParams: effectiveModel, + OutputContract: effectiveContract, InputHashes: inputHashes, - Usage: genResp.Usage, + RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), + Messages: renderedPrompt.Messages, StartTime: start, EndTime: end, - Duration: end.Sub(start), + DurationMS: end.Sub(start).Milliseconds(), }, nil } diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 9216b7f..7f11b6e 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -6,12 +6,17 @@ import ( "encoding/hex" "errors" "fmt" + "path/filepath" + "reflect" "regexp" "strings" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" + "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/validate" ) @@ -81,10 +86,16 @@ type fakeLLM struct { resp *domain.GenerateResponse err error lastReq domain.GenerateRequest + calls int + forbid bool } func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + f.calls++ f.lastReq = req + if f.forbid { + return nil, errors.New("llm should not be called") + } if f.err != nil { return nil, f.err } @@ -126,6 +137,280 @@ func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.G return f.responses[idx], nil } +func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) { + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} + reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ + "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, + "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, + }} + renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} + llmClient := &fakeLLM{forbid: true} + + runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + PromptVersion: "1", + ProfileID: "exec", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, + "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, + }, + Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if prepared.PromptID != "p" || prepared.PromptVersion != "1" { + t.Fatalf("unexpected prepared prompt metadata: %+v", prepared) + } + if prepared.SelectedProfileID != "exec" { + t.Fatalf("expected selected profile exec, got %q", prepared.SelectedProfileID) + } + if prepared.PromptHash == "" || prepared.RenderedPromptHash == "" { + t.Fatal("expected prompt hashes") + } + if prepared.EffectiveModelParams.Model != "m" || prepared.EffectiveModelParams.Endpoint != "http://override/v1" { + t.Fatalf("unexpected model params: %+v", prepared.EffectiveModelParams) + } + if prepared.OutputContract.Format != domain.FormatMarkdown { + t.Fatalf("expected output format markdown, got %q", prepared.OutputContract.Format) + } + if len(prepared.InputHashes) != 2 || prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { + t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) + } + if len(prepared.Messages) != 2 { + t.Fatalf("expected two messages, got %d", len(prepared.Messages)) + } + if llmClient.calls != 0 { + t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) + } +} + +func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) { + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + promptRepo.def.DefaultProfile = "from-prompt" + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, + }} + + runner := newMinimalRunner(promptRepo, execRepo) + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + Inputs: singleInputRef(), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if execRepo.lastID != "from-prompt" { + t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID) + } + if prepared.SelectedProfileID != "from-prompt" { + t.Fatalf("expected selected profile from-prompt, got %q", prepared.SelectedProfileID) + } +} + +func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) { + repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + repo.def.DefaultProfile = "" + runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}) + _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } +} + +func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { + repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + repo.def.DefaultProfile = "does-not-exist" + runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}}) + _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) + if !errors.Is(err, ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } +} + +func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": { + ID: "exec", + Endpoint: "http://profile/v1", + Model: "profile-model", + Temperature: 0.2, + MaxTokens: 500, + TopP: 0.9, + TimeoutSeconds: 120, + }, + }} + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: singleInputRef(), + Execution: &domain.ExecutionTarget{ + Endpoint: "http://override/v1", + Model: "override-model", + Temperature: 0.7, + TimeoutSeconds: 30, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if prepared.EffectiveModelParams.Endpoint != "http://override/v1" || prepared.EffectiveModelParams.Model != "override-model" { + t.Fatalf("expected endpoint/model override to win, got %+v", prepared.EffectiveModelParams) + } + if prepared.EffectiveModelParams.TopP != 0.9 { + t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP) + } +} + +func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": { + ID: "exec", + Endpoint: "http://profile/v1", + Model: "profile-model", + TopP: 0.8, + TimeoutSeconds: 90, + }, + }} + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: singleInputRef(), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if prepared.EffectiveModelParams.TopP != 0.8 { + t.Fatalf("expected profile top_p to beat default, got %v", prepared.EffectiveModelParams.TopP) + } + if prepared.EffectiveModelParams.TimeoutSeconds != 90 { + t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds) + } +} + +func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { + promptDir := filepath.Join("..", "promptdef", "testdata") + profileDir := filepath.Join("..", "profile", "testdata") + + reader := &fakeArtifactReader{ + artifactsByURI: map[string]*domain.Artifact{ + "a://transcript": { + Name: "transcript", + Body: []byte("Session transcript body."), + Hash: hashString("Session transcript body."), + }, + }, + } + llmClient := &fakeLLM{forbid: true} + runner := NewRunner( + promptdef.NewFilesystemRepository(promptDir), + profile.NewFilesystemRepository(profileDir), + reader, + prompt.NewGoRenderer(), + llmClient, + nil, + ) + + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "valid-file-backed", + ProfileID: "local-default", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"}, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(prepared.Messages) != 2 { + t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages)) + } + if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") { + t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content) + } +} + +func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { + def := promptDef(domain.FormatText, domain.ValidationNone, 0) + def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}} + + runner := NewRunner( + &fakePromptRepo{def: def}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), + prompt.NewGoRenderer(), + &fakeLLM{forbid: true}, + nil, + ) + _, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: map[string]domain.ArtifactRef{}, + }) + if !errors.Is(err, ErrPromptRender) { + t.Fatalf("expected ErrPromptRender, got %v", err) + } + if !errors.Is(err, prompt.ErrMissingRequiredInput) { + t.Fatalf("expected ErrMissingRequiredInput, got %v", err) + } +} + +func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { + def := promptDef(domain.FormatText, domain.ValidationNone, 0) + def.Inputs = []domain.PromptInput{{Name: "transcript", Required: false}} + def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "ghost"}}`}} + + runner := NewRunner( + &fakePromptRepo{def: def}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), + prompt.NewGoRenderer(), + &fakeLLM{forbid: true}, + nil, + ) + _, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: map[string]domain.ArtifactRef{}, + }) + if !errors.Is(err, ErrPromptRender) { + t.Fatalf("expected ErrPromptRender, got %v", err) + } + if !errors.Is(err, prompt.ErrUnknownInput) { + t.Fatalf("expected ErrUnknownInput, got %v", err) + } +} + +func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { + const envName = "SCRIPTORIUM_TEST_API_KEY" + const secret = "top-secret-value" + t.Setenv(envName, secret) + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, + }} + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if prepared.EffectiveModelParams.APIKeyEnv != envName { + t.Fatalf("expected api key env name, got %q", prepared.EffectiveModelParams.APIKeyEnv) + } + metadataDump := fmt.Sprintf("%+v|%s|%s", prepared.EffectiveModelParams, prepared.PromptHash, prepared.RenderedPromptHash) + if strings.Contains(metadataDump, secret) { + t.Fatalf("unexpected api key value in prepared metadata dump: %s", metadataDump) + } +} + func TestRunnerRunSuccessful(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} @@ -182,6 +467,52 @@ func TestRunnerRunSuccessful(t *testing.T) { } } +func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} + execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} + reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ + "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, + }} + renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} + llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} + runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) + + req := domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, + }, + Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, + } + + prepared, err := runner.Prepare(context.Background(), req) + if err != nil { + t.Fatalf("prepare should succeed, got %v", err) + } + + res, err := runner.Run(context.Background(), req) + if err != nil { + t.Fatalf("run should succeed, got %v", err) + } + + if res.SelectedProfileID != prepared.SelectedProfileID { + t.Fatalf("expected selected profile to match prepare, run=%q prepare=%q", res.SelectedProfileID, prepared.SelectedProfileID) + } + if !reflect.DeepEqual(res.EffectiveModelParams, prepared.EffectiveModelParams) { + t.Fatalf("effective model params mismatch:\nrun=%+v\nprepare=%+v", res.EffectiveModelParams, prepared.EffectiveModelParams) + } + if !reflect.DeepEqual(res.InputHashes, prepared.InputHashes) { + t.Fatalf("input hashes mismatch:\nrun=%#v\nprepare=%#v", res.InputHashes, prepared.InputHashes) + } + if res.RenderedPromptHash != prepared.RenderedPromptHash { + t.Fatalf("expected rendered prompt hash to match prepare, run=%q prepare=%q", res.RenderedPromptHash, prepared.RenderedPromptHash) + } + if !reflect.DeepEqual(llmClient.lastReq.Prompt.Messages, prepared.Messages) { + t.Fatalf("expected run to send prepare-rendered messages to llm") + } +} + func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "default-prof"