eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Add Runner.Prepare and refactor Runner.Run to reuse pre-LLM prepare flow

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-06 15:12:58 +00:00
parent 48c06218dc
commit 44056d7b8f
3 changed files with 420 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/domain/domain.go
View File

@@ -83,6 +83,7 @@ type PreparedRun struct {
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`

149
internal/usecase/runner.go
View File

@@ -74,10 +74,6 @@ func NewRunnerWithRepairer(
}
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
@@ -85,6 +81,85 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
prepared, err := r.Prepare(ctx, req)
if err != nil {
return nil, err
}
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
Target: prepared.EffectiveModelParams,
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
}
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
attemptsUsed := 0
for attemptsUsed < prepared.OutputContract.RepairAttempts && validationResult.Status == domain.ValidationFailed {
attemptsUsed++
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
Target: prepared.EffectiveModelParams,
Attempt: attemptsUsed,
MaxAttempts: prepared.OutputContract.RepairAttempts,
Mode: prepared.OutputContract.ValidationMode,
})
if repairErr != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr)
}
if repairResp == nil {
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation)
}
genResp = repairResp
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
}
}
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
InputHashes: prepared.InputHashes,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) {
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
start := time.Now().UTC()
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
@@ -93,6 +168,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
}
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
@@ -100,10 +176,12 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
}
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel := resolveExecutionTarget(execProfile, req.Execution)
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
@@ -114,6 +192,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveContract := resolveOutputContract(def, req.Validation)
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
@@ -135,72 +214,20 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
}
renderedPromptHash := hashRenderedPrompt(*renderedPrompt)
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: *renderedPrompt,
Target: effectiveModel,
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
}
outputArtifact := buildOutputArtifact(genResp.Content, effectiveContract.Format)
validationResult, err := r.validateOutput(ctx, &outputArtifact, effectiveContract, 0)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
if r.shouldAttemptRepair(effectiveContract, validationResult) {
attemptsUsed := 0
for attemptsUsed < effectiveContract.RepairAttempts && validationResult.Status == domain.ValidationFailed {
attemptsUsed++
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
Target: effectiveModel,
Attempt: attemptsUsed,
MaxAttempts: effectiveContract.RepairAttempts,
Mode: effectiveContract.ValidationMode,
})
if repairErr != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr)
}
if repairResp == nil {
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation)
}
genResp = repairResp
outputArtifact = buildOutputArtifact(genResp.Content, effectiveContract.Format)
validationResult, err = r.validateOutput(ctx, &outputArtifact, effectiveContract, attemptsUsed)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
}
}
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
return &domain.PreparedRun{
PromptID: def.ID,
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
RenderedPromptHash: renderedPromptHash,
SelectedProfileID: selectedProfileID,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
EffectiveModelParams: effectiveModel,
OutputContract: effectiveContract,
InputHashes: inputHashes,
Usage: genResp.Usage,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
DurationMS: end.Sub(start).Milliseconds(),
}, nil
}

331
internal/usecase/runner_test.go
View File

@@ -6,12 +6,17 @@ import (
"encoding/hex"
"errors"
"fmt"
"path/filepath"
"reflect"
"regexp"
"strings"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
@@ -81,10 +86,16 @@ type fakeLLM struct {
resp *domain.GenerateResponse
err error
lastReq domain.GenerateRequest
calls int
forbid bool
}
func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
f.calls++
f.lastReq = req
if f.forbid {
return nil, errors.New("llm should not be called")
}
if f.err != nil {
return nil, f.err
}
@@ -126,6 +137,280 @@ func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.G
return f.responses[idx], nil
}
func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if prepared.PromptID != "p" || prepared.PromptVersion != "1" {
t.Fatalf("unexpected prepared prompt metadata: %+v", prepared)
}
if prepared.SelectedProfileID != "exec" {
t.Fatalf("expected selected profile exec, got %q", prepared.SelectedProfileID)
}
if prepared.PromptHash == "" || prepared.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if prepared.EffectiveModelParams.Model != "m" || prepared.EffectiveModelParams.Endpoint != "http://override/v1" {
t.Fatalf("unexpected model params: %+v", prepared.EffectiveModelParams)
}
if prepared.OutputContract.Format != domain.FormatMarkdown {
t.Fatalf("expected output format markdown, got %q", prepared.OutputContract.Format)
}
if len(prepared.InputHashes) != 2 || prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
}
if len(prepared.Messages) != 2 {
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
}
if llmClient.calls != 0 {
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
}
}
func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "from-prompt"
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"},
}}
runner := newMinimalRunner(promptRepo, execRepo)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if execRepo.lastID != "from-prompt" {
t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID)
}
if prepared.SelectedProfileID != "from-prompt" {
t.Fatalf("expected selected profile from-prompt, got %q", prepared.SelectedProfileID)
}
}
func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
repo.def.DefaultProfile = ""
runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}})
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
repo.def.DefaultProfile = "does-not-exist"
runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}})
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
}
func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 500,
TopP: 0.9,
TimeoutSeconds: 120,
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{
Endpoint: "http://override/v1",
Model: "override-model",
Temperature: 0.7,
TimeoutSeconds: 30,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if prepared.EffectiveModelParams.Endpoint != "http://override/v1" || prepared.EffectiveModelParams.Model != "override-model" {
t.Fatalf("expected endpoint/model override to win, got %+v", prepared.EffectiveModelParams)
}
if prepared.EffectiveModelParams.TopP != 0.9 {
t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP)
}
}
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
TopP: 0.8,
TimeoutSeconds: 90,
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if prepared.EffectiveModelParams.TopP != 0.8 {
t.Fatalf("expected profile top_p to beat default, got %v", prepared.EffectiveModelParams.TopP)
}
if prepared.EffectiveModelParams.TimeoutSeconds != 90 {
t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds)
}
}
func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
promptDir := filepath.Join("..", "promptdef", "testdata")
profileDir := filepath.Join("..", "profile", "testdata")
reader := &fakeArtifactReader{
artifactsByURI: map[string]*domain.Artifact{
"a://transcript": {
Name: "transcript",
Body: []byte("Session transcript body."),
Hash: hashString("Session transcript body."),
},
},
}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
reader,
prompt.NewGoRenderer(),
llmClient,
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed",
ProfileID: "local-default",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(prepared.Messages) != 2 {
t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages))
}
if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") {
t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content)
}
}
func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{},
})
if !errors.Is(err, ErrPromptRender) {
t.Fatalf("expected ErrPromptRender, got %v", err)
}
if !errors.Is(err, prompt.ErrMissingRequiredInput) {
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
}
}
func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.Inputs = []domain.PromptInput{{Name: "transcript", Required: false}}
def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "ghost"}}`}}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{},
})
if !errors.Is(err, ErrPromptRender) {
t.Fatalf("expected ErrPromptRender, got %v", err)
}
if !errors.Is(err, prompt.ErrUnknownInput) {
t.Fatalf("expected ErrUnknownInput, got %v", err)
}
}
func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
const envName = "SCRIPTORIUM_TEST_API_KEY"
const secret = "top-secret-value"
t.Setenv(envName, secret)
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != envName {
t.Fatalf("expected api key env name, got %q", prepared.EffectiveModelParams.APIKeyEnv)
}
metadataDump := fmt.Sprintf("%+v|%s|%s", prepared.EffectiveModelParams, prepared.PromptHash, prepared.RenderedPromptHash)
if strings.Contains(metadataDump, secret) {
t.Fatalf("unexpected api key value in prepared metadata dump: %s", metadataDump)
}
}
func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -182,6 +467,52 @@ func TestRunnerRunSuccessful(t *testing.T) {
}
}
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
req := domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
},
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
}
prepared, err := runner.Prepare(context.Background(), req)
if err != nil {
t.Fatalf("prepare should succeed, got %v", err)
}
res, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("run should succeed, got %v", err)
}
if res.SelectedProfileID != prepared.SelectedProfileID {
t.Fatalf("expected selected profile to match prepare, run=%q prepare=%q", res.SelectedProfileID, prepared.SelectedProfileID)
}
if !reflect.DeepEqual(res.EffectiveModelParams, prepared.EffectiveModelParams) {
t.Fatalf("effective model params mismatch:\nrun=%+v\nprepare=%+v", res.EffectiveModelParams, prepared.EffectiveModelParams)
}
if !reflect.DeepEqual(res.InputHashes, prepared.InputHashes) {
t.Fatalf("input hashes mismatch:\nrun=%#v\nprepare=%#v", res.InputHashes, prepared.InputHashes)
}
if res.RenderedPromptHash != prepared.RenderedPromptHash {
t.Fatalf("expected rendered prompt hash to match prepare, run=%q prepare=%q", res.RenderedPromptHash, prepared.RenderedPromptHash)
}
if !reflect.DeepEqual(llmClient.lastReq.Prompt.Messages, prepared.Messages) {
t.Fatalf("expected run to send prepare-rendered messages to llm")
}
}
func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "default-prof"