package usecase import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "path/filepath" "reflect" "regexp" "strings" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/validate" ) type fakePromptRepo struct { def *domain.PromptDefinition err error lastID string lastVersion string } type fakeExecutionProfileRepo struct { profiles map[string]*domain.ExecutionProfile err error lastID string } func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { f.lastID = id if f.err != nil { return nil, f.err } if p, ok := f.profiles[id]; ok { cp := *p return &cp, nil } return nil, errors.New("profile not found") } func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { f.lastID = id f.lastVersion = version if f.err != nil { return nil, f.err } return f.def, nil } type fakeArtifactReader struct { artifactsByURI map[string]*domain.Artifact errByURI map[string]error } func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { if err, ok := f.errByURI[ref.URI]; ok { return nil, err } if art, ok := f.artifactsByURI[ref.URI]; ok { cp := *art return &cp, nil } return nil, errors.New("artifact not found") } type fakeRenderer struct { rendered *domain.RenderedPrompt err error } func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { if f.err != nil { return nil, f.err } return f.rendered, nil } type fakeLLM struct { resp *domain.GenerateResponse err error lastReq domain.GenerateRequest calls int forbid bool } func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { f.calls++ f.lastReq = req if f.forbid { return nil, errors.New("llm should not be called") } if f.err != nil { return nil, f.err } return f.resp, nil } type fakeValidator struct { result domain.ValidationResult err error } func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { if f.err != nil { return domain.ValidationResult{}, f.err } return f.result, nil } type fakeRepairer struct { responses []*domain.GenerateResponse err error calls int reqs []RepairRequest } func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { f.calls++ f.reqs = append(f.reqs, req) if f.err != nil { return nil, f.err } if len(f.responses) == 0 { return nil, errors.New("no repair response configured") } idx := f.calls - 1 if idx >= len(f.responses) { idx = len(f.responses) - 1 } return f.responses[idx], nil } func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{forbid: true} runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, }, Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if prepared.PromptID != "p" || prepared.PromptVersion != "1" { t.Fatalf("unexpected prepared prompt metadata: %+v", prepared) } if prepared.SelectedProfileID != "exec" { t.Fatalf("expected selected profile exec, got %q", prepared.SelectedProfileID) } if prepared.PromptHash == "" || prepared.RenderedPromptHash == "" { t.Fatal("expected prompt hashes") } if prepared.EffectiveModelParams.Model != "m" || prepared.EffectiveModelParams.Endpoint != "http://override/v1" { t.Fatalf("unexpected model params: %+v", prepared.EffectiveModelParams) } if prepared.OutputContract.Format != domain.FormatMarkdown { t.Fatalf("expected output format markdown, got %q", prepared.OutputContract.Format) } if len(prepared.InputHashes) != 2 || prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) } if len(prepared.Messages) != 2 { t.Fatalf("expected two messages, got %d", len(prepared.Messages)) } if llmClient.calls != 0 { t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) } } func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "from-prompt" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, }} runner := newMinimalRunner(promptRepo, execRepo) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if execRepo.lastID != "from-prompt" { t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID) } if prepared.SelectedProfileID != "from-prompt" { t.Fatalf("expected selected profile from-prompt, got %q", prepared.SelectedProfileID) } } func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} repo.def.DefaultProfile = "" runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) if !errors.Is(err, ErrInvalidRequest) { t.Fatalf("expected ErrInvalidRequest, got %v", err) } } func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} repo.def.DefaultProfile = "does-not-exist" runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}}) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", Temperature: 0.2, MaxTokens: 500, TopP: 0.9, TimeoutSeconds: 120, }, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Execution: &domain.ExecutionTarget{ Endpoint: "http://override/v1", Model: "override-model", Temperature: 0.7, TimeoutSeconds: 30, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if prepared.EffectiveModelParams.Endpoint != "http://override/v1" || prepared.EffectiveModelParams.Model != "override-model" { t.Fatalf("expected endpoint/model override to win, got %+v", prepared.EffectiveModelParams) } if prepared.EffectiveModelParams.TopP != 0.9 { t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP) } } func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TopP: 0.8, TimeoutSeconds: 90, }, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if prepared.EffectiveModelParams.TopP != 0.8 { t.Fatalf("expected profile top_p to beat default, got %v", prepared.EffectiveModelParams.TopP) } if prepared.EffectiveModelParams.TimeoutSeconds != 90 { t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds) } } func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { promptDir := filepath.Join("..", "promptdef", "testdata") profileDir := filepath.Join("..", "profile", "testdata") reader := &fakeArtifactReader{ artifactsByURI: map[string]*domain.Artifact{ "a://transcript": { Name: "transcript", Body: []byte("Session transcript body."), Hash: hashString("Session transcript body."), }, }, } llmClient := &fakeLLM{forbid: true} runner := NewRunner( promptdef.NewFilesystemRepository(promptDir), profile.NewFilesystemRepository(profileDir), reader, prompt.NewGoRenderer(), llmClient, nil, ) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "valid-file-backed", ProfileID: "local-default", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if len(prepared.Messages) != 2 { t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages)) } if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") { t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content) } } func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { def := promptDef(domain.FormatText, domain.ValidationNone, 0) def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}} runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, nil, ) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{}, }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) } if !errors.Is(err, prompt.ErrMissingRequiredInput) { t.Fatalf("expected ErrMissingRequiredInput, got %v", err) } } func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { def := promptDef(domain.FormatText, domain.ValidationNone, 0) def.Inputs = []domain.PromptInput{{Name: "transcript", Required: false}} def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "ghost"}}`}} runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, nil, ) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{}, }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) } if !errors.Is(err, prompt.ErrUnknownInput) { t.Fatalf("expected ErrUnknownInput, got %v", err) } } func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { const envName = "SCRIPTORIUM_TEST_API_KEY" const secret = "top-secret-value" t.Setenv(envName, secret) promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { t.Fatalf("expected no error, got %v", err) } if prepared.EffectiveModelParams.APIKeyEnv != envName { t.Fatalf("expected api key env name, got %q", prepared.EffectiveModelParams.APIKeyEnv) } metadataDump := fmt.Sprintf("%+v|%s|%s", prepared.EffectiveModelParams, prepared.PromptHash, prepared.RenderedPromptHash) if strings.Contains(metadataDump, secret) { t.Fatalf("unexpected api key value in prepared metadata dump: %s", metadataDump) } } func TestRunnerRunSuccessful(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, }, Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.PromptID != "p" || res.PromptVersion != "1" { t.Fatalf("unexpected prompt metadata: %+v", res) } if res.SelectedProfileID != "exec" { t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID) } if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok { t.Fatalf("invalid run id: %q", res.RunID) } if res.PromptHash == "" || res.RenderedPromptHash == "" { t.Fatal("expected prompt hashes") } if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" { t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams) } if res.Artifact.Name != defaults.OutputArtifactName { t.Fatalf("expected default output artifact name %q, got %q", defaults.OutputArtifactName, res.Artifact.Name) } if res.Artifact.ContentType != defaults.ContentTypeTextMarkdown { t.Fatalf("expected markdown content type %q, got %q", defaults.ContentTypeTextMarkdown, res.Artifact.ContentType) } if res.RawOutput != "# recap" { t.Fatalf("expected raw output, got %q", res.RawOutput) } if res.Validation.Status != domain.ValidationSkipped { t.Fatalf("expected skipped validation, got %q", res.Validation.Status) } if llmClient.lastReq.Target.TimeoutSeconds != 90 { t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) } } func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) req := domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, }, Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, } prepared, err := runner.Prepare(context.Background(), req) if err != nil { t.Fatalf("prepare should succeed, got %v", err) } res, err := runner.Run(context.Background(), req) if err != nil { t.Fatalf("run should succeed, got %v", err) } if res.SelectedProfileID != prepared.SelectedProfileID { t.Fatalf("expected selected profile to match prepare, run=%q prepare=%q", res.SelectedProfileID, prepared.SelectedProfileID) } if !reflect.DeepEqual(res.EffectiveModelParams, prepared.EffectiveModelParams) { t.Fatalf("effective model params mismatch:\nrun=%+v\nprepare=%+v", res.EffectiveModelParams, prepared.EffectiveModelParams) } if !reflect.DeepEqual(res.InputHashes, prepared.InputHashes) { t.Fatalf("input hashes mismatch:\nrun=%#v\nprepare=%#v", res.InputHashes, prepared.InputHashes) } if res.RenderedPromptHash != prepared.RenderedPromptHash { t.Fatalf("expected rendered prompt hash to match prepare, run=%q prepare=%q", res.RenderedPromptHash, prepared.RenderedPromptHash) } if !reflect.DeepEqual(llmClient.lastReq.Prompt.Messages, prepared.Messages) { t.Fatalf("expected run to send prepare-rendered messages to llm") } } func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "default-prof" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "explicit-prof": {ID: "explicit-prof", Endpoint: "http://explicit/v1", Model: "explicit"}, "default-prof": {ID: "default-prof", Endpoint: "http://default/v1", Model: "default"}, }} runner := newMinimalRunner(promptRepo, execRepo) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "explicit-prof", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if execRepo.lastID != "explicit-prof" { t.Fatalf("expected explicit profile lookup, got %q", execRepo.lastID) } if res.SelectedProfileID != "explicit-prof" { t.Fatalf("expected selected profile explicit-prof, got %q", res.SelectedProfileID) } } func TestRunnerRunPromptDefaultProfileIsUsedWhenNoExplicitProfileID(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "from-prompt" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, }} runner := newMinimalRunner(promptRepo, execRepo) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if execRepo.lastID != "from-prompt" { t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID) } if res.SelectedProfileID != "from-prompt" { t.Fatalf("expected selected profile from-prompt, got %q", res.SelectedProfileID) } } func TestRunnerRunMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} repo.def.DefaultProfile = "" runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) if !errors.Is(err, ErrInvalidRequest) { t.Fatalf("expected ErrInvalidRequest, got %v", err) } } func TestRunnerRunInvalidDefaultProfileFails(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} repo.def.DefaultProfile = "does-not-exist" runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}}) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) { repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected profile load failure, got %v", err) } } func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", Temperature: 0.2, MaxTokens: 500, TopP: 0.9, TimeoutSeconds: 120, }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Execution: &domain.ExecutionTarget{ Endpoint: "http://override/v1", Model: "override-model", Temperature: 0.7, TimeoutSeconds: 30, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Endpoint != "http://override/v1" || res.ModelName != "override-model" { t.Fatalf("expected endpoint/model override to win, got endpoint=%q model=%q", res.Endpoint, res.ModelName) } if res.EffectiveModelParams.Temperature != 0.7 || res.EffectiveModelParams.TimeoutSeconds != 30 { t.Fatalf("expected numeric override to win, got %+v", res.EffectiveModelParams) } if res.EffectiveModelParams.TopP != 0.9 { t.Fatalf("expected non-overridden profile top_p to remain, got %v", res.EffectiveModelParams.TopP) } } func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TopP: 0.8, TimeoutSeconds: 90, }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.TopP != 0.8 { t.Fatalf("expected profile top_p to beat default, got %v", res.EffectiveModelParams.TopP) } if res.EffectiveModelParams.TimeoutSeconds != 90 { t.Fatalf("expected profile timeout to beat default, got %d", res.EffectiveModelParams.TimeoutSeconds) } } func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.Temperature != defaults.ExecutionDefaultTemperature { t.Fatalf("expected default temperature %v, got %v", defaults.ExecutionDefaultTemperature, res.EffectiveModelParams.Temperature) } if res.EffectiveModelParams.TopP != defaults.ExecutionDefaultTopP { t.Fatalf("expected default top_p %v, got %v", defaults.ExecutionDefaultTopP, res.EffectiveModelParams.TopP) } if res.EffectiveModelParams.MaxTokens != defaults.ExecutionDefaultMaxTokens { t.Fatalf("expected default max_tokens %d, got %d", defaults.ExecutionDefaultMaxTokens, res.EffectiveModelParams.MaxTokens) } if res.EffectiveModelParams.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds { t.Fatalf("expected default timeout_seconds %d, got %d", defaults.ExecutionDefaultTimeoutSeconds, res.EffectiveModelParams.TimeoutSeconds) } } func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) { t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret") promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY"}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.APIKeyEnv != "SCRIPTORIUM_TEST_API_KEY" { t.Fatalf("expected api_key_env name in effective params, got %q", res.EffectiveModelParams.APIKeyEnv) } } func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if !errors.Is(err, ErrInvalidRequest) { t.Fatalf("expected ErrInvalidRequest, got %v", err) } if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") { t.Fatalf("expected missing env name in error, got %v", err) } } func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) { const envName = "SCRIPTORIUM_RUNTIME_API_KEY" t.Setenv(envName, "runtime-secret") promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Execution: &domain.ExecutionTarget{APIKeyEnv: envName}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.APIKeyEnv != envName { t.Fatalf("expected runtime api_key_env override in effective params, got %q", res.EffectiveModelParams.APIKeyEnv) } } func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) { const profileEnv = "SCRIPTORIUM_PROFILE_API_KEY" const runtimeEnv = "SCRIPTORIUM_RUNTIME_API_KEY" t.Setenv(runtimeEnv, "runtime-secret") promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.APIKeyEnv != runtimeEnv { t.Fatalf("expected runtime override to beat profile api_key_env, got %q", res.EffectiveModelParams.APIKeyEnv) } } func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { const envName = "SCRIPTORIUM_TEST_API_KEY" const secret = "top-secret-value" t.Setenv(envName, secret) promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, }} runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.EffectiveModelParams.APIKeyEnv != envName { t.Fatalf("expected api key env name, got %q", res.EffectiveModelParams.APIKeyEnv) } metadataDump := fmt.Sprintf("%+v|%s|%s|%s|%s", res.EffectiveModelParams, res.Endpoint, res.ModelName, res.PromptHash, res.RenderedPromptHash) if strings.Contains(metadataDump, secret) { t.Fatalf("unexpected api key value in metadata dump: %s", metadataDump) } } func TestRunnerRunPromptLoadFailure(t *testing.T) { runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}}, }) if !errors.Is(err, ErrArtifactLoad) { t.Fatalf("expected ErrArtifactLoad, got %v", err) } } func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), &fakeRenderer{err: errors.New("render failed")}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) } } func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{err: errors.New("llm failed")}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if !errors.Is(err, ErrLLMGenerate) { t.Fatalf("expected ErrLLMGenerate, got %v", err) } } func TestRunnerRunValidationStillWorks(t *testing.T) { validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, validator, ) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" { t.Fatalf("unexpected validation/raw output: %+v", res) } } func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t *testing.T) { repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}} runner := NewRunnerWithRepairer( &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55}, }}, defaultArtifactReader(), defaultRenderer(), llmClient, validate.NewStandardValidator("."), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if repairer.calls != 1 || res.Validation.RepairAttempts != 1 { t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts) } if len(repairer.reqs) != 1 { t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) } if repairer.reqs[0].Target.Endpoint != "http://override/v1" || repairer.reqs[0].Target.Model != "override-model" { t.Fatalf("expected repair to use effective target, got %+v", repairer.reqs[0].Target) } if repairer.reqs[0].Target.TimeoutSeconds != 22 { t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds) } } func TestBuildOutputArtifactDefaults(t *testing.T) { tests := []struct { name string format domain.OutputFormat contentType string }{ {name: "text", format: domain.FormatText, contentType: defaults.ContentTypeTextPlain}, {name: "markdown", format: domain.FormatMarkdown, contentType: defaults.ContentTypeTextMarkdown}, {name: "json", format: domain.FormatJSON, contentType: defaults.ContentTypeApplicationJSON}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { art := buildOutputArtifact("body", tc.format) if art.Name != defaults.OutputArtifactName { t.Fatalf("expected artifact name %q, got %q", defaults.OutputArtifactName, art.Name) } if art.ContentType != tc.contentType { t.Fatalf("expected content type %q, got %q", tc.contentType, art.ContentType) } }) } } func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition { return &domain.PromptDefinition{ ID: "p", Version: "1", DefaultProfile: "exec", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: format, Validation: domain.OutputContract{ ValidationMode: mode, RepairAttempts: attempts, Format: format, }, } } func hashString(s string) string { sum := sha256.Sum256([]byte(s)) return hex.EncodeToString(sum[:]) } func defaultExecutionProfile() *domain.ExecutionProfile { return &domain.ExecutionProfile{ ID: "exec", Endpoint: "http://llm/v1", Model: "model-from-profile", } } func defaultArtifactReader() *fakeArtifactReader { return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://ok": {Body: []byte("x"), Hash: hashString("x")}, }} } func defaultRenderer() *fakeRenderer { return &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}} } func singleInputRef() map[string]domain.ArtifactRef { return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}} } func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { return NewRunner( promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) }