371 lines
14 KiB
Go
371 lines
14 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
|
)
|
|
|
|
type selectionTestCollaborators struct {
|
|
artifactReads int
|
|
renderCalls int
|
|
generationCalls int
|
|
validationCalls int
|
|
}
|
|
|
|
type selectionTestArtifactReader struct{ state *selectionTestCollaborators }
|
|
|
|
func (r selectionTestArtifactReader) Read(context.Context, domain.ArtifactRef) (*domain.Artifact, error) {
|
|
r.state.artifactReads++
|
|
return nil, errors.New("artifact access must not occur during target selection")
|
|
}
|
|
|
|
type selectionTestRenderer struct{ state *selectionTestCollaborators }
|
|
|
|
func (r selectionTestRenderer) Render(context.Context, *domain.PromptDefinition, map[string]*domain.Artifact, map[string]string) (*domain.RenderedPrompt, error) {
|
|
r.state.renderCalls++
|
|
return nil, errors.New("prompt rendering must not occur during target selection")
|
|
}
|
|
|
|
type selectionTestLLM struct{ state *selectionTestCollaborators }
|
|
|
|
func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
c.state.generationCalls++
|
|
return nil, errors.New("generation must not occur during target selection")
|
|
}
|
|
|
|
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
|
|
|
func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
|
v.state.validationCalls++
|
|
return domain.ValidationResult{}, errors.New("validation must not occur during target selection")
|
|
}
|
|
|
|
type selectionTestRepositories struct {
|
|
profiles profile.Repository
|
|
backends smokeBackendLookup
|
|
profileCalls int
|
|
}
|
|
|
|
func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
r.profileCalls++
|
|
return r.profiles.GetProfile(ctx, id)
|
|
}
|
|
|
|
func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) {
|
|
return r.backends.GetBackend(id)
|
|
}
|
|
|
|
func TestResolveSmokeTestTargetSelectionAndRouting(t *testing.T) {
|
|
runner, collaborators := newSmokeSelectionRunner(t)
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
request domain.SmokeTestRequest
|
|
profileID string
|
|
backendID string
|
|
selectionSource string
|
|
defaultSourceBackend string
|
|
endpoint string
|
|
settingsEndpoint string
|
|
wantModel string
|
|
}{
|
|
{
|
|
name: "explicit profile",
|
|
request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile"},
|
|
profileID: "alpha-profile",
|
|
backendID: "alpha",
|
|
selectionSource: smokeSelectionSourceProfile,
|
|
endpoint: "https://alpha.example/v1",
|
|
settingsEndpoint: "https://alpha.example/v1",
|
|
wantModel: "alpha-model",
|
|
},
|
|
{
|
|
name: "prompt default",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt",
|
|
},
|
|
profileID: "prompt-profile",
|
|
backendID: "beta",
|
|
selectionSource: smokeSelectionSourcePrompt,
|
|
endpoint: "https://beta.example/v1",
|
|
settingsEndpoint: "https://beta.example/v1",
|
|
wantModel: "prompt-model",
|
|
},
|
|
{
|
|
name: "explicit profile overrides prompt default",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt",
|
|
ProfileID: "alpha-profile", BackendID: "beta",
|
|
},
|
|
profileID: "alpha-profile",
|
|
backendID: "alpha",
|
|
selectionSource: smokeSelectionSourceProfile,
|
|
endpoint: "https://alpha.example/v1",
|
|
settingsEndpoint: "https://alpha.example/v1",
|
|
wantModel: "alpha-model",
|
|
},
|
|
{
|
|
name: "backend default routes to a different backend",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestStructuredOutput, BackendID: "alpha",
|
|
},
|
|
profileID: "cross-backend-profile",
|
|
backendID: "beta",
|
|
selectionSource: smokeSelectionSourceBackendDefault,
|
|
defaultSourceBackend: "alpha",
|
|
endpoint: "https://beta.example/v1",
|
|
settingsEndpoint: "https://beta.example/v1",
|
|
wantModel: "cross-model",
|
|
},
|
|
{
|
|
name: "backend default may select endpoint-only profile",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, BackendID: "epsilon",
|
|
},
|
|
profileID: "endpoint-profile",
|
|
selectionSource: smokeSelectionSourceBackendDefault,
|
|
defaultSourceBackend: "epsilon",
|
|
endpoint: "https://endpoint-only.example/v1",
|
|
settingsEndpoint: "https://endpoint-only.example/v1",
|
|
wantModel: "endpoint-model",
|
|
},
|
|
{
|
|
name: "prompt profile supplies backend-only target over backend hint",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, PromptID: "selection.prompt", BackendID: "alpha",
|
|
},
|
|
profileID: "prompt-profile",
|
|
backendID: "beta",
|
|
selectionSource: smokeSelectionSourcePrompt,
|
|
endpoint: "https://beta.example/v1",
|
|
settingsEndpoint: "https://beta.example/v1",
|
|
wantModel: "prompt-model",
|
|
},
|
|
{
|
|
name: "backend-only probe uses backend when prompt has no profile default",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, PromptID: "backend.prompt", BackendID: "delta",
|
|
},
|
|
backendID: "delta",
|
|
selectionSource: smokeSelectionSourceBackend,
|
|
endpoint: "https://delta.example/v1",
|
|
settingsEndpoint: "https://delta.example/v1",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
selected, err := runner.resolveSmokeTestTarget(context.Background(), test.request)
|
|
if err != nil {
|
|
t.Fatalf("resolve target: %v", err)
|
|
}
|
|
if selected.target.ProfileID != test.profileID || selected.target.BackendID != test.backendID ||
|
|
selected.target.SelectionSource != test.selectionSource ||
|
|
selected.target.DefaultSourceBackendID != test.defaultSourceBackend ||
|
|
selected.target.Endpoint != test.endpoint ||
|
|
selected.target.EffectiveSettings.Endpoint != test.settingsEndpoint ||
|
|
selected.target.EffectiveSettings.Model != test.wantModel {
|
|
t.Fatalf("unexpected target: %+v", selected.target)
|
|
}
|
|
if test.request.PromptID != "" && selected.target.PromptID != test.request.PromptID {
|
|
t.Fatalf("prompt identity was not retained: %+v", selected.target)
|
|
}
|
|
})
|
|
}
|
|
|
|
if collaborators.artifactReads != 0 || collaborators.renderCalls != 0 ||
|
|
collaborators.generationCalls != 0 || collaborators.validationCalls != 0 {
|
|
t.Fatalf("selection entered application preparation: %+v", collaborators)
|
|
}
|
|
}
|
|
|
|
func TestResolveSmokeTestTargetBackendOnlyDoesNotLoadDefaultProfile(t *testing.T) {
|
|
loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repositories := &selectionTestRepositories{profiles: loaded, backends: loaded}
|
|
runner := NewRunner(nil, repositories, nil, nil, nil, nil, nil)
|
|
|
|
zeroTemperature := 0.0
|
|
selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, BackendID: "delta",
|
|
Execution: &domain.ExecutionTargetOverride{Temperature: &zeroTemperature},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("backend-only probe should not resolve its dangling default profile: %v", err)
|
|
}
|
|
if selected.target.SelectionSource != smokeSelectionSourceBackend || selected.target.BackendID != "delta" ||
|
|
selected.target.ProfileID != "" || selected.target.Endpoint != "https://delta.example/v1" {
|
|
t.Fatalf("unexpected direct backend target: %+v", selected.target)
|
|
}
|
|
if !selected.target.SettingsPresence.Temperature || selected.target.EffectiveSettings.Temperature != zeroTemperature {
|
|
t.Fatalf("explicit zero override presence was not retained: %+v", selected.target)
|
|
}
|
|
if repositories.profileCalls != 0 {
|
|
t.Fatalf("backend-only selection loaded %d profiles", repositories.profileCalls)
|
|
}
|
|
}
|
|
|
|
func TestResolveSmokeTestTargetPreservesDiagnosticAndBaseEndpoints(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
override := domain.ExecutionTargetOverride{Endpoint: " https://override.example/v1 "}
|
|
selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, ProfileID: "incomplete-profile", Execution: &override,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("HTTP diagnostic should not require a model: %v", err)
|
|
}
|
|
if selected.target.BackendID != "alpha" || selected.target.EffectiveSettings.Model != "" || selected.target.EffectiveSettings.Endpoint != "https://override.example/v1" ||
|
|
selected.target.Endpoint != "https://probe.example/ready?check=1" {
|
|
t.Fatalf("diagnostic destination and base settings were not kept separately: %+v", selected.target)
|
|
}
|
|
|
|
selected, err = runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestAuthentication, BackendID: "beta",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if selected.target.Endpoint != "https://beta.example/v1/key" {
|
|
t.Fatalf("authentication destination = %q", selected.target.Endpoint)
|
|
}
|
|
}
|
|
|
|
func TestResolveSmokeTestTargetSelectionAndSettingsErrors(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
negativeRetries := -1
|
|
negativeTimeout := -1
|
|
for _, test := range []struct {
|
|
name string
|
|
request domain.SmokeTestRequest
|
|
wantErr error
|
|
}{
|
|
{
|
|
name: "unknown kind",
|
|
request: domain.SmokeTestRequest{Kind: "unknown", BackendID: "epsilon"},
|
|
wantErr: ErrInvalidRequest,
|
|
},
|
|
{
|
|
name: "negative retry override",
|
|
request: domain.SmokeTestRequest{Kind: domain.SmokeTestReachability, BackendID: "epsilon", TransientRetries: &negativeRetries},
|
|
wantErr: ErrInvalidRequest,
|
|
},
|
|
{
|
|
name: "explicit selection does not fall back",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: "missing-profile", BackendID: "epsilon",
|
|
},
|
|
wantErr: ErrProfileLoad,
|
|
},
|
|
{
|
|
name: "invalid prompt selection does not fall back",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, PromptID: "missing.prompt", BackendID: "epsilon",
|
|
},
|
|
wantErr: ErrPromptLoad,
|
|
},
|
|
{
|
|
name: "blank explicit profile does not fall back",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: " ", BackendID: "epsilon",
|
|
},
|
|
wantErr: ErrInvalidRequest,
|
|
},
|
|
{
|
|
name: "dangling backend default fails only when selected",
|
|
request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, BackendID: "delta"},
|
|
wantErr: ErrProfileLoad,
|
|
},
|
|
{
|
|
name: "unused negative setting is still validated for HTTP",
|
|
request: domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, BackendID: "epsilon",
|
|
Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &negativeTimeout},
|
|
},
|
|
wantErr: ErrInvalidRequest,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if _, err := runner.resolveSmokeTestTarget(context.Background(), test.request); !errors.Is(err, test.wantErr) {
|
|
t.Fatalf("selection error = %v, want errors.Is(_, %v)", err, test.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func newSmokeSelectionRunner(t *testing.T) (*Runner, *selectionTestCollaborators) {
|
|
t.Helper()
|
|
loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil)
|
|
if err != nil {
|
|
t.Fatalf("load captured selection configuration: %v", err)
|
|
}
|
|
collaborators := &selectionTestCollaborators{}
|
|
return NewRunner(
|
|
loaded,
|
|
loaded,
|
|
selectionTestArtifactReader{state: collaborators},
|
|
selectionTestRenderer{state: collaborators},
|
|
selectionTestLLM{state: collaborators},
|
|
selectionTestValidator{state: collaborators},
|
|
nil,
|
|
), collaborators
|
|
}
|
|
|
|
func smokeSelectionConfig() map[string]any {
|
|
return map[string]any{
|
|
"backends": map[string]any{
|
|
"alpha": map[string]any{
|
|
"endpoint": "https://alpha.example/v1",
|
|
"smoke_tests": map[string]any{
|
|
"default_profile": "cross-backend-profile",
|
|
"reachability": map[string]any{"endpoint": "https://probe.example/ready?check=1"},
|
|
"authentication": map[string]any{"kind": "openrouter_key"},
|
|
},
|
|
},
|
|
"beta": map[string]any{
|
|
"endpoint": "https://beta.example/v1",
|
|
"smoke_tests": map[string]any{"authentication": map[string]any{"kind": "openrouter_key"}},
|
|
},
|
|
"delta": map[string]any{
|
|
"endpoint": "https://delta.example/v1",
|
|
"smoke_tests": map[string]any{"default_profile": "not-yet-defined"},
|
|
},
|
|
"epsilon": map[string]any{
|
|
"endpoint": "https://epsilon.example/v1",
|
|
"smoke_tests": map[string]any{"default_profile": "endpoint-profile"},
|
|
},
|
|
},
|
|
"profiles": map[string]any{
|
|
"alpha-profile": map[string]any{"backend": "alpha", "model": "alpha-model"},
|
|
"prompt-profile": map[string]any{"backend": "beta", "model": "prompt-model"},
|
|
"cross-backend-profile": map[string]any{"backend": "beta", "model": "cross-model"},
|
|
"endpoint-profile": map[string]any{"endpoint": "https://endpoint-only.example/v1", "model": "endpoint-model"},
|
|
"incomplete-profile": map[string]any{"backend": "alpha"},
|
|
},
|
|
"prompts": map[string]any{
|
|
"selection.prompt": map[string]any{
|
|
"version": "v1",
|
|
"default_profile": "prompt-profile",
|
|
"inputs": []any{map[string]any{"name": "required_input", "required": true}},
|
|
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
|
|
"session_id": "{{.vars.session}}",
|
|
"output": map[string]any{
|
|
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
|
},
|
|
},
|
|
"backend.prompt": map[string]any{
|
|
"version": "v1",
|
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
|
"output": map[string]any{
|
|
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|