375 lines
16 KiB
Go
375 lines
16 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
)
|
|
|
|
func TestInspectPromptExecutionSelectsDefaultAndOverrideProfiles(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
executor := &inspectionExecutor{
|
|
prompt: validPromptInspection(resolved.Definition),
|
|
profiles: map[string]promptexec.ProfileInspection{
|
|
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "default-model"},
|
|
"override-profile": {ProfileID: "override-profile", BackendID: "cloud", ModelName: "override-model"},
|
|
},
|
|
}
|
|
defaultResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor})
|
|
if err != nil {
|
|
t.Fatalf("InspectPromptExecution(default) error = %v", err)
|
|
}
|
|
if defaultResult.ProfileID != "default-profile" || defaultResult.ModelName != "default-model" {
|
|
t.Fatalf("default result = %#v", defaultResult)
|
|
}
|
|
overrideResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{
|
|
Resolved: resolved, Executor: executor, Promptkit: config.PromptkitConfig{Profile: "override-profile"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("InspectPromptExecution(override) error = %v", err)
|
|
}
|
|
if overrideResult.ProfileID != "override-profile" || overrideResult.ModelName != "override-model" {
|
|
t.Fatalf("override result = %#v", overrideResult)
|
|
}
|
|
if len(executor.promptRequests) != 2 || executor.promptRequests[0].version != resolved.Definition.PromptVersion || executor.profileRequests[0] != "default-profile" || executor.profileRequests[1] != "override-profile" {
|
|
t.Fatalf("inspection requests = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
|
|
}
|
|
}
|
|
|
|
func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
basePrompt := validPromptInspection(resolved.Definition)
|
|
tests := []struct {
|
|
name string
|
|
prompt promptexec.PromptInspection
|
|
profile promptexec.ProfileInspection
|
|
lookupEnv func(string) (string, bool)
|
|
wantCategory promptexec.ErrorCategory
|
|
}{
|
|
{
|
|
name: "extra input",
|
|
prompt: func() promptexec.PromptInspection {
|
|
value := basePrompt
|
|
value.Inputs = append(value.Inputs, promptexec.InputDefinition{Name: "unexpected"})
|
|
return value
|
|
}(),
|
|
wantCategory: promptexec.InvalidConfiguration,
|
|
},
|
|
{
|
|
name: "wrong schema",
|
|
prompt: func() promptexec.PromptInspection {
|
|
value := basePrompt
|
|
value.Output.SchemaPath = "unexpected.schema.json"
|
|
return value
|
|
}(),
|
|
wantCategory: promptexec.InvalidConfiguration,
|
|
},
|
|
{
|
|
name: "missing prompt hash",
|
|
prompt: func() promptexec.PromptInspection {
|
|
value := basePrompt
|
|
value.PromptHash = ""
|
|
return value
|
|
}(),
|
|
wantCategory: promptexec.InvalidConfiguration,
|
|
},
|
|
{
|
|
name: "missing profile backend",
|
|
prompt: basePrompt,
|
|
profile: promptexec.ProfileInspection{ProfileID: "default-profile", ModelName: "model"},
|
|
wantCategory: promptexec.InvalidConfiguration,
|
|
},
|
|
{
|
|
name: "direct key",
|
|
prompt: basePrompt,
|
|
profile: promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true},
|
|
wantCategory: promptexec.MissingCredential,
|
|
},
|
|
{
|
|
name: "missing environment credential",
|
|
prompt: basePrompt,
|
|
profile: promptexec.ProfileInspection{ProfileID: "default-profile", APIKeyEnv: "PROMPT_API_KEY"},
|
|
lookupEnv: func(string) (string, bool) { return "", false },
|
|
wantCategory: promptexec.MissingCredential,
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
executor := &inspectionExecutor{prompt: test.prompt, profiles: map[string]promptexec.ProfileInspection{"default-profile": test.profile}}
|
|
_, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor, LookupEnv: test.lookupEnv})
|
|
if err == nil || promptexec.CategoryOf(err) != test.wantCategory {
|
|
t.Fatalf("error/category = %v/%q, want %q", err, promptexec.CategoryOf(err), test.wantCategory)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
executor := &inspectionExecutor{promptErr: errors.New("provider response contains resolved-secret-value")}
|
|
_, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor})
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
|
|
t.Fatalf("error/category = %v/%q", err, promptexec.CategoryOf(err))
|
|
}
|
|
if strings.Contains(err.Error(), "resolved-secret-value") {
|
|
t.Fatalf("inspection error leaks provider value: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
|
|
first := inspectionResolved(t)
|
|
second := inspectionResolvedFor(t, report.Today)
|
|
executor := &inspectionExecutor{
|
|
prompt: validPromptInspection(first.Definition),
|
|
profiles: map[string]promptexec.ProfileInspection{
|
|
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
|
|
},
|
|
}
|
|
executor.prompts = map[string]promptexec.PromptInspection{
|
|
first.Definition.PromptID: validPromptInspection(first.Definition),
|
|
second.Definition.PromptID: validPromptInspection(second.Definition),
|
|
}
|
|
results, err := InspectPromptExecutions(context.Background(), PromptExecutionsInspectionRequest{Resolved: []report.Resolved{first, second}, Executor: executor})
|
|
if err != nil {
|
|
t.Fatalf("InspectPromptExecutions() error = %v", err)
|
|
}
|
|
if len(results) != 2 || len(executor.profileRequests) != 1 {
|
|
t.Fatalf("results/profile requests = %#v/%#v, want two results and one profile inspection", results, executor.profileRequests)
|
|
}
|
|
}
|
|
|
|
func TestPromptInspectionRejectsIncompatibleGeneratedTextCatalogBeforeExecutorWork(t *testing.T) {
|
|
base := inspectionResolved(t)
|
|
tests := []struct {
|
|
name string
|
|
resolved report.Resolved
|
|
inspect func(context.Context, report.Resolved, *inspectionExecutor) error
|
|
}{
|
|
{
|
|
name: "single report unknown template",
|
|
resolved: func() report.Resolved {
|
|
resolved := base
|
|
resolved.Definition.TemplateID = "unknown"
|
|
return resolved
|
|
}(),
|
|
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
|
|
_, err := InspectPromptExecution(ctx, PromptInspectionRequest{Resolved: resolved, Executor: executor})
|
|
return err
|
|
},
|
|
},
|
|
{
|
|
name: "batch known pair for another report",
|
|
resolved: func() report.Resolved {
|
|
resolved := base
|
|
resolved.Definition.GeneratedTextSchemaID = "today"
|
|
resolved.Definition.TemplateID = "today"
|
|
return resolved
|
|
}(),
|
|
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
|
|
_, err := InspectPromptExecutions(ctx, PromptExecutionsInspectionRequest{Resolved: []report.Resolved{resolved}, Executor: executor})
|
|
return err
|
|
},
|
|
},
|
|
{
|
|
name: "comparison known pair for another report",
|
|
resolved: func() report.Resolved {
|
|
resolved := base
|
|
resolved.Definition.GeneratedTextSchemaID = "today"
|
|
resolved.Definition.TemplateID = "today"
|
|
return resolved
|
|
}(),
|
|
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
|
|
_, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor})
|
|
return err
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
executor := &inspectionExecutor{}
|
|
err := test.inspect(context.Background(), test.resolved, executor)
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
|
|
t.Fatalf("inspection error/category = %v/%q, want invalid configuration", err, promptexec.CategoryOf(err))
|
|
}
|
|
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
|
t.Fatalf("incompatible catalog performed executor work: prompts %#v profiles %#v executions %d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
executor := &inspectionExecutor{
|
|
prompt: validPromptInspection(resolved.Definition),
|
|
profiles: map[string]promptexec.ProfileInspection{
|
|
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
|
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
|
},
|
|
}
|
|
profileIDs := []string{"weather-light", "weather-deep"}
|
|
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
|
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("InspectComparisonExecution() error = %v", err)
|
|
}
|
|
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash != "prompt-hash" {
|
|
t.Fatalf("prompt result = %#v", result)
|
|
}
|
|
if !reflect.DeepEqual(executor.profileRequests, profileIDs) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
|
}
|
|
wantProfiles := []ComparisonProfileInspection{
|
|
{ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
|
{ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
|
}
|
|
if !reflect.DeepEqual(result.Profiles, wantProfiles) {
|
|
t.Fatalf("profiles = %#v, want %#v", result.Profiles, wantProfiles)
|
|
}
|
|
}
|
|
|
|
func TestInspectComparisonExecutionRejectsInvalidProfilesBeforeInspection(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
for _, profileIDs := range [][]string{
|
|
{"weather-light"},
|
|
{"weather-light", " \t"},
|
|
{"weather-light", "weather-light"},
|
|
} {
|
|
t.Run(strings.Join(profileIDs, ","), func(t *testing.T) {
|
|
executor := &inspectionExecutor{prompt: validPromptInspection(resolved.Definition)}
|
|
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
|
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
|
})
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidRequest {
|
|
t.Fatalf("error/category = %v/%q, want invalid request", err, promptexec.CategoryOf(err))
|
|
}
|
|
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
|
t.Fatalf("invalid profile selection performed prompt/profile/execution work: %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
executor := &inspectionExecutor{
|
|
prompt: validPromptInspection(resolved.Definition),
|
|
profiles: map[string]promptexec.ProfileInspection{
|
|
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
|
"missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"},
|
|
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
|
},
|
|
}
|
|
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
|
Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor,
|
|
LookupEnv: func(string) (string, bool) { return "", false },
|
|
})
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential {
|
|
t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err))
|
|
}
|
|
if !reflect.DeepEqual(executor.profileRequests, []string{"weather-light", "missing-key"}) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
|
}
|
|
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash == "" || len(result.Profiles) != 1 || result.Profiles[0].ProfileID != "weather-light" {
|
|
t.Fatalf("partial inspection result = %#v", result)
|
|
}
|
|
}
|
|
|
|
func TestInspectComparisonExecutionStopsBeforeProfileInspectionWhenPromptFails(t *testing.T) {
|
|
resolved := inspectionResolved(t)
|
|
executor := &inspectionExecutor{promptErr: promptexec.NewError(promptexec.PromptNotFound, "prompt is unavailable", nil)}
|
|
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
|
Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor,
|
|
})
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.PromptNotFound || !strings.Contains(err.Error(), "comparison prompt") {
|
|
t.Fatalf("error/category = %v/%q, want prompt-context prompt not found", err, promptexec.CategoryOf(err))
|
|
}
|
|
if len(executor.promptRequests) != 1 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
|
}
|
|
}
|
|
|
|
type inspectionPromptRequest struct {
|
|
id string
|
|
version string
|
|
}
|
|
|
|
type inspectionExecutor struct {
|
|
prompt promptexec.PromptInspection
|
|
prompts map[string]promptexec.PromptInspection
|
|
profiles map[string]promptexec.ProfileInspection
|
|
promptErr error
|
|
promptRequests []inspectionPromptRequest
|
|
profileRequests []string
|
|
executeRequests int
|
|
}
|
|
|
|
func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
|
|
e.promptRequests = append(e.promptRequests, inspectionPromptRequest{id: id, version: version})
|
|
if e.promptErr != nil {
|
|
return promptexec.PromptInspection{}, e.promptErr
|
|
}
|
|
if prompt, ok := e.prompts[id]; ok {
|
|
return prompt, nil
|
|
}
|
|
return e.prompt, nil
|
|
}
|
|
|
|
func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
e.profileRequests = append(e.profileRequests, id)
|
|
value, ok := e.profiles[id]
|
|
if !ok {
|
|
return promptexec.ProfileInspection{}, errors.New("profile missing")
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
e.executeRequests++
|
|
return nil, errors.New("unexpected execution")
|
|
}
|
|
|
|
func inspectionResolved(t *testing.T) report.Resolved {
|
|
return inspectionResolvedFor(t, report.Daily)
|
|
}
|
|
|
|
func inspectionResolvedFor(t *testing.T, id report.ID) report.Resolved {
|
|
t.Helper()
|
|
request := report.ResolveRequest{Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), Location: time.UTC}
|
|
if id == report.Daily {
|
|
request.Date = time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC)
|
|
}
|
|
resolved, err := report.DefaultRegistry().Resolve(id, request)
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v", err)
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
func validPromptInspection(definition report.Definition) promptexec.PromptInspection {
|
|
return promptexec.PromptInspection{
|
|
PromptID: definition.PromptID, PromptVersion: definition.PromptVersion, PromptHash: "prompt-hash", DefaultProfileID: "default-profile",
|
|
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
|
|
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"},
|
|
}
|
|
}
|
|
|
|
func logicalPromptInspection(definition report.Definition) promptexec.PromptInspection {
|
|
inspection := validPromptInspection(definition)
|
|
if definition.ID == report.Hourly {
|
|
inspection.DefaultProfileID = "weather-light"
|
|
} else {
|
|
inspection.DefaultProfileID = "weather-balanced"
|
|
}
|
|
return inspection
|
|
}
|