258 lines
9.3 KiB
Go
258 lines
9.3 KiB
Go
package promptassets_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io/fs"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptassets"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type promptDefinition struct {
|
|
ID string `yaml:"id"`
|
|
Version string `yaml:"version"`
|
|
DefaultProfile string `yaml:"default_profile"`
|
|
Inputs []struct {
|
|
Name string `yaml:"name"`
|
|
Required bool `yaml:"required"`
|
|
ContentType string `yaml:"content_type"`
|
|
} `yaml:"inputs"`
|
|
Output struct {
|
|
Format string `yaml:"format"`
|
|
ValidationMode string `yaml:"validation_mode"`
|
|
SchemaPath string `yaml:"schema_path"`
|
|
RepairAttempts *int `yaml:"repair_attempts"`
|
|
} `yaml:"output"`
|
|
}
|
|
|
|
func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) {
|
|
tests := []struct {
|
|
path string
|
|
id string
|
|
schemaID string
|
|
profile string
|
|
}{
|
|
{"daily/daily_generated_text.yml", "weather.daily_generated_text", "daily", "weather-balanced"},
|
|
{"today/today_generated_text.yml", "weather.today_generated_text", "today", "weather-balanced"},
|
|
{"tomorrow/tomorrow_generated_text.yml", "weather.tomorrow_generated_text", "tomorrow", "weather-balanced"},
|
|
{"hourly/hourly_generated_text.yml", "weather.hourly_generated_text", "hourly", "weather-light"},
|
|
}
|
|
|
|
definitions := 0
|
|
if err := fs.WalkDir(promptassets.PromptFS(), ".", func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !entry.IsDir() && strings.HasSuffix(path, ".yml") {
|
|
definitions++
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk embedded prompts: %v", err)
|
|
}
|
|
if definitions != len(tests) {
|
|
t.Fatalf("prompt definitions = %d, want %d", definitions, len(tests))
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.id, func(t *testing.T) {
|
|
data, err := fs.ReadFile(promptassets.PromptFS(), tc.path)
|
|
if err != nil {
|
|
t.Fatalf("read prompt definition: %v", err)
|
|
}
|
|
var definition promptDefinition
|
|
if err := yaml.Unmarshal(data, &definition); err != nil {
|
|
t.Fatalf("decode prompt definition: %v", err)
|
|
}
|
|
if definition.ID != tc.id || definition.Version != "2.0.0" || definition.DefaultProfile != tc.profile {
|
|
t.Fatalf("definition = %#v, want %s version 2.0.0 and profile %s", definition, tc.id, tc.profile)
|
|
}
|
|
if len(definition.Inputs) != 1 || definition.Inputs[0].Name != "data_package" || !definition.Inputs[0].Required || definition.Inputs[0].ContentType != "application/yaml" {
|
|
t.Fatalf("inputs = %#v, want one required YAML data_package", definition.Inputs)
|
|
}
|
|
if definition.Output.Format != "json" || definition.Output.ValidationMode != "json_schema" || definition.Output.SchemaPath != tc.schemaID+".generated_text.schema.json" || definition.Output.RepairAttempts != nil {
|
|
t.Fatalf("output = %#v, want JSON schema output without repair attempts", definition.Output)
|
|
}
|
|
if _, err := promptassets.Schema(tc.schemaID); err != nil {
|
|
t.Fatalf("Schema(%q) error = %v", tc.schemaID, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSchemasAreCanonicalAndIndependent(t *testing.T) {
|
|
for _, id := range []string{"daily", "today", "tomorrow", "hourly"} {
|
|
t.Run(id, func(t *testing.T) {
|
|
data, err := promptassets.Schema(id)
|
|
if err != nil {
|
|
t.Fatalf("Schema() error = %v", err)
|
|
}
|
|
var schema struct {
|
|
ID string `json:"$id"`
|
|
Title string `json:"title"`
|
|
Type string `json:"type"`
|
|
AdditionalProperties bool `json:"additionalProperties"`
|
|
Required []string `json:"required"`
|
|
Properties map[string]any `json:"properties"`
|
|
}
|
|
if err := json.Unmarshal(data, &schema); err != nil {
|
|
t.Fatalf("decode schema: %v", err)
|
|
}
|
|
if schema.Type != "object" || schema.AdditionalProperties || strings.Join(schema.Required, ",") != "summary,forecast_discussion,precipitation_timing" {
|
|
t.Fatalf("schema = %#v, want strict generated-text object", schema)
|
|
}
|
|
if _, ok := schema.Properties["confidence"]; ok {
|
|
t.Fatalf("schema properties = %#v, do not want retired confidence field", schema.Properties)
|
|
}
|
|
if id == "daily" && (schema.ID != "weatherreporter.daily.generated_text.schema.json" || schema.Title != "Daily GeneratedText") {
|
|
t.Fatalf("daily schema identity = %q/%q, want corrected Daily identity", schema.ID, schema.Title)
|
|
}
|
|
data[0] = 'x'
|
|
fresh, err := promptassets.Schema(id)
|
|
if err != nil || fresh[0] != '{' {
|
|
t.Fatalf("Schema() returned shared data or error: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPromptkitInspectsEmbeddedPromptsOffline(t *testing.T) {
|
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
|
promptkit.WithPromptFS(promptassets.PromptFS(), "."),
|
|
promptkit.WithSchemaFS(promptassets.SchemaFS(), "."),
|
|
promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewEngine() error = %v", err)
|
|
}
|
|
for _, want := range []struct {
|
|
id string
|
|
profile string
|
|
model string
|
|
}{
|
|
{"weather.daily_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
|
{"weather.today_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
|
{"weather.tomorrow_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
|
{"weather.hourly_generated_text", "weather-light", "deepseek/deepseek-v4-flash"},
|
|
} {
|
|
t.Run(want.id, func(t *testing.T) {
|
|
inspection, err := engine.InspectPrompt(context.Background(), want.id, "2.0.0")
|
|
if err != nil {
|
|
t.Fatalf("InspectPrompt() error = %v", err)
|
|
}
|
|
if inspection.PromptID != want.id || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != want.profile {
|
|
t.Fatalf("inspection = %#v", inspection)
|
|
}
|
|
profile, err := engine.InspectProfile(context.Background(), inspection.DefaultProfileID)
|
|
if err != nil || profile.EffectiveModelParams.Model != want.model {
|
|
t.Fatalf("profile/error = %#v/%v, want model %q", profile, err, want.model)
|
|
}
|
|
})
|
|
}
|
|
profile, err := engine.InspectProfile(context.Background(), "weather-deep")
|
|
if err != nil || profile.EffectiveModelParams.Model != "~anthropic/claude-sonnet-latest" {
|
|
t.Fatalf("weather-deep profile/error = %#v/%v", profile, err)
|
|
}
|
|
}
|
|
|
|
func TestEmbeddedProfilesAreCompleteAndInspectable(t *testing.T) {
|
|
wantPaths := map[string]bool{
|
|
"weather-balanced.yml": false,
|
|
"weather-deep.yml": false,
|
|
"weather-light.yml": false,
|
|
}
|
|
if err := fs.WalkDir(promptassets.ProfileFS(), ".", func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil || entry.IsDir() {
|
|
return err
|
|
}
|
|
if _, ok := wantPaths[path]; !ok {
|
|
t.Fatalf("unexpected embedded profile asset %q", path)
|
|
}
|
|
wantPaths[path] = true
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk embedded profiles: %v", err)
|
|
}
|
|
for path, found := range wantPaths {
|
|
if !found {
|
|
t.Errorf("missing embedded profile asset %q", path)
|
|
}
|
|
}
|
|
|
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
|
promptkit.WithPromptFS(promptassets.PromptFS(), "."),
|
|
promptkit.WithSchemaFS(promptassets.SchemaFS(), "."),
|
|
promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewEngine() error = %v", err)
|
|
}
|
|
profiles := []struct {
|
|
id string
|
|
model string
|
|
timeoutSeconds int
|
|
reasoningEffort string
|
|
}{
|
|
{"weather-light", "deepseek/deepseek-v4-flash", 180, ""},
|
|
{"weather-balanced", "~google/gemini-flash-latest", 240, "high"},
|
|
{"weather-deep", "~anthropic/claude-sonnet-latest", 240, "high"},
|
|
}
|
|
for _, want := range profiles {
|
|
t.Run(want.id, func(t *testing.T) {
|
|
inspection, err := engine.InspectProfile(context.Background(), want.id)
|
|
if err != nil {
|
|
t.Fatalf("InspectProfile() error = %v", err)
|
|
}
|
|
got := inspection.EffectiveModelParams
|
|
if inspection.ProfileID != want.id || got.BackendID != "openrouter" || got.Model != want.model || got.TimeoutSeconds != want.timeoutSeconds || got.ServiceTier != "flex" || got.ReasoningEffort != want.reasoningEffort {
|
|
t.Fatalf("inspection = %#v, want %q using openrouter model %q", inspection, want.id, want.model)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEmbeddedProfilesExcludeUnsafeOrIncidentalSettings(t *testing.T) {
|
|
forbidden := []string{"endpoint:", "api_key", "credential", "temperature:", "top_p:", "max_tokens:"}
|
|
if err := fs.WalkDir(promptassets.ProfileFS(), ".", func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil || entry.IsDir() {
|
|
return err
|
|
}
|
|
data, err := fs.ReadFile(promptassets.ProfileFS(), path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, setting := range forbidden {
|
|
if strings.Contains(string(data), setting) {
|
|
t.Fatalf("%s contains forbidden profile setting %q", path, setting)
|
|
}
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk embedded profiles: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPromptAssetsExcludeRetiredRuntimeSettings(t *testing.T) {
|
|
if err := fs.WalkDir(promptassets.PromptFS(), ".", func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil || entry.IsDir() {
|
|
return err
|
|
}
|
|
data, err := fs.ReadFile(promptassets.PromptFS(), path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, unwanted := range []string{"local-heavy", "pipeline-weather/", "application/json", "repair_attempts:", "weather.daily_report"} {
|
|
if strings.Contains(string(data), unwanted) {
|
|
t.Fatalf("%s contains retired runtime setting %q", path, unwanted)
|
|
}
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk embedded prompts: %v", err)
|
|
}
|
|
}
|