Adopt logical prompt profile defaults
This commit is contained in:
@@ -68,11 +68,11 @@ func (client *fakeClient) request() promptkit.GenerateRequest {
|
|||||||
|
|
||||||
func TestInspectPromptAndProfile(t *testing.T) {
|
func TestInspectPromptAndProfile(t *testing.T) {
|
||||||
adapter := newTestAdapter(t, &fakeClient{})
|
adapter := newTestAdapter(t, &fakeClient{})
|
||||||
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "1.0.1")
|
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "1.1.0")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("InspectPrompt() error = %v", err)
|
t.Fatalf("InspectPrompt() error = %v", err)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "1.0.1" || inspection.DefaultProfileID != "gemini-flash-latest" {
|
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "1.1.0" || inspection.DefaultProfileID != "weather-balanced" {
|
||||||
t.Fatalf("inspection = %#v", inspection)
|
t.Fatalf("inspection = %#v", inspection)
|
||||||
}
|
}
|
||||||
if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" {
|
if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" {
|
||||||
@@ -482,7 +482,7 @@ func writeProfileFile(t *testing.T, profile string) string {
|
|||||||
func testExecuteRequest() promptexec.ExecuteRequest {
|
func testExecuteRequest() promptexec.ExecuteRequest {
|
||||||
return promptexec.ExecuteRequest{
|
return promptexec.ExecuteRequest{
|
||||||
PromptID: "weather.daily_generated_text",
|
PromptID: "weather.daily_generated_text",
|
||||||
PromptVersion: "1.0.1",
|
PromptVersion: "1.1.0",
|
||||||
ProfileID: "test-profile",
|
ProfileID: "test-profile",
|
||||||
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
|
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
|
||||||
DataPackagePath: "data-packages/daily/data_package.yaml",
|
DataPackagePath: "data-packages/daily/data_package.yaml",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
id: weather.daily_generated_text
|
id: weather.daily_generated_text
|
||||||
version: "1.0.1"
|
version: "1.1.0"
|
||||||
default_profile: gemini-flash-latest
|
default_profile: weather-balanced
|
||||||
description: Daily weather report analysis prompt.
|
description: Daily weather report analysis prompt.
|
||||||
inputs:
|
inputs:
|
||||||
- name: data_package
|
- name: data_package
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
id: weather.hourly_generated_text
|
id: weather.hourly_generated_text
|
||||||
version: "1.0.1"
|
version: "1.1.0"
|
||||||
default_profile: gemini-flash-latest
|
default_profile: weather-light
|
||||||
description: Hourly weather report analysis prompt.
|
description: Hourly weather report analysis prompt.
|
||||||
inputs:
|
inputs:
|
||||||
- name: data_package
|
- name: data_package
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
id: weather.today_generated_text
|
id: weather.today_generated_text
|
||||||
version: "1.0.1"
|
version: "1.1.0"
|
||||||
default_profile: gemini-flash-latest
|
default_profile: weather-balanced
|
||||||
description: Today's weather report analysis prompt.
|
description: Today's weather report analysis prompt.
|
||||||
inputs:
|
inputs:
|
||||||
- name: data_package
|
- name: data_package
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
id: weather.tomorrow_generated_text
|
id: weather.tomorrow_generated_text
|
||||||
version: "1.0.1"
|
version: "1.1.0"
|
||||||
default_profile: gemini-flash-latest
|
default_profile: weather-balanced
|
||||||
description: Tomorrow's weather report analysis prompt.
|
description: Tomorrow's weather report analysis prompt.
|
||||||
inputs:
|
inputs:
|
||||||
- name: data_package
|
- name: data_package
|
||||||
|
|||||||
@@ -34,11 +34,12 @@ func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) {
|
|||||||
path string
|
path string
|
||||||
id string
|
id string
|
||||||
schemaID string
|
schemaID string
|
||||||
|
profile string
|
||||||
}{
|
}{
|
||||||
{"daily/daily_generated_text.yml", "weather.daily_generated_text", "daily"},
|
{"daily/daily_generated_text.yml", "weather.daily_generated_text", "daily", "weather-balanced"},
|
||||||
{"today/today_generated_text.yml", "weather.today_generated_text", "today"},
|
{"today/today_generated_text.yml", "weather.today_generated_text", "today", "weather-balanced"},
|
||||||
{"tomorrow/tomorrow_generated_text.yml", "weather.tomorrow_generated_text", "tomorrow"},
|
{"tomorrow/tomorrow_generated_text.yml", "weather.tomorrow_generated_text", "tomorrow", "weather-balanced"},
|
||||||
{"hourly/hourly_generated_text.yml", "weather.hourly_generated_text", "hourly"},
|
{"hourly/hourly_generated_text.yml", "weather.hourly_generated_text", "hourly", "weather-light"},
|
||||||
}
|
}
|
||||||
|
|
||||||
definitions := 0
|
definitions := 0
|
||||||
@@ -67,8 +68,8 @@ func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) {
|
|||||||
if err := yaml.Unmarshal(data, &definition); err != nil {
|
if err := yaml.Unmarshal(data, &definition); err != nil {
|
||||||
t.Fatalf("decode prompt definition: %v", err)
|
t.Fatalf("decode prompt definition: %v", err)
|
||||||
}
|
}
|
||||||
if definition.ID != tc.id || definition.Version != "1.0.1" || definition.DefaultProfile != "gemini-flash-latest" {
|
if definition.ID != tc.id || definition.Version != "1.1.0" || definition.DefaultProfile != tc.profile {
|
||||||
t.Fatalf("definition = %#v, want %s version 1.0.1 and gemini-flash-latest", definition, tc.id)
|
t.Fatalf("definition = %#v, want %s version 1.1.0 and profile %s", definition, tc.id, tc.profile)
|
||||||
}
|
}
|
||||||
if len(definition.Inputs) != 1 || definition.Inputs[0].Name != "data_package" || !definition.Inputs[0].Required || definition.Inputs[0].ContentType != "application/yaml" {
|
if len(definition.Inputs) != 1 || definition.Inputs[0].Name != "data_package" || !definition.Inputs[0].Required || definition.Inputs[0].ContentType != "application/yaml" {
|
||||||
t.Fatalf("inputs = %#v, want one required YAML data_package", definition.Inputs)
|
t.Fatalf("inputs = %#v, want one required YAML data_package", definition.Inputs)
|
||||||
@@ -123,21 +124,39 @@ func TestPromptkitInspectsEmbeddedPromptsOffline(t *testing.T) {
|
|||||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
promptkit.WithPromptFS(promptassets.PromptFS(), "."),
|
promptkit.WithPromptFS(promptassets.PromptFS(), "."),
|
||||||
promptkit.WithSchemaFS(promptassets.SchemaFS(), "."),
|
promptkit.WithSchemaFS(promptassets.SchemaFS(), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewEngine() error = %v", err)
|
t.Fatalf("NewEngine() error = %v", err)
|
||||||
}
|
}
|
||||||
for _, id := range []string{"weather.daily_generated_text", "weather.today_generated_text", "weather.tomorrow_generated_text", "weather.hourly_generated_text"} {
|
for _, want := range []struct {
|
||||||
t.Run(id, func(t *testing.T) {
|
id string
|
||||||
inspection, err := engine.InspectPrompt(context.Background(), id, "1.0.1")
|
profile string
|
||||||
|
model string
|
||||||
|
}{
|
||||||
|
{"weather.daily_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
||||||
|
{"weather.today_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
||||||
|
{"weather.tomorrow_generated_text", "weather-balanced", "~google/gemini-flash-latest"},
|
||||||
|
{"weather.hourly_generated_text", "weather-light", "deepseek/deepseek-v4-flash"},
|
||||||
|
} {
|
||||||
|
t.Run(want.id, func(t *testing.T) {
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), want.id, "1.1.0")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("InspectPrompt() error = %v", err)
|
t.Fatalf("InspectPrompt() error = %v", err)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != id || inspection.PromptVersion != "1.0.1" || inspection.DefaultProfileID != "gemini-flash-latest" {
|
if inspection.PromptID != want.id || inspection.PromptVersion != "1.1.0" || inspection.DefaultProfileID != want.profile {
|
||||||
t.Fatalf("inspection = %#v", inspection)
|
t.Fatalf("inspection = %#v", inspection)
|
||||||
}
|
}
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), inspection.DefaultProfileID)
|
||||||
|
if err != nil || profile.EffectiveModelParams.Model != want.model {
|
||||||
|
t.Fatalf("profile/error = %#v/%v, want model %q", profile, err, want.model)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), "weather-deep")
|
||||||
|
if err != nil || profile.EffectiveModelParams.Model != "~anthropic/claude-sonnet-latest" {
|
||||||
|
t.Fatalf("weather-deep profile/error = %#v/%v", profile, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEmbeddedProfilesAreCompleteAndInspectable(t *testing.T) {
|
func TestEmbeddedProfilesAreCompleteAndInspectable(t *testing.T) {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ func dailyDefinition() Definition {
|
|||||||
ID: Daily,
|
ID: Daily,
|
||||||
Name: "Daily Report",
|
Name: "Daily Report",
|
||||||
PromptID: "weather.daily_generated_text",
|
PromptID: "weather.daily_generated_text",
|
||||||
PromptVersion: "1.0.1",
|
PromptVersion: "1.1.0",
|
||||||
TemplateID: "daily",
|
TemplateID: "daily",
|
||||||
GeneratedTextSchemaID: "daily",
|
GeneratedTextSchemaID: "daily",
|
||||||
ComparisonStrategy: CompareSameValidDate,
|
ComparisonStrategy: CompareSameValidDate,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ func hourlyDefinition() Definition {
|
|||||||
ID: Hourly,
|
ID: Hourly,
|
||||||
Name: "Hourly Report",
|
Name: "Hourly Report",
|
||||||
PromptID: "weather.hourly_generated_text",
|
PromptID: "weather.hourly_generated_text",
|
||||||
PromptVersion: "1.0.1",
|
PromptVersion: "1.1.0",
|
||||||
TemplateID: "hourly",
|
TemplateID: "hourly",
|
||||||
GeneratedTextSchemaID: "hourly",
|
GeneratedTextSchemaID: "hourly",
|
||||||
ComparisonStrategy: CompareRollingWindow,
|
ComparisonStrategy: CompareRollingWindow,
|
||||||
|
|||||||
@@ -51,8 +51,11 @@ func TestRegistryContainsOnlyPromptBackedReports(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, definition := range definitions {
|
for _, definition := range definitions {
|
||||||
if definition.PromptVersion != "1.0.1" {
|
if definition.PromptVersion != "1.1.0" {
|
||||||
t.Fatalf("%s PromptVersion = %q, want 1.0.1", definition.ID, definition.PromptVersion)
|
t.Fatalf("%s PromptVersion = %q, want 1.1.0", definition.ID, definition.PromptVersion)
|
||||||
|
}
|
||||||
|
if definition.PromptID == "" {
|
||||||
|
t.Fatalf("%s PromptID is empty", definition.ID)
|
||||||
}
|
}
|
||||||
if definition.TemplateID == "" || definition.GeneratedTextSchemaID == "" {
|
if definition.TemplateID == "" || definition.GeneratedTextSchemaID == "" {
|
||||||
t.Fatalf("%s template/schema = %q/%q, want both set", definition.ID, definition.TemplateID, definition.GeneratedTextSchemaID)
|
t.Fatalf("%s template/schema = %q/%q, want both set", definition.ID, definition.TemplateID, definition.GeneratedTextSchemaID)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ func todayDefinition() Definition {
|
|||||||
ID: Today,
|
ID: Today,
|
||||||
Name: "Today Report",
|
Name: "Today Report",
|
||||||
PromptID: "weather.today_generated_text",
|
PromptID: "weather.today_generated_text",
|
||||||
PromptVersion: "1.0.1",
|
PromptVersion: "1.1.0",
|
||||||
TemplateID: "today",
|
TemplateID: "today",
|
||||||
GeneratedTextSchemaID: "today",
|
GeneratedTextSchemaID: "today",
|
||||||
ComparisonStrategy: CompareSameValidDate,
|
ComparisonStrategy: CompareSameValidDate,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ func tomorrowDefinition() Definition {
|
|||||||
ID: Tomorrow,
|
ID: Tomorrow,
|
||||||
Name: "Tomorrow Report",
|
Name: "Tomorrow Report",
|
||||||
PromptID: "weather.tomorrow_generated_text",
|
PromptID: "weather.tomorrow_generated_text",
|
||||||
PromptVersion: "1.0.1",
|
PromptVersion: "1.1.0",
|
||||||
TemplateID: "tomorrow",
|
TemplateID: "tomorrow",
|
||||||
GeneratedTextSchemaID: "tomorrow",
|
GeneratedTextSchemaID: "tomorrow",
|
||||||
ComparisonStrategy: CompareSameValidDate,
|
ComparisonStrategy: CompareSameValidDate,
|
||||||
|
|||||||
@@ -172,9 +172,9 @@ func validPreparationArtifact() PromptPreparationArtifact {
|
|||||||
return PromptPreparationArtifact{
|
return PromptPreparationArtifact{
|
||||||
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
|
||||||
PromptVersion: "1.0.1", DataPackagePath: "/workspace/data.yaml",
|
PromptVersion: "1.1.0", DataPackagePath: "/workspace/data.yaml",
|
||||||
Preparation: &promptexec.Preparation{
|
Preparation: &promptexec.Preparation{
|
||||||
PromptID: "weather.daily_generated_text", PromptVersion: "1.0.1",
|
PromptID: "weather.daily_generated_text", PromptVersion: "1.1.0",
|
||||||
DataPackagePath: "/workspace/data.yaml",
|
DataPackagePath: "/workspace/data.yaml",
|
||||||
},
|
},
|
||||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||||
@@ -186,9 +186,9 @@ func validExecutionArtifact() PromptExecutionArtifact {
|
|||||||
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
||||||
return PromptExecutionArtifact{
|
return PromptExecutionArtifact{
|
||||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.1",
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.1.0",
|
||||||
Provenance: &PromptExecutionProvenance{
|
Provenance: &PromptExecutionProvenance{
|
||||||
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.1",
|
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.1.0",
|
||||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
|
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
|
||||||
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
|
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
|
||||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||||
@@ -201,7 +201,7 @@ func validFailedExecutionArtifact() PromptExecutionArtifact {
|
|||||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
return PromptExecutionArtifact{
|
return PromptExecutionArtifact{
|
||||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
|
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.1",
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.1.0",
|
||||||
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
|
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user