Update examples and benchmarks for identified resources
This commit is contained in:
@@ -71,7 +71,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
message := "Hello {{.name}}"
|
||||
response := "ok"
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var expectedMessage string
|
||||
|
||||
@@ -99,9 +99,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"profiles": map[string]any{"profile": profile},
|
||||
"prompts": map[string]any{"workflow": map[string]any{
|
||||
"version": "v1",
|
||||
"profiles": []any{profile},
|
||||
"prompts": []any{map[string]any{"id": "workflow",
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||
"output": output,
|
||||
@@ -128,7 +127,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||
if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
|
||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||
}
|
||||
|
||||
@@ -38,10 +38,11 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
}
|
||||
cfg := simpleConfig("hello")
|
||||
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
||||
endpoint, model := "http://example.test/v1", "model"
|
||||
lowerName := name
|
||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||
{ID: "profile", APIKeyEnv: &lowerName},
|
||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
||||
}),
|
||||
}}})
|
||||
env := " " + name + " "
|
||||
@@ -49,7 +50,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
env = ""
|
||||
}
|
||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||
{ID: "profile", APIKeyEnv: &env},
|
||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
||||
})}
|
||||
var calls int
|
||||
var auth string
|
||||
|
||||
@@ -60,11 +60,10 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
|
||||
func simpleConfig(content string) promptkit.Config {
|
||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile",
|
||||
"endpoint": "http://example.test/v1", "model": "model",
|
||||
}},
|
||||
"prompts": map[string]any{"prompt": map[string]any{
|
||||
"version": "1",
|
||||
"prompts": []any{map[string]any{"id": "prompt",
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
|
||||
@@ -803,6 +803,13 @@ Validation: run all three examples through the semantic checker once runnable.
|
||||
Use a short focused construction benchmark if needed to investigate a suspected
|
||||
repeated-scan regression, not to claim unmeasured performance improvements.
|
||||
|
||||
Completion: the three maintained examples and benchmark/shared fixtures now use
|
||||
identified resources and ordered builders. The preparation example includes a
|
||||
headerless item import and an inherited replacement profile. The semantic example
|
||||
checker, full public contract suite, and short configuration benchmark fixture
|
||||
runs passed on Go 1.27.0. The public run also verified the previously deferred
|
||||
preparation and observation tests after updating their shared fixture.
|
||||
|
||||
## Stage 20: Document Formats And Internal Handoff
|
||||
|
||||
Scope: `docs/formats.md`, `docs/internal/sources.md`, and other current internal
|
||||
|
||||
@@ -84,13 +84,15 @@ func run() (summary, error) {
|
||||
return summary{}, err
|
||||
}
|
||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||
backend.ID = backendID
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
backend,
|
||||
})},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
|
||||
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
|
||||
})},
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.assessment:
|
||||
version: "1.0.0"
|
||||
- id: example.assessment
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.proposal:
|
||||
version: "1.0.0"
|
||||
- id: example.proposal
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
backends:
|
||||
offline:
|
||||
- id: offline
|
||||
endpoint: https://review.example.invalid/v1
|
||||
profiles:
|
||||
audit:
|
||||
- id: audit
|
||||
backend: review/offline
|
||||
model: review-model
|
||||
prompts:
|
||||
check:
|
||||
- id: check
|
||||
default_profile: review/audit
|
||||
description: Inspect a review item without contacting a model provider.
|
||||
messages:
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
version: "1"
|
||||
version: "2"
|
||||
config:
|
||||
imports:
|
||||
- path: fragments/resources.yaml
|
||||
required: true
|
||||
- path: fragments/prepare.yaml
|
||||
destination: [prompts]
|
||||
placement: sequence-item
|
||||
required: true
|
||||
release_version: "1.4.0"
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
id: prepare
|
||||
default_profile: support/fast
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content_file: ../messages/note.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -1,28 +1,11 @@
|
||||
backends:
|
||||
offline:
|
||||
- id: offline
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
base:
|
||||
- id: base
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
timeout_seconds: 90
|
||||
fast:
|
||||
extends: base
|
||||
- id: fast
|
||||
extends: {id: base}
|
||||
timeout_seconds: 60
|
||||
prompts:
|
||||
prepare:
|
||||
version: "1.0.0"
|
||||
default_profile: support/fast
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content_file: ../messages/note.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# A later input intentionally replaces only the inherited profile timeout.
|
||||
# A later input replaces the profile with an inherited variant.
|
||||
profiles:
|
||||
fast:
|
||||
- id: fast
|
||||
extends: {id: base}
|
||||
timeout_seconds: 30
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
type summary struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
@@ -70,9 +69,7 @@ func main() {
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys(
|
||||
"collections", "support", "profiles", "fast", "timeout_seconds",
|
||||
)); !found {
|
||||
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||
exit(fmt.Errorf("collection profile override was not found"))
|
||||
}
|
||||
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
@@ -148,7 +145,6 @@ func main() {
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(summary{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
SelectedProfile: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
|
||||
@@ -41,8 +41,8 @@ type summary struct {
|
||||
func main() {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.run:
|
||||
version: "1.0.0"
|
||||
- id: example.run
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
|
||||
@@ -31,13 +31,26 @@ type configurationShape struct {
|
||||
|
||||
func benchmarkPrompt(profileID string) map[string]any {
|
||||
return map[string]any{
|
||||
"version": "1",
|
||||
"default_profile": profileID,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkRecords(values map[string]any) []any {
|
||||
records := make([]any, 0, len(values))
|
||||
for _, id := range sortedKeys(values) {
|
||||
value := values[id].(map[string]any)
|
||||
record := make(map[string]any, len(value)+1)
|
||||
for key, field := range value {
|
||||
record[key] = field
|
||||
}
|
||||
record["id"] = id
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func independentDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
@@ -48,7 +61,7 @@ func independentDefinitions(count int) configurationShape {
|
||||
prompts[promptID] = benchmarkPrompt(profileID)
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
|
||||
inputEntries: count * 2,
|
||||
effectiveEntries: count * 2,
|
||||
tiers: 1,
|
||||
@@ -71,7 +84,7 @@ func fileBackedDefinitions(count int) configurationShape {
|
||||
})
|
||||
return configurationShape{
|
||||
main: configkit.Input{
|
||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
},
|
||||
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
||||
@@ -85,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape {
|
||||
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
||||
}
|
||||
for index := range count {
|
||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
|
||||
"profiles": benchmarkRecords(profiles),
|
||||
"prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
|
||||
})},
|
||||
inputEntries: count + 2,
|
||||
effectiveEntries: count + 2,
|
||||
@@ -106,19 +119,19 @@ func layeredDefinitions(count int) configurationShape {
|
||||
for index := range count {
|
||||
id := fmt.Sprintf("profile-%03d", index)
|
||||
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
||||
operator[id] = map[string]any{"model": "operator-model"}
|
||||
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
||||
operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
|
||||
programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||
final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
|
||||
},
|
||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||
"profiles": final,
|
||||
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
|
||||
"profiles": benchmarkRecords(final),
|
||||
"prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
|
||||
})},
|
||||
inputEntries: count*4 + 1,
|
||||
effectiveEntries: count + 1,
|
||||
@@ -132,8 +145,8 @@ func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
|
||||
collections := make(map[string]any, count)
|
||||
for index := range count {
|
||||
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
|
||||
"prompts": map[string]any{
|
||||
"example": map[string]any{
|
||||
"prompts": []any{
|
||||
map[string]any{"id": "example",
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
},
|
||||
@@ -240,8 +253,8 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": prompts,
|
||||
"profiles": benchmarkRecords(profiles),
|
||||
"prompts": benchmarkRecords(prompts),
|
||||
})},
|
||||
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||
}
|
||||
|
||||
@@ -38,8 +38,8 @@ func BenchmarkRequirementInspection(b *testing.B) {
|
||||
}, "", noValidationOutput)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
||||
"prompts": oneVersionPrompt("inspection", "1", definition),
|
||||
"profiles": profiles,
|
||||
"prompts": identifiedRequirementRecords(onePrompt("inspection", definition)),
|
||||
"profiles": identifiedRequirementRecords(profiles),
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
@@ -51,7 +51,7 @@ func BenchmarkRequirementInspection(b *testing.B) {
|
||||
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
inspection, err := engine.InspectRequirements(ctx, request)
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
||||
if err != nil || inspection.PromptID != "inspection" ||
|
||||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
||||
|
||||
Reference in New Issue
Block a user