eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Benchmark configuration workflows and reuse captured content origins

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-30 04:25:48 +00:00
parent 48343300e2
commit 66c55b7397
4 changed files with 194 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/roadmap/implementation.md
View File

@@ -600,6 +600,8 @@ the reduced implementation/test inventory.
## Stage 12 — Validate Scale And The Complete Consumer Workflow
**Completed:** 2026-09-30.
Scope: final integration evidence and high-confidence simplification, not another
feature expansion or release gate.

179
internal/configuration/benchmark_test.go Normal file
View File

@@ -0,0 +1,179 @@
package configuration
import (
"context"
"fmt"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const benchmarkDefinitionCount = 128
var (
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedProfile *domain.ExecutionProfile
)
type configurationShape struct {
main configkit.Input
underlays []configkit.Underlay
inputEntries int
effectiveEntries int
tiers int
}
func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{
"default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
func independentDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
for index := range count {
profileID := fmt.Sprintf("profile-%03d", index)
promptID := fmt.Sprintf("prompt-%03d", index)
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
inputEntries: count * 2,
effectiveEntries: count * 2,
tiers: 1,
}
}
func fileBackedDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
for index := range count {
profileID := fmt.Sprintf("profile-%03d", index)
promptID := fmt.Sprintf("prompt-%03d", index)
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompt := benchmarkPrompt(profileID)
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
prompts[promptID] = map[string]any{"1": prompt}
}
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
})
return configurationShape{
main: configkit.Input{
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
},
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
}
}
func sharedBaseDefinitions(count int) configurationShape {
profiles := make(map[string]any, count+1)
profiles["base"] = map[string]any{
"endpoint": "https://example.test/v1", "model": "shared-model",
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
}
for index := range count {
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": profiles,
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
})},
inputEntries: count + 2,
effectiveEntries: count + 2,
tiers: 1,
}
}
func layeredDefinitions(count int) configurationShape {
base := make(map[string]any, count)
operator := make(map[string]any, count)
programmatic := make(map[string]any, count)
final := make(map[string]any, count)
for index := range count {
id := fmt.Sprintf("profile-%03d", index)
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
operator[id] = map[string]any{"model": "operator-model"}
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"temperature": float64(index%10) / 10}
}
return configurationShape{
underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
},
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": final,
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
})},
inputEntries: count*4 + 1,
effectiveEntries: count + 1,
tiers: 4,
}
}
func BenchmarkConfigurationConstruction(b *testing.B) {
for _, scenario := range []struct {
name string
shape configurationShape
}{
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
{name: "file_backed", shape: fileBackedDefinitions(64)},
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
} {
b.Run(scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
result, err := load(scenario.shape.main, scenario.shape.underlays)
if err != nil {
b.Fatal(err)
}
constructionResult = result
}
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
})
}
}
func BenchmarkConfigurationSelection(b *testing.B) {
shape := independentDefinitions(benchmarkDefinitionCount)
result, err := load(shape.main, nil)
if err != nil {
b.Fatal(err)
}
b.Run("prompt", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, _, err := result.prompt("prompt-127", "1")
if err != nil {
b.Fatal(err)
}
selectedPrompt = definition
}
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
b.Run("profile", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
profile, err := result.GetProfile(context.Background(), "profile-127")
if err != nil {
b.Fatal(err)
}
selectedProfile = profile
}
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
}

18
internal/configuration/prompts.go
View File

@@ -108,6 +108,14 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
func (state *loaded) materializePrompts() error {
state.prompts = make(map[string]map[string]*promptEntry, len(state.result.Config.Prompts))
state.ambiguous = make(map[string]bool)
var metadata *configkit.Metadata
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
if metadata == nil {
captured := state.result.Metadata()
metadata = &captured
}
return metadata.Lookup(path)
}
for _, id := range sortedKeys(state.result.Config.Prompts) {
if strings.TrimSpace(id) == "" {
return fmt.Errorf("%s: prompt ID must not be blank", configkit.Keys("prompts", id))
@@ -122,7 +130,7 @@ func (state *loaded) materializePrompts() error {
if strings.TrimSpace(version) == "" {
return fmt.Errorf("%s: prompt version must not be blank", configkit.Keys("prompts", id, version))
}
entry, err := state.materializePrompt(id, version, versions[version])
entry, err := state.materializePrompt(id, version, versions[version], lookupOrigin)
if err != nil {
return err
}
@@ -132,7 +140,7 @@ func (state *loaded) materializePrompts() error {
return nil
}
func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*promptEntry, error) {
func (state *loaded) materializePrompt(id, version string, raw promptConfig, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
base := configkit.Keys("prompts", id, version)
if len(raw.Messages) == 0 {
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
@@ -180,7 +188,7 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*p
content = *message.Content
} else {
var err error
content, err = state.readContent(path.Key("content_file"), *message.ContentFile)
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
if err != nil {
return nil, err
}
@@ -249,11 +257,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*p
return entry, nil
}
func (state *loaded) readContent(fieldPath configkit.Path, name string) (string, error) {
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
if strings.TrimSpace(name) == "" {
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
}
field, found := state.result.Metadata().Lookup(fieldPath)
field, found := lookupOrigin(fieldPath)
if !found || len(field.Origin.Locations) != 1 {
return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath)
}

3
internal/configuration/targets.go
View File

@@ -185,6 +185,3 @@ func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy
}
return state.registry.CapacityPolicies()
}
// ExactIDs indicates that ConfigKit definition keys are literal identities.
func (state *loaded) ExactIDs() bool { return true }