diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 130c751..8a3a26c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -600,6 +600,8 @@ the reduced implementation/test inventory. ## Stage 12 — Validate Scale And The Complete Consumer Workflow +**Completed:** 2026-09-30. + Scope: final integration evidence and high-confidence simplification, not another feature expansion or release gate. diff --git a/internal/configuration/benchmark_test.go b/internal/configuration/benchmark_test.go new file mode 100644 index 0000000..d01433e --- /dev/null +++ b/internal/configuration/benchmark_test.go @@ -0,0 +1,179 @@ +package configuration + +import ( + "context" + "fmt" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +const benchmarkDefinitionCount = 128 + +var ( + constructionResult *loaded + selectedPrompt *domain.PromptDefinition + selectedProfile *domain.ExecutionProfile +) + +type configurationShape struct { + main configkit.Input + underlays []configkit.Underlay + inputEntries int + effectiveEntries int + tiers int +} + +func benchmarkPrompt(profileID string) map[string]any { + return map[string]any{ + "default_profile": profileID, + "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + } +} + +func independentDefinitions(count int) configurationShape { + profiles := make(map[string]any, count) + prompts := make(map[string]any, count) + for index := range count { + profileID := fmt.Sprintf("profile-%03d", index) + promptID := fmt.Sprintf("prompt-%03d", index) + profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} + prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)} + } + return configurationShape{ + main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})}, + inputEntries: count * 2, + effectiveEntries: count * 2, + tiers: 1, + } +} + +func fileBackedDefinitions(count int) configurationShape { + profiles := make(map[string]any, count) + prompts := make(map[string]any, count) + for index := range count { + profileID := fmt.Sprintf("profile-%03d", index) + promptID := fmt.Sprintf("prompt-%03d", index) + profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} + prompt := benchmarkPrompt(profileID) + prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}} + prompts[promptID] = map[string]any{"1": prompt} + } + namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{ + "assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")}, + }) + return configurationShape{ + main: configkit.Input{ + Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}), + Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"}, + }, + inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, + } +} + +func sharedBaseDefinitions(count int) configurationShape { + profiles := make(map[string]any, count+1) + profiles["base"] = map[string]any{ + "endpoint": "https://example.test/v1", "model": "shared-model", + "extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}}, + } + for index := range count { + profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10} + } + return configurationShape{ + main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{ + "profiles": profiles, + "prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}}, + })}, + inputEntries: count + 2, + effectiveEntries: count + 2, + tiers: 1, + } +} + +func layeredDefinitions(count int) configurationShape { + base := make(map[string]any, count) + operator := make(map[string]any, count) + programmatic := make(map[string]any, count) + final := make(map[string]any, count) + for index := range count { + id := fmt.Sprintf("profile-%03d", index) + base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"} + operator[id] = map[string]any{"model": "operator-model"} + programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} + final[id] = map[string]any{"temperature": float64(index%10) / 10} + } + return configurationShape{ + underlays: []configkit.Underlay{ + {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}}, + {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}}, + {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}}, + }, + main: configkit.Input{Source: configkit.Mapping("main", map[string]any{ + "profiles": final, + "prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}}, + })}, + inputEntries: count*4 + 1, + effectiveEntries: count + 1, + tiers: 4, + } +} + +func BenchmarkConfigurationConstruction(b *testing.B) { + for _, scenario := range []struct { + name string + shape configurationShape + }{ + {name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)}, + {name: "file_backed", shape: fileBackedDefinitions(64)}, + {name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)}, + {name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)}, + } { + b.Run(scenario.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + result, err := load(scenario.shape.main, scenario.shape.underlays) + if err != nil { + b.Fatal(err) + } + constructionResult = result + } + b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op") + b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op") + b.ReportMetric(float64(scenario.shape.tiers), "tiers/op") + }) + } +} + +func BenchmarkConfigurationSelection(b *testing.B) { + shape := independentDefinitions(benchmarkDefinitionCount) + result, err := load(shape.main, nil) + if err != nil { + b.Fatal(err) + } + b.Run("prompt", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + definition, _, err := result.prompt("prompt-127", "1") + if err != nil { + b.Fatal(err) + } + selectedPrompt = definition + } + b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") + }) + b.Run("profile", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + profile, err := result.GetProfile(context.Background(), "profile-127") + if err != nil { + b.Fatal(err) + } + selectedProfile = profile + } + b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") + }) +} diff --git a/internal/configuration/prompts.go b/internal/configuration/prompts.go index c18fd25..b8f0f89 100644 --- a/internal/configuration/prompts.go +++ b/internal/configuration/prompts.go @@ -108,6 +108,14 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf func (state *loaded) materializePrompts() error { state.prompts = make(map[string]map[string]*promptEntry, len(state.result.Config.Prompts)) state.ambiguous = make(map[string]bool) + var metadata *configkit.Metadata + lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) { + if metadata == nil { + captured := state.result.Metadata() + metadata = &captured + } + return metadata.Lookup(path) + } for _, id := range sortedKeys(state.result.Config.Prompts) { if strings.TrimSpace(id) == "" { return fmt.Errorf("%s: prompt ID must not be blank", configkit.Keys("prompts", id)) @@ -122,7 +130,7 @@ func (state *loaded) materializePrompts() error { if strings.TrimSpace(version) == "" { return fmt.Errorf("%s: prompt version must not be blank", configkit.Keys("prompts", id, version)) } - entry, err := state.materializePrompt(id, version, versions[version]) + entry, err := state.materializePrompt(id, version, versions[version], lookupOrigin) if err != nil { return err } @@ -132,7 +140,7 @@ func (state *loaded) materializePrompts() error { return nil } -func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*promptEntry, error) { +func (state *loaded) materializePrompt(id, version string, raw promptConfig, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) { base := configkit.Keys("prompts", id, version) if len(raw.Messages) == 0 { return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages")) @@ -180,7 +188,7 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*p content = *message.Content } else { var err error - content, err = state.readContent(path.Key("content_file"), *message.ContentFile) + content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin) if err != nil { return nil, err } @@ -249,11 +257,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig) (*p return entry, nil } -func (state *loaded) readContent(fieldPath configkit.Path, name string) (string, error) { +func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) { if strings.TrimSpace(name) == "" { return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath) } - field, found := state.result.Metadata().Lookup(fieldPath) + field, found := lookupOrigin(fieldPath) if !found || len(field.Origin.Locations) != 1 { return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath) } diff --git a/internal/configuration/targets.go b/internal/configuration/targets.go index adf34e1..7d11943 100644 --- a/internal/configuration/targets.go +++ b/internal/configuration/targets.go @@ -185,6 +185,3 @@ func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy } return state.registry.CapacityPolicies() } - -// ExactIDs indicates that ConfigKit definition keys are literal identities. -func (state *loaded) ExactIDs() bool { return true }