From 91ae4fa7eff1571e7c05de01f9e922d75df7453d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 00:14:19 +0000 Subject: [PATCH] Measure startup configuration loading --- docs/roadmap/startup-benchmark.md | 321 ++++++++++++++++++++++++++++++ docs/roadmap/startup.md | 15 +- startup_bench_test.go | 159 +++++++++++++++ 3 files changed, 491 insertions(+), 4 deletions(-) create mode 100644 docs/roadmap/startup-benchmark.md create mode 100644 startup_bench_test.go diff --git a/docs/roadmap/startup-benchmark.md b/docs/roadmap/startup-benchmark.md new file mode 100644 index 0000000..db82b3c --- /dev/null +++ b/docs/roadmap/startup-benchmark.md @@ -0,0 +1,321 @@ +# Startup construction baseline and first optimization proposal + +Status: Stages 1–3 complete; optimization has not started. + +Measured on 2026-10-09 under the [startup roadmap](startup.md). The +[consumer request](promptkit-performance-request.md) owns the submitted workload +and measurements. This document records the reproduced baselines, maintained +benchmark, profiles, and proposed first change. It introduces no performance +contract or release gate. + +## Environment and revisions + +- PromptKit checkout: `eb830bb61346672f0f6c5bce6119b76ccc6d61ac` for both + downstream runs. The adapter and benchmark are unchanged from its earlier + review revision, `de6443f3aef4a66169e323e9f9a5f41594fbface`. +- Submitted ConfigKit dependency: v0.5.0, source commit + `dc09959cca4e105f49ec316c6d4ba585bd2340bd`. +- Current ConfigKit baseline: `f1722274c02c08ff28556dd66f29e0da07a9f2c3`. + The benchmark added with this report changes no production implementation. +- Go: `/snap/bin/go`, `go1.26.8`, `linux/amd64`. +- CPU: Intel Core i5-7260U at 2.20 GHz; two physical cores, four logical CPUs. +- `GOMAXPROCS=4`, `GOTOOLCHAIN=local`, `GOWORK=off`, `GOPROXY=off`, and + readonly module mode. `GOGC` and `GOMEMLIMIT` were unset (runtime defaults). +- Dependencies were already cached. No consumer repository's committed + dependencies were changed, and no adoption was published. Separate temporary + clones excluded consumer worktree edits; the current-code comparison used a + temporary module replacement. +- Logs, test binaries, and profiles are outside both repositories under + `/tmp/configkit-startup.L9vnUo`. These are disposable evidence, not repository + dependencies; the commands below regenerate them. + +The processor and toolchain match the submitted environment. The machine was +shared, with no exclusive CPU allocation or frequency control. An initial cold +downstream pass overlapped discovery work and ranged from 90.95 to 256.19 ms; +it was excluded from the warmed comparison below. Subsequent local runs also +showed substantial timing variation. All final-run observations, including +outliers, are recorded below. Allocation counts were much steadier than elapsed +time. These measurements do not establish statistical timing significance. + +## Stage 1: Downstream reproduction + +PromptKit's `BenchmarkConfigurationConstruction/independent` prepares 128 +profiles and 128 prompt versions outside timing, including the captured +`configkit.Mapping` source. Each profile supplies an endpoint and model. Each +prompt has version key `1`, one user message, a profile reference, and text output +with validation disabled. There are 256 definitions, one tier, and no files, +imports, inheritance, credentials, or inference calls. + +Each timed iteration invokes the real adapter's `load` function. Within +`LoadLayered`, it captures optional-field presence with `Resolution.InputInfo`, +normalizes profile values, performs domain validation, and supplies a path-based +secret classifier. The supplied endpoints are already canonical, so this +fixture records no normalization changes. After ConfigKit returns, the adapter +materializes prompts, compiles templates, computes content hashes, and builds +execution profiles. Those operations are included in the downstream timing. + +The comparison used an isolated PromptKit checkout at the revision above. First +it retained the declared v0.5.0 dependency. Then its temporary `go.mod` received +a replacement pointing at an isolated ConfigKit checkout at the current +baseline. Both used: + +```sh +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOMAXPROCS=4 \ + go test -mod=readonly -run '^$' \ + -bench '^BenchmarkConfigurationConstruction/independent$' \ + -benchmem -benchtime=500ms -count=5 ./internal/configuration +``` + +To reproduce the temporary wiring, clone with `--no-hardlinks --local`, check +out the recorded revisions, and run this only in the disposable PromptKit copy: + +```sh +go mod edit -replace=gitea.maximumdirect.net/eric/ConfigKit=/absolute/path/to/configkit-copy +``` + +| ConfigKit | Run | ms/construction | B/construction | allocations/construction | +| --- | ---: | ---: | ---: | ---: | +| v0.5.0 | 1 | 74.339 | 50,155,869 | 538,590 | +| v0.5.0 | 2 | 69.340 | 50,159,666 | 538,541 | +| v0.5.0 | 3 | 71.453 | 50,131,114 | 538,542 | +| v0.5.0 | 4 | 70.164 | 50,113,244 | 538,541 | +| v0.5.0 | 5 | 72.495 | 50,165,238 | 538,540 | +| current | 1 | 73.567 | 50,147,688 | 538,593 | +| current | 2 | 70.959 | 50,152,860 | 538,540 | +| current | 3 | 67.509 | 50,167,353 | 538,541 | +| current | 4 | 71.491 | 50,181,043 | 538,543 | +| current | 5 | 68.786 | 50,131,684 | 538,541 | + +The per-metric medians are 71.453 ms / 50,155,869 B / 538,541 allocations for +v0.5.0 and 70.959 ms / 50,152,860 B / 538,541 allocations for current code. +The submitted timing median was approximately 67.740 ms; its approximately +50.16 MB and 538,542 allocations are reproduced. The modest warmed timing +difference and the cold-run outlier are consistent with machine/runtime +variation; they do not establish a code regression. The production loading +code is identical between these ConfigKit revisions: their Go-file differences +are release/development tests. No startup improvement is claimed. + +## Stage 2: Maintained public-API benchmark + +[startup_bench_test.go](../../startup_bench_test.go) adds +`BenchmarkStartupConstruction` in the external `configkit_test` package. It +measures `LoadLayered` through the supported root facade at 32, 128, and 512 +profile/prompt pairs (64, 256, and 1,024 definitions). + +The input has two root collections. Profiles contain endpoint and model scalars; +prompts contain a numeric mapping key `1`, a profile reference, a message +sequence with role/content, and an output mapping with format/validation mode. +The source tree has `13 * pairs + 3` nodes (419, 1,667, and 6,659 respectively), +before typed projection adds omitted-field metadata. Typed profiles have twelve +optional fields; typed prompts retain optional metadata, inputs, messages, and +output fields. This records configuration shape as well as definition count. + +Fixture generation, `Mapping` capture, and one successful lifecycle/metadata +check are outside timing. Timed work includes typed decoding, each callback, +resolution auditing, validation, final publication, and retention of the latest +result. The resolver captures all twelve profile field states and parses their +canonical endpoints. Validation visits sorted profile IDs. A simple path-based +secret predicate is installed, although the fixture contains no credentials. +The preflight checks callback execution, collection sizes, explicit versus +omitted profile fields, and metadata for the message reached through mapping +key `1` and sequence index 0. + +This is representative rather than an exact copy of the consumer adapter: it +omits the unused backend collection and message cache-control type, simplifies +domain validation/classification, and does not compile templates, compute +hashes, or build runtime registries. Its lower allocation total must not be +treated as an exact subtraction of PromptKit overhead. It adds no imports of +PromptKit and runs offline without credentials. Its measurements are not +pass/fail thresholds. + +Run from ConfigKit: + +```sh +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOMAXPROCS=4 \ + go test -mod=readonly -run '^$' -bench '^BenchmarkStartupConstruction$' \ + -benchmem -benchtime=500ms -count=5 +``` + +| Pairs | Run | ms/construction | B/construction | allocations/construction | +| ---: | ---: | ---: | ---: | ---: | +| 32 | 1 | 18.327 | 11,999,640 | 129,136 | +| 32 | 2 | 18.418 | 11,999,423 | 129,136 | +| 32 | 3 | 50.366 | 11,999,534 | 129,137 | +| 32 | 4 | 55.028 | 11,999,548 | 129,137 | +| 32 | 5 | 25.824 | 11,999,468 | 129,136 | +| 128 | 1 | 67.610 | 47,408,046 | 513,722 | +| 128 | 2 | 65.090 | 47,451,718 | 513,724 | +| 128 | 3 | 65.682 | 47,584,876 | 513,727 | +| 128 | 4 | 62.574 | 47,324,919 | 513,722 | +| 128 | 5 | 123.565 | 47,273,050 | 513,721 | +| 512 | 1 | 276.663 | 188,313,648 | 2,051,961 | +| 512 | 2 | 284.979 | 189,649,296 | 2,052,000 | +| 512 | 3 | 497.075 | 189,234,128 | 2,051,985 | +| 512 | 4 | 257.534 | 189,084,032 | 2,051,981 | +| 512 | 5 | 265.167 | 188,679,712 | 2,051,973 | + +Per-metric medians: + +| Pairs | ms | B | Allocations | +| ---: | ---: | ---: | ---: | +| 32 | 25.824 | 11,999,534 | 129,136 | +| 128 | 65.682 | 47,408,046 | 513,722 | +| 512 | 276.663 | 189,084,032 | 2,051,981 | + +Increasing the catalog fourfold multiplies allocation counts by approximately +3.98 and 3.99, and allocated bytes by approximately 3.95 and 3.99. At these +fixed-depth shapes, allocations scale approximately with catalog size; the +measurements do not demonstrate a superlinear defect or establish a general +complexity guarantee. Timing is too variable for a precise scaling claim. +The original exploratory fixture lacked the classifier and had lower allocation +counts; only the final fixture's results above are the maintained baseline. + +### Complementary existing benchmarks + +The following existing benchmarks were run sequentially, with +`GOMAXPROCS=4`, local/offline toolchain settings, readonly modules, +`-benchtime=200ms`, and `-count=3`: + +```sh +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOMAXPROCS=4 \ + go test ./internal/engine -mod=readonly -run '^$' \ + -bench '^(BenchmarkComposeWideMappings|BenchmarkUnderlayBroadCatalog|BenchmarkUnderlayRenderedHistory|BenchmarkInheritanceInspectedLoad)$' \ + -benchmem -benchtime=200ms -count=3 +``` + +These per-metric medians provide comparison points for later changes. They have +different timing boundaries and must not be compared as equivalent loads. + +| Benchmark / size | ms | B | Allocations | +| --- | ---: | ---: | ---: | +| ComposeWideMappings / 128 | 1.815 | 848,512 | 9,658 | +| ComposeWideMappings / 256 | 3.531 | 1,703,590 | 19,652 | +| ComposeWideMappings / 512 | 9.532 | 3,626,298 | 39,643 | +| UnderlayBroadCatalog / 128 | 3.299 | 1,265,032 | 15,904 | +| UnderlayBroadCatalog / 256 | 3.436 | 2,581,000 | 33,473 | +| UnderlayBroadCatalog / 512 | 8.472 | 5,268,937 | 68,581 | +| InheritanceInspectedLoad / 16 | 2.571 | 876,236 | 11,050 | +| InheritanceInspectedLoad / 32 | 4.066 | 1,545,216 | 19,943 | +| InheritanceInspectedLoad / 64 | 8.358 | 2,906,067 | 37,779 | +| UnderlayRenderedHistory / 32 | 0.336 | 504,546 | 3,294 | +| UnderlayRenderedHistory / 64 | 0.661 | 1,006,053 | 6,529 | +| UnderlayRenderedHistory / 128 | 1.709 | 2,031,853 | 12,997 | + +Composition measures document combination; broad underlays measure an engine +load; inspected inheritance includes parsing, loading, and an explanation query; +rendered history measures explanation rendering after an untimed load. The +existing inspection benchmark supplies coverage without adding another suite. + +## Stage 3: Profiles and implementation trace + +CPU and sampled allocation profiles were collected in separate two-second runs +at 128 pairs. The current-code downstream run measured 71.877 ms, 50,125,624 B, +and 538,552 allocations. The final local fixture's profiled run measured +66.153 ms, 47,342,971 B, and 513,721 allocations. Profiled timings are not used +in the ordinary timing comparison. + +To regenerate local profiles outside the checkout: + +```sh +profile_dir=$(mktemp -d) +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOMAXPROCS=4 \ + go test -mod=readonly -run '^$' -bench '^BenchmarkStartupConstruction/128$' \ + -benchtime=2s -cpuprofile="$profile_dir/startup.cpu" \ + -memprofile="$profile_dir/startup.mem" -o "$profile_dir/startup.test" +go tool pprof -top -nodecount=12 -alloc_space "$profile_dir/startup.test" "$profile_dir/startup.mem" +go tool pprof -top -nodecount=12 "$profile_dir/startup.test" "$profile_dir/startup.cpu" +go tool pprof -list='engine.Path.String' -alloc_space "$profile_dir/startup.test" "$profile_dir/startup.mem" +``` + +For downstream profiles, use the stage 1 benchmark selector and package, the +same two-second settings, and separate output filenames in a temporary directory. + +| Allocation site (flat sampled bytes) | Local | Downstream | +| --- | ---: | ---: | +| `clonePathSegments` | 31.00% | 30.18% | +| `strings.Builder.WriteString` | 10.61% | 11.02% | +| `collectOrigins` | 5.04% | 4.86% | +| `collectFieldInfo` | 4.16% | 5.17% | +| `ParsePath` | 4.11% | 4.37% | +| `effectivePathIndex.func1` | 3.69% | 3.13% | +| `strings.Builder.WriteByte` | 3.30% | 3.48% | + +`Path.String` accounts cumulatively for 13.90% of sampled allocated bytes and +23.03% of CPU samples locally, versus 14.51% and 21.80% downstream. The local +line profile attributes 48.50 MB to writing separators and 156.01 MB to writing +tokens across the profiled run, exposing repeated builder growth. These are +sampled totals across many operations, not bytes per construction. Flat and +cumulative shares overlap and must not be added together. GC/scanning, +allocation, and map operations also occupy substantial CPU samples. + +The source trace identifies these responsibilities: + +- [Path construction](../../internal/engine/path.go): `Key` and `Index` copy + segments to preserve immutable parents and contextual mapping/sequence intent. + `Keys` builds through repeated `Key` calls. `Path.String` starts an ungrown + builder and formats every segment on every invocation. +- [Metadata collection](../../internal/engine/load_metadata.go): + `collectFieldInfo` formats each visited path and detaches origin records. + Decode's `collectOrigins` likewise traverses and formats paths. +- [Path indexes](../../internal/engine/metadata.go): `effectivePathIndex` + walks and retains contextual paths; `finalizeMetadataPaths` repeats walks for + effective and historical trees. These indexes serve different query roles. +- [Sensitivity classification](../../internal/engine/secret.go): + `classifySecretPaths` indexes several trees and repeatedly parses textual + pointers through `addSecretPath`. It also evaluates the caller's classifier. +- [Publication](../../internal/engine/load_result.go): final typed detachment, + metadata finalization, and sensitivity classification happen before return. + Detachment protects against equal borrowed objects supplied by callbacks. +- [Root conversion](../../lifecycle.go): callback adapters delegate to live + engine contexts; result publication wraps the detached configuration and + inspection. Paths delegate to engine paths. The dominant sampled work is + inside the engine, not a second root-owned tree conversion. + +The profiles support reducing redundant allocation and formatting. They do not +justify dropping ownership copies, historical trees, indexes, or sensitivity +processing. The consumer callback work is part of the measured lifecycle; +post-return materialization remains downstream work. + +### Proposed first optimization + +Try pre-sizing the local `strings.Builder` in engine `Path.String` using the +encoded pointer length before writing tokens. Keep the existing segment kinds, +canonical escaping, index formatting, root behavior, and independent returned +strings. A straightforward length calculation plus `Grow` should reduce buffer +growth without changing the `Path` representation or introducing shared state. +Avoid a cache or buffer pool for this first experiment. This is a hypothesis, +not an implemented improvement or a promise of one allocation per path. + +Although segment cloning is the largest allocation site, changing it requires +more ownership analysis across branches and captured records. Buffer sizing is +a smaller first change supported by both profiles, with a clear measurement +signal: fewer allocated bytes/allocations at all catalog sizes and less sampled +builder-growth work. CPU benefit must be measured because length calculation +also adds work. Escaped tokens and larger indexes require particular care; +preserve ordinary handling of invalid paths rather than adding validation here. + +Use existing `TestPathRoundTripsSemanticKeys`, +`TestPathPreservesSegmentIntentAndImmutability`, and +`TestParsePathRejectsMalformedPointers`, plus schema/metadata tests for numeric +keys versus sequence indexes. Verify root-facade integration, history, +explanation, and secret tests through the ordinary development suite. Add a +focused case only for a consequential boundary not covered already. Keep +inspection/rendering measurements alongside construction measurements; run +race validation if a later change affects ownership or concurrency. No runtime +ownership or concurrency behavior changes are proposed for buffer sizing. + +Then compare repeated matching before/after runs at all three sizes and the +downstream fixture; reprofile to confirm the attributed reduction. Keep the +change only if repeatable benefit justifies its complexity. If it does not, +return to profiling and assess pointer reparsing or repeated traversal rather +than assuming the next optimization in advance. + +## Validation and checkpoint + +The maintained fixture's preflight and measured runs succeeded at every size. +`./scripts/check.sh` passed formatting and whitespace checks, the offline suite +(including local documentation links and facade boundaries), full vet, and +package builds. Stages 4–6 remain unstarted. No loading semantics, public API, +dependency, release gate, or consumer adoption decision changes in this work. +The checkpoint has sufficient evidence for one focused optimization experiment; +wider representation changes would need their own measured justification. diff --git a/docs/roadmap/startup.md b/docs/roadmap/startup.md index bd21fee..ac8c671 100644 --- a/docs/roadmap/startup.md +++ b/docs/roadmap/startup.md @@ -1,6 +1,7 @@ # Startup Loading Performance Roadmap -Status: Approach accepted; execution has not started. +Status: Stages 1–3 complete; optimization has not started. Findings and +reproduction commands are in [startup-benchmark.md](startup-benchmark.md). Audience: ConfigKit maintainers implementing and reviewing this work. @@ -39,7 +40,9 @@ throughout execution. ### 1. Establish the downstream and current-code baselines -Status: Not started. +Status: Complete. The recorded PromptKit revision was measured with v0.5.0 and +current ConfigKit; allocation costs were reproduced. See the +[downstream results](startup-benchmark.md#stage-1-downstream-reproduction). 1. Inspect PromptKit's recorded benchmark and configuration adapter at the revision identified in its request. Identify which work is timed, including @@ -63,7 +66,9 @@ evidence. Do not attribute all adapter costs to ConfigKit. ### 2. Add a representative ConfigKit construction benchmark -Status: Not started. +Status: Complete. A maintained root-API benchmark measures 32, 128, and 512 +profile/prompt pairs, with complementary existing benchmarks. See the +[local results](startup-benchmark.md#stage-2-maintained-public-api-benchmark). 1. Add a maintained benchmark through the root public API, using a synthetic configuration shape representative of PromptKit's profiles and versioned @@ -89,7 +94,9 @@ measurements, not machine-specific pass/fail assertions. ### 3. Profile and choose the first optimization -Status: Not started. +Status: Complete. CPU/allocation profiles and the source trace identify +`Path.String` buffer sizing as the first experiment. See the +[profile evidence and proposal](startup-benchmark.md#stage-3-profiles-and-implementation-trace). 1. Collect CPU and allocation profiles for the representative local fixture. Compare them with the downstream profile where available, distinguishing diff --git a/startup_bench_test.go b/startup_bench_test.go new file mode 100644 index 0000000..1e92c25 --- /dev/null +++ b/startup_bench_test.go @@ -0,0 +1,159 @@ +package configkit_test + +import ( + "fmt" + "net/url" + "slices" + "strings" + "testing" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" +) + +// These synthetic types retain the optional-field shape of a catalog consumer +// without importing its adapter, template compiler, or runtime registries. +type startupCatalog struct { + Profiles map[string]startupProfile `yaml:"profiles"` + Prompts map[string]map[string]startupPrompt `yaml:"prompts"` +} + +type startupProfile struct { + Backend *string `yaml:"backend"` + Endpoint *string `yaml:"endpoint"` + Model *string `yaml:"model"` + Temperature *float64 `yaml:"temperature"` + TopP *float64 `yaml:"top_p"` + MaxTokens *int `yaml:"max_tokens"` + TimeoutSeconds *int `yaml:"timeout_seconds"` + ServiceTier *string `yaml:"service_tier"` + ReasoningEffort *string `yaml:"reasoning_effort"` + APIKeyEnv *string `yaml:"api_key_env"` + APIKeyRequired *bool `yaml:"api_key_required"` + ExtraParams map[string]any `yaml:"extra_params"` +} + +type startupPrompt struct { + DefaultProfile *string `yaml:"default_profile"` + Description *string `yaml:"description"` + SessionID *string `yaml:"session_id"` + Inputs []startupInput `yaml:"inputs"` + Messages []startupMessage `yaml:"messages"` + Output *startupOutput `yaml:"output"` +} + +type startupInput struct { + Name string `yaml:"name"` + Required bool `yaml:"required"` + ContentType *string `yaml:"content_type"` + Description *string `yaml:"description"` +} + +type startupMessage struct { + Role *string `yaml:"role"` + Content *string `yaml:"content"` + ContentFile *string `yaml:"content_file"` +} + +type startupOutput struct { + Format *string `yaml:"format"` + ValidationMode *string `yaml:"validation_mode"` + SchemaPath *string `yaml:"schema_path"` + RepairAttempts int `yaml:"repair_attempts"` +} + +type startupPresence struct{ explicit, null bool } + +var startupResult *configkit.Result[startupCatalog] + +func BenchmarkStartupConstruction(b *testing.B) { + for _, count := range []int{32, 128, 512} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + profiles, prompts := make(map[string]any, count), make(map[string]any, count) + for index := range count { + profileID := fmt.Sprintf("profile-%03d", index) + promptID := fmt.Sprintf("prompt-%03d", index) + profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} + prompts[promptID] = map[string]any{"1": map[string]any{ + "default_profile": profileID, + "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }} + } + // Mapping captures the source once outside timing, matching the + // downstream benchmark's reusable input. Each load still owns its result. + main := configkit.Input{Source: configkit.Mapping("startup catalog", map[string]any{ + "profiles": profiles, "prompts": prompts, + })} + var captured map[string]map[string]startupPresence + validated := false + options := configkit.LayeredOptions[startupCatalog]{Definition: configkit.Definition[startupCatalog]{ + // Include a simple path-based classifier, as catalog consumers use + // this callback even when the representative input has no credentials. + Secret: func(path configkit.Path) bool { + return strings.HasSuffix(path.String(), "/extra_params/token") + }, + Resolve: func(catalog *startupCatalog, resolution *configkit.Resolution) error { + captured = make(map[string]map[string]startupPresence, len(catalog.Profiles)) + for id, profile := range catalog.Profiles { + fields := make(map[string]startupPresence, 12) + for _, field := range []string{ + "backend", "endpoint", "model", "temperature", "top_p", "max_tokens", + "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", + "api_key_required", "extra_params", + } { + info, exists := resolution.InputInfo(configkit.Keys("profiles", id, field)) + fields[field] = startupPresence{explicit: exists && info.Explicit, null: exists && info.Null} + } + captured[id] = fields + // The representative endpoints are already canonical, so this + // callback, like the consumer's, records no normalization change. + if _, err := url.Parse(*profile.Endpoint); err != nil { + return err + } + } + return nil + }, + Validate: func(catalog *startupCatalog, validation *configkit.Validation) { + validated = true + ids := make([]string, 0, len(catalog.Profiles)) + for id := range catalog.Profiles { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + profile := catalog.Profiles[id] + if profile.Model == nil || strings.TrimSpace(*profile.Model) == "" { + validation.Error(configkit.Keys("profiles", id, "model"), "model is required") + } + } + }, + }} + // Check the measured workload before timing, including numeric mapping + // keys, sequence paths, callback presence capture, and published metadata. + result, err := configkit.LoadLayered(main, options) + if err != nil { + b.Fatal(err) + } + if !validated || len(result.Config.Profiles) != count || len(result.Config.Prompts) != count || len(captured) != count { + b.Fatal("startup fixture did not exercise the expected lifecycle") + } + fields := captured["profile-000"] + if !fields["endpoint"].explicit || !fields["model"].explicit || fields["temperature"].explicit { + b.Fatal("startup fixture presence capture disagrees with input") + } + path := configkit.Keys("prompts", "prompt-000", "1", "messages").Index(0).Key("content") + if info, exists := result.Metadata().Lookup(path); !exists || !info.Explicit || info.Null { + b.Fatal("startup fixture message provenance is missing") + } + b.ReportAllocs() + for b.Loop() { + result, err := configkit.LoadLayered(main, options) + if err != nil { + b.Fatal(err) + } + startupResult = result + } + b.ReportMetric(float64(count*2), "definitions/op") + }) + } +}