diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index aa2405f..c5f2ba4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -166,7 +166,12 @@ receive independent public values with the selected error behavior. ## Stage 3 — Verify Effective Membership, Ownership, And Concurrent Use -**Status: Pending.** Depends on Stage 2. +**Status: Complete (2026-10-09).** Verified with +`go test -race ./internal/configuration ./internal/engine .` and +`git diff --check`. External tests cover layered effective membership, +maintained profiles, empty inventories, caller ownership, concurrent listing +and inspection, captured sources, and the absence of execution work. Depends on +Stage 2. **Scope:** focused additions to the external-package listing contract tests. Production edits are limited to defects those tests reveal. diff --git a/listing_contract_test.go b/listing_contract_test.go index 5b422f5..2c52db1 100644 --- a/listing_contract_test.go +++ b/listing_contract_test.go @@ -3,7 +3,11 @@ package promptkit_test import ( "context" "errors" + "fmt" + "reflect" + "sync" "testing" + "testing/fstest" "time" "gitea.maximumdirect.net/eric/ConfigKit" @@ -108,6 +112,218 @@ func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) } } +func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) { + underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{ + "profiles": map[string]any{ + "base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}, + "child/profile": map[string]any{"extends": "base/profile"}, + "overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"}, + }, + "prompts": map[string]any{ + "shared/prompt": map[string]any{ + "v1": listingContractPrompt(), + "v2": listingContractPrompt(), + }, + "underlay-only": map[string]any{"v1": listingContractPrompt()}, + }, + })} + main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{ + "profiles": map[string]any{ + "overlay/profile": map[string]any{"model": "effective-model"}, + }, + "prompts": map[string]any{ + "shared/prompt": map[string]any{"v1": listingContractPrompt()}, + "main-only": map[string]any{"v3": listingContractPrompt()}, + }, + })} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}, + Main: &main, + }}) + if err != nil { + t.Fatal(err) + } + + profiles, err := engine.ListProfiles(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{ + "base/profile", "child/profile", "overlay/profile", + "promptkit-openrouter-standard", "promptkit-openrouter-auto", + } { + if !containsListingProfile(profiles, id) { + t.Errorf("profile listing %#v does not include %q", profiles, id) + } + } + for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} { + if got := countListingProfile(profiles, id); got != 1 { + t.Errorf("profile %q appears %d times, want 1", id, got) + } + } + + overlay, err := engine.InspectProfile(context.Background(), "overlay/profile") + if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" { + t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err) + } + child, err := engine.InspectProfile(context.Background(), "child/profile") + if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" { + t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err) + } + + prompts, err := engine.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + wantPrompts := []promptkit.PromptIdentity{ + {ID: "main-only", Version: "v3"}, + {ID: "shared/prompt", Version: "v1"}, + {ID: "shared/prompt", Version: "v2"}, + {ID: "underlay-only", Version: "v1"}, + } + if !reflect.DeepEqual(prompts, wantPrompts) { + t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts) + } + if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 { + t.Fatalf("overlaid prompt identity appears %d times, want 1", got) + } +} + +func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) { + main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{ + "profiles": nil, + "prompts": nil, + })} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}}) + if err != nil { + t.Fatal(err) + } + profiles, err := engine.ListProfiles(context.Background()) + if err != nil || len(profiles) != 0 { + t.Fatalf("cleared profile listing = %#v, %v", profiles, err) + } + prompts, err := engine.ListPrompts(context.Background()) + if err != nil || len(prompts) != 0 { + t.Fatalf("cleared prompt listing = %#v, %v", prompts, err) + } +} + +func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) { + engine, err := newListingContractEngine(t, map[string]any{ + "profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}}, + "prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}}, + }) + if err != nil { + t.Fatal(err) + } + + profiles, err := engine.ListProfiles(context.Background()) + if err != nil { + t.Fatal(err) + } + prompts, err := engine.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + for i := range profiles { + if profiles[i] == "profile" { + profiles[i] = "caller mutation" + } + } + for i := range prompts { + if prompts[i].ID == "prompt" { + prompts[i].ID = "caller mutation" + prompts[i].Version = "caller mutation" + } + } + profiles, err = engine.ListProfiles(context.Background()) + if err != nil || !containsListingProfile(profiles, "profile") { + t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err) + } + prompts, err = engine.ListPrompts(context.Background()) + if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 { + t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err) + } + + const workers = 4 + const iterations = 25 + start := make(chan struct{}) + results := make(chan error, workers) + var wait sync.WaitGroup + wait.Add(workers) + for worker := 0; worker < workers; worker++ { + go func() { + defer wait.Done() + <-start + for iteration := 0; iteration < iterations; iteration++ { + profiles, err := engine.ListProfiles(context.Background()) + if err != nil { + results <- fmt.Errorf("list profiles: %w", err) + return + } + if len(profiles) == 0 { + results <- errors.New("profile listing is empty") + return + } + profiles[0] = "worker mutation" + + prompts, err := engine.ListPrompts(context.Background()) + if err != nil { + results <- fmt.Errorf("list prompts: %w", err) + return + } + if len(prompts) == 0 { + results <- errors.New("prompt listing is empty") + return + } + prompts[0].ID = "worker mutation" + prompts[0].Version = "worker mutation" + + if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil { + results <- fmt.Errorf("inspect profile: %w", err) + return + } + if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil { + results <- fmt.Errorf("inspect prompt: %w", err) + return + } + } + }() + } + close(start) + wait.Wait() + close(results) + for err := range results { + t.Error(err) + } +} + +func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) { + files := fstest.MapFS{ + "config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")}, + } + inputs, err := promptkit.FilesystemInputs("application listing", files, "config") + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: inputs}}, + }}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{})) + if err != nil { + t.Fatal(err) + } + files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n") + + profiles, err := engine.ListProfiles(context.Background()) + if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") { + t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err) + } + prompts, err := engine.ListPrompts(context.Background()) + if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 { + t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err) + } +} + func listingContractPrompt() map[string]any { return map[string]any{ "messages": []any{map[string]any{"role": "user", "content": "Hello."}}, @@ -123,3 +339,41 @@ func containsListingProfile(profiles []string, id string) bool { } return false } + +func countListingProfile(profiles []string, id string) int { + count := 0 + for _, profile := range profiles { + if profile == id { + count++ + } + } + return count +} + +func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int { + count := 0 + for _, prompt := range prompts { + if prompt.ID == id && prompt.Version == version { + count++ + } + } + return count +} + +func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) { + t.Helper() + input := configkit.Input{Source: configkit.Mapping("listing contract", values)} + return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...) +} + +type rejectingListingLLMClient struct{} + +func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, errors.New("unexpected model generation") +} + +type rejectingListingArtifactReader struct{} + +func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + return nil, errors.New("unexpected artifact read") +}