eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Verify effective listing ownership and concurrency

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-09 04:31:31 +00:00
parent a27ee8f7da
commit f2aeb5eb30
2 changed files with 260 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
listing_contract_test.go
View File

@@ -3,7 +3,11 @@ package promptkit_test
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
@@ -108,6 +112,218 @@ func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T)
}
}
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
"profiles": map[string]any{
"base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"},
"child/profile": map[string]any{"extends": "base/profile"},
"overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"},
},
"prompts": map[string]any{
"shared/prompt": map[string]any{
"v1": listingContractPrompt(),
"v2": listingContractPrompt(),
},
"underlay-only": map[string]any{"v1": listingContractPrompt()},
},
})}
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
"profiles": map[string]any{
"overlay/profile": map[string]any{"model": "effective-model"},
},
"prompts": map[string]any{
"shared/prompt": map[string]any{"v1": listingContractPrompt()},
"main-only": map[string]any{"v3": listingContractPrompt()},
},
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}},
Main: &main,
}})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
for _, id := range []string{
"base/profile", "child/profile", "overlay/profile",
"promptkit-openrouter-standard", "promptkit-openrouter-auto",
} {
if !containsListingProfile(profiles, id) {
t.Errorf("profile listing %#v does not include %q", profiles, id)
}
}
for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} {
if got := countListingProfile(profiles, id); got != 1 {
t.Errorf("profile %q appears %d times, want 1", id, got)
}
}
overlay, err := engine.InspectProfile(context.Background(), "overlay/profile")
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
}
child, err := engine.InspectProfile(context.Background(), "child/profile")
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []promptkit.PromptIdentity{
{ID: "main-only", Version: "v3"},
{ID: "shared/prompt", Version: "v1"},
{ID: "shared/prompt", Version: "v2"},
{ID: "underlay-only", Version: "v1"},
}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
}
if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 {
t.Fatalf("overlaid prompt identity appears %d times, want 1", got)
}
}
func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) {
main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{
"profiles": nil,
"prompts": nil,
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil || len(profiles) != 0 {
t.Fatalf("cleared profile listing = %#v, %v", profiles, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("cleared prompt listing = %#v, %v", prompts, err)
}
}
func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
engine, err := newListingContractEngine(t, map[string]any{
"profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}},
"prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}},
})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
for i := range profiles {
if profiles[i] == "profile" {
profiles[i] = "caller mutation"
}
}
for i := range prompts {
if prompts[i].ID == "prompt" {
prompts[i].ID = "caller mutation"
prompts[i].Version = "caller mutation"
}
}
profiles, err = engine.ListProfiles(context.Background())
if err != nil || !containsListingProfile(profiles, "profile") {
t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err)
}
prompts, err = engine.ListPrompts(context.Background())
if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 {
t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err)
}
const workers = 4
const iterations = 25
start := make(chan struct{})
results := make(chan error, workers)
var wait sync.WaitGroup
wait.Add(workers)
for worker := 0; worker < workers; worker++ {
go func() {
defer wait.Done()
<-start
for iteration := 0; iteration < iterations; iteration++ {
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
results <- fmt.Errorf("list profiles: %w", err)
return
}
if len(profiles) == 0 {
results <- errors.New("profile listing is empty")
return
}
profiles[0] = "worker mutation"
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
results <- fmt.Errorf("list prompts: %w", err)
return
}
if len(prompts) == 0 {
results <- errors.New("prompt listing is empty")
return
}
prompts[0].ID = "worker mutation"
prompts[0].Version = "worker mutation"
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
results <- fmt.Errorf("inspect profile: %w", err)
return
}
if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil {
results <- fmt.Errorf("inspect prompt: %w", err)
return
}
}
}()
}
close(start)
wait.Wait()
close(results)
for err := range results {
t.Error(err)
}
}
func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
files := fstest.MapFS{
"config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")},
}
inputs, err := promptkit.FilesystemInputs("application listing", files, "config")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}},
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
if err != nil {
t.Fatal(err)
}
files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n")
profiles, err := engine.ListProfiles(context.Background())
if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") {
t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 {
t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err)
}
}
func listingContractPrompt() map[string]any {
return map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
@@ -123,3 +339,41 @@ func containsListingProfile(profiles []string, id string) bool {
}
return false
}
func countListingProfile(profiles []string, id string) int {
count := 0
for _, profile := range profiles {
if profile == id {
count++
}
}
return count
}
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
count := 0
for _, prompt := range prompts {
if prompt.ID == id && prompt.Version == version {
count++
}
}
return count
}
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
t.Helper()
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
}
type rejectingListingLLMClient struct{}
func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, errors.New("unexpected model generation")
}
type rejectingListingArtifactReader struct{}
func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
return nil, errors.New("unexpected artifact read")
}