Add scoped identity listings and backend inventory
This commit is contained in:
@@ -60,6 +60,185 @@ func TestListIdentitiesBasicContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedIdentityListingsApplyNamespaceFilters(t *testing.T) {
|
||||
engine := newNamespaceListingEngine(t)
|
||||
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||
Prompt: "team", Profile: "team", Backend: "team",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := promptkit.NamespaceFilter{Mode: promptkit.NamespaceAll}
|
||||
named := promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team-blue"}
|
||||
unqualified := promptkit.NamespaceFilter{Mode: promptkit.NamespaceUnqualified}
|
||||
defaultOverride := promptkit.NamespaceFilter{Mode: promptkit.NamespaceDefault, Name: "ignored"}
|
||||
absent := promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "missing"}
|
||||
|
||||
type listing struct {
|
||||
name string
|
||||
list func(*promptkit.Engine, ...promptkit.NamespaceFilter) ([]string, error)
|
||||
all []string
|
||||
team []string
|
||||
blue []string
|
||||
root []string
|
||||
}
|
||||
listings := []listing{
|
||||
{name: "profiles", list: func(e *promptkit.Engine, f ...promptkit.NamespaceFilter) ([]string, error) {
|
||||
return e.ListProfiles(context.Background(), f...)
|
||||
}, all: []string{"root-profile", "team-blue/profile", "team/profile"}, team: []string{"team/profile"}, blue: []string{"team-blue/profile"}, root: []string{"root-profile"}},
|
||||
{name: "prompts", list: func(e *promptkit.Engine, f ...promptkit.NamespaceFilter) ([]string, error) {
|
||||
prompts, err := e.ListPrompts(context.Background(), f...)
|
||||
ids := make([]string, len(prompts))
|
||||
for index := range prompts {
|
||||
ids[index] = prompts[index].ID
|
||||
}
|
||||
return ids, err
|
||||
}, all: []string{"root-prompt", "team-blue/prompt", "team/prompt"}, team: []string{"team/prompt"}, blue: []string{"team-blue/prompt"}, root: []string{"root-prompt"}},
|
||||
{name: "backends", list: func(e *promptkit.Engine, f ...promptkit.NamespaceFilter) ([]string, error) {
|
||||
return e.ListBackends(context.Background(), f...)
|
||||
}, all: []string{"root-backend", "team-blue/backend", "team/backend"}, team: []string{"team/backend"}, blue: []string{"team-blue/backend"}, root: []string{"root-backend"}},
|
||||
}
|
||||
for _, listing := range listings {
|
||||
t.Run(listing.name, func(t *testing.T) {
|
||||
for _, check := range []struct {
|
||||
name string
|
||||
engine *promptkit.Engine
|
||||
filter []promptkit.NamespaceFilter
|
||||
want []string
|
||||
}{
|
||||
{name: "scoped default", engine: scoped, want: listing.team},
|
||||
{name: "explicit default override", engine: scoped, filter: []promptkit.NamespaceFilter{defaultOverride}, want: listing.team},
|
||||
{name: "named override", engine: scoped, filter: []promptkit.NamespaceFilter{named}, want: listing.blue},
|
||||
{name: "unqualified override", engine: scoped, filter: []promptkit.NamespaceFilter{unqualified}, want: listing.root},
|
||||
{name: "all override", engine: scoped, filter: []promptkit.NamespaceFilter{all}, want: listing.all},
|
||||
{name: "absent namespace", engine: scoped, filter: []promptkit.NamespaceFilter{absent}, want: []string{}},
|
||||
{name: "unscoped default", engine: engine, want: listing.all},
|
||||
} {
|
||||
got, err := listing.list(check.engine, check.filter...)
|
||||
if err != nil || (check.name == "absent namespace" && len(got) != 0) ||
|
||||
(check.name != "absent namespace" && !reflect.DeepEqual(got, check.want)) {
|
||||
t.Errorf("%s = %#v, %v; want %#v", check.name, got, err, check.want)
|
||||
}
|
||||
}
|
||||
for _, invalid := range [][]promptkit.NamespaceFilter{
|
||||
{{Mode: promptkit.NamespaceNamed}},
|
||||
{{Mode: promptkit.NamespaceNamed, Name: "team/child"}},
|
||||
{{Mode: promptkit.NamespaceFilterMode(99)}},
|
||||
{{Mode: promptkit.NamespaceAll}, {Mode: promptkit.NamespaceUnqualified}},
|
||||
} {
|
||||
got, err := listing.list(scoped, invalid...)
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) || len(got) != 0 {
|
||||
t.Errorf("invalid filter %#v = %#v, %v; want ErrInvalidRequest without results", invalid, got, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
prompts, err := scoped.ListPrompts(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(prompts) != 1 || prompts[0].ID != "team/prompt" || prompts[0].Version == nil || *prompts[0].Version != "team-v1" {
|
||||
t.Fatalf("named prompt identity = %#v, %v", prompts, err)
|
||||
}
|
||||
if _, err := scoped.InspectPrompt(context.Background(), prompts[0].ID); err != nil {
|
||||
t.Fatalf("listed full prompt ID did not round trip: %v", err)
|
||||
}
|
||||
*prompts[0].Version = "caller change"
|
||||
prompts[0].ID = "caller change"
|
||||
prompts, err = scoped.ListPrompts(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(prompts) != 1 || prompts[0].ID != "team/prompt" || prompts[0].Version == nil || *prompts[0].Version != "team-v1" {
|
||||
t.Fatalf("mutating filtered prompt inventory escaped: %#v, %v", prompts, err)
|
||||
}
|
||||
profiles, err := scoped.ListProfiles(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(profiles) != 1 || profiles[0] != "team/profile" {
|
||||
t.Fatalf("named profile inventory = %#v, %v", profiles, err)
|
||||
}
|
||||
if _, err := scoped.InspectProfile(context.Background(), profiles[0]); err != nil {
|
||||
t.Fatalf("listed full profile ID did not round trip: %v", err)
|
||||
}
|
||||
profiles[0] = "caller change"
|
||||
profiles, err = scoped.ListProfiles(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(profiles) != 1 || profiles[0] != "team/profile" {
|
||||
t.Fatalf("mutating filtered profile inventory escaped: %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := scoped.ListBackends(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(backends) != 1 || backends[0] != "team/backend" {
|
||||
t.Fatalf("named backend inventory = %#v, %v", backends, err)
|
||||
}
|
||||
if report, err := scoped.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
|
||||
Kind: promptkit.SmokeTestReachability, BackendID: backends[0],
|
||||
}); err != nil || report.Target.BackendID != backends[0] {
|
||||
t.Fatalf("listed full backend ID did not round trip: %+v, %v", report, err)
|
||||
}
|
||||
backends[0] = "caller change"
|
||||
backends, err = scoped.ListBackends(context.Background(), promptkit.NamespaceFilter{Mode: promptkit.NamespaceNamed, Name: "team"})
|
||||
if err != nil || len(backends) != 1 || backends[0] != "team/backend" {
|
||||
t.Fatalf("mutating filtered backend inventory escaped: %#v, %v", backends, err)
|
||||
}
|
||||
rootPrompts, err := engine.ListPrompts(context.Background(), unqualified)
|
||||
if err != nil || len(rootPrompts) != 1 || rootPrompts[0].ID != "root-prompt" {
|
||||
t.Fatalf("unqualified prompt listing = %#v, %v", rootPrompts, err)
|
||||
}
|
||||
if _, err := engine.InspectPrompt(context.Background(), rootPrompts[0].ID); err != nil {
|
||||
t.Fatalf("unqualified prompt did not select through original engine: %v", err)
|
||||
}
|
||||
rootProfiles, err := engine.ListProfiles(context.Background(), unqualified)
|
||||
if err != nil || len(rootProfiles) != 1 || rootProfiles[0] != "root-profile" {
|
||||
t.Fatalf("unqualified profile listing = %#v, %v", rootProfiles, err)
|
||||
}
|
||||
if _, err := engine.InspectProfile(context.Background(), rootProfiles[0]); err != nil {
|
||||
t.Fatalf("unqualified profile did not select through original engine: %v", err)
|
||||
}
|
||||
rootBackends, err := engine.ListBackends(context.Background(), unqualified)
|
||||
if err != nil || len(rootBackends) != 1 || rootBackends[0] != "root-backend" {
|
||||
t.Fatalf("unqualified backend listing = %#v, %v", rootBackends, err)
|
||||
}
|
||||
if report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
|
||||
Kind: promptkit.SmokeTestReachability, BackendID: rootBackends[0],
|
||||
}); err != nil || report.Target.BackendID != rootBackends[0] {
|
||||
t.Fatalf("unqualified backend did not select through original engine: %+v, %v", report, err)
|
||||
}
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if got, err := scoped.ListProfiles(canceled); !errors.Is(err, context.Canceled) || len(got) != 0 {
|
||||
t.Errorf("canceled profile list = %#v, %v", got, err)
|
||||
}
|
||||
if got, err := scoped.ListPrompts(canceled); !errors.Is(err, context.Canceled) || len(got) != 0 {
|
||||
t.Errorf("canceled prompt list = %#v, %v", got, err)
|
||||
}
|
||||
if got, err := scoped.ListBackends(canceled); !errors.Is(err, context.Canceled) || len(got) != 0 {
|
||||
t.Errorf("canceled backend list = %#v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func newNamespaceListingEngine(t *testing.T) *promptkit.Engine {
|
||||
t.Helper()
|
||||
collection := func(namespace string) map[string]any {
|
||||
return map[string]any{
|
||||
"backends": map[string]any{"backend": map[string]any{"endpoint": "https://example.test/v1"}},
|
||||
"profiles": map[string]any{"profile": map[string]any{"backend": namespace + "/backend", "model": "example-model"}},
|
||||
"prompts": map[string]any{"prompt": promptDefinitionWithVersion("team-v1", listingContractPrompt())},
|
||||
}
|
||||
}
|
||||
input := configkit.Input{Source: configkit.Mapping("namespaced listing", map[string]any{
|
||||
"collections": map[string]any{
|
||||
"team": collection("team"),
|
||||
"team-blue": collection("team-blue"),
|
||||
},
|
||||
"backends": map[string]any{"root-backend": map[string]any{"endpoint": "https://root.example.test/v1"}},
|
||||
"profiles": map[string]any{"root-profile": map[string]any{"backend": "root-backend", "model": "root-model"}},
|
||||
"prompts": map[string]any{"root-prompt": promptDefinitionWithVersion("root-v1", listingContractPrompt())},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if err != nil {
|
||||
var loadErr *configkit.LoadError
|
||||
if errors.As(err, &loadErr) {
|
||||
t.Fatalf("construct namespaced listing engine: %v: %+v", err, loadErr.Diagnostics())
|
||||
}
|
||||
t.Fatalf("construct namespaced listing engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestListIdentitiesInvalidEngine(t *testing.T) {
|
||||
var nilEngine *promptkit.Engine
|
||||
var zeroEngine promptkit.Engine
|
||||
@@ -70,6 +249,9 @@ func TestListIdentitiesInvalidEngine(t *testing.T) {
|
||||
if _, err := engine.ListPrompts(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListPrompts on invalid engine: %v", err)
|
||||
}
|
||||
if _, err := engine.ListBackends(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListBackends on invalid engine: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user