218 lines
9.1 KiB
Go
218 lines
9.1 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestWithNamespacesRoutesOrdinarySelectionsAndPreservesViews(t *testing.T) {
|
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
|
engine := newNamespacedContractEngine(t, client)
|
|
alpha, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
beta, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "beta", Profile: "beta"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared"}
|
|
prepared, err := alpha.Prepare(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("prepare scoped request: %v", err)
|
|
}
|
|
assertNamespacedSelection(t, prepared.PromptID, prepared.SelectedProfileID, prepared.SelectedBackendID, "alpha")
|
|
|
|
betaPrepared, err := beta.Prepare(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("prepare independent view: %v", err)
|
|
}
|
|
assertNamespacedSelection(t, betaPrepared.PromptID, betaPrepared.SelectedProfileID, betaPrepared.SelectedBackendID, "beta")
|
|
if err := alpha.ValidateRequest(context.Background(), request); err != nil {
|
|
t.Fatalf("validate scoped request: %v", err)
|
|
}
|
|
|
|
qualified, err := alpha.Prepare(context.Background(), promptkit.RunRequest{PromptID: "beta/shared", ProfileID: "beta/shared"})
|
|
if err != nil {
|
|
t.Fatalf("prepare qualified bypass: %v", err)
|
|
}
|
|
assertNamespacedSelection(t, qualified.PromptID, qualified.SelectedProfileID, qualified.SelectedBackendID, "beta")
|
|
|
|
preparedExecution, err := alpha.PrepareExecution(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
details := preparedExecution.Details()
|
|
assertNamespacedSelection(t, details.PromptID, details.SelectedProfileID, details.SelectedBackendID, "alpha")
|
|
// A handle captures its qualified selection and remains runnable through the
|
|
// original facade because the views share the same assembled runner.
|
|
result, err := engine.RunPrepared(context.Background(), preparedExecution)
|
|
if err != nil {
|
|
t.Fatalf("run scoped prepared handle through original view: %v", err)
|
|
}
|
|
assertNamespacedSelection(t, result.PromptID, result.SelectedProfileID, result.SelectedBackendID, "alpha")
|
|
|
|
var events []promptkit.ExecutionEvent
|
|
runResult, err := alpha.Run(context.Background(), promptkit.RunRequest{
|
|
PromptID: "shared", ProfileID: "shared",
|
|
Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) },
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("run scoped request: %v", err)
|
|
}
|
|
assertNamespacedSelection(t, runResult.PromptID, runResult.SelectedProfileID, runResult.SelectedBackendID, "alpha")
|
|
var sawScopedEvent bool
|
|
for _, event := range events {
|
|
if event.PromptID == "alpha/shared" && event.ProfileID == "alpha/shared" && event.BackendID == "alpha/backend" {
|
|
sawScopedEvent = true
|
|
}
|
|
}
|
|
if !sawScopedEvent {
|
|
t.Fatalf("observer events did not retain qualified identities: %+v", events)
|
|
}
|
|
|
|
// The original engine remains unscoped and can still select the exact IDs.
|
|
original, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "root-only", ProfileID: "root-only"})
|
|
if err != nil {
|
|
t.Fatalf("original engine was changed by scoped views: %v", err)
|
|
}
|
|
if original.PromptID != "root-only" || original.SelectedProfileID != "root-only" || original.EffectiveModelParams.Model != "root-model" {
|
|
t.Fatalf("original engine selection = %+v, want unqualified root resources", original)
|
|
}
|
|
}
|
|
|
|
func TestWithNamespacesHasNoFallbackAndValidatesAtTheCorrectBoundary(t *testing.T) {
|
|
engine := newNamespacedContractEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}})
|
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := scoped.Prepare(context.Background(), promptkit.RunRequest{PromptID: "root-only", ProfileID: "alpha/shared"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("root prompt fallback error = %v, want ErrPromptNotFound", err)
|
|
}
|
|
if _, err := scoped.Prepare(context.Background(), promptkit.RunRequest{PromptID: "alpha/shared", ProfileID: "root-only"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
|
t.Fatalf("root profile fallback error = %v, want ErrProfileNotFound", err)
|
|
}
|
|
unknown, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "not-configured"})
|
|
if err != nil {
|
|
t.Fatalf("well-formed absent namespace rejected at view creation: %v", err)
|
|
}
|
|
if _, err := unknown.Prepare(context.Background(), promptkit.RunRequest{PromptID: "root-only", ProfileID: "alpha/shared"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("selection from absent namespace error = %v, want ErrPromptNotFound", err)
|
|
}
|
|
if _, err := scoped.Prepare(context.Background(), promptkit.RunRequest{PromptID: "alpha/beta/gamma", ProfileID: "alpha/shared"}); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
t.Fatalf("malformed caller prompt ID error = %v, want ErrInvalidRequest", err)
|
|
}
|
|
for _, invalid := range []promptkit.Namespaces{
|
|
{Prompt: "alpha/beta"},
|
|
{Profile: "/alpha"},
|
|
{Backend: " "},
|
|
} {
|
|
if _, err := engine.WithNamespaces(invalid); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
t.Errorf("invalid namespaces %+v error = %v, want ErrInvalidConfig", invalid, err)
|
|
}
|
|
}
|
|
if _, err := (*promptkit.Engine)(nil).WithNamespaces(promptkit.Namespaces{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("nil engine error = %v, want ErrInvalidConfig", err)
|
|
}
|
|
cleared, err := scoped.WithNamespaces(promptkit.Namespaces{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := cleared.Prepare(context.Background(), promptkit.RunRequest{PromptID: "shared", ProfileID: "alpha/shared"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("cleared view unexpectedly retained prior scope: %v", err)
|
|
}
|
|
if _, err := scoped.Prepare(context.Background(), promptkit.RunRequest{PromptID: "shared", ProfileID: "shared"}); err != nil {
|
|
t.Fatalf("replacing scopes mutated the prior view: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestWithNamespacesViewsCanPrepareConcurrently(t *testing.T) {
|
|
engine := newNamespacedContractEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}})
|
|
alpha, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
beta, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "beta", Profile: "beta"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wait sync.WaitGroup
|
|
for _, view := range []*promptkit.Engine{alpha, beta} {
|
|
view := view
|
|
wait.Add(1)
|
|
go func() {
|
|
defer wait.Done()
|
|
prepared, err := view.Prepare(context.Background(), promptkit.RunRequest{PromptID: "shared", ProfileID: "shared"})
|
|
if err != nil {
|
|
t.Errorf("concurrent prepare: %v", err)
|
|
return
|
|
}
|
|
want := "alpha"
|
|
if view == beta {
|
|
want = "beta"
|
|
}
|
|
assertNamespacedSelection(t, prepared.PromptID, prepared.SelectedProfileID, prepared.SelectedBackendID, want)
|
|
}()
|
|
}
|
|
wait.Wait()
|
|
}
|
|
|
|
func newNamespacedContractEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
|
t.Helper()
|
|
var inputs []configkit.Input
|
|
for _, namespace := range []string{"alpha", "beta"} {
|
|
endpoint := "https://" + namespace + ".example/v1"
|
|
input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{
|
|
"backends": map[string]any{"backend": map[string]any{"endpoint": endpoint}},
|
|
"profiles": map[string]any{"shared": map[string]any{
|
|
"backend": namespace + "/backend", "model": namespace + "-model",
|
|
}},
|
|
"prompts": map[string]any{"shared": map[string]any{
|
|
"version": "1",
|
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
|
}},
|
|
}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
input.Imports = false
|
|
inputs = append(inputs, input)
|
|
}
|
|
inputs = append(inputs, configkit.Input{Source: configkit.Mapping("root resources", map[string]any{
|
|
"profiles": map[string]any{"root-only": map[string]any{
|
|
"endpoint": "https://root.example/v1", "model": "root-model",
|
|
}},
|
|
"prompts": map[string]any{"root-only": map[string]any{
|
|
"version": "1",
|
|
"messages": []any{map[string]any{"role": "user", "content": "Root prompt"}},
|
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
|
}},
|
|
})})
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
|
}}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
var loadErr *configkit.LoadError
|
|
if errors.As(err, &loadErr) {
|
|
t.Fatalf("construct namespaced engine: %v: %+v", err, loadErr.Diagnostics())
|
|
}
|
|
t.Fatalf("construct namespaced engine: %v", err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func assertNamespacedSelection(t *testing.T, promptID, profileID, backendID, namespace string) {
|
|
t.Helper()
|
|
if promptID != namespace+"/shared" || profileID != namespace+"/shared" || backendID != namespace+"/backend" {
|
|
t.Fatalf("selected IDs = (%q, %q, %q), want namespace %q", promptID, profileID, backendID, namespace)
|
|
}
|
|
}
|