Remove superseded configuration loaders and consolidate runtime tests
This commit is contained in:
@@ -4,247 +4,169 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/testfixture"
|
||||
)
|
||||
|
||||
func aliasPrompt() map[string]any {
|
||||
return map[string]any{"prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{
|
||||
"default_profile": "child",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}}}}
|
||||
}
|
||||
|
||||
func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
|
||||
engine, err := testfixture.NewEngine(testfixture.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
}, testfixture.WithProfiles(testfixture.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "promptkit-openrouter-standard",
|
||||
ReasoningEffort: "high",
|
||||
TimeoutSeconds: 120,
|
||||
}))
|
||||
standardID, reasoning, timeout := "promptkit-openrouter-standard", "high", 120
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", map[string]promptkit.ProfileConfig{
|
||||
"child": {Extends: &standardID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct alias engine: %v", err)
|
||||
}
|
||||
|
||||
base, err := engine.InspectProfile(context.Background(), "promptkit-openrouter-standard")
|
||||
base, err := engine.InspectProfile(context.Background(), standardID)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect base: %v", err)
|
||||
}
|
||||
child, err := engine.InspectProfile(context.Background(), "weather-light")
|
||||
child, err := engine.InspectProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect alias: %v", err)
|
||||
}
|
||||
if child.ProfileID != "weather-light" ||
|
||||
child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
|
||||
if child.ProfileID != "child" || child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
|
||||
child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
|
||||
child.EffectiveModelParams.ReasoningEffort != "high" ||
|
||||
child.EffectiveModelParams.TimeoutSeconds != 120 {
|
||||
child.EffectiveModelParams.ReasoningEffort != reasoning || child.EffectiveModelParams.TimeoutSeconds != timeout {
|
||||
t.Fatalf("alias inspection = %+v, base = %+v", child, base)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: "weather-light",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare alias: %v", err)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil || prepared.SelectedProfileID != "child" {
|
||||
t.Fatalf("prepare alias = (%+v, %v)", prepared, err)
|
||||
}
|
||||
if prepared.SelectedProfileID != "weather-light" {
|
||||
t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID)
|
||||
}
|
||||
|
||||
timeout := 15
|
||||
reasoning := "low"
|
||||
overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: "weather-light",
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
TimeoutSeconds: &timeout,
|
||||
ReasoningEffort: &reasoning,
|
||||
},
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare override: %v", err)
|
||||
}
|
||||
if overridden.EffectiveModelParams.TimeoutSeconds != timeout ||
|
||||
overridden.EffectiveModelParams.ReasoningEffort != reasoning {
|
||||
t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams)
|
||||
shorter, lower := 15, "low"
|
||||
overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared", Execution: &promptkit.ExecutionTargetOverride{
|
||||
TimeoutSeconds: &shorter, ReasoningEffort: &lower,
|
||||
}})
|
||||
if err != nil || overridden.EffectiveModelParams.TimeoutSeconds != shorter || overridden.EffectiveModelParams.ReasoningEffort != lower {
|
||||
t.Fatalf("runtime override = (%+v, %v)", overridden, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
|
||||
engine, err := testfixture.NewEngine(testfixture.Config{},
|
||||
testfixture.WithPromptFS(fstest.MapFS{}, "."),
|
||||
testfixture.WithProfileFS(fstest.MapFS{
|
||||
"alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: promptkit-openrouter-standard\n")},
|
||||
}, "."),
|
||||
)
|
||||
inputs, err := promptkit.FilesystemInputs("profile alias", fstest.MapFS{
|
||||
"config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n yaml-alias:\n extends: promptkit-openrouter-standard\n")},
|
||||
}, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("construct YAML alias engine: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect YAML alias: %v", err)
|
||||
}
|
||||
if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
|
||||
t.Fatalf("YAML alias inspection = %+v", inspection)
|
||||
if err != nil || inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
|
||||
t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
|
||||
engine, err := testfixture.NewEngine(testfixture.Config{},
|
||||
testfixture.WithPromptFS(fstest.MapFS{}, "."),
|
||||
testfixture.WithBackend(testfixture.Backend{
|
||||
ID: "credential-backend",
|
||||
Endpoint: "https://credential.example/v1",
|
||||
APIKeyEnv: "OPTIONAL_BACKEND_KEY",
|
||||
}),
|
||||
testfixture.WithProfiles(
|
||||
testfixture.Profile{
|
||||
ID: "credential-base",
|
||||
BackendID: "credential-backend",
|
||||
Model: "model",
|
||||
APIKeyRequired: true,
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("credentials", map[string]any{
|
||||
"backends": map[string]any{"credential-backend": map[string]any{
|
||||
"endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY",
|
||||
}},
|
||||
"profiles": map[string]any{
|
||||
"credential-base": map[string]any{"backend": "credential-backend", "model": "model", "api_key_required": true},
|
||||
"credential-child": map[string]any{"extends": "credential-base"},
|
||||
},
|
||||
testfixture.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
|
||||
),
|
||||
)
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct credential inheritance engine: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "credential-child")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect credential child: %v", err)
|
||||
}
|
||||
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
||||
t.Fatalf("credential inspection = %+v", inspection)
|
||||
if err != nil || !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
||||
t.Fatalf("credential inspection = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
|
||||
newEngine := func(t *testing.T, profiles fs.FS) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
options := []any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}
|
||||
if profiles != nil {
|
||||
options = append(options, testfixture.WithProfileFS(profiles, "."))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection, err := engine.InspectProfile(context.Background(), "missing"); inspection != nil || !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile = (%+v, %v)", inspection, err)
|
||||
}
|
||||
for _, document := range []map[string]any{
|
||||
{"profiles": map[string]any{"child": map[string]any{"extends": "missing"}}},
|
||||
{"profiles": map[string]any{"a": map[string]any{"extends": "b"}, "b": map[string]any{"extends": "a"}}},
|
||||
} {
|
||||
invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid inheritance", document),
|
||||
}}})
|
||||
if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err)
|
||||
}
|
||||
return testfixture.NewEngine(testfixture.Config{}, options...)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
profiles fs.FS
|
||||
profile string
|
||||
contains []string
|
||||
want error
|
||||
wantNot error
|
||||
}{
|
||||
{
|
||||
name: "missing selected profile",
|
||||
profile: "missing",
|
||||
want: promptkit.ErrProfileNotFound,
|
||||
wantNot: promptkit.ErrProfileLoad,
|
||||
},
|
||||
{
|
||||
name: "missing base",
|
||||
profiles: fstest.MapFS{
|
||||
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")},
|
||||
},
|
||||
profile: "child",
|
||||
contains: []string{"child", "missing"},
|
||||
want: promptkit.ErrProfileLoad,
|
||||
wantNot: promptkit.ErrProfileNotFound,
|
||||
},
|
||||
{
|
||||
name: "cycle",
|
||||
profiles: fstest.MapFS{
|
||||
"a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")},
|
||||
"b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")},
|
||||
},
|
||||
profile: "a",
|
||||
contains: []string{"a", "b"},
|
||||
want: promptkit.ErrProfileLoad,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine, err := newEngine(t, tc.profiles)
|
||||
if tc.profiles != nil {
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid configuration", engine, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
result, err := engine.InspectProfile(context.Background(), tc.profile)
|
||||
if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) {
|
||||
t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot)
|
||||
}
|
||||
for _, fragment := range tc.contains {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %v, want %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
|
||||
profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{
|
||||
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")},
|
||||
"base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")},
|
||||
"config/profiles.yaml": &fstest.MapFile{Data: []byte("profiles:\n base:\n endpoint: https://base.example/v1\n model: first-model\n child:\n extends: base\n")},
|
||||
}}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine, err := testfixture.NewEngine(testfixture.Config{},
|
||||
testfixture.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
|
||||
testfixture.WithProfileFS(profiles, "."),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
inputs, err := promptkit.FilesystemInputs("profiles", profiles, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &recordingProfileClient{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n")
|
||||
|
||||
profiles.set("config/profiles.yaml", "profiles:\n base:\n endpoint: https://base.example/v1\n model: second-model\n child:\n extends: base\n")
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" {
|
||||
t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests)
|
||||
if err != nil || result == nil || client.model != "first-model" {
|
||||
t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "child")
|
||||
if err != nil || inspection.EffectiveModelParams.Model != "first-model" {
|
||||
t.Fatalf("existing engine inspection=(%+v, %v)", inspection, err)
|
||||
t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingProfileClient struct{ model string }
|
||||
|
||||
func (client *recordingProfileClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.model = request.Target.Model
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, ctx.Err()
|
||||
}
|
||||
|
||||
type mutableInheritanceProfileFS struct {
|
||||
mu sync.RWMutex
|
||||
files fstest.MapFS
|
||||
}
|
||||
|
||||
func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
return f.files.Open(name)
|
||||
func (source *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
|
||||
source.mu.RLock()
|
||||
defer source.mu.RUnlock()
|
||||
return source.files.Open(name)
|
||||
}
|
||||
|
||||
func (f *mutableInheritanceProfileFS) set(name, content string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.files[name] = &fstest.MapFile{Data: []byte(content)}
|
||||
func (source *mutableInheritanceProfileFS) set(name, content string) {
|
||||
source.mu.Lock()
|
||||
defer source.mu.Unlock()
|
||||
source.files[name] = &fstest.MapFile{Data: []byte(content)}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user