Expose profile inspection through the engine
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -16,6 +17,163 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type inspectionCountingFS struct {
|
||||
opens atomic.Int64
|
||||
}
|
||||
|
||||
func (f *inspectionCountingFS) Open(string) (fs.File, error) {
|
||||
f.opens.Add(1)
|
||||
return nil, fs.ErrNotExist
|
||||
}
|
||||
|
||||
func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
|
||||
const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
|
||||
t.Setenv(environmentName, "")
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(fstest.MapFS{}, "."),
|
||||
promptkit.WithProfileFS(fstest.MapFS{
|
||||
"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
|
||||
endpoint: http://environment.example/v1
|
||||
model: environment-model
|
||||
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
|
||||
`)},
|
||||
}, "."),
|
||||
promptkit.WithProfiles(
|
||||
promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
|
||||
promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
|
||||
),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct inspection engine: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
profileID string
|
||||
wantEnv string
|
||||
wantDirectKey bool
|
||||
wantEndpoint string
|
||||
}{
|
||||
{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
|
||||
{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
|
||||
{profileID: "none", wantEndpoint: "http://none.example/v1"},
|
||||
} {
|
||||
t.Run(tc.profileID, func(t *testing.T) {
|
||||
inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect profile: %v", err)
|
||||
}
|
||||
if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
|
||||
inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
|
||||
inspection.EffectiveModelParams.BackendID != "" ||
|
||||
inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
|
||||
inspection.APIKeyRequired != tc.wantDirectKey {
|
||||
t.Fatalf("unexpected inspection: %#v", inspection)
|
||||
}
|
||||
})
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
|
||||
newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct inspection engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
var nilEngine *promptkit.Engine
|
||||
if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
|
||||
}
|
||||
|
||||
valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
|
||||
}))
|
||||
if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
|
||||
t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
|
||||
}
|
||||
|
||||
malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
|
||||
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")},
|
||||
}, "."))
|
||||
if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrProfileLoad) {
|
||||
t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
|
||||
}
|
||||
|
||||
unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "unknown-backend", BackendID: "unknown", Model: "model",
|
||||
}))
|
||||
if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrProfileLoad) {
|
||||
t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
|
||||
}
|
||||
|
||||
countingFS := &inspectionCountingFS{}
|
||||
canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
|
||||
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
|
||||
extraParams := map[string]any{
|
||||
"nested": map[string]any{"value": "original"},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct inspection engine: %v", err)
|
||||
}
|
||||
|
||||
first, err := engine.InspectProfile(context.Background(), "profile")
|
||||
if err != nil {
|
||||
t.Fatalf("first inspection: %v", err)
|
||||
}
|
||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
first.EffectiveModelParams.ExtraParams["later"] = true
|
||||
|
||||
second, err := engine.InspectProfile(context.Background(), "profile")
|
||||
if err != nil {
|
||||
t.Fatalf("second inspection: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare after inspection mutation: %v", err)
|
||||
}
|
||||
for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
|
||||
if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
|
||||
t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
|
||||
t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
||||
payload, err := json.Marshal(promptkit.PreparedRun{})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user