Add fallback PromptKit profile assets
This commit is contained in:
@@ -431,6 +431,84 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
||||
assets := newTestPromptKitAssets(t)
|
||||
const profileID = "fallback-profile"
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
ProfileID: profileID,
|
||||
SessionID: "fallback-profile-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||
}
|
||||
if response.ProfileID != profileID || response.Model != "fallback-model" {
|
||||
t.Fatalf("completion response = %#v, want fallback profile", response)
|
||||
}
|
||||
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: assets})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v, want nil", err)
|
||||
}
|
||||
inspection, err := inspector.InspectProfile(context.Background(), profileID)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v, want nil", err)
|
||||
}
|
||||
if inspection.ProfileID != profileID || inspection.Model != "fallback-model" {
|
||||
t.Fatalf("profile inspection = %#v, want fallback profile", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
assets := newTestPromptKitAssets(t)
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte(content)},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: assets})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
|
||||
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
|
||||
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
|
||||
if first == second {
|
||||
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
|
||||
}
|
||||
if strings.Contains(first.Value, "model-one") || strings.Contains(first.Value, "fallback.yaml") {
|
||||
t.Fatalf("checkpoint fingerprint leaked fallback source details: %#v", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
||||
const (
|
||||
firstEndpoint = "http://localhost:8000/v1"
|
||||
|
||||
Reference in New Issue
Block a user