Add application fallback profile sources

2026-08-01 12:37:01 +00:00
parent 01ca5430bd
commit 9354d2b373
7 changed files with 341 additions and 52 deletions
--- a/public_contract_test.go
+++ b/public_contract_test.go
@@ -973,6 +973,24 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
 		}
 	})

+	t.Run("fallback profile source", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+		if err != nil {
+			t.Fatalf("prepare from last fallback profile source: %v", err)
+		}
+		if prepared.EffectiveModelParams.Model != "second-model" {
+			t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model)
+		}
+	})
+
 	t.Run("in-memory profiles", func(t *testing.T) {
 		first := profile
 		first.Model = "first-model"
@@ -1057,6 +1075,204 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
 	})
 }

+func TestFallbackProfileSourcePrecedence(t *testing.T) {
+	const profileID = "application-profile"
+
+	prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string {
+		t.Helper()
+		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID})
+		if err != nil {
+			t.Fatalf("prepare: %v", err)
+		}
+		return prepared.EffectiveModelParams.Model
+	}
+
+	t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+			promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
+			promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if model := prepareModel(t, engine, "prompt"); model != "memory-model" {
+			t.Fatalf("expected in-memory profile, got %q", model)
+		}
+	})
+
+	t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+			promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" {
+			t.Fatalf("expected ordinary profile, got %q", model)
+		}
+	})
+
+	t.Run("configured directory overrides fallback profile", func(t *testing.T) {
+		profileDir := t.TempDir()
+		writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
+		engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
+			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if model := prepareModel(t, engine, "prompt"); model != "directory-model" {
+			t.Fatalf("expected configured directory profile, got %q", model)
+		}
+	})
+
+	t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if model := prepareModel(t, engine, "prompt"); model != "fallback-model" {
+			t.Fatalf("expected fallback profile, got %q", model)
+		}
+	})
+
+	t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
+		t.Setenv("OPENROUTER_API_KEY", "test-key")
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if model := prepareModel(t, engine, "prompt"); model != "mistralai/mistral-small-3.2-24b-instruct" {
+			t.Fatalf("expected built-in profile, got %q", model)
+		}
+	})
+}
+
+func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
+	const profileID = "application-profile"
+
+	t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
+		_, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(fstest.MapFS{}, "."),
+			promptkit.WithFallbackProfileFS(fstest.MapFS{
+				"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
+			}, "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine with malformed fallback profile: %v", err)
+		}
+	})
+
+	t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+			promptkit.WithFallbackProfileFS(fstest.MapFS{
+				"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
+				"valid.yaml":  &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
+			}, "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+		if err != nil {
+			t.Fatalf("prepare from valid fallback profile: %v", err)
+		}
+		if prepared.EffectiveModelParams.Model != "fallback-model" {
+			t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
+		}
+	})
+
+	t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
+			promptkit.WithFallbackProfileFS(fstest.MapFS{
+				"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
+			}, "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
+			t.Fatalf("expected ErrProfileLoad, got %v", err)
+		}
+	})
+
+	t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
+		engine, err := promptkit.NewEngine(promptkit.Config{},
+			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+			promptkit.WithProfileFS(fstest.MapFS{
+				"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
+			}, "."),
+			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+		)
+		if err != nil {
+			t.Fatalf("construct engine: %v", err)
+		}
+		if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
+			t.Fatalf("expected ErrProfileLoad, got %v", err)
+		}
+	})
+}
+
+func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
+	const profileID = "application-profile"
+	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
+	engine, err := promptkit.NewEngine(promptkit.Config{},
+		promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
+		promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
+		promptkit.WithLLMClient(client),
+	)
+	if err != nil {
+		t.Fatalf("construct engine: %v", err)
+	}
+
+	inspection, err := engine.InspectProfile(context.Background(), profileID)
+	if err != nil {
+		t.Fatalf("inspect fallback profile: %v", err)
+	}
+	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+	if err != nil {
+		t.Fatalf("prepare fallback profile: %v", err)
+	}
+	preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+	if err != nil {
+		t.Fatalf("prepare execution with fallback profile: %v", err)
+	}
+	preparedDetails := preparedExecution.Details()
+	preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution)
+	if err != nil {
+		t.Fatalf("run prepared fallback profile: %v", err)
+	}
+	runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+	if err != nil {
+		t.Fatalf("run fallback profile: %v", err)
+	}
+
+	for name, model := range map[string]string{
+		"inspection":         inspection.EffectiveModelParams.Model,
+		"preparation":        prepared.EffectiveModelParams.Model,
+		"prepared execution": preparedDetails.EffectiveModelParams.Model,
+		"prepared result":    preparedResult.EffectiveModelParams.Model,
+		"run result":         runResult.EffectiveModelParams.Model,
+	} {
+		if model != "fallback-model" {
+			t.Fatalf("%s model=%q, want fallback-model", name, model)
+		}
+	}
+}
+
 func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
 	engine, err := promptkit.NewEngine(promptkit.Config{},
 		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),