From 75a3f51ceee52d0f743a8663daa4fa4415b70718 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 25 Aug 2026 19:31:00 +0000 Subject: [PATCH] Support inherited PromptKit profiles --- docs/config.md | 6 +- docs/integrations/pkg-promptkit.md | 6 ++ docs/internal/llm.md | 2 +- docs/operations.md | 3 + docs/roadmap/implementation.md | 2 +- internal/cli/promptkit_profiles_test.go | 22 +++++ .../framework/llm/promptkit_client_test.go | 98 +++++++++++++++++++ 7 files changed, 136 insertions(+), 3 deletions(-) diff --git a/docs/config.md b/docs/config.md index 7eb49b36..7e8b2068 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,7 +132,11 @@ model: example-model Keep credentials out of the local-backend object. A PromptKit profile may name its credential environment variable through `api_key_env`; set that variable only in the run environment. PromptKit owns the -[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md). +[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md), +including `base_profile` inheritance. Notarius passes profiles through without +merging them. Filesystem profiles cannot express PromptKit's in-memory +`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the +provider without authorization. The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the supported package API, and [Operations](operations.md#operational-limits) describes the effective concurrency layers. diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index e0a0971f..48c9ba46 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -84,6 +84,12 @@ configuration and deployment workflow are defined in [Configuration](../config.md#promptkit-profiles) and [Operations](../operations.md#promptkit-profile-deployment). +PromptKit owns `base_profile` resolution under its +[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md). +Notarius records the selected leaf identity and resolved target without parsing +or merging inheritance. An unset filesystem `api_key_env` is optional and may +reach the provider without authorization, which can result in a 401 or 403. + Notarius supports this boundary against PromptKit v0.8.0. Its fallback source, prepared-execution, inspection, and typed capacity APIs are used as public upstream contracts; other PromptKit APIs or file-format behavior are not diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 410ac381..58bc000f 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -75,7 +75,7 @@ backend membership as runtime without performing generation. Fallback assets are mounted only when at least one source is registered. The production D&D registrar contributes its `dnd-extraction` fallback, and the maintained D&D prompts select that logical ID by default. PromptKit owns source precedence and -profile parsing: an operator-provided matching profile takes precedence over a +profile parsing and inheritance: an operator-provided matching profile takes precedence over a fallback profile without Notarius merging either document. When the registration is absent, a profile selecting `backend: local` fails inspection instead of falling back to a built-in or endpoint-only target. diff --git a/docs/operations.md b/docs/operations.md index ff98e28d..54e0c83b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -89,6 +89,9 @@ provider call or credentials: notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session ~~~ +An unset optional `api_key_env` reaches the provider without authorization and +may receive a 401 or 403 response. + Profile paths are currently resolved from the process working directory, not from the configuration file. The complete example's `./examples/profiles/dnd-extraction.yml` path is valid for a repository-root diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d918038e..1b8537ad 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -195,7 +195,7 @@ not reproduce PromptKit's internal path, JSON-depth, or response-size matrices. boundary. - Ordinary and race-enabled tests pass. -## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials +## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ✅ ### Goal diff --git a/internal/cli/promptkit_profiles_test.go b/internal/cli/promptkit_profiles_test.go index a5b9e417..9b5fbb0d 100644 --- a/internal/cli/promptkit_profiles_test.go +++ b/internal/cli/promptkit_profiles_test.go @@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) { t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) } } + +func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) { + for _, profiles := range []string{ + "id: child\nbase_profile: missing\n", + "id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n", + } { + t.Run("invalid inheritance", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil { + t.Fatal(err) + } + profileID := "child" + if strings.Contains(profiles, "id: first") { + profileID = "first" + } + err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil) + if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) { + t.Fatalf("profile preflight error = %v", err) + } + }) + } +} diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 41ad4c7c..9076bca5 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -456,6 +456,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second) } }) + + t.Run("inherited parent", func(t *testing.T) { + profileDir := t.TempDir() + parentPath := filepath.Join(profileDir, "parent.yaml") + leafPath := filepath.Join(profileDir, "leaf.yaml") + if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil { + t.Fatal(err) + } + first, err := promptKitProfileFingerprint(profileDir, "", "") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil { + t.Fatal(err) + } + second, err := promptKitProfileFingerprint(profileDir, "", "") + if err != nil { + t.Fatal(err) + } + if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) { + t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second) + } + }) } func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) { @@ -531,6 +557,78 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing } } +func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) { + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil { + t.Fatal(err) + } + fake := &fakePromptKitLLM{content: `{"ok":true}`} + client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}}) + if err != nil { + t.Fatal(err) + } + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}}) + if err != nil { + t.Fatal(err) + } + inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile") + if err != nil { + t.Fatal(err) + } + if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" { + t.Fatalf("inspection = %#v, want resolved leaf target", inspection) + } + var out map[string]any + response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out) + if err != nil { + t.Fatal(err) + } + if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" { + t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection) + } +} + +func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) { + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{}) + if err != nil { + t.Fatal(err) + } + inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b") + if err != nil { + t.Fatal(err) + } + if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" { + t.Fatalf("inspection = %#v", inspection) + } +} + +func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want omitted", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)) + })) + defer server.Close() + t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "") + profilePath := filepath.Join(t.TempDir(), "optional.yaml") + if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil { + t.Fatal(err) + } + client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath}) + if err != nil { + t.Fatal(err) + } + var out map[string]any + if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil { + t.Fatal(err) + } +} + func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) { fingerprintFor := func(content string) CheckpointFingerprint { t.Helper()