Support inherited PromptKit profiles

This commit is contained in:
2026-08-25 19:31:00 +00:00
parent 2be999ebd3
commit 75a3f51cee
7 changed files with 136 additions and 3 deletions

View File

@@ -132,7 +132,11 @@ model: example-model
Keep credentials out of the local-backend object. A PromptKit profile may name Keep credentials out of the local-backend object. A PromptKit profile may name
its credential environment variable through `api_key_env`; set that variable its credential environment variable through `api_key_env`; set that variable
only in the run environment. PromptKit owns the only in the run environment. PromptKit owns the
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md). [pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md),
including `base_profile` inheritance. Notarius passes profiles through without
merging them. Filesystem profiles cannot express PromptKit's in-memory
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
provider without authorization.
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
supported package API, and [Operations](operations.md#operational-limits) supported package API, and [Operations](operations.md#operational-limits)
describes the effective concurrency layers. describes the effective concurrency layers.

View File

@@ -84,6 +84,12 @@ configuration and deployment workflow are defined in
[Configuration](../config.md#promptkit-profiles) and [Configuration](../config.md#promptkit-profiles) and
[Operations](../operations.md#promptkit-profile-deployment). [Operations](../operations.md#promptkit-profile-deployment).
PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403.
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source, Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not upstream contracts; other PromptKit APIs or file-format behavior are not

View File

@@ -75,7 +75,7 @@ backend membership as runtime without performing generation. Fallback assets
are mounted only when at least one source is registered. The production D&D are mounted only when at least one source is registered. The production D&D
registrar contributes its `dnd-extraction` fallback, and the maintained D&D registrar contributes its `dnd-extraction` fallback, and the maintained D&D
prompts select that logical ID by default. PromptKit owns source precedence and prompts select that logical ID by default. PromptKit owns source precedence and
profile parsing: an operator-provided matching profile takes precedence over a profile parsing and inheritance: an operator-provided matching profile takes precedence over a
fallback profile without Notarius merging either document. fallback profile without Notarius merging either document.
When the registration is absent, a profile selecting `backend: local` fails When the registration is absent, a profile selecting `backend: local` fails
inspection instead of falling back to a built-in or endpoint-only target. inspection instead of falling back to a built-in or endpoint-only target.

View File

@@ -89,6 +89,9 @@ provider call or credentials:
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
~~~ ~~~
An unset optional `api_key_env` reaches the provider without authorization and
may receive a 401 or 403 response.
Profile paths are currently resolved from the process working directory, not Profile paths are currently resolved from the process working directory, not
from the configuration file. The complete example's from the configuration file. The complete example's
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root `./examples/profiles/dnd-extraction.yml` path is valid for a repository-root

View File

@@ -195,7 +195,7 @@ not reproduce PromptKit's internal path, JSON-depth, or response-size matrices.
boundary. boundary.
- Ordinary and race-enabled tests pass. - Ordinary and race-enabled tests pass.
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials
### Goal ### Goal

View File

@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
} }
} }
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
for _, profiles := range []string{
"id: child\nbase_profile: missing\n",
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
} {
t.Run("invalid inheritance", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
t.Fatal(err)
}
profileID := "child"
if strings.Contains(profiles, "id: first") {
profileID = "first"
}
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
t.Fatalf("profile preflight error = %v", err)
}
})
}
}

View File

@@ -456,6 +456,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second) t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
} }
}) })
t.Run("inherited parent", func(t *testing.T) {
profileDir := t.TempDir()
parentPath := filepath.Join(profileDir, "parent.yaml")
leafPath := filepath.Join(profileDir, "leaf.yaml")
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
}
})
} }
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) { func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
@@ -531,6 +557,78 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
} }
} }
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
if err != nil {
t.Fatal(err)
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
if err != nil {
t.Fatal(err)
}
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
if err != nil {
t.Fatal(err)
}
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
if err != nil {
t.Fatal(err)
}
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
}
}
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
if err != nil {
t.Fatal(err)
}
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
if err != nil {
t.Fatal(err)
}
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
t.Fatalf("inspection = %#v", inspection)
}
}
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "" {
t.Fatalf("Authorization = %q, want omitted", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
if err != nil {
t.Fatal(err)
}
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
t.Fatal(err)
}
}
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) { func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
fingerprintFor := func(content string) CheckpointFingerprint { fingerprintFor := func(content string) CheckpointFingerprint {
t.Helper() t.Helper()