Support inherited PromptKit profiles
This commit is contained in:
@@ -132,7 +132,11 @@ model: example-model
|
|||||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||||
its credential environment variable through `api_key_env`; set that variable
|
its credential environment variable through `api_key_env`; set that variable
|
||||||
only in the run environment. PromptKit owns the
|
only in the run environment. PromptKit owns the
|
||||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md),
|
||||||
|
including `base_profile` inheritance. Notarius passes profiles through without
|
||||||
|
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||||
|
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||||
|
provider without authorization.
|
||||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||||
supported package API, and [Operations](operations.md#operational-limits)
|
supported package API, and [Operations](operations.md#operational-limits)
|
||||||
describes the effective concurrency layers.
|
describes the effective concurrency layers.
|
||||||
|
|||||||
@@ -84,6 +84,12 @@ configuration and deployment workflow are defined in
|
|||||||
[Configuration](../config.md#promptkit-profiles) and
|
[Configuration](../config.md#promptkit-profiles) and
|
||||||
[Operations](../operations.md#promptkit-profile-deployment).
|
[Operations](../operations.md#promptkit-profile-deployment).
|
||||||
|
|
||||||
|
PromptKit owns `base_profile` resolution under its
|
||||||
|
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.8.0/docs/formats.md).
|
||||||
|
Notarius records the selected leaf identity and resolved target without parsing
|
||||||
|
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||||
|
reach the provider without authorization, which can result in a 401 or 403.
|
||||||
|
|
||||||
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
|
Notarius supports this boundary against PromptKit v0.8.0. Its fallback source,
|
||||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ backend membership as runtime without performing generation. Fallback assets
|
|||||||
are mounted only when at least one source is registered. The production D&D
|
are mounted only when at least one source is registered. The production D&D
|
||||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||||
profile parsing: an operator-provided matching profile takes precedence over a
|
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||||
fallback profile without Notarius merging either document.
|
fallback profile without Notarius merging either document.
|
||||||
When the registration is absent, a profile selecting `backend: local` fails
|
When the registration is absent, a profile selecting `backend: local` fails
|
||||||
inspection instead of falling back to a built-in or endpoint-only target.
|
inspection instead of falling back to a built-in or endpoint-only target.
|
||||||
|
|||||||
@@ -89,6 +89,9 @@ provider call or credentials:
|
|||||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
|
An unset optional `api_key_env` reaches the provider without authorization and
|
||||||
|
may receive a 401 or 403 response.
|
||||||
|
|
||||||
Profile paths are currently resolved from the process working directory, not
|
Profile paths are currently resolved from the process working directory, not
|
||||||
from the configuration file. The complete example's
|
from the configuration file. The complete example's
|
||||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||||
|
|||||||
@@ -195,7 +195,7 @@ not reproduce PromptKit's internal path, JSON-depth, or response-size matrices.
|
|||||||
boundary.
|
boundary.
|
||||||
- Ordinary and race-enabled tests pass.
|
- Ordinary and race-enabled tests pass.
|
||||||
|
|
||||||
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials
|
## Stage 3: Adopt Profile Inheritance, Rakestrawhome, And Optional Credentials ✅
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -157,3 +157,25 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
|||||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||||
|
for _, profiles := range []string{
|
||||||
|
"id: child\nbase_profile: missing\n",
|
||||||
|
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||||
|
} {
|
||||||
|
t.Run("invalid inheritance", func(t *testing.T) {
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profileID := "child"
|
||||||
|
if strings.Contains(profiles, "id: first") {
|
||||||
|
profileID = "first"
|
||||||
|
}
|
||||||
|
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
|
||||||
|
t.Fatalf("profile preflight error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -456,6 +456,32 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("inherited parent", func(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||||
|
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||||
|
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||||
@@ -531,6 +557,78 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileDir: profileDir, EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Source: PromptKitProfileSourceConfig{ProfileDir: profileDir}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "inherited-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != "inherited-profile" || inspection.Model != "leaf-model" {
|
||||||
|
t.Fatalf("inspection = %#v, want resolved leaf target", inspection)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "inherited-profile", SessionID: "inheritance-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if response.ProfileID != inspection.ProfileID || response.Model != inspection.Model || fake.lastRequest().Target.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("response=%#v target=%#v inspection=%#v", response, fake.lastRequest().Target, inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitProfileInspectorExposesRakestrawhomeBuiltIn(t *testing.T) {
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inspection.BackendID != promptkit.BackendRakestrawHome || inspection.Model != "google/gemma-4-31b-it" {
|
||||||
|
t.Fatalf("inspection = %#v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.Header.Get("Authorization"); got != "" {
|
||||||
|
t.Fatalf("Authorization = %q, want omitted", got)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
||||||
|
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "optional-profile", SessionID: "optional-credential-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
Reference in New Issue
Block a user