From b92f83e49bb874e0db0ce7b71c31d54ebe1862df Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 25 Aug 2026 19:38:16 +0000 Subject: [PATCH] Adopt PromptKit profile inheritance --- docs/config.md | 15 +- docs/integrations/promptkit.md | 25 +++- docs/internal/app-orchestration.md | 2 +- docs/internal/promptkit-adapter.md | 4 +- docs/policy/architecture.md | 6 +- docs/roadmap/implementation.md | 2 + internal/adapters/promptkit/adapter_test.go | 131 +++++++++++++++--- internal/app/comparison.go | 3 +- internal/app/profile_execution.go | 2 +- internal/app/prompt_inspection.go | 22 +-- internal/app/prompt_inspection_test.go | 17 +-- .../app/prompt_profile_integration_test.go | 53 ++++++- .../assets/profiles/weather-balanced.yml | 6 +- .../assets/profiles/weather-deep.yml | 6 +- .../assets/profiles/weather-light.yml | 5 +- internal/promptassets/promptassets_test.go | 23 +++ 16 files changed, 241 insertions(+), 81 deletions(-) diff --git a/docs/config.md b/docs/config.md index f67cf1e..1d9e8bb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -209,10 +209,17 @@ definitions. Promptkit resolves a selected profile definition from a test or embedding consumer's explicit in-memory profile, then the configured `profile_file` or `profile_dir`, then Weatherreporter's embedded catalog, and finally Promptkit's -built-in catalog. Sources provide complete definitions; fields are never -merged. A matching malformed external profile fails rather than using the -embedded definition. The [Promptkit integration guide](integrations/promptkit.md) -owns the catalog and precedence details. +built-in catalog. Weatherreporter's embedded `weather-*` definitions are small +aliases of Promptkit's maintained base profiles, so Promptkit also resolves +their inherited target and settings. A configured definition with the same ID +as either a selected profile or an inherited base takes precedence. A matching +malformed external profile fails rather than using the embedded definition. The +[Promptkit integration guide](integrations/promptkit.md) owns the catalog and +precedence details. + +An endpoint-only profile may intentionally have no backend identity. Profiles +that require a direct API key are rejected before collection, while Promptkit +resolves optional environment credential sources during execution. To replace the default Hourly definition with a local OpenAI-compatible endpoint, set `profile_file` to a copy of diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index 7f4f4a5..54bb432 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -4,7 +4,9 @@ Weatherreporter uses Promptkit for all generated-text reports. The four logical ## Logical Profile Catalog -Prompt definitions select a stable Weatherreporter profile ID. The embedded definitions currently use Promptkit's `openrouter` backend: +Prompt definitions select a stable Weatherreporter profile ID. Each embedded +definition contains only its ID and one Promptkit base-profile reference; the +effective execution settings resolve from Promptkit's maintained catalog: | Profile ID | Model | Reasoning effort | Timeout | Service tier | Default reports | | --- | --- | --- | --- | --- | --- | @@ -12,20 +14,33 @@ Prompt definitions select a stable Weatherreporter profile ID. The embedded defi | `weather-balanced` | `~google/gemini-flash-latest` | `high` | 240 seconds | `flex` | Daily, Today, Tomorrow | | `weather-deep` | `~anthropic/claude-sonnet-latest` | `high` | 240 seconds | `flex` | None | -The `~` prefix is part of each OpenRouter rolling-alias model ID. The embedded profiles intentionally omit endpoints, credentials, temperature, `top_p`, and output-token limits. +The `~` prefix is part of each OpenRouter rolling-alias model ID. The embedded +profiles intentionally omit endpoints, credentials, and execution settings; +Promptkit owns inherited resolution and its provider-native defaults. +Promptkit's built-in `rakestrawhome-gemma-4-31b` is also available for ordinary +and comparison selection and reports the `rakestrawhome` backend without +Weatherreporter-specific configuration. ## Selection And Active Execution -Before weather collection, Weatherreporter validates the report's exact generated-text report/schema/template catalog binding, prompt version and hash, output contract, and selected profile. Active profiles must resolve a nonblank backend and model identity. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: +Before weather collection, Weatherreporter validates the report's exact generated-text report/schema/template catalog binding, prompt version and hash, output contract, and selected profile. Active profiles must resolve a nonblank model; an endpoint-only profile may intentionally have no backend identity. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: 1. explicit in-memory profiles used by an embedding consumer or test; 2. the configured `profile_file` or `profile_dir`; 3. Weatherreporter's embedded fallback profiles; and 4. Promptkit's built-in catalog. -A source falls through only when the selected ID is absent. Each source supplies a complete definition, so profile fields are not merged. A malformed matching operator definition is an error and does not fall back. +A source falls through only when the selected ID is absent. Promptkit resolves a +derived profile's base with the same source precedence, so a configured base +can shadow a built-in base. A missing, cyclic, malformed, or incomplete +selected inheritance chain is an error and does not fall back. -Profiles that require a direct API key are unsupported; a profile that reports `APIKeyEnv` requires a nonblank value in that environment variable. Active results retain the selected logical profile ID and resolved backend and model. Ordinary errors, summaries, logs, and outputs exclude endpoints, credentials, rendered messages, schemas, request bodies, response bodies, and complete parameter maps. +Profiles that require a direct API key are unsupported. Optional environment +credential sources are Promptkit runtime concerns and are not checked by +Weatherreporter during profile inspection. Active results retain the selected +logical profile ID and resolved backend and model. Ordinary errors, summaries, +logs, and outputs exclude endpoints, credentials, rendered messages, schemas, +request bodies, response bodies, and complete parameter maps. Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index ba822ab..89e0f70 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -7,7 +7,7 @@ is owned by the [CLI reference](../cli.md) and [operations guide](../operations. ## Single-Report Flow -`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. Output preflight validates the final filename, permits only an absent or regular final destination, and validates the bounded same-directory temporary form without creating a missing parent. It then validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result. +`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. Output preflight validates the final filename, permits only an absent or regular final destination, and validates the bounded same-directory temporary form without creating a missing parent. It then validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. Profile inspection requires a model, permits an empty backend identity for endpoint-only profiles, and leaves inherited resolution and optional credential sources to Promptkit. The resolved profile, backend, and model are carried in the active result. The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit only against the inspected prompt and profile, reconciles the preparation callback and completed result with that identity and the prepared report schema, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` writes the completed Markdown through a same-directory temporary file, rechecks the final destination and context after close and immediately before the atomic rename. Only after that write succeeds does single-report notification run. diff --git a/docs/internal/promptkit-adapter.md b/docs/internal/promptkit-adapter.md index cd7d902..7708e2f 100644 --- a/docs/internal/promptkit-adapter.md +++ b/docs/internal/promptkit-adapter.md @@ -2,9 +2,9 @@ `internal/adapters/promptkit` maps Weatherreporter's project-owned executor contract to Promptkit. The CLI maps `promptkit` configuration to a `PromptExecutorConfig` and constructs one executor per action. Promptkit dependency types do not escape the adapter. -The adapter supplies Weatherreporter's embedded prompt, schema, and fallback profile filesystems to each engine. Promptkit resolves configured operator profile sources, the embedded fallback catalog, and its built-in catalog; the adapter does not parse profile YAML, merge sources, or probe endpoints. +The adapter supplies Weatherreporter's embedded prompt, schema, and fallback profile filesystems to each engine. Promptkit resolves configured operator profile sources, embedded profile aliases and their bases, and its built-in catalog; the adapter does not parse profile YAML, resolve inheritance, merge sources, inspect optional environment credentials, or probe endpoints. -The adapter exposes exact prompt and profile validation plus prepared execution. It maps safe prompt identity, logical profile, effective backend/model, preparation, execution, validation, and optional debug values into `promptexec`. `Execute` passes the YAML package as an inline Promptkit input; it does not construct a filesystem URI or write a package file. +The adapter exposes exact prompt and profile validation plus prepared execution. It maps safe prompt identity, logical profile, effective backend/model, preparation, execution, validation, and optional debug values into `promptexec`. An empty backend identity remains valid for an endpoint-only profile; a nonblank model is required. `Execute` passes the YAML package as an inline Promptkit input; it does not construct a filesystem URI or write a package file. The application uses the preparation callback to record active safe provenance in memory and optionally writes content-rich diagnostics only through an explicit debug writer. The adapter returns raw output for application validation and rendering. It does not retain application state, render Markdown, choose report definitions, or send Distributor notifications. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 1b0d62c..24d7378 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -50,8 +50,10 @@ directly. - Prompts receive curated module packages, never unbounded raw weather payloads. - Every execution validates the exact prompt version and output contract before collection. The selected profile is configured explicitly or declared by the - prompt; unsupported direct-key profiles and missing reported credentials fail - before collection. + prompt; profiles requiring unsupported direct API keys fail before collection. + A profile may have an empty backend identity when it supplies an endpoint; + PromptKit resolves inherited profiles and optional credential sources when it + executes them. - Prompt and profile validation completes before weather collection. Raw output is validated before template rendering. - Comparison validates every explicit profile before collection, prepares one diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a24c48d..38c63c3 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -121,6 +121,8 @@ GOWORK=off go test -race -count=1 ./internal/adapters/promptkit ### Stage 2: Adopt Profile Inheritance And Current Credential Routing +Status: Complete. + Purpose: adopt v0.7.0 profile composition, endpoint-only routing, optional credential semantics, and the Rakestrawhome built-in without changing the model ladder. diff --git a/internal/adapters/promptkit/adapter_test.go b/internal/adapters/promptkit/adapter_test.go index 90eae54..ef2860e 100644 --- a/internal/adapters/promptkit/adapter_test.go +++ b/internal/adapters/promptkit/adapter_test.go @@ -175,15 +175,38 @@ model: file-light } assertProfile(t, fileAdapter, "weather-light", "", "file-light") - directory := testProfileDirectory(t, `id: weather-light + directory := testProfileDirectory(t, map[string]string{"profile.yml": `id: weather-light backend: local model: directory-light -`) +`}) directoryAdapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"}) if err != nil { t.Fatalf("New(profile directory) error = %v", err) } assertProfile(t, directoryAdapter, "weather-light", promptkit.BackendLocal, "directory-light") + + derived := writeProfileFile(t, `id: weather-light +base_profile: gemini-flash-latest +`) + derivedAdapter, err := New(Config{ProfileFile: derived}) + if err != nil { + t.Fatalf("New(derived profile) error = %v", err) + } + assertProfile(t, derivedAdapter, "weather-light", "openrouter", "~google/gemini-flash-latest") +} + +func TestConfiguredBaseProfileOverridesEmbeddedProfileTarget(t *testing.T) { + directory := testProfileDirectory(t, map[string]string{ + "deepseek.yml": `id: deepseek-4-flash +backend: local +model: shadowed-deepseek +`, + }) + adapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + assertProfile(t, adapter, "weather-light", promptkit.BackendLocal, "shadowed-deepseek") } func TestMaintainedWeatherLightLocalProfileExampleInspectsOffline(t *testing.T) { @@ -195,18 +218,18 @@ func TestMaintainedWeatherLightLocalProfileExampleInspectsOffline(t *testing.T) } func TestProfileResolutionFallsThroughOnlyWhenTheConfiguredIDIsAbsent(t *testing.T) { - absentAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: other-profile + absentAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, map[string]string{"profile.yml": `id: other-profile backend: openrouter model: other-model -`)}) +`})}) if err != nil { t.Fatalf("New(absent profile) error = %v", err) } assertProfile(t, absentAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash") - malformedAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: weather-light + malformedAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, map[string]string{"profile.yml": `id: weather-light backend: openrouter -`)}) +`})}) if err != nil { t.Fatalf("New(malformed profile) error = %v", err) } @@ -215,6 +238,78 @@ backend: openrouter } } +func TestProfileResolutionReturnsConfiguredInheritanceFailures(t *testing.T) { + tests := []struct { + name string + profile string + profiles map[string]string + }{ + { + name: "missing base", + profile: "missing-base", + profiles: map[string]string{"missing.yml": `id: missing-base +base_profile: unavailable +`}, + }, + { + name: "cyclic bases", + profile: "first", + profiles: map[string]string{ + "first.yml": `id: first +base_profile: second +`, + "second.yml": `id: second +base_profile: first +`, + }, + }, + { + name: "malformed base", + profile: "child", + profiles: map[string]string{ + "child.yml": `id: child +base_profile: malformed +`, + "malformed.yml": `id: malformed +base_profile: [not-a-profile] +`, + }, + }, + { + name: "incomplete target", + profile: "incomplete", + profiles: map[string]string{"incomplete.yml": `id: incomplete +backend: openrouter +`}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + adapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, test.profiles)}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if _, err := adapter.InspectProfile(context.Background(), test.profile); err == nil { + t.Fatal("InspectProfile() error = nil, want configured inheritance error") + } + }) + } +} + +func TestRakestrawhomeBuiltInProfileInspectsOffline(t *testing.T) { + adapter, err := New(Config{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + profile, err := adapter.InspectProfile(context.Background(), "rakestrawhome-gemma-4-31b") + if err != nil { + t.Fatalf("InspectProfile() error = %v", err) + } + if profile.ProfileID != "rakestrawhome-gemma-4-31b" || profile.BackendID != "rakestrawhome" || profile.ModelName == "" { + t.Fatalf("profile = %#v", profile) + } +} + func TestProfileResolutionPreservesBuiltInAndExplicitPrecedence(t *testing.T) { adapter, err := New(Config{}) if err != nil { @@ -509,10 +604,10 @@ func TestNewValidatesConfiguration(t *testing.T) { func TestLocalBackendAndOptionalCredentialSourceBehavior(t *testing.T) { t.Setenv("WEATHERREPORTER_TEST_MISSING_KEY", "") - profiles := testProfileDirectory(t, `id: local-profile + profiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: local-profile backend: local model: local-model -`) +`}) adapter, err := newAdapterForTest(Config{ ProfileDirectory: profiles, LocalEndpoint: "https://local.example/v1", @@ -529,11 +624,11 @@ model: local-model t.Fatalf("capacity classification = %v", got) } - credentialProfiles := testProfileDirectory(t, `id: credential-profile + credentialProfiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: credential-profile endpoint: https://profile.example/v1 model: test-model api_key_env: WEATHERREPORTER_TEST_MISSING_KEY -`) +`}) client := &fakeClient{response: validResponse()} credentialAdapter, err := newAdapterForTest(Config{ProfileDirectory: credentialProfiles}, client) if err != nil { @@ -568,14 +663,14 @@ func assertProfile(t *testing.T, adapter *Adapter, id string, backend string, mo func newTestAdapterWithOptions(t *testing.T, client promptkit.LLMClient, options ...promptkit.Option) *Adapter { t.Helper() - profiles := testProfileDirectory(t, `id: test-profile + profiles := testProfileDirectory(t, map[string]string{"profile.yml": `id: test-profile endpoint: https://profile.example/v1 model: test-model temperature: 0.2 max_tokens: 300 top_p: 1 timeout_seconds: 30 -`) +`}) options = append(options, promptkit.WithLLMClient(client)) adapter, err := newAdapter(Config{ProfileDirectory: profiles, Timeout: time.Second}, options...) if err != nil { @@ -584,13 +679,15 @@ timeout_seconds: 30 return adapter } -func testProfileDirectory(t *testing.T, profile string) string { +func testProfileDirectory(t *testing.T, profiles map[string]string) string { t.Helper() - profiles := t.TempDir() - if err := os.WriteFile(filepath.Join(profiles, "profile.yml"), []byte(profile), 0o600); err != nil { - t.Fatalf("write profile: %v", err) + directory := t.TempDir() + for name, profile := range profiles { + if err := os.WriteFile(filepath.Join(directory, name), []byte(profile), 0o600); err != nil { + t.Fatalf("write profile: %v", err) + } } - return profiles + return directory } func writeProfileFile(t *testing.T, profile string) string { diff --git a/internal/app/comparison.go b/internal/app/comparison.go index 3b1f330..e4ceaaa 100644 --- a/internal/app/comparison.go +++ b/internal/app/comparison.go @@ -3,7 +3,6 @@ package app import ( "context" "fmt" - "os" "path/filepath" "time" @@ -121,7 +120,7 @@ func compareDetailed(ctx context.Context, req ComparisonRequest, publish compari } defer func() { _ = debugWriter.Close() }() inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{ - Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv, + Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, }) result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash if err != nil { diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go index 191512e..567d12c 100644 --- a/internal/app/profile_execution.go +++ b/internal/app/profile_execution.go @@ -157,7 +157,7 @@ func validatePreparedExecutionRequest(req profileExecutionRequest) error { if req.Prompt.ProfileID != "" && (req.Prompt.ProfileID != req.Profile.ProfileID || req.Prompt.BackendID != req.Profile.BackendID || req.Prompt.ModelName != req.Profile.ModelName) { return promptProvenanceError() } - if req.Prompt.PromptHash == "" || req.Profile.ProfileID == "" || req.Profile.BackendID == "" || req.Profile.ModelName == "" { + if req.Prompt.PromptHash == "" || req.Profile.ProfileID == "" || req.Profile.ModelName == "" { return promptProvenanceError() } return nil diff --git a/internal/app/prompt_inspection.go b/internal/app/prompt_inspection.go index 7e30ab1..42970a5 100644 --- a/internal/app/prompt_inspection.go +++ b/internal/app/prompt_inspection.go @@ -2,7 +2,6 @@ package app import ( "context" - "os" "strings" "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" @@ -18,7 +17,6 @@ type PromptInspectionRequest struct { Resolved report.Resolved Executor promptexec.Executor Promptkit config.PromptkitConfig - LookupEnv func(string) (string, bool) } // PromptInspectionResult contains only safe identity and provenance from a @@ -39,7 +37,6 @@ type PromptExecutionsInspectionRequest struct { Resolved []report.Resolved Executor promptexec.Executor Promptkit config.PromptkitConfig - LookupEnv func(string) (string, bool) } // ComparisonInspectionRequest contains the explicit profile selection for one @@ -48,7 +45,6 @@ type ComparisonInspectionRequest struct { Resolved report.Resolved ProfileIDs []string Executor promptexec.Executor - LookupEnv func(string) (string, bool) } // ComparisonInspectionResult contains the safe, shared prompt identity and @@ -76,7 +72,6 @@ func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (P Resolved: []report.Resolved{req.Resolved}, Executor: req.Executor, Promptkit: req.Promptkit, - LookupEnv: req.LookupEnv, }) if err != nil { return PromptInspectionResult{}, err @@ -111,7 +106,7 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection } profile, ok := profiles[profileID] if !ok { - profile, err = inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv) + profile, err = inspectPromptProfile(ctx, req.Executor, profileID) if err != nil { return nil, err } @@ -154,7 +149,7 @@ func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionReq handler: handler, } for _, profileID := range req.ProfileIDs { - profile, err := inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv) + profile, err := inspectPromptProfile(ctx, req.Executor, profileID) if err != nil { return result, comparisonInspectionError("comparison profile inspection failed", err) } @@ -190,7 +185,7 @@ func inspectPromptContract(ctx context.Context, executor promptexec.Executor, de return inspection, nil } -func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) { +func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string) (promptexec.ProfileInspection, error) { profile, err := executor.InspectProfile(ctx, profileID) if err != nil { return promptexec.ProfileInspection{}, promptInspectionError("profile inspection failed", err) @@ -201,16 +196,7 @@ func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, pro if profile.CredentialRequired { return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.MissingCredential, "selected profile requires an unsupported direct API key", nil) } - if strings.TrimSpace(profile.APIKeyEnv) != "" { - if lookupEnv == nil { - lookupEnv = os.LookupEnv - } - value, present := lookupEnv(profile.APIKeyEnv) - if !present || strings.TrimSpace(value) == "" { - return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.MissingCredential, "selected profile credential is unavailable", nil) - } - } - if strings.TrimSpace(profile.BackendID) == "" || strings.TrimSpace(profile.ModelName) == "" { + if strings.TrimSpace(profile.ModelName) == "" { return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "profile inspection did not return a complete execution identity", nil) } return profile, nil diff --git a/internal/app/prompt_inspection_test.go b/internal/app/prompt_inspection_test.go index 28789a7..0ec609b 100644 --- a/internal/app/prompt_inspection_test.go +++ b/internal/app/prompt_inspection_test.go @@ -50,7 +50,6 @@ func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing. name string prompt promptexec.PromptInspection profile promptexec.ProfileInspection - lookupEnv func(string) (string, bool) wantCategory promptexec.ErrorCategory }{ { @@ -81,9 +80,9 @@ func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing. wantCategory: promptexec.InvalidConfiguration, }, { - name: "missing profile backend", + name: "missing profile model", prompt: basePrompt, - profile: promptexec.ProfileInspection{ProfileID: "default-profile", ModelName: "model"}, + profile: promptexec.ProfileInspection{ProfileID: "default-profile", BackendID: "backend"}, wantCategory: promptexec.InvalidConfiguration, }, { @@ -92,18 +91,11 @@ func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing. profile: promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}, wantCategory: promptexec.MissingCredential, }, - { - name: "missing environment credential", - prompt: basePrompt, - profile: promptexec.ProfileInspection{ProfileID: "default-profile", APIKeyEnv: "PROMPT_API_KEY"}, - lookupEnv: func(string) (string, bool) { return "", false }, - wantCategory: promptexec.MissingCredential, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { executor := &inspectionExecutor{prompt: test.prompt, profiles: map[string]promptexec.ProfileInspection{"default-profile": test.profile}} - _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor, LookupEnv: test.lookupEnv}) + _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor}) if err == nil || promptexec.CategoryOf(err) != test.wantCategory { t.Fatalf("error/category = %v/%q, want %q", err, promptexec.CategoryOf(err), test.wantCategory) } @@ -265,13 +257,12 @@ func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) { prompt: validPromptInspection(resolved.Definition), profiles: map[string]promptexec.ProfileInspection{ "weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"}, - "missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"}, + "missing-key": {ProfileID: "missing-key", CredentialRequired: true}, "weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"}, }, } result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{ Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor, - LookupEnv: func(string) (string, bool) { return "", false }, }) if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential { t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err)) diff --git a/internal/app/prompt_profile_integration_test.go b/internal/app/prompt_profile_integration_test.go index a885dba..ff05d1e 100644 --- a/internal/app/prompt_profile_integration_test.go +++ b/internal/app/prompt_profile_integration_test.go @@ -2,8 +2,10 @@ package app_test import ( "context" + "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -14,14 +16,12 @@ import ( ) func TestPromptInspectionResolvesEmbeddedAndOverriddenProfilesOffline(t *testing.T) { - lookupEnv := func(string) (string, bool) { return "test-key", true } inspect := func(t *testing.T, adapter *promptkitadapter.Adapter, id report.ID, profile string, wantID string, wantBackend string, wantModel string) { t.Helper() result, err := app.InspectPromptExecution(context.Background(), app.PromptInspectionRequest{ Resolved: resolvedPromptProfile(t, id), Executor: adapter, Promptkit: config.PromptkitConfig{Profile: profile}, - LookupEnv: lookupEnv, }) if err != nil { t.Fatalf("InspectPromptExecution() error = %v", err) @@ -50,6 +50,55 @@ model: local-weather inspect(t, override, report.Hourly, "", "weather-light", "openrouter", "local-weather") } +func TestPromptInspectionAcceptsMaintainedEndpointOnlyProfile(t *testing.T) { + adapter, err := promptkitadapter.New(promptkitadapter.Config{ProfileFile: filepath.Join("..", "..", "examples", "weather-light-local-profile.yml")}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + result, err := app.InspectPromptExecution(context.Background(), app.PromptInspectionRequest{ + Resolved: resolvedPromptProfile(t, report.Hourly), Executor: adapter, + }) + if err != nil { + t.Fatalf("InspectPromptExecution() error = %v", err) + } + if result.ProfileID != "weather-light" || result.BackendID != "" || result.ModelName != "weather-local" { + t.Fatalf("inspection = %#v", result) + } + if strings.Contains(fmt.Sprintf("%#v", result), "127.0.0.1") { + t.Fatalf("inspection leaks endpoint: %#v", result) + } +} + +func TestPromptInspectionSupportsRakestrawhomeProfileOffline(t *testing.T) { + adapter, err := promptkitadapter.New(promptkitadapter.Config{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + prompt, err := app.InspectPromptExecution(context.Background(), app.PromptInspectionRequest{ + Resolved: resolvedPromptProfile(t, report.Hourly), + Executor: adapter, + Promptkit: config.PromptkitConfig{Profile: "rakestrawhome-gemma-4-31b"}, + }) + if err != nil { + t.Fatalf("InspectPromptExecution() error = %v", err) + } + if prompt.ProfileID != "rakestrawhome-gemma-4-31b" || prompt.BackendID != "rakestrawhome" || prompt.ModelName == "" { + t.Fatalf("prompt inspection = %#v", prompt) + } + + comparison, err := app.InspectComparisonExecution(context.Background(), app.ComparisonInspectionRequest{ + Resolved: resolvedPromptProfile(t, report.Hourly), + ProfileIDs: []string{"rakestrawhome-gemma-4-31b", "weather-deep"}, + Executor: adapter, + }) + if err != nil { + t.Fatalf("InspectComparisonExecution() error = %v", err) + } + if len(comparison.Profiles) != 2 || comparison.Profiles[0].ProfileID != "rakestrawhome-gemma-4-31b" || comparison.Profiles[0].BackendID != "rakestrawhome" || comparison.Profiles[0].ModelName == "" { + t.Fatalf("comparison inspection = %#v", comparison) + } +} + func resolvedPromptProfile(t *testing.T, id report.ID) report.Resolved { t.Helper() now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) diff --git a/internal/promptassets/assets/profiles/weather-balanced.yml b/internal/promptassets/assets/profiles/weather-balanced.yml index bc1c776..dd2bf63 100644 --- a/internal/promptassets/assets/profiles/weather-balanced.yml +++ b/internal/promptassets/assets/profiles/weather-balanced.yml @@ -1,6 +1,2 @@ id: weather-balanced -backend: openrouter -model: "~google/gemini-flash-latest" -reasoning_effort: high -timeout_seconds: 240 -service_tier: flex +base_profile: gemini-flash-latest diff --git a/internal/promptassets/assets/profiles/weather-deep.yml b/internal/promptassets/assets/profiles/weather-deep.yml index 6ba301a..b18b801 100644 --- a/internal/promptassets/assets/profiles/weather-deep.yml +++ b/internal/promptassets/assets/profiles/weather-deep.yml @@ -1,6 +1,2 @@ id: weather-deep -backend: openrouter -model: "~anthropic/claude-sonnet-latest" -reasoning_effort: high -timeout_seconds: 240 -service_tier: flex +base_profile: claude-sonnet-latest diff --git a/internal/promptassets/assets/profiles/weather-light.yml b/internal/promptassets/assets/profiles/weather-light.yml index de000aa..e26a7db 100644 --- a/internal/promptassets/assets/profiles/weather-light.yml +++ b/internal/promptassets/assets/profiles/weather-light.yml @@ -1,5 +1,2 @@ id: weather-light -backend: openrouter -model: deepseek/deepseek-v4-flash -timeout_seconds: 180 -service_tier: flex +base_profile: deepseek-4-flash diff --git a/internal/promptassets/promptassets_test.go b/internal/promptassets/promptassets_test.go index 18d0485..692e22b 100644 --- a/internal/promptassets/promptassets_test.go +++ b/internal/promptassets/promptassets_test.go @@ -361,6 +361,29 @@ func TestEmbeddedProfilesAreCompleteAndInspectable(t *testing.T) { } } +func TestEmbeddedProfilesAreMinimalBaseAliases(t *testing.T) { + wantBases := map[string]string{ + "weather-balanced.yml": "gemini-flash-latest", + "weather-deep.yml": "claude-sonnet-latest", + "weather-light.yml": "deepseek-4-flash", + } + for path, wantBase := range wantBases { + t.Run(path, func(t *testing.T) { + data, err := fs.ReadFile(promptassets.ProfileFS(), path) + if err != nil { + t.Fatalf("read profile: %v", err) + } + var definition map[string]string + if err := yaml.Unmarshal(data, &definition); err != nil { + t.Fatalf("unmarshal profile: %v", err) + } + if definition["id"] != strings.TrimSuffix(path, ".yml") || definition["base_profile"] != wantBase || len(definition) != 2 { + t.Fatalf("profile definition = %#v, want only its id and base profile %q", definition, wantBase) + } + }) + } +} + func TestEmbeddedProfilesExcludeUnsafeOrIncidentalSettings(t *testing.T) { forbidden := []string{"endpoint:", "api_key", "credential", "temperature:", "top_p:", "max_tokens:"} if err := fs.WalkDir(promptassets.ProfileFS(), ".", func(path string, entry fs.DirEntry, err error) error {