Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9178f3eb91 | |||
| a70fd7bb65 | |||
| 7091b51c87 | |||
| 8627c751a4 | |||
| 6fff3d822c | |||
| a1f12f3185 | |||
| 165cef7aec | |||
| 0221cd4610 | |||
| dfd8da843e | |||
| 29f104fbc0 | |||
| 8669294cf6 | |||
| c9b623b374 | |||
| 15f79ce7b1 | |||
| aa54b8d1c4 | |||
| 60c58260ec | |||
| 8dfe35a1f0 | |||
| 13b60a72ac | |||
| ee40a1bdd2 | |||
| 473fc1f6d7 | |||
| a10a9053b7 | |||
| ec3245c97e | |||
| 8cbb7c0daf | |||
| d40a010538 | |||
| 0cd833513c | |||
| 5e58b6944b | |||
| 0a85ac8cd1 | |||
| 89af2a1281 | |||
| 683689a4f8 | |||
| ddda2f8767 | |||
| 190ec69345 | |||
| b5866dd749 | |||
| c7490b783b | |||
| c7d1b0dca7 | |||
| 9e5ed08635 | |||
| 4bddbbe8f4 | |||
| 1516631a30 | |||
| cf997c2034 | |||
| a0ec67c256 | |||
| 75891b6561 | |||
| 5dc6370a97 | |||
| 4cda3674c4 | |||
| 5ea6f06a6d | |||
| 116bd0e39b | |||
| 208ff42915 | |||
| 972a103a97 | |||
| 2f735aedf9 | |||
| f4c721d934 | |||
| b7a951685b | |||
| abb2461244 | |||
| b268c55783 | |||
| f0182fb2c9 | |||
| 8d05b594e2 | |||
| edaaecc982 | |||
| 2e81c9e61b | |||
| 2b723760d1 | |||
| c5aed7a451 | |||
| d91afc61a4 | |||
| 6eeb0ee9c0 | |||
| b9c232ee63 | |||
| 7291d23587 | |||
| da1d196dc5 | |||
| b0de49a87a | |||
| 27ff580dae | |||
| ece764f19e | |||
| f91a02548e | |||
| 8fbd058b90 | |||
| c8f9a4def9 | |||
| d8b6c0b471 | |||
| 52cd94fcd1 | |||
| f1f72dfe8f | |||
| d4676bc870 | |||
| 64769883ae |
21
README.md
21
README.md
@@ -21,15 +21,21 @@ Run the maintained offline preparation example from the repository root:
|
|||||||
go run ./examples/go-library/prepare
|
go run ./examples/go-library/prepare
|
||||||
```
|
```
|
||||||
|
|
||||||
It loads a repository-local prompt, supplies a programmatic profile and inline
|
It composes two explicitly selected synthetic collections, imports
|
||||||
input, inspects the captured configuration, and prints deterministic
|
source-relative prompt content, supplies an inline input, and prepares through
|
||||||
preparation metadata without contacting a provider or requiring credentials.
|
a scoped view. The example also demonstrates collection inventories and
|
||||||
|
namespace-filtered discovery, without contacting a provider or requiring
|
||||||
|
credentials.
|
||||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||||
engine when source files change. Read the
|
engine when source files change. Applications adapting the prior keyed
|
||||||
[example source](examples/go-library/prepare/main.go), the
|
configuration or versioned-prompt format can consult the
|
||||||
|
[configuration migration guide](docs/consumers/configuration-migration.md).
|
||||||
|
See the [example source](examples/go-library/prepare/main.go), the
|
||||||
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
||||||
[framework format reference](docs/formats.md) to build a consumer workflow.
|
[framework format reference](docs/formats.md) to build a consumer workflow.
|
||||||
|
The consumer guide also covers explicit, live smoke diagnostics and interpreting
|
||||||
|
their factual reports; they are separate from the offline quickstart.
|
||||||
|
|
||||||
Contributors should start with the [development guide](docs/development.md).
|
Contributors should start with the [development guide](docs/development.md).
|
||||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||||
@@ -37,6 +43,11 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
|
||||||
|
diagnostics, configuration collections, scoped engine views, and discovery.
|
||||||
|
Upgrading consumers should review the breaking changes to bundled configuration
|
||||||
|
and prompt version selection and follow the linked migration guide.
|
||||||
|
|
||||||
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
||||||
validation of concrete requests with `Engine.ValidateRequest`.
|
validation of concrete requests with `Engine.ValidateRequest`.
|
||||||
|
|
||||||
|
|||||||
@@ -100,11 +100,12 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
|
|
||||||
config := map[string]any{
|
config := map[string]any{
|
||||||
"profiles": map[string]any{"profile": profile},
|
"profiles": map[string]any{"profile": profile},
|
||||||
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
|
"prompts": map[string]any{"workflow": map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||||
"output": output,
|
"output": output,
|
||||||
}}},
|
}},
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
@@ -113,7 +114,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
request := promptkit.RunRequest{
|
request := promptkit.RunRequest{
|
||||||
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
|
||||||
}
|
}
|
||||||
return assembledWorkflowFixture{
|
return assembledWorkflowFixture{
|
||||||
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||||
@@ -127,7 +128,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||||
}
|
}
|
||||||
|
|||||||
70
backends.go
70
backends.go
@@ -5,10 +5,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BackendOpenRouter identifies the maintained OpenRouter backend.
|
// BackendOpenRouter is the conventional ID for an OpenRouter backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
|
// BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
|
||||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||||
|
|
||||||
// BackendLocal is a conventional ID for a caller-configured local backend.
|
// BackendLocal is a conventional ID for a caller-configured local backend.
|
||||||
@@ -17,8 +17,9 @@ const BackendLocal = "local"
|
|||||||
// BackendConfig is a presence-aware convenience value for a keyed backend
|
// BackendConfig is a presence-aware convenience value for a keyed backend
|
||||||
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
||||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||||
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
|
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||||
// explicit null. BackendConfig has no stable JSON representation.
|
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||||
|
// inputs for explicit null. BackendConfig has no stable JSON representation.
|
||||||
type BackendConfig struct {
|
type BackendConfig struct {
|
||||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||||
Endpoint *string
|
Endpoint *string
|
||||||
@@ -31,6 +32,43 @@ type BackendConfig struct {
|
|||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
// ExtraParams supplies JSON-compatible provider request defaults.
|
// ExtraParams supplies JSON-compatible provider request defaults.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
|
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
|
||||||
|
SmokeTests *BackendSmokeTestConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendSmokeTestConfig contains optional diagnostic settings associated with
|
||||||
|
// a backend. Nil nested fields are omitted by BackendSource.
|
||||||
|
type BackendSmokeTestConfig struct {
|
||||||
|
// DefaultProfile is the exact profile ID selected as the backend's
|
||||||
|
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||||
|
// is resolved only when that default is selected.
|
||||||
|
DefaultProfile *string
|
||||||
|
// Reachability optionally selects a full HTTP destination for reachability
|
||||||
|
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||||
|
Reachability *ReachabilityProbeConfig
|
||||||
|
// Authentication optionally declares a named authentication diagnostic.
|
||||||
|
// A supplied object requires Kind.
|
||||||
|
Authentication *AuthenticationProbeConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||||
|
type ReachabilityProbeConfig struct {
|
||||||
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
|
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||||
|
// invalid.
|
||||||
|
Endpoint *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuthenticationProbeConfig configures an optional named authentication
|
||||||
|
// diagnostic.
|
||||||
|
type AuthenticationProbeConfig struct {
|
||||||
|
// Kind selects the authentication adapter. Supported values are "none" and
|
||||||
|
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||||
|
Kind *string
|
||||||
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
|
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||||
|
// value is invalid.
|
||||||
|
Endpoint *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
||||||
@@ -57,6 +95,30 @@ func BackendSource(label string, backends map[string]BackendConfig) configkit.So
|
|||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
||||||
}
|
}
|
||||||
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := make(map[string]any)
|
||||||
|
if config.SmokeTests.DefaultProfile != nil {
|
||||||
|
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
reachability := make(map[string]any)
|
||||||
|
if config.SmokeTests.Reachability.Endpoint != nil {
|
||||||
|
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["reachability"] = reachability
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
authentication := make(map[string]any)
|
||||||
|
if config.SmokeTests.Authentication.Kind != nil {
|
||||||
|
authentication["kind"] = *config.SmokeTests.Authentication.Kind
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication.Endpoint != nil {
|
||||||
|
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["authentication"] = authentication
|
||||||
|
}
|
||||||
|
fields["smoke_tests"] = smokeTests
|
||||||
|
}
|
||||||
definitions[id] = fields
|
definitions[id] = fields
|
||||||
}
|
}
|
||||||
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
||||||
|
|||||||
@@ -267,7 +267,7 @@ func newBackendCapacityEngine(
|
|||||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||||
"backends": map[string]any{"limited": backend},
|
"backends": map[string]any{"limited": backend},
|
||||||
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
||||||
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
|
"prompts": map[string]any{promptID: promptDefinitionWithVersion("1", definition)},
|
||||||
}),
|
}),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -23,13 +23,19 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||||
}},
|
}},
|
||||||
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"endpoint": "https://example.invalid/v1",
|
||||||
|
"extra_params": map[string]any{"token": "namespaced-private-sentinel"},
|
||||||
|
}},
|
||||||
"profiles": map[string]any{
|
"profiles": map[string]any{
|
||||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
||||||
"literal/id": map[string]any{"extends": "base.v1"},
|
"id": map[string]any{"extends": "base.v1"},
|
||||||
},
|
},
|
||||||
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||||
"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
|
"collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
@@ -40,17 +46,17 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
modelPath := configkit.Keys("profiles", "base.v1", "model")
|
modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model")
|
||||||
model, found := view.Explain(modelPath)
|
model, found := view.Explain(modelPath)
|
||||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
||||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||||
}
|
}
|
||||||
inheritedPath := configkit.Keys("profiles", "literal/id", "model")
|
inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model")
|
||||||
inherited, found := view.Explain(inheritedPath)
|
inherited, found := view.Explain(inheritedPath)
|
||||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||||
}
|
}
|
||||||
if inheritedPath.String() != "/profiles/literal~1id/model" {
|
if inheritedPath.String() != "/collections/literal/profiles/id/model" {
|
||||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||||
}
|
}
|
||||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
||||||
@@ -58,6 +64,11 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||||
}
|
}
|
||||||
|
namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token")
|
||||||
|
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||||
|
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||||
|
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||||
|
}
|
||||||
var effective bytes.Buffer
|
var effective bytes.Buffer
|
||||||
if err := view.WriteEffective(&effective); err != nil {
|
if err := view.WriteEffective(&effective); err != nil {
|
||||||
t.Fatalf("write effective configuration: %v", err)
|
t.Fatalf("write effective configuration: %v", err)
|
||||||
@@ -66,7 +77,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
t.Fatalf("unexpected effective report: %s", effective.String())
|
t.Fatalf("unexpected effective report: %s", effective.String())
|
||||||
}
|
}
|
||||||
var explanation bytes.Buffer
|
var explanation bytes.Buffer
|
||||||
if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
|
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
|
||||||
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +104,28 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("main", map[string]any{"version": "1"}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
|
}
|
||||||
|
path := configkit.Keys("version")
|
||||||
|
field, found := inspection.Explain(path)
|
||||||
|
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") {
|
||||||
|
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
|
||||||
|
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{ err error }
|
type failingInspectionWriter struct{ err error }
|
||||||
|
|
||||||
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||||
@@ -117,7 +150,13 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("zero explanation writer = %v", err)
|
t.Fatalf("zero explanation writer = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
valid, err := promptkit.NewEngine(promptkit.Config{})
|
endpoint, model := "https://configured.example/v1", "configured-model"
|
||||||
|
profile := configkit.Input{Source: promptkit.ProfileSource("inspection test profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"inspection-profile": {Endpoint: &endpoint, Model: &model},
|
||||||
|
})}
|
||||||
|
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -130,7 +169,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("effective writer error = %v", err)
|
t.Fatalf("effective writer error = %v", err)
|
||||||
}
|
}
|
||||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
|
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "inspection-profile", "model")); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("explanation writer error = %v", err)
|
t.Fatalf("explanation writer error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -199,7 +238,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
var report bytes.Buffer
|
var report bytes.Buffer
|
||||||
|
|||||||
@@ -8,9 +8,26 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CollectionInput mounts source at the selected collection's configuration
|
||||||
|
// destination, collections/<namespace>, and enables ConfigKit imports for that
|
||||||
|
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
|
||||||
|
// source is not read or validated until ConfigKit loads it. Callers may adjust
|
||||||
|
// the returned input, including its filesystem context or imports setting.
|
||||||
|
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
|
||||||
|
if !domain.IsValidResourceNamespace(namespace) {
|
||||||
|
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
|
||||||
|
}
|
||||||
|
return configkit.Input{
|
||||||
|
Source: source,
|
||||||
|
Destination: configkit.Keys("collections", namespace),
|
||||||
|
Imports: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||||
// directory as one strict-composition group. The returned order is deterministic;
|
// directory as one strict-composition group. The returned order is deterministic;
|
||||||
// it does not assign precedence among those files. Discovery is recursive over
|
// it does not assign precedence among those files. Discovery is recursive over
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -16,6 +18,78 @@ type sourceDocument struct {
|
|||||||
Second string `yaml:"second"`
|
Second string `yaml:"second"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if input.Destination.String() != "/collections/docs" || !input.Imports {
|
||||||
|
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
|
||||||
|
}
|
||||||
|
for _, namespace := range []string{"", " ", "docs/api"} {
|
||||||
|
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
mainPath := filepath.Join(directory, "collection.yaml")
|
||||||
|
defaultsPath := filepath.Join(directory, "defaults.yaml")
|
||||||
|
if err := os.WriteFile(mainPath, []byte("configkit:\n imports:\n - path: defaults.yaml\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(defaultsPath, []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
|
||||||
|
Destination: configkit.Keys("collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with native collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"collection.yaml": {Data: []byte("configkit:\n imports:\n - path: defaults.yaml\n")},
|
||||||
|
"defaults.yaml": {Data: []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
|
||||||
|
}
|
||||||
|
filesystem := configkit.NewFilesystem("embedded collection", files)
|
||||||
|
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
|
||||||
|
Destination: configkit.Keys("collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
@@ -72,11 +73,12 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||||
backendID, model := "workstation", "offline-model"
|
backendID, model := "workstation", "offline-model"
|
||||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
||||||
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
|
"prompts": map[string]any{"hello": map[string]any{
|
||||||
|
"version": "1",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})}
|
})}
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
@@ -105,6 +107,36 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||||
|
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
|
||||||
|
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
|
||||||
|
model := "configured-model"
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
|
{Source: promptkit.BackendSource("application backends", map[string]promptkit.BackendConfig{
|
||||||
|
openRouterID: {Endpoint: &openRouterEndpoint},
|
||||||
|
rakestrawhomeID: {Endpoint: &rakestrawhomeEndpoint},
|
||||||
|
})},
|
||||||
|
{Source: promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
|
||||||
|
"openrouter-profile": {Backend: &openRouterID, Model: &model},
|
||||||
|
"rakestrawhome-profile": {Backend: &rakestrawhomeID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
|
||||||
|
}
|
||||||
|
for profileID, backendID := range map[string]string{
|
||||||
|
"openrouter-profile": openRouterID,
|
||||||
|
"rakestrawhome-profile": rakestrawhomeID,
|
||||||
|
} {
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
|
||||||
|
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
||||||
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
||||||
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
@@ -114,7 +146,7 @@ profiles:
|
|||||||
model: default-model
|
model: default-model
|
||||||
prompts:
|
prompts:
|
||||||
hello:
|
hello:
|
||||||
"1":
|
version: "1"
|
||||||
default_profile: application
|
default_profile: application
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -189,43 +221,94 @@ func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
|
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
|
||||||
endpoint := "http://local-openrouter.example/v1"
|
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
|
{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
||||||
})}}}},
|
backendID: {Endpoint: &endpoint},
|
||||||
|
})},
|
||||||
|
{Source: promptkit.ProfileSource("operator profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"application-profile": {Backend: &backendID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine with backend overlay: %v", err)
|
t.Fatalf("construct engine with backend overlay: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
|
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect catalog profile: %v", err)
|
t.Fatalf("inspect configured profile: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
||||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
|
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||||
definition := func(content string) map[string]any {
|
backendEndpoint := "https://backend.example/v1"
|
||||||
return map[string]any{
|
defaultProfile := "not-yet-listed"
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
reachabilityEndpoint := " https://probe.example/ready?check=1 "
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
authenticationKind := " OPENROUTER_KEY "
|
||||||
}
|
authenticationEndpoint := "https://probe.example/key?scope=current"
|
||||||
}
|
source := promptkit.BackendSource("diagnostic backend", map[string]promptkit.BackendConfig{
|
||||||
|
"local": {
|
||||||
|
Endpoint: &backendEndpoint,
|
||||||
|
SmokeTests: &promptkit.BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &defaultProfile,
|
||||||
|
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &promptkit.AuthenticationProbeConfig{
|
||||||
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
backendEndpoint = "https://mutated.example/v1"
|
||||||
|
defaultProfile = "mutated-profile"
|
||||||
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
|
authenticationKind = "none"
|
||||||
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
|
||||||
"prompts": map[string]any{"hello": map[string]any{
|
|
||||||
"1": definition("first"), "2": definition("second"),
|
|
||||||
}},
|
|
||||||
})},
|
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
contents := effective.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"https://backend.example/v1",
|
||||||
|
"not-yet-listed",
|
||||||
|
"https://probe.example/ready?check=1",
|
||||||
|
"openrouter_key",
|
||||||
|
"https://probe.example/key?scope=current",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(contents, want) {
|
||||||
|
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{
|
||||||
|
"mutated.example",
|
||||||
|
"mutated-profile",
|
||||||
|
} {
|
||||||
|
if strings.Contains(contents, unwanted) {
|
||||||
|
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
|
||||||
}
|
}
|
||||||
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
|
}
|
||||||
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
|
}
|
||||||
|
|
||||||
|
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "selected")
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
|
||||||
|
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,11 +63,12 @@ func simpleConfig(content string) promptkit.Config {
|
|||||||
"profiles": map[string]any{"profile": map[string]any{
|
"profiles": map[string]any{"profile": map[string]any{
|
||||||
"endpoint": "http://example.test/v1", "model": "model",
|
"endpoint": "http://example.test/v1", "model": "model",
|
||||||
}},
|
}},
|
||||||
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
|
"prompts": map[string]any{"prompt": map[string]any{
|
||||||
|
"version": "1",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|||||||
15
convert.go
15
convert.go
@@ -25,7 +25,6 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
}
|
}
|
||||||
return domain.RunRequest{
|
return domain.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
@@ -73,7 +72,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
}
|
}
|
||||||
return &PreparedRun{
|
return &PreparedRun{
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: copyStringPtr(prepared.PromptVersion),
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
@@ -100,7 +99,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
RawOutput: result.RawOutput,
|
RawOutput: result.RawOutput,
|
||||||
Validation: fromDomainValidationResult(result.Validation),
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
PromptVersion: copyStringPtr(result.PromptVersion),
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
SessionID: result.SessionID,
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
@@ -211,6 +210,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
|
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
|
||||||
|
// ownership of extraParams, which the caller has already copied.
|
||||||
|
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
BackendID: target.BackendID,
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
@@ -222,7 +227,7 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|||||||
ServiceTier: target.ServiceTier,
|
ServiceTier: target.ServiceTier,
|
||||||
ReasoningEffort: target.ReasoningEffort,
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
APIKeyEnv: target.APIKeyEnv,
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
ExtraParams: extraParams,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,7 +257,7 @@ func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInsp
|
|||||||
}
|
}
|
||||||
return &PromptInspection{
|
return &PromptInspection{
|
||||||
PromptID: inspection.PromptID,
|
PromptID: inspection.PromptID,
|
||||||
PromptVersion: inspection.PromptVersion,
|
PromptVersion: copyStringPtr(inspection.PromptVersion),
|
||||||
PromptHash: inspection.PromptHash,
|
PromptHash: inspection.PromptHash,
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
DefaultProfileID: inspection.DefaultProfileID,
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
|
|||||||
165
docs/consumers/configuration-migration.md
Normal file
165
docs/consumers/configuration-migration.md
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
# Configuration And API Migration
|
||||||
|
|
||||||
|
Promptkit no longer installs a provider catalog or selects prompt definitions
|
||||||
|
by descriptive version labels. Applications now choose configuration inputs
|
||||||
|
explicitly, and execution selects one prompt by its exact ID. This guide
|
||||||
|
explains the shape changes and decisions an application must make; it is not an
|
||||||
|
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||||
|
the current configuration contract, and the root package GoDoc owns exact API
|
||||||
|
signatures and errors.
|
||||||
|
|
||||||
|
## Select And Compose The Configuration You Need
|
||||||
|
|
||||||
|
Supply each backend, profile, prompt, schema, and collection source that the
|
||||||
|
application intends to use. Promptkit does not discover directories or install
|
||||||
|
provider definitions implicitly. An application may select an explicit
|
||||||
|
collection input with `CollectionInput`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
input, err := promptkit.CollectionInput(
|
||||||
|
"openrouter",
|
||||||
|
configkit.File("config/openrouter.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Collection resource keys are local keys, while their runtime identities are
|
||||||
|
qualified. For example, `collections.openrouter.profiles.gemini-flash-latest`
|
||||||
|
is selected as `openrouter/gemini-flash-latest`. Stored backend, prompt-default,
|
||||||
|
and other resource references must use their exact intended IDs; Promptkit does
|
||||||
|
not rewrite references when a source is mounted. Profile `extends` remains a
|
||||||
|
sibling local key in the same collection, not a cross-collection reference.
|
||||||
|
Review collisions and all references when moving resources into collections.
|
||||||
|
|
||||||
|
The root `version` is an optional ConfigKit source header. A collection's
|
||||||
|
`release_version` and a prompt's `version` are optional descriptive labels;
|
||||||
|
neither selects a resource or changes execution. Preserve a label only when it
|
||||||
|
is useful to your application. A formerly version-keyed prompt mapping must be
|
||||||
|
reduced to one deliberately chosen definition per prompt ID, with its old label
|
||||||
|
retained as prompt metadata if desired. Promptkit cannot choose a winner from a
|
||||||
|
consumer's version history.
|
||||||
|
|
||||||
|
For example, change a version-keyed definition:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
report:
|
||||||
|
"2.0":
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
to one direct definition. Its optional `version` is only a label:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
report:
|
||||||
|
version: "2.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
## Identity Mapping
|
||||||
|
|
||||||
|
The reviewed catalog identities have slash-free profile keys. Their documented
|
||||||
|
mapping is:
|
||||||
|
|
||||||
|
| Previous identity | Namespaced identity or action |
|
||||||
|
| --- | --- |
|
||||||
|
| Backend `openrouter` | `openrouter/default` |
|
||||||
|
| Backend `rakestrawhome` | `rakestrawhome/default` |
|
||||||
|
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
|
||||||
|
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
|
||||||
|
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
|
||||||
|
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
|
||||||
|
|
||||||
|
These are conversion examples, not aliases or bundled resources. Mounting old
|
||||||
|
`/v3` assets does not update backend references, profile inheritance, prompt
|
||||||
|
defaults, or diagnostic defaults inside them. Edit those values to the exact
|
||||||
|
qualified IDs you chose.
|
||||||
|
|
||||||
|
For example, an optional caller-owned standard profile may extend a selected
|
||||||
|
Gemini profile in the same `openrouter` collection:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
standard:
|
||||||
|
extends: gemini-flash-latest
|
||||||
|
```
|
||||||
|
|
||||||
|
An optional auto profile may instead select the exact backend and model values:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
auto:
|
||||||
|
backend: openrouter/default
|
||||||
|
model: openrouter/auto
|
||||||
|
```
|
||||||
|
|
||||||
|
Neither optional profile name is a Promptkit contract. Keep provider model
|
||||||
|
strings and settings as selected by the application, including any meaningful
|
||||||
|
prefixes. If a backend declares diagnostic defaults, update them to exact
|
||||||
|
qualified IDs; an unselected diagnostic default is not automatically made
|
||||||
|
resolvable by migration.
|
||||||
|
|
||||||
|
## Prompt Selection API
|
||||||
|
|
||||||
|
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
|
||||||
|
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
|
||||||
|
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
|
||||||
|
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
|
||||||
|
explicitly in the request rather than setting a version selector:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "support/report",
|
||||||
|
ProfileID: "support/standard",
|
||||||
|
}
|
||||||
|
_, err = engine.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If the former format contained multiple versions under one ID, choose the one
|
||||||
|
definition the application should install, and optionally keep its label in
|
||||||
|
the prompt's `version` field. That label remains descriptive metadata and is
|
||||||
|
reported by inspection or results where applicable; it never chooses the
|
||||||
|
definition.
|
||||||
|
|
||||||
|
The public inventory APIs can help an operator inspect a captured effective
|
||||||
|
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
|
||||||
|
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
|
||||||
|
namespaces and effective collection declarations. Inventory does not select
|
||||||
|
which configuration version the application should deploy.
|
||||||
|
|
||||||
|
## Deployment Selection Remains With The Application
|
||||||
|
|
||||||
|
Applications decide which checked-in, embedded, operator-supplied, or
|
||||||
|
programmatic inputs to compose for each deployment. They also decide when to
|
||||||
|
construct a replacement engine after changing those sources. Promptkit does
|
||||||
|
not fetch catalog updates, select provider availability, validate upstream
|
||||||
|
catalog release status, or automatically migrate arbitrary consumer files.
|
||||||
|
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
|
||||||
|
shows explicit source selection, local imports, layering, a scoped view, and
|
||||||
|
operator inventory without a provider call.
|
||||||
@@ -15,6 +15,12 @@ Import the package as:
|
|||||||
import "gitea.maximumdirect.net/eric/promptkit"
|
import "gitea.maximumdirect.net/eric/promptkit"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
|
demonstrates explicit collection composition and inventory. Applications
|
||||||
|
adapting older provider or versioned-prompt configuration can use the
|
||||||
|
[configuration migration guide](configuration-migration.md) for the required
|
||||||
|
caller-owned choices.
|
||||||
|
|
||||||
The following Go fragments are illustrative and omit surrounding package,
|
The following Go fragments are illustrative and omit surrounding package,
|
||||||
import, and error-handling code. Use the maintained examples for complete
|
import, and error-handling code. Use the maintained examples for complete
|
||||||
programs.
|
programs.
|
||||||
@@ -63,6 +69,34 @@ new engine to observe configuration edits. See the [constructor GoDoc](../../eng
|
|||||||
for the exact public contract and the [framework format reference](../formats.md)
|
for the exact public contract and the [framework format reference](../formats.md)
|
||||||
for definition fields.
|
for definition fields.
|
||||||
|
|
||||||
|
### Use Scoped Engine Views
|
||||||
|
|
||||||
|
Create a view when an application wants short request IDs to resolve within a
|
||||||
|
selected collection:
|
||||||
|
|
||||||
|
```go
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||||
|
Prompt: "support",
|
||||||
|
Profile: "support",
|
||||||
|
})
|
||||||
|
if err != nil { return err }
|
||||||
|
|
||||||
|
prepared, err := scoped.Prepare(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "answer",
|
||||||
|
ProfileID: "concise",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Execution, preparation, prompt/profile inspection, requirement inspection, and
|
||||||
|
smoke diagnostics qualify slashless caller selections with their respective
|
||||||
|
namespaces. Already-qualified IDs pass through unchanged; configured defaults
|
||||||
|
and references are not rewritten. Each call replaces the view's scope defaults,
|
||||||
|
including clearing a scope with an empty field, without changing the receiver
|
||||||
|
or rebuilding its captured configuration and execution collaborators.
|
||||||
|
Namespace syntax is checked when creating the view; resource existence is
|
||||||
|
checked when selected. Configuration inspection remains global. See
|
||||||
|
[`WithNamespaces`](../../namespaces.go) for the exact public contract.
|
||||||
|
|
||||||
## Supply Embedded Application Defaults
|
## Supply Embedded Application Defaults
|
||||||
|
|
||||||
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
||||||
@@ -109,8 +143,8 @@ input := configkit.Input{Source: document}
|
|||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||||
is not needed. A caller underlay can overlay catalog backend fields just as it
|
is not needed. Backend and profile definitions are supplied by the caller;
|
||||||
can overlay profiles; there is no registration-only backend namespace.
|
there is no registration-only backend namespace.
|
||||||
|
|
||||||
When configuration or message files change, build and validate a new engine,
|
When configuration or message files change, build and validate a new engine,
|
||||||
then switch the application to that engine for subsequent work. Existing
|
then switch the application to that engine for subsequent work. Existing
|
||||||
@@ -118,11 +152,15 @@ engines and prepared handles retain their captured definitions; schema and
|
|||||||
request artifact resources still follow their runtime lifecycle. Inspection
|
request artifact resources still follow their runtime lifecycle. Inspection
|
||||||
does not watch or reload sources automatically.
|
does not watch or reload sources automatically.
|
||||||
|
|
||||||
## Discover Available Profiles And Prompts
|
## Discover Available Resources And Collections
|
||||||
|
|
||||||
Use [`Engine.ListProfiles`](../../listing.go) and
|
Use [`Engine.ListProfiles`](../../listing.go),
|
||||||
[`Engine.ListPrompts`](../../listing.go) to discover exact identities from the
|
[`Engine.ListPrompts`](../../listing.go), and
|
||||||
engine's final captured configuration, then inspect a selected prompt:
|
[`Engine.ListBackends`](../../listing.go) to discover exact identities from the
|
||||||
|
engine's final captured configuration. A scoped view filters by its respective
|
||||||
|
namespace by default; pass `NamespaceNamed`, `NamespaceUnqualified`, or
|
||||||
|
`NamespaceAll` to override that listing filter without changing execution
|
||||||
|
scope. Returned IDs remain full identities and can be used for exact selection:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
profiles, err := engine.ListProfiles(ctx)
|
profiles, err := engine.ListProfiles(ctx)
|
||||||
@@ -141,7 +179,7 @@ if err != nil {
|
|||||||
}
|
}
|
||||||
if len(prompts) > 0 {
|
if len(prompts) > 0 {
|
||||||
selected := prompts[0]
|
selected := prompts[0]
|
||||||
inspection, err := engine.InspectPrompt(ctx, selected.ID, selected.Version)
|
inspection, err := engine.InspectPrompt(ctx, selected.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -149,6 +187,12 @@ if len(prompts) > 0 {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Use [`Engine.ListNamespaces`](../../listing.go) to discover resource-bearing
|
||||||
|
namespaces across the entire captured configuration, independent of view
|
||||||
|
scopes. [`Engine.ListCollections`](../../listing.go) reports effective
|
||||||
|
collection declarations, including metadata-only entries; it does not infer
|
||||||
|
collection metadata from resource membership or source history.
|
||||||
|
|
||||||
These lists describe effective membership, not ConfigKit contribution history,
|
These lists describe effective membership, not ConfigKit contribution history,
|
||||||
and do not promise that a profile is ready to execute. Use inspection for
|
and do not promise that a profile is ready to execute. Use inspection for
|
||||||
details and preparation to resolve request resources. The linked method GoDoc
|
details and preparation to resolve request resources. The linked method GoDoc
|
||||||
@@ -208,7 +252,7 @@ for _, difference := range differences {
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
|
and ConfigKit's [v0.8.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs)
|
||||||
for the exact comparison contract. The operation compares only frozen effective
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
configuration values: it does not reread external files or compare their
|
configuration values: it does not reread external files or compare their
|
||||||
contents, resolved credential contents, later execution preparation, or
|
contents, resolved credential contents, later execution preparation, or
|
||||||
@@ -216,6 +260,69 @@ request-specific overrides. An empty diff is not proof of equivalent execution.
|
|||||||
Records can contain ordinary application data, so use the application's
|
Records can contain ordinary application data, so use the application's
|
||||||
disclosure policy when presenting them.
|
disclosure policy when presenting them.
|
||||||
|
|
||||||
|
## Run Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
[`Engine.SmokeTest`](../../smoke_diagnostics.go) runs one explicit diagnostic
|
||||||
|
using a target resolved from the engine's captured configuration. It does not
|
||||||
|
execute the selected application prompt or its inputs, session, artifacts, or
|
||||||
|
output schema. Unlike the offline examples and default test suite, calling it
|
||||||
|
can contact the selected provider and use configured credentials. It is a
|
||||||
|
consumer operation, not an automatic readiness verdict or a release check.
|
||||||
|
|
||||||
|
For example, a caller can request the built-in text inference check and treat
|
||||||
|
its optional fact as an observation rather than assuming that a nil Go error
|
||||||
|
means success:
|
||||||
|
|
||||||
|
```go
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
|
||||||
|
Kind: promptkit.SmokeTestTextInference,
|
||||||
|
ProfileID: profileID,
|
||||||
|
APIKey: apiKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if report != nil {
|
||||||
|
retainDisclosureSafeSummary(report)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if report != nil {
|
||||||
|
matched := report.Facts.ExpectedTextMatched
|
||||||
|
switch {
|
||||||
|
case matched == nil:
|
||||||
|
// The comparison was not evaluated or is unknown/inapplicable.
|
||||||
|
case *matched:
|
||||||
|
// The private expected-text comparison matched.
|
||||||
|
default:
|
||||||
|
// It did not match; the operation may still have concluded normally.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A nil Go error means the diagnostic concluded, not that its finding was
|
||||||
|
positive. Fact pointers are tri-state: nil means unknown, not evaluated, or
|
||||||
|
inapplicable. Reports contain factual observations and effective settings, not
|
||||||
|
generated text or provider bodies; identifiers and settings can still be
|
||||||
|
sensitive. The
|
||||||
|
[`SmokeTest` GoDoc](../../smoke_diagnostics.go) owns selection precedence,
|
||||||
|
exact fields, timeout and retry behavior, supported transports, and disclosure
|
||||||
|
details.
|
||||||
|
|
||||||
|
Caller cancellation can return both an error and a partial report. Preserve the
|
||||||
|
context error identity and inspect any available observations before returning:
|
||||||
|
|
||||||
|
```go
|
||||||
|
report, err := engine.SmokeTest(ctx, request)
|
||||||
|
if report != nil {
|
||||||
|
retainDisclosureSafeSummary(report)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
// The report may describe work completed before interruption.
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## Inspect A Prompt Before Preparation
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||||
@@ -223,7 +330,7 @@ declared inputs and output workflow without creating placeholder inputs or
|
|||||||
resolving a profile:
|
resolving a profile:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
inspection, err := engine.InspectPrompt(ctx, "meeting.summary")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -530,96 +637,28 @@ are keys in configuration, not registrations. Choose caller underlays in
|
|||||||
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
||||||
highest-priority input.
|
highest-priority input.
|
||||||
|
|
||||||
Move embedded `id` and `version` fields into keyed `profiles` and `prompts`
|
Move embedded IDs into keyed `profiles` and `prompts` maps; each prompt ID now
|
||||||
maps; convert `base_profile` to ConfigKit `extends`. IDs and references are
|
contains one installed definition, with any descriptive `version` label inside
|
||||||
exact literal keys, without trimming. A supplied zero, false, or empty string
|
that definition. Convert `base_profile` to ConfigKit `extends`. For the current
|
||||||
is now distinguishable from omission; null can clear or suppress an inherited
|
collection identity mapping, ID-only prompt selection, and required
|
||||||
field according to the [field table](../formats.md#defaults-and-overrides).
|
caller-owned choices, see the
|
||||||
Maps merge recursively across inheritance and layers. `{}` does not clear an
|
[configuration migration guide](configuration-migration.md). Exact current
|
||||||
inherited map: use null to clear the whole map. At request time, a literal
|
format, reference, inheritance, and presence semantics belong to the
|
||||||
`extra_params["extends"]` is ordinary provider data, not an inheritance
|
[framework format reference](../formats.md). Prompt text and static templates
|
||||||
instruction; a nonempty request ExtraParams map replaces configured params.
|
are captured at construction, not automatically reloaded between runs. Replace
|
||||||
Changing a model or backend does not clear unrelated explicit fields, though
|
the engine to pick up changed definitions or message files.
|
||||||
backend-derived defaults are applied after inheritance to the final backend.
|
|
||||||
|
|
||||||
Invalid final definitions, including unused ones, now fail `NewEngine`; only
|
|
||||||
request-dependent failures remain at preparation or execution. Prompt text
|
|
||||||
and static templates are captured at construction, not automatically reloaded
|
|
||||||
between runs. Replace the engine to pick up changed definitions or message
|
|
||||||
files. The [format reference](../formats.md) owns exact field and presence
|
|
||||||
rules; the maintained examples show complete runnable workflows.
|
|
||||||
|
|
||||||
### Choose A Profile Ownership Model
|
### Choose A Profile Ownership Model
|
||||||
|
|
||||||
Choose the profile source that matches the application's continuity needs:
|
Supply backend and profile definitions through the configuration sources your
|
||||||
|
application selects. PromptKit does not install provider catalogs or built-in
|
||||||
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
|
profile inventories; conventional backend ID constants are names only. Define
|
||||||
the maintained literal name is the desired consumer contract. Their current
|
the model, endpoint, credentials lookup, and settings that your application
|
||||||
selections are in the [format reference](../formats.md#built-in-profile-catalog).
|
intends to support. A profile may inherit a sibling profile by its local key in
|
||||||
- Select an external catalog profile directly when the application accepts that
|
the same root or collection mapping; its backend reference uses the exact
|
||||||
profile's catalog volatility. The selected external inventories are linked
|
configured identity. Caller-selected source layers determine the final values. The
|
||||||
from the [format reference](../formats.md#external-profile-inventories).
|
[format reference](../formats.md) describes the definition fields and
|
||||||
- Define a complete application-owned profile with an explicit backend or
|
composition rules.
|
||||||
endpoint, model, and settings when the application owns those choices.
|
|
||||||
|
|
||||||
An application profile that inherits an external base follows later changes to
|
|
||||||
that base and fails if the base disappears; it is not a frozen copy. Promptkit
|
|
||||||
tests the catalog versions selected in its module graph. Independently advancing
|
|
||||||
a catalog dependency can remove an inherited base and produce an untested
|
|
||||||
combination. Pinning the resolved module graph preserves the embedded data for
|
|
||||||
a build, but cannot preserve provider availability. Running binaries neither
|
|
||||||
download catalog updates nor automatically replace withdrawn models. See the
|
|
||||||
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
|
|
||||||
cross-version guarantees.
|
|
||||||
|
|
||||||
### Alias A Built-In Profile
|
|
||||||
|
|
||||||
Give an application-owned profile ID a built-in base when prompts should select
|
|
||||||
the application ID while inheriting the built-in target. The child can override
|
|
||||||
only the setting it owns:
|
|
||||||
|
|
||||||
```go
|
|
||||||
profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
|
|
||||||
"weather-light": {
|
|
||||||
Extends: &standardID,
|
|
||||||
ReasoningEffort: &high,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
Place `profile` in a ConfigKit underlay, with `standardID` set to
|
|
||||||
`"promptkit-openrouter-standard"` and `high` to `"high"`. Select
|
|
||||||
`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
|
||||||
reported selected profile. Definition IDs and inheritance references are exact
|
|
||||||
keys. If the base is an external profile, apply the volatility guidance
|
|
||||||
above when deciding whether to use an application-owned complete definition
|
|
||||||
instead.
|
|
||||||
|
|
||||||
### Use The Rakestrawhome Built-In Profile
|
|
||||||
|
|
||||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
|
||||||
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
|
||||||
prepared result identifies the selected built-in through
|
|
||||||
`BackendRakestrawHome`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
|
||||||
PromptID: "meeting.summary",
|
|
||||||
ProfileID: "rakestrawhome-gemma-4-31b",
|
|
||||||
Inputs: inputs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
|
||||||
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The maintained catalog supplies this backend, and callers may overlay its
|
|
||||||
effective configuration through a later ConfigKit source. Direct request credentials
|
|
||||||
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
|
||||||
format contracts.
|
|
||||||
|
|
||||||
### Inspect A Profile Before Prompt Work
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
|
|||||||
@@ -43,13 +43,13 @@ Before starting work:
|
|||||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
| Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
|
||||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
| Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||||
|
|
||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
|
|||||||
202
docs/formats.md
202
docs/formats.md
@@ -10,12 +10,14 @@ owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/Co
|
|||||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||||
This reference describes what Promptkit does with the final effective values.
|
This reference describes what Promptkit does with the final effective values.
|
||||||
|
|
||||||
A configuration document has keyed top-level `backends`, `profiles`, and
|
A configuration document may contain keyed top-level `backends`, `profiles`,
|
||||||
`prompts` mappings. Their keys are literal IDs, not filename-derived identities.
|
and `prompts` mappings, plus a `collections` mapping of namespaced resource
|
||||||
Prompt versions are a second keyed mapping. Keys and references must be
|
groups. Resource keys are literal IDs, not filename-derived identities. Each
|
||||||
nonblank and remain exact: Promptkit does not trim or re-key them. Old embedded
|
prompt ID maps directly to one definition; an optional `version` string is
|
||||||
`id`, `version`, and `base_profile` fields are invalid. Unknown fields are
|
descriptive metadata, not a selection key. Keys and references must be nonblank
|
||||||
errors.
|
and remain exact: Promptkit does not trim or re-key them. Old embedded `id` and
|
||||||
|
`base_profile` fields are invalid. Unknown fields are errors. Promptkit installs
|
||||||
|
no provider or profile catalog; callers select the definitions they need.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backends:
|
backends:
|
||||||
@@ -28,7 +30,7 @@ profiles:
|
|||||||
model: example-model
|
model: example-model
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
meeting.summary:
|
||||||
"1.0.0":
|
version: "1.0.0"
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -36,8 +38,66 @@ prompts:
|
|||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
|
collections:
|
||||||
|
support:
|
||||||
|
release_version: "1.4.0"
|
||||||
|
backends:
|
||||||
|
offline:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
base:
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
|
fast:
|
||||||
|
extends: base
|
||||||
|
timeout_seconds: 30
|
||||||
|
prompts:
|
||||||
|
answer:
|
||||||
|
default_profile: support/fast
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Answer the selected caller input.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Source Headers, Collections, And Resource IDs
|
||||||
|
|
||||||
|
An input may omit a source header. When present, the source-root `version` is
|
||||||
|
reserved for ConfigKit's specification header and accepts only the exact string
|
||||||
|
`"1"` or strict decimal integer `1`; null and other values are invalid. This
|
||||||
|
header is distinct from a prompt's nested `version` label and a collection's
|
||||||
|
`release_version`. ConfigKit checks and consumes accepted headers on non-main
|
||||||
|
inputs; a declaration on the main input remains visible in explicit
|
||||||
|
configuration inspection. Headerless inputs still undergo ordinary final-value
|
||||||
|
validation. ConfigKit owns generic import and source-history behavior.
|
||||||
|
|
||||||
|
The root resource mappings contain unqualified IDs. Each `collections` entry
|
||||||
|
has a namespace key and may contain `release_version` plus the same three
|
||||||
|
resource mappings. Collection resource keys are local keys; their runtime IDs
|
||||||
|
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
|
||||||
|
namespace and local key must be nonblank and slash-free, and references must
|
||||||
|
name their intended exact ID; mounting a source does not rewrite references.
|
||||||
|
For example, the `support/fast` profile above stores `support/offline` as its
|
||||||
|
backend ID. In `collections.support.profiles.fast`, ConfigKit `extends: base`
|
||||||
|
refers to a sibling profile in that same mapping. It is not a qualified or
|
||||||
|
cross-collection reference.
|
||||||
|
|
||||||
|
`release_version` and prompt `version` are optional opaque labels. Absent or
|
||||||
|
null means no label; an explicitly empty string is retained as a declared
|
||||||
|
label. Neither label selects a resource or changes execution. Runtime lookups
|
||||||
|
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||||
|
explicit operator view of effective settings and provenance; source history is
|
||||||
|
not a second runtime resolver, and the retained effective shape is not flattened
|
||||||
|
or rewritten into those indexes.
|
||||||
|
|
||||||
|
For examples of explicitly selecting native collection inputs and inspecting
|
||||||
|
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
|
||||||
|
[maintained preparation example](../examples/go-library/prepare/main.go). For
|
||||||
|
breaking identity and prompt-selection changes, see the
|
||||||
|
[configuration migration guide](consumers/configuration-migration.md).
|
||||||
|
|
||||||
Every effective backend, profile, and prompt definition is validated when
|
Every effective backend, profile, and prompt definition is validated when
|
||||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
||||||
be partial: an endpoint and model need only be complete after request overrides
|
be partial: an endpoint and model need only be complete after request overrides
|
||||||
@@ -52,16 +112,16 @@ Schema and request artifact reads still happen during preparation. Use
|
|||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||||
for request-specific rendered details. For identity discovery before inspection,
|
for request-specific rendered details. For identity discovery before inspection,
|
||||||
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
|
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The `prompts` mapping selects a literal prompt ID, then a string version key:
|
The `prompts` mapping selects a literal prompt ID whose value is the definition:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
meeting.summary:
|
||||||
"1.0.0":
|
version: "1.0.0"
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
description: Summarize a synthetic meeting note.
|
description: Summarize a synthetic meeting note.
|
||||||
session_id: '{{.session}}'
|
session_id: '{{.session}}'
|
||||||
@@ -89,9 +149,8 @@ prompts:
|
|||||||
fields may be absent or cleared with null where appropriate. A null whole
|
fields may be absent or cleared with null where appropriate. A null whole
|
||||||
definition, required role, output format, or validation mode is invalid.
|
definition, required role, output format, or validation mode is invalid.
|
||||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
`required` and `repair_attempts` default to false and zero, respectively, and
|
||||||
are not nullable. A request that omits a prompt version succeeds only if that
|
are not nullable. `InspectPrompt` selects the same captured definition without
|
||||||
prompt ID has exactly one effective version. `InspectPrompt` selects from the
|
rendering or resolving a profile.
|
||||||
same captured definitions without rendering or resolving a profile.
|
|
||||||
|
|
||||||
### Inputs
|
### Inputs
|
||||||
|
|
||||||
@@ -156,28 +215,64 @@ last candidate and its failed validation result. Operational errors remain
|
|||||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
errors. A request-level `OutputContract` replaces the whole configured one,
|
||||||
not individual fields; an empty request format defaults to `text`.
|
not individual fields; an empty request format defaults to `text`.
|
||||||
|
|
||||||
## Built-In Backends
|
## Backend Definitions
|
||||||
|
|
||||||
The selected catalog modules contribute these backend definitions as the
|
Backend definitions are supplied through caller-selected configuration
|
||||||
earliest underlay. Their settings can be overlaid by caller sources; these IDs
|
sources. The conventional backend IDs are names only; PromptKit does not
|
||||||
are not reserved against configuration changes.
|
install definitions for them.
|
||||||
|
|
||||||
| ID | Catalog endpoint | API-key environment name | Active generation limit |
|
|
||||||
| --- | --- | --- | ---: |
|
|
||||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 |
|
|
||||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 |
|
|
||||||
|
|
||||||
A backend definition may have `endpoint`, `api_key_env`,
|
A backend definition may have `endpoint`, `api_key_env`,
|
||||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. The final endpoint
|
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||||
is required and must be an absolute HTTP(S) URL with a host, no user
|
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||||
information, query, or fragment. An omitted or null API-key environment name
|
user information, query, or fragment. An omitted or null API-key environment
|
||||||
means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
||||||
negative value is invalid. For positive concurrency, omitted/null queue
|
negative value is invalid. For positive concurrency, omitted/null queue
|
||||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
||||||
|
|
||||||
|
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||||
|
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backends:
|
||||||
|
openrouter:
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: application-profile
|
||||||
|
reachability:
|
||||||
|
endpoint: https://status.example/ready?check=1
|
||||||
|
authentication:
|
||||||
|
kind: openrouter_key
|
||||||
|
endpoint: https://openrouter.ai/api/v1/key
|
||||||
|
```
|
||||||
|
|
||||||
|
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||||
|
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||||
|
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||||
|
`default_profile` means there is no inference default. Omitted/null
|
||||||
|
`reachability` or its endpoint uses the effective backend endpoint. An explicit
|
||||||
|
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||||
|
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||||
|
means fallback, while an explicit blank value is invalid.
|
||||||
|
|
||||||
|
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||||
|
non-null object requires `kind`, which is trimmed and case-normalized; the
|
||||||
|
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
|
||||||
|
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||||
|
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||||
|
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||||
|
These values are configuration only: engine construction does not contact a
|
||||||
|
backend.
|
||||||
|
|
||||||
|
These fields are resolved through the ordinary ConfigKit source layers before
|
||||||
|
Promptkit validates the final effective values. A higher-priority mapping can
|
||||||
|
override individual nested fields; an explicit null can clear the whole
|
||||||
|
`smoke_tests` block or one of its nested objects. An endpoint set to null uses
|
||||||
|
its documented fallback, while an explicit blank endpoint is invalid. The
|
||||||
|
metadata does not change ordinary generation credential requirements or
|
||||||
|
provider settings.
|
||||||
|
|
||||||
## Profile Definitions
|
## Profile Definitions
|
||||||
|
|
||||||
Profiles are keyed by literal ID and may inherit a named parent with
|
Profiles are keyed by literal ID and may inherit a named parent with
|
||||||
@@ -238,57 +333,36 @@ does not disable the caller context or HTTP transport timeout.
|
|||||||
|
|
||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
`NewEngine` composes catalog definitions first, Promptkit-owned definitions
|
`NewEngine` composes caller underlays in their given order, followed by an
|
||||||
next, then caller underlays in their given order, with an optional main input
|
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||||
last. Later tiers overlay earlier ones. Inputs within one underlay group are
|
underlay group are strictly composed: conflicting contributions are errors,
|
||||||
strictly composed: conflicting contributions are errors, not priority
|
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||||
signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
|
||||||
in sorted order, but that order does not give one file higher precedence.
|
in sorted order, but that order does not give one file higher precedence.
|
||||||
An explicitly selected missing/unreadable root fails; an empty root contributes
|
An explicitly selected missing/unreadable root fails; an empty root contributes
|
||||||
nothing. Imports are disabled by these helpers unless the caller enables them
|
nothing. Imports are disabled by these helpers unless the caller enables them
|
||||||
on an input. Single files can use ConfigKit inputs directly.
|
on an input. Single files can use ConfigKit inputs directly.
|
||||||
|
|
||||||
Backends, profiles, and prompts all follow these same source rules. A later
|
Root backends, profiles, and prompts, as well as collection declarations and
|
||||||
profile or backend can change individual fields of a lower definition. A
|
their resource mappings, follow these same source rules. `CollectionInput`
|
||||||
request `ProfileID` takes precedence over a prompt `default_profile`; if
|
mounts one explicitly selected source at a collection destination and enables
|
||||||
|
ConfigKit imports for that input; it does not discover or select sources.
|
||||||
|
A later profile or backend can change individual fields of a lower definition.
|
||||||
|
A request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||||
neither is selected, preparation fails. Exact profile inspection selects one
|
neither is selected, preparation fails. Exact profile inspection selects one
|
||||||
explicit ID and does not consult a prompt default.
|
explicit ID and does not consult a prompt default.
|
||||||
|
|
||||||
### Profile Inheritance
|
### Profile Inheritance
|
||||||
|
|
||||||
ConfigKit resolves `extends` on effective profile definitions. One profile
|
ConfigKit resolves `extends` on effective profile definitions. One profile
|
||||||
may inherit another keyed profile; presence, explicit null, and recursively
|
may inherit a sibling profile by local key in the same root or collection
|
||||||
merged maps determine the final fields. References are literal IDs, including
|
mapping; collection inheritance cannot cross namespaces. Presence, explicit
|
||||||
IDs containing punctuation. There is no Promptkit-specific depth limit or
|
null, and recursively merged maps determine the final fields. Other resource
|
||||||
second base-chain resolver. Missing references and cycles fail at construction.
|
references are exact IDs and use their qualified form when they select a
|
||||||
For generic inheritance and layer mechanics, consult
|
collection resource. There is no Promptkit-specific depth limit or second
|
||||||
|
base-chain resolver. Missing references and cycles fail at construction. For
|
||||||
|
generic inheritance and layer mechanics, consult
|
||||||
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
||||||
|
|
||||||
## Built-In Profile Catalog
|
|
||||||
|
|
||||||
Promptkit contributes two opt-in definitions; neither is selected by default:
|
|
||||||
|
|
||||||
| ID | Current definition |
|
|
||||||
| --- | --- |
|
|
||||||
| `promptkit-openrouter-standard` | Extends the selected OpenRouter catalog's `gemini-flash-latest` profile. |
|
|
||||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
|
||||||
|
|
||||||
Select one through request `ProfileID` or prompt `default_profile`.
|
|
||||||
Caller layers may overlay these definitions or their catalog bases. The
|
|
||||||
[release policy](policy/release.md#catalog-profile-compatibility) owns their
|
|
||||||
cross-version compatibility boundary.
|
|
||||||
|
|
||||||
## External Profile Inventories
|
|
||||||
|
|
||||||
The selected catalog modules, not this document, own their profile inventories:
|
|
||||||
|
|
||||||
- [OpenRouter catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
- [Rakestrawhome catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
|
|
||||||
These are tested local data, not a promise of live provider availability.
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns
|
|
||||||
the compatibility boundary.
|
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||||
|
|||||||
@@ -45,6 +45,45 @@ precedence over the backend default. Only the name reaches prepared metadata;
|
|||||||
the environment value is read just before the provider call and is never added
|
the environment value is read just before the provider call and is never added
|
||||||
to the JSON body.
|
to the JSON body.
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
Promptkit's explicit smoke diagnostics are separate consumer operations; they
|
||||||
|
are not part of ordinary `Run`, the offline examples, or release validation.
|
||||||
|
With the built-in client, inference diagnostics use the same configured client
|
||||||
|
and `/chat/completions` request as generation. Their content cases are private
|
||||||
|
to Promptkit and are not application prompt, schema, or provider protocol
|
||||||
|
extensions. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
|
||||||
|
report facts, and injected-client behavior.
|
||||||
|
|
||||||
|
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||||
|
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||||
|
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||||
|
is selected from backend metadata, not inferred from the host. The
|
||||||
|
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||||
|
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||||
|
the [backend format reference](../formats.md#backend-definitions)). It may send
|
||||||
|
the available bearer credential, but can dispatch without one. A 401 is
|
||||||
|
recognized as rejection. A 200 is recognized as acceptance only when its JSON
|
||||||
|
object has a `data` object containing string `label`, Boolean `is_free_tier`,
|
||||||
|
and numeric `usage`; unrelated fields and account limits are ignored. This is
|
||||||
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
|
200 responses do not establish acceptance.
|
||||||
|
|
||||||
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
|
selected destination without URL user information, response status, and
|
||||||
|
credential presence when observed. Redirects can change the last observed
|
||||||
|
destination or strip credentials. This evidence does not prove that bytes
|
||||||
|
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
|
||||||
|
reports do not retain headers, credentials, provider
|
||||||
|
bodies, or generated text. A consumer-injected `LLMClient` can support inference
|
||||||
|
diagnostics, but does not establish HTTP status, destination, reachability, or
|
||||||
|
upstream credential-supply facts; Promptkit does not fall back to a built-in
|
||||||
|
transport for HTTP-only diagnostics.
|
||||||
|
|
||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
|
|||||||
@@ -88,6 +88,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
|
|||||||
the logical admission lease but no active permit. Observer callbacks must not
|
the logical admission lease but no active permit. Observer callbacks must not
|
||||||
block on work that requires the same run's admission slot.
|
block on work that requires the same run's admission slot.
|
||||||
|
|
||||||
|
## Smoke-Test Capacity
|
||||||
|
|
||||||
|
The engine's smoke tester shares this manager and generation coordinator with
|
||||||
|
the ordinary runner. Inference diagnostics reserve the same logical admission
|
||||||
|
slot and acquire the same backend-scoped active permits for actual model
|
||||||
|
attempts, including retries. They therefore participate in the same configured
|
||||||
|
backend limits as `Run`. Reachability and authentication HTTP probes do not
|
||||||
|
enter generation admission or consume active permits. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
|
||||||
|
operation and injected-client limits.
|
||||||
|
|
||||||
## Cancellation And Release
|
## Cancellation And Release
|
||||||
|
|
||||||
Admission checks the caller context before reserving a slot. A call canceled
|
Admission checks the caller context before reserving a slot. A call canceled
|
||||||
@@ -127,4 +138,7 @@ cover shared and independent backend pools during retry and repair. The
|
|||||||
own the public prepared-capacity boundary. The
|
own the public prepared-capacity boundary. The
|
||||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
the public generation category and context identity when generation is
|
the public generation category and context identity when generation is
|
||||||
canceled.
|
canceled. The assembled
|
||||||
|
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
|
||||||
|
verify that inference diagnostics share the engine's Run admission and recover
|
||||||
|
capacity after cancellation or failure.
|
||||||
|
|||||||
@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
|
|||||||
with terminal observations without parsing diagnostic text or retrying injected
|
with terminal observations without parsing diagnostic text or retrying injected
|
||||||
clients.
|
clients.
|
||||||
|
|
||||||
|
The diagnostic path uses the same private attempt and retry implementation as
|
||||||
|
ordinary generation. It retains invocation-local transport, response,
|
||||||
|
generation, usage, capacity-wait, and retry-delay evidence before normal
|
||||||
|
result-discard and cancellation checks. Built-in preparation also supplies its
|
||||||
|
frozen request URL to the evidence path before attempts begin; injected
|
||||||
|
generation has no equivalent HTTP destination. This evidence is returned to
|
||||||
|
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||||
|
observer accounting. The [model-client document](llm.md) describes the
|
||||||
|
transport-observation boundary, and the root
|
||||||
|
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
|
||||||
|
|
||||||
Focused coordinator tests use controlled transport and wait functions. They
|
Focused coordinator tests use controlled transport and wait functions. They
|
||||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||||
release, and the injected-client path. The capacity manager tests own permit
|
release, and the injected-client path. The
|
||||||
|
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
|
||||||
|
attempt retention and retry evidence. The capacity manager tests own permit
|
||||||
acquisition and idempotent release. Public
|
acquisition and idempotent release. Public
|
||||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||||
|
|||||||
@@ -59,6 +59,33 @@ This client does not schedule retries; the [generation coordinator](generation.m
|
|||||||
does so around its prepared attempts. The client has no tool-call support,
|
does so around its prepared attempts. The client has no tool-call support,
|
||||||
provider catalog, inbound HTTP behavior, or durable session store.
|
provider catalog, inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Diagnostic HTTP Evidence
|
||||||
|
|
||||||
|
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
|
||||||
|
invocation using a shallow client copy, preserving its timeout, redirect policy,
|
||||||
|
cookie jar, and underlying transport without mutating the configured client.
|
||||||
|
Evidence distinguishes a diagnostic dispatch invocation from an observed
|
||||||
|
request and response. A newly observed request replaces the previous
|
||||||
|
destination-specific response evidence, so redirects retain only the last
|
||||||
|
observed hop. A response is recorded before envelope decoding or later context
|
||||||
|
checks can discard a generation result.
|
||||||
|
|
||||||
|
The collector keeps only the last observed destination (without URL user
|
||||||
|
information), status, credential-presence, response-availability, decoded
|
||||||
|
generation, and presence-aware usage facts needed by the diagnostic. It does
|
||||||
|
not retain headers, credentials, raw request or response bodies, or a redirect
|
||||||
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
|
Reachability and named authentication probes use the same configured built-in
|
||||||
|
client but discard their response bodies after local recognition. The
|
||||||
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
|
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
|
||||||
|
response can establish response and inference facts, but not an HTTP
|
||||||
|
destination, status, reachability, or upstream credential-supply fact. The
|
||||||
|
facade does not silently install a built-in transport for those diagnostics.
|
||||||
|
|
||||||
## Prepared Generation
|
## Prepared Generation
|
||||||
|
|
||||||
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
@@ -137,7 +164,10 @@ own configuration, client cloning, deterministic deadline precedence,
|
|||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||||
bounded single-document successful-response framing, closure, and
|
bounded single-document successful-response framing, closure, and
|
||||||
response-body suppression. The focused
|
response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
|
||||||
|
own invocation-local last-hop request/response facts, while the
|
||||||
|
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
|
||||||
|
own retention across decoding and cancellation. The focused
|
||||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||||
own envelope parsing, normalization, and bounded-reader cases; their
|
own envelope parsing, normalization, and bounded-reader cases; their
|
||||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||||
|
|||||||
@@ -23,6 +23,10 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
|||||||
parses an error string or receives provider bodies, credentials, headers, or
|
parses an error string or receives provider bodies, credentials, headers, or
|
||||||
other content. Events are values with no mutable containers. Callback panics
|
other content. Events are values with no mutable containers. Callback panics
|
||||||
propagate to the caller.
|
propagate to the caller.
|
||||||
|
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||||
|
the invocation, including namespace qualification when supplied by an engine
|
||||||
|
view. A prompt's optional version label is descriptive metadata, not a
|
||||||
|
selection key.
|
||||||
Per-operation callbacks are ordered and synchronous; different operations may
|
Per-operation callbacks are ordered and synchronous; different operations may
|
||||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||||
end before delivery, so callback work does not inflate provider duration.
|
end before delivery, so callback work does not inflate provider duration.
|
||||||
@@ -41,6 +45,11 @@ attempts above the first in a generation, and repair generations with at least
|
|||||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||||
attempt events.
|
attempt events.
|
||||||
|
|
||||||
|
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||||
|
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||||
|
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||||
|
enclosing run's observation totals.
|
||||||
|
|
||||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||||
|
|||||||
@@ -11,17 +11,16 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, concrete-request validation, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
@@ -30,9 +29,9 @@ contributor workflow and validation.
|
|||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -32,6 +32,13 @@ The runner has no durable run or session store.
|
|||||||
|
|
||||||
## Shared Prompt Selection
|
## Shared Prompt Selection
|
||||||
|
|
||||||
|
The internal engine applies each namespace view to slashless caller selections
|
||||||
|
before they reach the runner. The runner therefore selects captured prompts and
|
||||||
|
profiles by the resulting exact runtime IDs. Already-qualified selections pass
|
||||||
|
through, while configured defaults and stored references are used as authored
|
||||||
|
and are not requalified. Prompt version labels are returned as optional
|
||||||
|
descriptive metadata; they are not prompt selectors.
|
||||||
|
|
||||||
Ordinary preparation selects the full prompt definition and construction-time
|
Ordinary preparation selects the full prompt definition and construction-time
|
||||||
hash. It retains its early request-ID check before direct-session
|
hash. It retains its early request-ID check before direct-session
|
||||||
normalization. Referenced message content and static template syntax were
|
normalization. Referenced message content and static template syntax were
|
||||||
@@ -46,8 +53,8 @@ admission, or model collaborators. The root
|
|||||||
exact contract.
|
exact contract.
|
||||||
|
|
||||||
`InspectRequirements` uses the same metadata selection to resolve the
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
request-profile/default identity, then obtains the exact captured requirement
|
request-profile/default identity, then obtains the captured requirement
|
||||||
snapshot by the selected prompt ID and version. The analyzer receives only
|
snapshot by the selected prompt ID. The analyzer receives only
|
||||||
those identity fields rather than an execution definition. It analyzes the
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
snapshot with the supplied caller variables and normalized direct-session
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
bypass; it does not resolve target readiness or enter the preparation
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
@@ -70,6 +77,37 @@ artifact, schema, rendering, admission, or model-client work. The root
|
|||||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||||
exact contract.
|
exact contract.
|
||||||
|
|
||||||
|
## Smoke-Test Diagnostics
|
||||||
|
|
||||||
|
The separate `SmokeTester` in `internal/usecase` selects from the same captured
|
||||||
|
configuration snapshot and backend/profile collaborators as the runner. The
|
||||||
|
internal engine applies namespace-view qualification to slashless caller IDs
|
||||||
|
before diagnostic selection; captured defaults and references remain exact.
|
||||||
|
Explicit profile selection takes precedence over prompt-default selection and a
|
||||||
|
backend diagnostic default; HTTP-only selection does not resolve a default
|
||||||
|
inference profile. Exact public selection rules belong to the root
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
|
||||||
|
They do not render application messages, load application artifacts, use
|
||||||
|
application schemas, or inspect session templates. Private text and structured
|
||||||
|
cases and the structured case's schema remain internal to the diagnostic
|
||||||
|
operation; they are not entries in prompt/profile listings or inspection.
|
||||||
|
`InspectConfiguration` remains a separate, explicit operator-oriented view of
|
||||||
|
effective settings and source history. A diagnostic report may include selected
|
||||||
|
effective settings and identifiers, so consumers still apply their disclosure
|
||||||
|
policy.
|
||||||
|
|
||||||
|
The engine assembles the smoke tester with the runner's generation coordinator
|
||||||
|
and capacity manager. Inference holds the ordinary logical admission lease and
|
||||||
|
uses the same active permits, retries, timeout, and request evidence path as
|
||||||
|
generation. Reachability and authentication probes bypass generation capacity
|
||||||
|
and do not require a model. An engine with an injected `LLMClient` can run
|
||||||
|
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
|
||||||
|
diagnostics report that limitation instead of falling back to another client.
|
||||||
|
The public GoDoc owns the caller-visible timeout, cancellation, report, and
|
||||||
|
transport semantics.
|
||||||
|
|
||||||
## Shared Preparation Pipeline
|
## Shared Preparation Pipeline
|
||||||
|
|
||||||
`Prepare` and `Run` share one private preparation pipeline split at the point
|
`Prepare` and `Run` share one private preparation pipeline split at the point
|
||||||
@@ -256,4 +294,8 @@ Changes to orchestration should continue to use the existing package
|
|||||||
interfaces, keep request state local to an invocation, and preserve the shared
|
interfaces, keep request state local to an invocation, and preserve the shared
|
||||||
resolution and completion pipeline. Source, renderer, validator, or
|
resolution and completion pipeline. Source, renderer, validator, or
|
||||||
model-client contract changes belong first in their owning package and
|
model-client contract changes belong first in their owning package and
|
||||||
document.
|
document. The
|
||||||
|
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
|
||||||
|
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
|
||||||
|
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
|
||||||
|
own diagnostic selection, report assembly, and inference findings.
|
||||||
|
|||||||
@@ -14,39 +14,63 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
effective configuration to execution-owned definitions. It performs one
|
effective configuration to execution-owned definitions. It performs one
|
||||||
layered load, captures effective presence (including explicit null), records
|
layered load, captures effective presence (including explicit null), records
|
||||||
normalization and defaults through resolution, and validates all effective
|
normalization and defaults through resolution, and validates all effective
|
||||||
backend, profile, and prompt values. Replaced lower-layer domain values are
|
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||||
not separately validated. It builds an immutable backend index and
|
domain values are not separately validated. It indexes root resources under
|
||||||
presence-aware profile targets, applying final backend defaults only after
|
their unqualified IDs and collection resources under their full
|
||||||
profile inheritance. It retains ConfigKit's result privately for explicit
|
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
|
||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
authored shape. It builds an immutable backend index and presence-aware profile
|
||||||
depend on ConfigKit.
|
targets, applying final backend defaults only after profile inheritance. Runtime
|
||||||
|
lookups use those final indexes and exact stored references; they do not rewrite
|
||||||
|
or resolve resources from ConfigKit source history. The runtime/domain packages
|
||||||
|
do not depend on ConfigKit.
|
||||||
|
|
||||||
|
ConfigKit owns source headers, imports, composition, and source evidence.
|
||||||
|
Promptkit does not preparse contributions: it validates the final effective
|
||||||
|
values once, captures the indexed identities and their original resource
|
||||||
|
contexts, and retains ConfigKit's result for explicit operator inspection.
|
||||||
|
Accepted non-main source headers are consumed during composition; the main
|
||||||
|
source declaration remains available in that inspection view. Prompt and
|
||||||
|
collection release labels are descriptive metadata, not selection inputs. The
|
||||||
|
[format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||||
|
owns the exact source-header and collection contracts.
|
||||||
|
|
||||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
once to ConfigKit v0.8.0 and returns its public difference records. Comparison
|
||||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
both results; reports may still contain ordinary application data. See the
|
both results; reports may still contain ordinary application data. See the
|
||||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
and ConfigKit's [v0.8.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
After prompt materialization and effective-profile snapshot construction, the
|
After resource materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered, identity-only inventories from the final
|
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
and prompts, per-kind resource namespace sets, and metadata for effective
|
||||||
of those inventories without consulting ConfigKit or copying definitions.
|
collection declarations. Listing methods return caller-owned copies from those
|
||||||
|
captured values without consulting ConfigKit or copying resource definitions.
|
||||||
|
|
||||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||||
files under an explicit root, provide resource context, and leave imports
|
files under an explicit root, provide resource context, and leave imports
|
||||||
disabled. Each returned slice belongs to one strict-composition group; file
|
disabled. Each returned slice belongs to one strict-composition group; file
|
||||||
order does not confer priority. The engine prepends selected catalog and
|
order does not confer priority. Caller underlays are composed in their given
|
||||||
Promptkit-owned definitions, then caller underlays and optional main. The
|
order, followed by the optional main input. The
|
||||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||||
real loader and final-value handoff.
|
real loader and final-value handoff.
|
||||||
|
|
||||||
|
The public `CollectionInput` helper mounts one caller-selected source at
|
||||||
|
`collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||||
|
helper does not read the source; ConfigKit retains responsibility for loading,
|
||||||
|
composition, imports, and source-local resource context. Imported or inherited
|
||||||
|
`content_file` values continue resolving relative to the source that contributed
|
||||||
|
the effective field, not to a flattened runtime identity. Explicit
|
||||||
|
`InspectConfiguration` views expose the retained effective configuration and
|
||||||
|
provenance for operator review; runtime selection and inventories do not derive
|
||||||
|
from source history.
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The configuration adapter indexes each prompt by exact ID and version,
|
The configuration adapter indexes each prompt by exact ID,
|
||||||
materializes file-backed message content using the exact effective field's
|
materializes file-backed message content using the exact effective field's
|
||||||
origin and resource context, and checks static message/session template syntax
|
origin and resource context, and checks static message/session template syntax
|
||||||
at construction. It stores captured content, compiled templates, declared
|
at construction. It stores captured content, compiled templates, declared
|
||||||
@@ -58,14 +82,17 @@ declaration; absent and null fields remain distinguishable from explicit
|
|||||||
strings, including empty strings. It closes content-file handles before
|
strings, including empty strings. It closes content-file handles before
|
||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
|
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||||
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Role and cache-control normalization is applied to execution-owned message
|
||||||
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
copies. ConfigKit v0.8.0 retains effective input message metadata and its
|
||||||
resource origins; recording a sequence-wide normalization in the retained
|
resource origins; recording a sequence-wide normalization in the retained
|
||||||
configuration would replace origins of unchanged content-file fields.
|
configuration would replace origins of unchanged content-file fields.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and uses
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||||
internal requirement snapshot lookup selects by exact prompt ID and version.
|
internal requirement snapshot lookup uses the selected prompt ID alone.
|
||||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
returns declared metadata without rendering or resolving profile, input, or
|
||||||
schema resources, and its public shape and prompt hashes do not include the
|
schema resources, and its public shape and prompt hashes do not include the
|
||||||
@@ -73,8 +100,8 @@ captured presence state. Runtime rendering still checks request variables and
|
|||||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
The configuration adapter shares exact ID/version selection across its
|
The configuration adapter shares ID-only selection across its internal prompt
|
||||||
internal prompt lookups. The execution snapshot lookup returns an owned prompt
|
lookups. The execution snapshot lookup returns an owned prompt
|
||||||
definition and cached hash; metadata inspection returns owned input metadata
|
definition and cached hash; metadata inspection returns owned input metadata
|
||||||
without copying execution templates. Profile selection likewise has an
|
without copying execution templates. Profile selection likewise has an
|
||||||
identity-only lookup for consumers that do not need the execution target,
|
identity-only lookup for consumers that do not need the execution target,
|
||||||
@@ -112,18 +139,12 @@ dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
|||||||
and bypassed session templates are not traversed. This internal walker does not
|
and bypassed session templates are not traversed. This internal walker does not
|
||||||
change rendering behavior.
|
change rendering behavior.
|
||||||
|
|
||||||
## Profiles And Built-Ins
|
## Profiles
|
||||||
|
|
||||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
|
`extends` names a sibling local key; it is not a cross-collection reference.
|
||||||
It discovers YAML inputs and supplies passive filesystem context without
|
The configuration adapter validates final effective resource references and
|
||||||
parsing or pre-validating catalog definitions. The configuration loader
|
field values, then builds target
|
||||||
validates only the final effective backend and profile values, including
|
|
||||||
consumer overlays of maintained bases or backends. There is no catalog-only
|
|
||||||
preflight, reserved source namespace, or replay of backend registration.
|
|
||||||
|
|
||||||
ConfigKit resolves profile inheritance before the handoff. The configuration
|
|
||||||
adapter validates effective references and field values, then builds target
|
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
snapshots with backend defaults and presence/suppression state. A profile can
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
that captured target without prompt, artifact, schema, credential lookup, or
|
||||||
@@ -131,8 +152,7 @@ provider work. Preparation applies runtime overrides and then checks target
|
|||||||
completeness. Identity-only profile selection can identify a partial captured
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
profile without copying its target or requiring readiness. The
|
profile without copying its target or requiring readiness. The
|
||||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
target interpretation.
|
||||||
cover selected released assets.
|
|
||||||
|
|
||||||
The internal engine delegates listing through a narrow identity-inventory
|
The internal engine delegates listing through a narrow identity-inventory
|
||||||
interface directly to the configuration snapshot. The execution runner has no
|
interface directly to the configuration snapshot. The execution runner has no
|
||||||
|
|||||||
@@ -26,47 +26,17 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.8.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
|
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog v3
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
|
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
|
||||||
|
|
||||||
## jsonschema
|
## jsonschema
|
||||||
|
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ The [internal component overview](../internal/overview.md) owns the inventory
|
|||||||
of implemented packages and their concrete responsibilities. This policy owns
|
of implemented packages and their concrete responsibilities. This policy owns
|
||||||
the boundaries and dependency rules those components must preserve.
|
the boundaries and dependency rules those components must preserve.
|
||||||
|
|
||||||
The internal engine composes selected catalog and caller ConfigKit inputs once,
|
The internal engine composes caller-selected ConfigKit inputs once, then
|
||||||
then assembles the captured definitions, capacity manager, renderer, validator,
|
assembles the captured definitions, capacity manager, renderer, validator,
|
||||||
outbound client, and use-case runner. The configuration adapter interprets and
|
outbound client, and use-case runner. The configuration adapter interprets and
|
||||||
validates only the final effective values, including presence, null, and
|
validates only the final effective values, including presence, null, and
|
||||||
resource context. It retains source history for explicit operator inspection,
|
resource context. It retains source history for explicit operator inspection,
|
||||||
@@ -97,7 +97,8 @@ on consumers or on Scriptorium.
|
|||||||
`internal/domain` owns source-neutral invariants for values shared across
|
`internal/domain` owns source-neutral invariants for values shared across
|
||||||
multiple input and execution boundaries, including execution-setting bounds,
|
multiple input and execution boundaries, including execution-setting bounds,
|
||||||
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||||
legality. Callers retain source parsing, required-field rules, other
|
legality, plus shared resource-identity syntax and qualification. Callers retain
|
||||||
|
source parsing, required-field rules, other
|
||||||
source-specific normalization, defaulting, error classification, and policy
|
source-specific normalization, defaulting, error classification, and policy
|
||||||
specific to their own boundary.
|
specific to their own boundary.
|
||||||
|
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ secret values.
|
|||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||||
|
|
||||||
Create supporting documents for existing interfaces and responsibilities when
|
Create supporting documents for existing interfaces and responsibilities when
|
||||||
substantive content is available.
|
substantive content is available.
|
||||||
|
|||||||
@@ -278,29 +278,3 @@ captured publication endpoint before retrying; the repository-specific
|
|||||||
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||||
instructions. Do not wait for CI or Go module availability. Licensing footer
|
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||||
URLs are preparation work, not a network gate.
|
URLs are preparation work, not a network gate.
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
|
||||||
|
|
||||||
External catalog profile data may be added, removed, renamed, or modified
|
|
||||||
without deprecation or advance notice. This exception is limited to profile
|
|
||||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
|
||||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
|
||||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
|
||||||
consumer-owned definitions.
|
|
||||||
|
|
||||||
Within the catalog versions selected and tested by a Promptkit release, and
|
|
||||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
|
||||||
profile IDs `promptkit-openrouter-standard` and
|
|
||||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
|
||||||
model, backend, routing behavior, settings, cost, output, or upstream
|
|
||||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
|
||||||
owns their current definitions.
|
|
||||||
|
|
||||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
|
||||||
Afterward, catalog-only updates and adjustments to the current selections may
|
|
||||||
ship in a patch release when all other compatibility obligations remain intact.
|
|
||||||
External profile removals can require consumer action; they are not automatic
|
|
||||||
migrations.
|
|
||||||
|
|
||||||
Catalog adoption and release validation remain local and offline. They do not
|
|
||||||
probe provider endpoints, request model lists, or make inference calls.
|
|
||||||
|
|||||||
@@ -97,11 +97,6 @@ the push outcome remains unknown. Successful publication needs no remote
|
|||||||
read-back, CI wait, or installation check. Provider probes and inference calls
|
read-back, CI wait, or installation check. Provider probes and inference calls
|
||||||
are not part of validation.
|
are not part of validation.
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
|
||||||
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
|
||||||
catalog compatibility boundary. This heading preserves older incoming links.
|
|
||||||
|
|
||||||
## Failure Recovery
|
## Failure Recovery
|
||||||
|
|
||||||
For status 1, use the diagnostic to distinguish a validation failure from a
|
For status 1, use the diagnostic to distinguish a validation failure from a
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ the existing Go API. This refactor requires no consumer code changes.
|
|||||||
|
|
||||||
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
||||||
`v2.20260922.0`; their exact inventories are linked from the
|
`v2.20260922.0`; their exact inventories are linked from the
|
||||||
[format reference](../formats.md#external-profile-inventories).
|
[migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|
||||||
The maintained names provide continuity only within the catalog versions
|
The maintained names provide continuity only within the catalog versions
|
||||||
selected and tested by a Promptkit release and when consumers do not override
|
selected and tested by a Promptkit release and when consumers do not override
|
||||||
@@ -37,8 +37,8 @@ an available profile or supply their own complete definition; the new
|
|||||||
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
||||||
aliases and need not preserve the previous settings.
|
aliases and need not preserve the previous settings.
|
||||||
|
|
||||||
See the [release policy](../release.md#catalog-profile-compatibility) for the
|
See the [release policy](../policy/release.md#versioning-and-compatibility) for the
|
||||||
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
|
compatibility boundary, the [migration identity mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for current Promptkit-owned definitions and selected external inventories, and
|
for current resource mappings, and
|
||||||
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
||||||
for configuration choices.
|
for configuration choices.
|
||||||
|
|||||||
@@ -72,5 +72,5 @@ for a structured proposal-and-assessment workflow.
|
|||||||
|
|
||||||
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
||||||
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility)
|
||||||
and [profile inventories](../formats.md#external-profile-inventories).
|
and [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ No public API, runtime behavior, or profile changes are introduced.
|
|||||||
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
||||||
See the [project grant](../../COPYRIGHT),
|
See the [project grant](../../COPYRIGHT),
|
||||||
[component attribution](../oss/attribution.md), and
|
[component attribution](../oss/attribution.md), and
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility).
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ for API and source-selection changes.
|
|||||||
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||||
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||||
assets directly must adopt the keyed format. The maintained Promptkit profile
|
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||||
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
IDs remain available under the [release compatibility policy](../policy/release.md#versioning-and-compatibility),
|
||||||
which does not promise fixed profile data.
|
which does not promise fixed profile data.
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ increases to 1.26.0.
|
|||||||
engine's captured configuration, are lexically ordered and caller-owned, and
|
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||||
include consumer definitions and maintained profiles that remain effective.
|
include consumer definitions and maintained profiles that remain effective.
|
||||||
Listing performs no provider or execution work. See the
|
Listing performs no provider or execution work. See the
|
||||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections)
|
||||||
and [GoDoc](../../listing.go).
|
and [GoDoc](../../listing.go).
|
||||||
|
|
||||||
## Efficiency And Tooling
|
## Efficiency And Tooling
|
||||||
|
|||||||
86
docs/releases/v0.16.0.md
Normal file
86
docs/releases/v0.16.0.md
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# Promptkit v0.16.0
|
||||||
|
|
||||||
|
This release adds explicit smoke diagnostics, configuration collections,
|
||||||
|
namespace-scoped engine views, and expanded resource discovery. It also makes
|
||||||
|
intentional breaking changes to configuration ownership and prompt selection.
|
||||||
|
The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Configuration Collections And Discovery
|
||||||
|
|
||||||
|
Applications can compose caller-selected collections of backends, profiles,
|
||||||
|
and prompts. Collection resources have qualified identities such as
|
||||||
|
`support/answer`; root resources retain unqualified identities. `CollectionInput`
|
||||||
|
provides a convenience for mounting a selected ConfigKit input into a collection.
|
||||||
|
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
|
||||||
|
header support.
|
||||||
|
|
||||||
|
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
|
||||||
|
and backend scopes for short caller-supplied IDs. Views share the underlying
|
||||||
|
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
|
||||||
|
qualified IDs and configured references keep their exact identities.
|
||||||
|
|
||||||
|
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
|
||||||
|
`ListNamespaces`, and `ListCollections` methods expose captured resource and
|
||||||
|
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
|
||||||
|
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
|
||||||
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
|
||||||
|
and structured-output inference diagnostics. Reports provide factual
|
||||||
|
observations, timing, and available usage rather than an overall readiness
|
||||||
|
verdict. Optional Boolean facts distinguish negative observations from unknown
|
||||||
|
or inapplicable checks; matching a JSON schema does not prove that a provider
|
||||||
|
enforced it.
|
||||||
|
|
||||||
|
A concluded diagnostic can report failure with a nil Go error. Invalid requests
|
||||||
|
return ordinary errors, and caller cancellation can return partial observations
|
||||||
|
with a recognizable context error. Diagnostics can contact providers and incur
|
||||||
|
inference costs only when explicitly invoked; they are not automatic execution
|
||||||
|
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
|
||||||
|
and [API GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
## Breaking Changes And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.15.0 should follow the
|
||||||
|
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||||
|
|
||||||
|
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
|
||||||
|
provider catalogs or the `promptkit-openrouter-standard` and
|
||||||
|
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
|
||||||
|
definitions automatically. Applications choose and maintain their sources.
|
||||||
|
- Convert version-keyed prompt mappings into one definition per prompt ID.
|
||||||
|
Prompt `version` is now an optional descriptive label, not a selector.
|
||||||
|
Choose deliberately which former definition to install.
|
||||||
|
- Remove `PromptVersion` selectors from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
|
||||||
|
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
|
||||||
|
consuming version-bearing inspection, identity, result, or observation fields
|
||||||
|
to handle optional labels, including nil and explicitly empty labels.
|
||||||
|
- When moving resources into collections, update stored backend, profile, and
|
||||||
|
diagnostic-default references to their intended qualified IDs. Mounting a
|
||||||
|
source does not rewrite references; profile inheritance remains within its
|
||||||
|
sibling mapping. Root configuration `version`, when supplied, is now a source
|
||||||
|
specification header, distinct from descriptive resource labels.
|
||||||
|
|
||||||
|
The migration guide includes identity mappings and before/after prompt examples.
|
||||||
|
The [maintained offline example](../../examples/go-library/prepare/main.go)
|
||||||
|
demonstrates explicit composition, source-relative content, scoped preparation,
|
||||||
|
and inventory without contacting a provider.
|
||||||
|
|
||||||
|
After adapting configuration and API calls:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Correctness And Efficiency
|
||||||
|
|
||||||
|
Collection prompt materialization shares one lazy origin-metadata snapshot,
|
||||||
|
avoiding repeated full metadata copies across collections. Internal prepared
|
||||||
|
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
|
||||||
|
cancellation and report actual inference destinations and known-zero usage when
|
||||||
|
no generation request was dispatched.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)
|
||||||
@@ -63,8 +63,8 @@ Every engine now includes the reserved `rakestrawhome` backend, identified by
|
|||||||
selects that backend. Consumers can use the maintained endpoint, credential,
|
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||||
capacity, and model defaults without registering either definition themselves.
|
capacity, and model defaults without registering either definition themselves.
|
||||||
|
|
||||||
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
See the [backend format reference](../formats.md#backend-definitions)
|
||||||
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
and the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for the current contracts.
|
for the current contracts.
|
||||||
|
|
||||||
## Structured Generation Errors
|
## Structured Generation Errors
|
||||||
@@ -127,7 +127,7 @@ freezes the fully resolved target. The selected leaf ID remains public while
|
|||||||
effective execution settings reflect the resolved chain.
|
effective execution settings reflect the resolved chain.
|
||||||
|
|
||||||
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
||||||
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping),
|
||||||
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||||
behavior.
|
behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -96,9 +96,8 @@ precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
|||||||
or hard-coded definitions for those maintained backends.
|
or hard-coded definitions for those maintained backends.
|
||||||
|
|
||||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||||
tested with this release. The [built-in backend and profile format
|
tested with this release. The [backend format reference](../formats.md#backend-definitions)
|
||||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
remains the canonical consumer contract, while the [internal source documentation](../internal/sources.md#profiles)
|
||||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
|
||||||
describes the dependency boundary.
|
describes the dependency boundary.
|
||||||
|
|
||||||
## Public API Changes
|
## Public API Changes
|
||||||
|
|||||||
65
engine.go
65
engine.go
@@ -24,9 +24,8 @@ var (
|
|||||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||||
// credentials, or effective settings are invalid.
|
// credentials, or effective settings are invalid.
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
// ErrPromptNotFound identifies a requested prompt ID that is not present in
|
||||||
// present in the selected prompt source. It does not also match
|
// the captured configuration. It does not also match ErrPromptLoad.
|
||||||
// ErrPromptLoad.
|
|
||||||
ErrPromptNotFound = errors.New("prompt not found")
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
// every configured profile source. It does not also match ErrProfileLoad.
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
@@ -85,13 +84,18 @@ var (
|
|||||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||||
// prepares and runs Promptkit prompt requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||||
|
// [Engine.InspectConfiguration],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectPrompt],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||||
// model generation. Injected collaborators may still be invoked concurrently
|
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
|
||||||
// across different backend pools or for unlimited backends.
|
// independent backend-capacity pools that coordinate Run, RunPrepared, and
|
||||||
|
// inference SmokeTest admission and model generation. Namespace views returned
|
||||||
|
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
|
||||||
|
// and prepared-execution owner. Injected collaborators may still be invoked
|
||||||
|
// concurrently across different backend pools or for unlimited backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
internal *internalengine.Engine
|
internal *internalengine.Engine
|
||||||
}
|
}
|
||||||
@@ -208,11 +212,11 @@ func WithSchemaFile(path string) Option {
|
|||||||
|
|
||||||
// NewEngine constructs an Engine from configuration and options.
|
// NewEngine constructs an Engine from configuration and options.
|
||||||
//
|
//
|
||||||
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
|
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||||
// caller underlays, and the optional main input in that order. Construction
|
// by the optional main input. Construction validates the final effective
|
||||||
// validates the final effective backend, profile, and prompt definitions and
|
// backend, profile, and prompt definitions and captures referenced prompt
|
||||||
// captures referenced prompt content. A missing prompt collection is allowed.
|
// content. A missing prompt collection is allowed. Schema documents and
|
||||||
// Schema documents and request artifacts remain runtime resources.
|
// request artifacts remain runtime resources.
|
||||||
//
|
//
|
||||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
// configuration or options, preserving a ConfigKit LoadError when loading
|
||||||
@@ -241,14 +245,13 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
return &Engine{internal: assembled}, nil
|
return &Engine{internal: assembled}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
// InspectPrompt resolves one prompt ID without selecting a profile or starting
|
||||||
// profile or starting execution work.
|
// execution work.
|
||||||
//
|
//
|
||||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
// InspectPrompt requires a nonblank promptID and performs an exact,
|
||||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
// case-sensitive lookup in the captured configuration snapshot. A
|
||||||
// prompt selection. An empty version succeeds only when that ID has one
|
// namespace-scoped view qualifies slashless caller IDs before lookup; already
|
||||||
// version; a nonempty version selects one exact ID/version pair from the
|
// qualified IDs pass through. Prompt version labels do not select definitions.
|
||||||
// captured configuration snapshot.
|
|
||||||
//
|
//
|
||||||
// A successful result proves that the selected definition and any referenced
|
// A successful result proves that the selected definition and any referenced
|
||||||
// message content files were structurally loaded. Inputs are returned in
|
// message content files were structurally loaded. Inputs are returned in
|
||||||
@@ -267,21 +270,19 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
// or content files; construct another Engine to observe source changes.
|
// or content files; construct another Engine to observe source changes.
|
||||||
//
|
//
|
||||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
// matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
|
||||||
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
|
// not ErrPromptLoad. Invalid definitions and resource failures reject
|
||||||
// ErrPromptLoad. Invalid definitions and resource failures reject construction
|
// construction instead. Cancellation during lookup preserves the context
|
||||||
// instead. Cancellation during lookup preserves the context error. InspectPrompt
|
// error. InspectPrompt returns no partial result on error.
|
||||||
// returns no partial result on error.
|
|
||||||
func (e *Engine) InspectPrompt(
|
func (e *Engine) InspectPrompt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
promptID string,
|
promptID string,
|
||||||
promptVersion string,
|
|
||||||
) (*PromptInspection, error) {
|
) (*PromptInspection, error) {
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
inspection, err := e.internal.InspectPrompt(ctx, promptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
@@ -393,8 +394,9 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
|
|||||||
// configured LLMClient or reserving backend admission capacity. Validated
|
// configured LLMClient or reserving backend admission capacity. Validated
|
||||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||||
//
|
//
|
||||||
// The returned opaque handle is bound to this Engine and permits one
|
// The returned opaque handle is bound to this Engine's underlying execution
|
||||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
// owner, shared with its namespace views, and permits one [Engine.RunPrepared]
|
||||||
|
// invocation in total. Preparation freezes the selected sources,
|
||||||
// rendered messages, effective settings, inputs, provider structured-output
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
// metadata, and validation resources needed by that invocation. The handle
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
// retains a direct RunRequest.APIKey only in private execution state;
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
@@ -506,6 +508,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
// RunPrepared atomically claims and executes a handle created by
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
// [Engine.PrepareExecution].
|
// [Engine.PrepareExecution].
|
||||||
//
|
//
|
||||||
|
// The originating Engine and namespace views sharing its underlying owner may
|
||||||
|
// claim the handle. Execution uses its captured selections without applying the
|
||||||
|
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
|
||||||
|
// is a foreign owner.
|
||||||
|
//
|
||||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
// credential revalidation, backend admission, generation, or validation.
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
// Cancellation, capacity rejection, generation failure, operational
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.assessment:
|
example.assessment:
|
||||||
"1.0.0":
|
version: "1.0.0"
|
||||||
default_profile: assessment-profile
|
default_profile: assessment-profile
|
||||||
description: Assess a structured proposal independently of schema validity.
|
description: Assess a structured proposal independently of schema validity.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.proposal:
|
example.proposal:
|
||||||
"1.0.0":
|
version: "1.0.0"
|
||||||
default_profile: proposal-profile
|
default_profile: proposal-profile
|
||||||
description: Propose a bounded improvement from a synthetic idea.
|
description: Propose a bounded improvement from a synthetic idea.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
backends:
|
||||||
|
offline:
|
||||||
|
endpoint: https://review.example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
audit:
|
||||||
|
backend: review/offline
|
||||||
|
model: review-model
|
||||||
|
prompts:
|
||||||
|
check:
|
||||||
|
default_profile: review/audit
|
||||||
|
description: Inspect a review item without contacting a model provider.
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review the supplied synthetic example.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
version: "1"
|
||||||
|
configkit:
|
||||||
|
imports:
|
||||||
|
- path: fragments/resources.yaml
|
||||||
|
release_version: "1.4.0"
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
backends:
|
||||||
|
offline:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
base:
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
|
timeout_seconds: 90
|
||||||
|
fast:
|
||||||
|
extends: base
|
||||||
|
timeout_seconds: 60
|
||||||
|
prompts:
|
||||||
|
prepare:
|
||||||
|
version: "1.0.0"
|
||||||
|
default_profile: support/fast
|
||||||
|
description: Prepare a prompt without contacting a model provider.
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Summarize the note in one sentence.
|
||||||
|
- role: user
|
||||||
|
content_file: ../messages/note.md
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: basic
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{{input "note"}}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# A later input intentionally replaces only the inherited profile timeout.
|
||||||
|
profiles:
|
||||||
|
fast:
|
||||||
|
timeout_seconds: 30
|
||||||
@@ -12,32 +12,71 @@ import (
|
|||||||
|
|
||||||
type summary struct {
|
type summary struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version"`
|
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||||
SelectedProfile string `json:"selected_profile"`
|
SelectedProfile string `json:"selected_profile"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
MessageCount int `json:"message_count"`
|
MessageCount int `json:"message_count"`
|
||||||
|
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||||
|
Namespaces []string `json:"namespaces"`
|
||||||
|
Collections []collectionSummary `json:"collections"`
|
||||||
|
ScopedPrompts []string `json:"scoped_prompts"`
|
||||||
|
ReviewPrompts []string `json:"review_prompts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type collectionSummary struct {
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
ReleaseVersion *string `json:"release_version"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
support, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File("examples/go-library/prepare/collections/support/collection.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
review, err := promptkit.CollectionInput(
|
||||||
|
"review",
|
||||||
|
configkit.File("examples/go-library/prepare/collections/review/collection.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
override, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File("examples/go-library/prepare/collections/support/operator-overrides.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{
|
||||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
{Inputs: []configkit.Input{support, review}},
|
||||||
})}}}},
|
{Inputs: []configkit.Input{override}},
|
||||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
|
},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||||
|
Prompt: "support", Profile: "support", Backend: "support",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
if _, found := view.Explain(configkit.Keys(
|
||||||
exit(fmt.Errorf("configured profile model was not found"))
|
"collections", "support", "profiles", "fast", "timeout_seconds",
|
||||||
|
)); !found {
|
||||||
|
exit(fmt.Errorf("collection profile override was not found"))
|
||||||
}
|
}
|
||||||
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "prepare",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
@@ -49,18 +88,61 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
request := promptkit.RunRequest{
|
request := promptkit.RunRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "prepare",
|
||||||
|
ProfileID: "fast",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if err := engine.ValidateRequest(context.Background(), request); err != nil {
|
if err := scoped.ValidateRequest(ctx, request); err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
prepared, err := engine.Prepare(context.Background(), request)
|
prepared, err := scoped.Prepare(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
|
profile, err := scoped.InspectProfile(ctx, "fast")
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
|
||||||
|
exit(fmt.Errorf(
|
||||||
|
"later profile override was not applied: timeout is %d seconds",
|
||||||
|
profile.EffectiveModelParams.TimeoutSeconds,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
namespaces, err := scoped.ListNamespaces(ctx)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
collections, err := scoped.ListCollections(ctx)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
scopedPrompts, err := scoped.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
|
||||||
|
Mode: promptkit.NamespaceNamed, Name: "review",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
scopedPromptIDs := make([]string, len(scopedPrompts))
|
||||||
|
for index := range scopedPrompts {
|
||||||
|
scopedPromptIDs[index] = scopedPrompts[index].ID
|
||||||
|
}
|
||||||
|
reviewPromptIDs := make([]string, len(reviewPrompts))
|
||||||
|
for index := range reviewPrompts {
|
||||||
|
reviewPromptIDs[index] = reviewPrompts[index].ID
|
||||||
|
}
|
||||||
|
collectionSummaries := make([]collectionSummary, len(collections))
|
||||||
|
for index := range collections {
|
||||||
|
collectionSummaries[index] = collectionSummary{
|
||||||
|
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
encoder.SetIndent("", " ")
|
encoder.SetIndent("", " ")
|
||||||
@@ -70,6 +152,11 @@ func main() {
|
|||||||
SelectedProfile: prepared.SelectedProfileID,
|
SelectedProfile: prepared.SelectedProfileID,
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
MessageCount: len(prepared.Messages),
|
MessageCount: len(prepared.Messages),
|
||||||
|
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
|
||||||
|
Namespaces: namespaces,
|
||||||
|
Collections: collectionSummaries,
|
||||||
|
ScopedPrompts: scopedPromptIDs,
|
||||||
|
ReviewPrompts: reviewPromptIDs,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
prompts:
|
|
||||||
example.prepare:
|
|
||||||
"1.0.0":
|
|
||||||
default_profile: offline-example
|
|
||||||
description: Prepare a prompt without contacting a model provider.
|
|
||||||
inputs:
|
|
||||||
- name: note
|
|
||||||
required: true
|
|
||||||
content_type: text/plain
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content: Summarize the note in one sentence.
|
|
||||||
- role: user
|
|
||||||
content: '{{input "note"}}'
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: basic
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.run:
|
example.run:
|
||||||
"1.0.0":
|
version: "1.0.0"
|
||||||
default_profile: offline-example
|
default_profile: offline-example
|
||||||
description: Run a prompt with a deterministic injected model client.
|
description: Run a prompt with a deterministic injected model client.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -18,9 +18,8 @@ func (r RunRequest) GoString() string {
|
|||||||
|
|
||||||
func (r RunRequest) redactedString() string {
|
func (r RunRequest) redactedString() string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
"promptkit.RunRequest{PromptID:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||||
r.PromptID,
|
r.PromptID,
|
||||||
r.PromptVersion,
|
|
||||||
r.ProfileID,
|
r.ProfileID,
|
||||||
r.APIKey != "",
|
r.APIKey != "",
|
||||||
len(r.Inputs),
|
len(r.Inputs),
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
const capturedPromptDocument = `
|
const capturedPromptDocument = `
|
||||||
prompts:
|
prompts:
|
||||||
captured:
|
captured:
|
||||||
"1":
|
version: "1"
|
||||||
default_profile: offline
|
default_profile: offline
|
||||||
session_id: '{{.session}}'
|
session_id: '{{.session}}'
|
||||||
messages:
|
messages:
|
||||||
@@ -149,12 +149,13 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
|||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
"offline": {Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||||
"prompts": map[string]any{"structured": map[string]any{"1": map[string]any{
|
"prompts": map[string]any{"structured": map[string]any{
|
||||||
|
"version": "1",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||||
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
||||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||||
}}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
4
go.mod
4
go.mod
@@ -3,9 +3,7 @@ module gitea.maximumdirect.net/eric/promptkit
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
gitea.maximumdirect.net/eric/ConfigKit v0.8.0
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
10
go.sum
10
go.sum
@@ -1,11 +1,7 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
gitea.maximumdirect.net/eric/ConfigKit v0.8.0 h1:xCIhYnOHjw7rFvB+vIremPKGeROOxKTomnUEUipuQno=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
gitea.maximumdirect.net/eric/ConfigKit v0.8.0/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
@@ -18,5 +14,3 @@ golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
|||||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
|
|||||||
@@ -7,13 +7,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// OpenRouterID identifies the maintained OpenRouter backend.
|
// OpenRouterID is the conventional OpenRouter backend identifier.
|
||||||
OpenRouterID = "openrouter"
|
OpenRouterID = "openrouter"
|
||||||
// RakestrawHomeID identifies the maintained Rakestrawhome backend.
|
// RakestrawHomeID is the conventional Rakestrawhome backend identifier.
|
||||||
RakestrawHomeID = "rakestrawhome"
|
RakestrawHomeID = "rakestrawhome"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -31,13 +30,12 @@ type Registry struct {
|
|||||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
||||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||||
for id, definition := range definitions {
|
for id, definition := range definitions {
|
||||||
params, err := jsonvalue.CopyMap(definition.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("backend %q extra parameters: %w", id, err)
|
|
||||||
}
|
|
||||||
definition.ID = id
|
definition.ID = id
|
||||||
definition.ExtraParams = params
|
cloned, err := domain.CloneBackend(definition)
|
||||||
registry.backends[id] = definition
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backend %q: %w", id, err)
|
||||||
|
}
|
||||||
|
registry.backends[id] = cloned
|
||||||
}
|
}
|
||||||
return registry, nil
|
return registry, nil
|
||||||
}
|
}
|
||||||
@@ -51,12 +49,11 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
}
|
}
|
||||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
cloned, err := domain.CloneBackend(definition)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||||
}
|
}
|
||||||
definition.ExtraParams = extraParams
|
return cloned, nil
|
||||||
return definition, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
|||||||
@@ -11,25 +11,61 @@ import (
|
|||||||
|
|
||||||
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||||
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
||||||
|
defaultProfile := "catalog-profile"
|
||||||
|
reachabilityEndpoint := "https://probe.example/ready"
|
||||||
|
authenticationKind := "openrouter_key"
|
||||||
|
authenticationEndpoint := "https://probe.example/key"
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &defaultProfile,
|
||||||
|
Reachability: &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
|
},
|
||||||
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||||
" local.v1 ": {
|
" local.v1 ": {
|
||||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
||||||
|
SmokeTests: smokeTests,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
params["nested"].(map[string]any)["enabled"] = false
|
params["nested"].(map[string]any)["enabled"] = false
|
||||||
|
defaultProfile = "mutated-profile"
|
||||||
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
|
authenticationKind = "none"
|
||||||
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
selected, err := registry.GetBackend(" local.v1 ")
|
selected, err := registry.GetBackend(" local.v1 ")
|
||||||
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||||
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
|
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
|
||||||
|
*selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
|
||||||
|
selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||||
|
selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
|
||||||
|
*selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
|
||||||
|
*selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||||
|
t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
|
||||||
|
}
|
||||||
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
||||||
|
*selected.SmokeTests.DefaultProfile = "mutated-return"
|
||||||
|
*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
|
||||||
|
*selected.SmokeTests.Authentication.Kind = "none"
|
||||||
|
*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
|
||||||
again, err := registry.GetBackend(" local.v1 ")
|
again, err := registry.GetBackend(" local.v1 ")
|
||||||
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||||
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
||||||
}
|
}
|
||||||
|
if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
|
||||||
|
*again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
|
||||||
|
again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||||
|
again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
|
||||||
|
*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
|
||||||
|
*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||||
|
t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
|
||||||
|
}
|
||||||
policies := registry.CapacityPolicies()
|
policies := registry.CapacityPolicies()
|
||||||
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
||||||
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
// Package catalog selects the maintained configuration assets supplied with
|
|
||||||
// PromptKit. ConfigKit owns their composition and effective validation.
|
|
||||||
package catalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3"
|
|
||||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Underlays selects external catalogs followed by PromptKit-owned profiles.
|
|
||||||
// Neither group is interpreted before the final effective configuration loads.
|
|
||||||
func Underlays() ([]configkit.Underlay, error) {
|
|
||||||
openrouterInputs, err := assetInputs("OpenRouter catalog", openrouter.FS(), openrouter.Root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rakestrawhomeInputs, err := assetInputs("Rakestrawhome catalog", rakestrawhome.FS(), rakestrawhome.Root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
catalogInputs := append(openrouterInputs, rakestrawhomeInputs...)
|
|
||||||
owned := configkit.Input{Source: configkit.Mapping("PromptKit-owned profiles", map[string]any{
|
|
||||||
"profiles": map[string]any{
|
|
||||||
"promptkit-openrouter-standard": map[string]any{"extends": "gemini-flash-latest"},
|
|
||||||
"promptkit-openrouter-auto": map[string]any{"backend": backend.OpenRouterID, "model": "openrouter/auto"},
|
|
||||||
},
|
|
||||||
})}
|
|
||||||
return []configkit.Underlay{{Inputs: catalogInputs}, {Inputs: []configkit.Input{owned}}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func assetInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
|
||||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("discover %s configuration: %w", label, err)
|
|
||||||
}
|
|
||||||
namespace := configkit.NewFilesystem(label, filesystem)
|
|
||||||
inputs := make([]configkit.Input, 0, len(files))
|
|
||||||
for _, name := range files {
|
|
||||||
inputs = append(inputs, configkit.Input{
|
|
||||||
Source: namespace.File(name),
|
|
||||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return inputs, nil
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package catalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPublishedCatalogsResolveThroughEffectiveConfiguration(t *testing.T) {
|
|
||||||
underlays, err := Underlays()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
state, err := configuration.Load(configkit.Input{Source: configkit.NoSource()}, underlays)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
openrouter, err := state.GetBackend("openrouter")
|
|
||||||
if err != nil || openrouter.Endpoint != "https://openrouter.ai/api/v1" || openrouter.ConcurrencyLimit != 16 || openrouter.QueueCapacity != 1024 {
|
|
||||||
t.Fatalf("OpenRouter backend: %+v, %v", openrouter, err)
|
|
||||||
}
|
|
||||||
rakestrawhome, err := state.GetBackend("rakestrawhome")
|
|
||||||
if err != nil || rakestrawhome.Endpoint != "https://inference.ai.rakestrawhome.com/v1" || rakestrawhome.ConcurrencyLimit != 4 {
|
|
||||||
t.Fatalf("Rakestrawhome backend: %+v, %v", rakestrawhome, err)
|
|
||||||
}
|
|
||||||
for id, want := range map[string]string{
|
|
||||||
"promptkit-openrouter-standard": "~google/gemini-flash-latest",
|
|
||||||
"promptkit-openrouter-auto": "openrouter/auto",
|
|
||||||
"rakestrawhome-gemma-4-31b": "google/gemma-4-31b-it",
|
|
||||||
} {
|
|
||||||
profile, err := state.GetProfile(context.Background(), id)
|
|
||||||
if err != nil || profile.ResolvedTarget.Model != want {
|
|
||||||
t.Fatalf("profile %s: %+v, %v", id, profile, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConsumerOverlayReplacesCatalogDefaultsBeforeValidation(t *testing.T) {
|
|
||||||
underlays, err := Underlays()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid lower value"}},
|
|
||||||
})}}})
|
|
||||||
main := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "https://alternate.example/v1"}},
|
|
||||||
"profiles": map[string]any{"gemini-flash-latest": map[string]any{"model": "consumer-model"}},
|
|
||||||
})}
|
|
||||||
state, err := configuration.Load(main, underlays)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("replaced lower-layer value failed final validation: %v", err)
|
|
||||||
}
|
|
||||||
profile, err := state.GetProfile(context.Background(), "promptkit-openrouter-standard")
|
|
||||||
if err != nil || profile.ResolvedTarget.Model != "consumer-model" || profile.ResolvedTarget.Endpoint != "https://alternate.example/v1" {
|
|
||||||
t.Fatalf("maintained profile did not observe final configuration: %+v, %v", profile, err)
|
|
||||||
}
|
|
||||||
_, err = configuration.Load(configkit.Input{Source: configkit.Mapping("invalid final", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid final value"}},
|
|
||||||
})}, underlays[:2])
|
|
||||||
var loadError *configkit.LoadError
|
|
||||||
if !errors.As(err, &loadError) {
|
|
||||||
t.Fatalf("invalid final backend did not produce a ConfigKit load error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -31,6 +31,7 @@ type configurationShape struct {
|
|||||||
|
|
||||||
func benchmarkPrompt(profileID string) map[string]any {
|
func benchmarkPrompt(profileID string) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
|
"version": "1",
|
||||||
"default_profile": profileID,
|
"default_profile": profileID,
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
@@ -44,7 +45,7 @@ func independentDefinitions(count int) configurationShape {
|
|||||||
profileID := fmt.Sprintf("profile-%03d", index)
|
profileID := fmt.Sprintf("profile-%03d", index)
|
||||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||||
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
|
prompts[promptID] = benchmarkPrompt(profileID)
|
||||||
}
|
}
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
||||||
@@ -63,7 +64,7 @@ func fileBackedDefinitions(count int) configurationShape {
|
|||||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||||
prompt := benchmarkPrompt(profileID)
|
prompt := benchmarkPrompt(profileID)
|
||||||
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
||||||
prompts[promptID] = map[string]any{"1": prompt}
|
prompts[promptID] = prompt
|
||||||
}
|
}
|
||||||
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
||||||
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
||||||
@@ -89,7 +90,7 @@ func sharedBaseDefinitions(count int) configurationShape {
|
|||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||||
"profiles": profiles,
|
"profiles": profiles,
|
||||||
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
|
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
|
||||||
})},
|
})},
|
||||||
inputEntries: count + 2,
|
inputEntries: count + 2,
|
||||||
effectiveEntries: count + 2,
|
effectiveEntries: count + 2,
|
||||||
@@ -117,7 +118,7 @@ func layeredDefinitions(count int) configurationShape {
|
|||||||
},
|
},
|
||||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||||
"profiles": final,
|
"profiles": final,
|
||||||
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
|
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
|
||||||
})},
|
})},
|
||||||
inputEntries: count*4 + 1,
|
inputEntries: count*4 + 1,
|
||||||
effectiveEntries: count + 1,
|
effectiveEntries: count + 1,
|
||||||
@@ -125,6 +126,40 @@ func layeredDefinitions(count int) configurationShape {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
|
||||||
|
for _, count := range []int{4, 16, 64} {
|
||||||
|
b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) {
|
||||||
|
collections := make(map[string]any, count)
|
||||||
|
for index := range count {
|
||||||
|
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"example": map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{
|
||||||
|
"body.md": {Data: []byte("Hello")},
|
||||||
|
})
|
||||||
|
main := configkit.Input{
|
||||||
|
Source: configkit.Mapping("collections", map[string]any{"collections": collections}),
|
||||||
|
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."},
|
||||||
|
}
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
result, err := load(main, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
constructionResult = result
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(count), "collections/op")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func BenchmarkConfigurationConstruction(b *testing.B) {
|
func BenchmarkConfigurationConstruction(b *testing.B) {
|
||||||
for _, scenario := range []struct {
|
for _, scenario := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -160,7 +195,7 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
|||||||
b.Run("prompt", func(b *testing.B) {
|
b.Run("prompt", func(b *testing.B) {
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for b.Loop() {
|
for b.Loop() {
|
||||||
definition, _, err := result.prompt("prompt-127", "1")
|
definition, _, err := result.prompt("prompt-127")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -201,7 +236,7 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
|||||||
prompt := benchmarkPrompt("small")
|
prompt := benchmarkPrompt("small")
|
||||||
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||||
prompts := map[string]any{
|
prompts := map[string]any{
|
||||||
"inspection": map[string]any{"1": prompt},
|
"inspection": prompt,
|
||||||
}
|
}
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||||
@@ -226,8 +261,8 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||||
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
|
if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" ||
|
||||||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||||
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||||
}
|
}
|
||||||
@@ -236,8 +271,8 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||||
}
|
}
|
||||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||||
}
|
}
|
||||||
@@ -249,7 +284,7 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for b.Loop() {
|
for b.Loop() {
|
||||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -261,7 +296,7 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for b.Loop() {
|
for b.Loop() {
|
||||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ func secretProviderPath(path configkit.Path) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
||||||
if len(parts) < 4 || (parts[0] != "backends" && parts[0] != "profiles") || parts[2] != "extra_params" {
|
if !providerExtraParamsPath(parts) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
||||||
@@ -64,3 +64,13 @@ func secretProviderPath(path configkit.Path) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func providerExtraParamsPath(parts []string) bool {
|
||||||
|
if len(parts) >= 4 && parts[1] != "" &&
|
||||||
|
(parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" &&
|
||||||
|
(parts[2] == "backends" || parts[2] == "profiles") &&
|
||||||
|
parts[3] != "" && parts[4] == "extra_params"
|
||||||
|
}
|
||||||
|
|||||||
32
internal/configuration/inspection_test.go
Normal file
32
internal/configuration/inspection_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path configkit.Path
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "backend nested secret", path: configkit.Keys("backends", "provider", "extra_params", "nested", "Token"), want: true},
|
||||||
|
{name: "profile nested secret", path: configkit.Keys("profiles", "assistant", "extra_params", "authorization"), want: true},
|
||||||
|
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends", "default", "extra_params", "nested", "Token"), want: true},
|
||||||
|
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles", "model", "extra_params", "authorization"), want: true},
|
||||||
|
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
|
||||||
|
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
|
||||||
|
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
|
||||||
|
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
|
||||||
|
{name: "ordinary provider value", path: configkit.Keys("profiles", "assistant", "extra_params", "provider", "mode"), want: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := secretProviderPath(test.path); got != test.want {
|
||||||
|
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,12 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||||
|
type CollectionIdentity struct {
|
||||||
|
Namespace string
|
||||||
|
ReleaseVersion *string
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -17,11 +23,63 @@ func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
|||||||
return profiles, nil
|
return profiles, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
backends := append([]string(nil), state.backendIDs...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return backends, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
selected := kinds
|
||||||
|
if len(selected) == 0 {
|
||||||
|
selected = []string{"backend", "profile", "prompt"}
|
||||||
|
}
|
||||||
|
set := make(map[string]struct{})
|
||||||
|
for _, kind := range selected {
|
||||||
|
for _, namespace := range state.namespaces[kind] {
|
||||||
|
set[namespace] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
namespaces := sortedKeys(set)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return namespaces, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
collections := make([]CollectionIdentity, len(state.collections))
|
||||||
|
for index, collection := range state.collections {
|
||||||
|
collections[index] = CollectionIdentity{
|
||||||
|
Namespace: collection.Namespace,
|
||||||
|
ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return collections, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||||
|
for index := range prompts {
|
||||||
|
prompts[index].Version = stringPointerValue(prompts[index].Version)
|
||||||
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,24 +15,24 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
"backends": map[string]any{
|
"backends": map[string]any{
|
||||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
||||||
},
|
},
|
||||||
"profiles": map[string]any{
|
"collections": map[string]any{
|
||||||
"z/profile": map[string]any{"backend": "local", "model": "z-model"},
|
"z": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "z-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
|
||||||
"partial/profile": map[string]any{"backend": "local"},
|
"partial": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local"}}},
|
||||||
"a/profile": map[string]any{"backend": "local", "model": "a-model"},
|
"a": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "a-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
|
||||||
" z/profile ": map[string]any{"backend": "local", "model": "spaced-model"},
|
" z": map[string]any{"profiles": map[string]any{"profile ": map[string]any{"backend": "local", "model": "spaced-model"}}, "prompts": map[string]any{"prompt ": listingPromptDefinition()}},
|
||||||
},
|
|
||||||
"prompts": map[string]any{
|
|
||||||
"z/prompt": map[string]any{"v1": listingPromptDefinition()},
|
|
||||||
"a/prompt": map[string]any{"v1": listingPromptDefinition()},
|
|
||||||
" z/prompt ": map[string]any{
|
|
||||||
"2": listingPromptDefinition(),
|
|
||||||
"10": listingPromptDefinition(),
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
backends, err := state.ListBackends(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantBackends := []string{"local"}
|
||||||
|
if !reflect.DeepEqual(backends, wantBackends) {
|
||||||
|
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||||
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -48,10 +48,9 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
wantPrompts := []domain.PromptIdentity{
|
wantPrompts := []domain.PromptIdentity{
|
||||||
{ID: " z/prompt ", Version: "10"},
|
{ID: " z/prompt ", Version: stringPointer("v1")},
|
||||||
{ID: " z/prompt ", Version: "2"},
|
{ID: "a/prompt", Version: stringPointer("v1")},
|
||||||
{ID: "a/prompt", Version: "v1"},
|
{ID: "z/prompt", Version: stringPointer("v1")},
|
||||||
{ID: "z/prompt", Version: "v1"},
|
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||||
@@ -60,8 +59,9 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
|
|
||||||
func TestLoadedListsAreIndependent(t *testing.T) {
|
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||||
state, err := Load(mapping("identities", map[string]any{
|
state, err := Load(mapping("identities", map[string]any{
|
||||||
|
"backends": map[string]any{"backend": map[string]any{"endpoint": "https://example.test/v1"}},
|
||||||
"profiles": map[string]any{"profile": map[string]any{}},
|
"profiles": map[string]any{"profile": map[string]any{}},
|
||||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
"prompts": map[string]any{"example": listingPromptDefinition()},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -73,13 +73,18 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
profiles = append(profiles, "caller profile")
|
profiles = append(profiles, "caller profile")
|
||||||
profiles[0] = "changed profile"
|
profiles[0] = "changed profile"
|
||||||
|
backends, err := state.ListBackends(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends[0] = "changed backend"
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
prompts[0].ID = "changed prompt"
|
prompts[0].ID = "changed prompt"
|
||||||
prompts[0].Version = "changed version"
|
*prompts[0].Version = "changed version"
|
||||||
|
|
||||||
profiles, err = state.ListProfiles(context.Background())
|
profiles, err = state.ListProfiles(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -88,11 +93,15 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||||
}
|
}
|
||||||
|
backends, err = state.ListBackends(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||||
|
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||||
|
}
|
||||||
prompts, err = state.ListPrompts(context.Background())
|
prompts, err = state.ListPrompts(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
|
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: stringPointer("v1")}}
|
||||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||||
}
|
}
|
||||||
@@ -107,6 +116,10 @@ func TestLoadedListsEmptyInventory(t *testing.T) {
|
|||||||
if err != nil || len(profiles) != 0 {
|
if err != nil || len(profiles) != 0 {
|
||||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
|
backends, err := state.ListBackends(context.Background())
|
||||||
|
if err != nil || len(backends) != 0 {
|
||||||
|
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||||
|
}
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background())
|
||||||
if err != nil || len(prompts) != 0 {
|
if err != nil || len(prompts) != 0 {
|
||||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||||
@@ -116,7 +129,7 @@ func TestLoadedListsEmptyInventory(t *testing.T) {
|
|||||||
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||||
state, err := Load(mapping("identities", map[string]any{
|
state, err := Load(mapping("identities", map[string]any{
|
||||||
"profiles": map[string]any{"partial": map[string]any{}},
|
"profiles": map[string]any{"partial": map[string]any{}},
|
||||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
"prompts": map[string]any{"example": listingPromptDefinition()},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -128,6 +141,10 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
|||||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
|
backends, err := state.ListBackends(ctx)
|
||||||
|
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||||
|
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||||
|
}
|
||||||
prompts, err := state.ListPrompts(ctx)
|
prompts, err := state.ListPrompts(ctx)
|
||||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||||
@@ -136,6 +153,7 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
|||||||
|
|
||||||
func listingPromptDefinition() map[string]any {
|
func listingPromptDefinition() map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -16,9 +17,18 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type document struct {
|
type document struct {
|
||||||
|
Version *string `yaml:"version"`
|
||||||
Backends map[string]backendConfig `yaml:"backends"`
|
Backends map[string]backendConfig `yaml:"backends"`
|
||||||
Profiles map[string]profileConfig `yaml:"profiles"`
|
Profiles map[string]profileConfig `yaml:"profiles"`
|
||||||
Prompts map[string]map[string]promptConfig `yaml:"prompts"`
|
Prompts map[string]promptConfig `yaml:"prompts"`
|
||||||
|
Collections map[string]collectionConfig `yaml:"collections"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type collectionConfig struct {
|
||||||
|
ReleaseVersion *string `yaml:"release_version"`
|
||||||
|
Backends map[string]backendConfig `yaml:"backends"`
|
||||||
|
Profiles map[string]profileConfig `yaml:"profiles"`
|
||||||
|
Prompts map[string]promptConfig `yaml:"prompts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type backendConfig struct {
|
type backendConfig struct {
|
||||||
@@ -27,6 +37,22 @@ type backendConfig struct {
|
|||||||
ConcurrencyLimit *int `yaml:"concurrency_limit"`
|
ConcurrencyLimit *int `yaml:"concurrency_limit"`
|
||||||
QueueCapacity *int `yaml:"queue_capacity"`
|
QueueCapacity *int `yaml:"queue_capacity"`
|
||||||
ExtraParams map[string]any `yaml:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params"`
|
||||||
|
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type backendSmokeTestConfig struct {
|
||||||
|
DefaultProfile *string `yaml:"default_profile"`
|
||||||
|
Reachability *reachabilityProbeConfig `yaml:"reachability"`
|
||||||
|
Authentication *authenticationProbeConfig `yaml:"authentication"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type reachabilityProbeConfig struct {
|
||||||
|
Endpoint *string `yaml:"endpoint"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type authenticationProbeConfig struct {
|
||||||
|
Kind *string `yaml:"kind"`
|
||||||
|
Endpoint *string `yaml:"endpoint"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type profileConfig struct {
|
type profileConfig struct {
|
||||||
@@ -45,6 +71,7 @@ type profileConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptConfig struct {
|
type promptConfig struct {
|
||||||
|
Version *string `yaml:"version"`
|
||||||
DefaultProfile *string `yaml:"default_profile"`
|
DefaultProfile *string `yaml:"default_profile"`
|
||||||
Description *string `yaml:"description"`
|
Description *string `yaml:"description"`
|
||||||
SessionID *string `yaml:"session_id"`
|
SessionID *string `yaml:"session_id"`
|
||||||
@@ -93,18 +120,98 @@ type loaded struct {
|
|||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
backends map[string]map[string]fieldState
|
backends map[string]map[string]fieldState
|
||||||
profiles map[string]map[string]fieldState
|
profiles map[string]map[string]fieldState
|
||||||
promptMetadata map[string]map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
prompts map[string]map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
ambiguous map[string]bool
|
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
indexedProfiles map[string]domain.ExecutionProfile
|
indexedProfiles map[string]domain.ExecutionProfile
|
||||||
|
backendIDs []string
|
||||||
profileIDs []string
|
profileIDs []string
|
||||||
promptIDs []domain.PromptIdentity
|
promptIDs []domain.PromptIdentity
|
||||||
|
namespaces map[string][]string
|
||||||
|
collections []CollectionIdentity
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceBases struct {
|
||||||
|
backends configkit.Path
|
||||||
|
profiles configkit.Path
|
||||||
|
prompts configkit.Path
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceSet struct {
|
||||||
|
isCollection bool
|
||||||
|
namespace string
|
||||||
|
bases resourceBases
|
||||||
|
backends map[string]backendConfig
|
||||||
|
profiles map[string]profileConfig
|
||||||
|
prompts map[string]promptConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
func rootResourceBases() resourceBases {
|
||||||
|
return resourceBases{
|
||||||
|
backends: configkit.Keys("backends"),
|
||||||
|
profiles: configkit.Keys("profiles"),
|
||||||
|
prompts: configkit.Keys("prompts"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourceSets(config *document) []resourceSet {
|
||||||
|
sets := make([]resourceSet, 0, len(config.Collections)+1)
|
||||||
|
sets = append(sets, resourceSet{
|
||||||
|
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
|
||||||
|
})
|
||||||
|
for _, namespace := range sortedKeys(config.Collections) {
|
||||||
|
collection := config.Collections[namespace]
|
||||||
|
base := configkit.Keys("collections", namespace)
|
||||||
|
sets = append(sets, resourceSet{
|
||||||
|
isCollection: true,
|
||||||
|
namespace: namespace,
|
||||||
|
bases: resourceBases{
|
||||||
|
backends: base.Key("backends"),
|
||||||
|
profiles: base.Key("profiles"),
|
||||||
|
prompts: base.Key("prompts"),
|
||||||
|
},
|
||||||
|
backends: collection.Backends,
|
||||||
|
profiles: collection.Profiles,
|
||||||
|
prompts: collection.Prompts,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return sets
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexedResourceID(namespace, localKey string) string {
|
||||||
|
qualified, valid := domain.QualifyLocalResourceID(namespace, localKey)
|
||||||
|
if valid {
|
||||||
|
return qualified
|
||||||
|
}
|
||||||
|
if namespace == "" {
|
||||||
|
return localKey
|
||||||
|
}
|
||||||
|
return namespace + "/" + localKey
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexedBackends(config *document) map[string]backendConfig {
|
||||||
|
backends := make(map[string]backendConfig)
|
||||||
|
for _, resources := range resourceSets(config) {
|
||||||
|
for localKey, definition := range resources.backends {
|
||||||
|
backends[indexedResourceID(resources.namespace, localKey)] = definition
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return backends
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexedProfiles(config *document) map[string]profileConfig {
|
||||||
|
profiles := make(map[string]profileConfig)
|
||||||
|
for _, resources := range resourceSets(config) {
|
||||||
|
for localKey, definition := range resources.profiles {
|
||||||
|
profiles[indexedResourceID(resources.namespace, localKey)] = definition
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return profiles
|
||||||
}
|
}
|
||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params"}
|
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
||||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||||
|
|
||||||
// Load captures one immutable effective configuration for internal consumers.
|
// Load captures one immutable effective configuration for internal consumers.
|
||||||
@@ -117,22 +224,31 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
state := &loaded{
|
state := &loaded{
|
||||||
backends: make(map[string]map[string]fieldState),
|
backends: make(map[string]map[string]fieldState),
|
||||||
profiles: make(map[string]map[string]fieldState),
|
profiles: make(map[string]map[string]fieldState),
|
||||||
promptMetadata: make(map[string]map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
|
SourceVersionMode: configkit.SourceVersionModeIfPresent,
|
||||||
Definition: configkit.Definition[document]{
|
Definition: configkit.Definition[document]{
|
||||||
|
Version: &configkit.VersionRule{
|
||||||
|
Path: configkit.Keys("version"),
|
||||||
|
Check: checkConfigurationVersion,
|
||||||
|
},
|
||||||
Secret: secretProviderPath,
|
Secret: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
for _, resources := range resourceSets(config) {
|
||||||
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
captureFields(resources.backends, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
||||||
capturePromptMetadata(config.Prompts, resolution, state.promptMetadata)
|
captureFields(resources.profiles, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
||||||
resolveBackends(config.Backends, resolution)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||||
resolveProfiles(config.Profiles, resolution)
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
resolvePrompts(config.Prompts, resolution)
|
resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
|
||||||
|
resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
Validate: validateEffective,
|
Validate: func(config *document, validation *configkit.Validation) {
|
||||||
|
validateEffective(config, validation)
|
||||||
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -145,29 +261,63 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
if err := state.buildExecutionProfiles(); err != nil {
|
if err := state.buildExecutionProfiles(); err != nil {
|
||||||
return nil, fmt.Errorf("load configuration: %w", err)
|
return nil, fmt.Errorf("load configuration: %w", err)
|
||||||
}
|
}
|
||||||
|
state.backendIDs = sortedKeys(indexedBackends(&result.Config))
|
||||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||||
for _, id := range sortedKeys(state.prompts) {
|
for _, id := range sortedKeys(state.prompts) {
|
||||||
for _, version := range sortedKeys(state.prompts[id]) {
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: stringPointerValue(state.prompts[id].definition.Version)})
|
||||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: version})
|
|
||||||
}
|
}
|
||||||
|
state.namespaces = map[string][]string{
|
||||||
|
"backend": resourceNamespaces(state.backendIDs),
|
||||||
|
"profile": resourceNamespaces(state.profileIDs),
|
||||||
|
"prompt": promptNamespaces(state.promptIDs),
|
||||||
|
}
|
||||||
|
for _, namespace := range sortedKeys(result.Config.Collections) {
|
||||||
|
collection := result.Config.Collections[namespace]
|
||||||
|
state.collections = append(state.collections, CollectionIdentity{
|
||||||
|
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resourceNamespaces(ids []string) []string {
|
||||||
|
namespaces := make(map[string]struct{})
|
||||||
|
for _, id := range ids {
|
||||||
|
if namespace, _, qualified := strings.Cut(id, "/"); qualified {
|
||||||
|
namespaces[namespace] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedKeys(namespaces)
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptNamespaces(prompts []domain.PromptIdentity) []string {
|
||||||
|
ids := make([]string, len(prompts))
|
||||||
|
for index := range prompts {
|
||||||
|
ids[index] = prompts[index].ID
|
||||||
|
}
|
||||||
|
return resourceNamespaces(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
|
||||||
|
if !version.Present {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if version.Null || version.Text != "1" {
|
||||||
|
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type promptInputPresence struct {
|
type promptInputPresence struct {
|
||||||
contentType fieldState
|
contentType fieldState
|
||||||
description fieldState
|
description fieldState
|
||||||
}
|
}
|
||||||
|
|
||||||
func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) {
|
func capturePromptMetadata(prompts map[string]promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
||||||
for _, id := range sortedKeys(prompts) {
|
for _, id := range sortedKeys(prompts) {
|
||||||
if destination[id] == nil {
|
inputs := prompts[id].Inputs
|
||||||
destination[id] = make(map[string][]promptInputPresence, len(prompts[id]))
|
|
||||||
}
|
|
||||||
for _, version := range sortedKeys(prompts[id]) {
|
|
||||||
inputs := prompts[id][version].Inputs
|
|
||||||
states := make([]promptInputPresence, len(inputs))
|
states := make([]promptInputPresence, len(inputs))
|
||||||
base := configkit.Keys("prompts", id, version, "inputs")
|
base := basePath.Key(id).Key("inputs")
|
||||||
for index := range inputs {
|
for index := range inputs {
|
||||||
path := base.Index(index)
|
path := base.Index(index)
|
||||||
states[index] = promptInputPresence{
|
states[index] = promptInputPresence{
|
||||||
@@ -175,8 +325,7 @@ func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolutio
|
|||||||
description: captureFieldState(path.Key("description"), resolution),
|
description: captureFieldState(path.Key("description"), resolution),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[id][version] = states
|
destination[indexedResourceID(namespace, id)] = states
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,11 +341,11 @@ func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
func captureFields[T any](entries map[string]T, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||||
for id := range entries {
|
for id := range entries {
|
||||||
states := make(map[string]fieldState, len(fields))
|
states := make(map[string]fieldState, len(fields))
|
||||||
for _, field := range fields {
|
for _, field := range fields {
|
||||||
info, exists := resolution.InputInfo(configkit.Keys(collection, id, field))
|
info, exists := resolution.InputInfo(basePath.Key(id).Key(field))
|
||||||
switch {
|
switch {
|
||||||
case !exists || !info.Explicit:
|
case !exists || !info.Explicit:
|
||||||
states[field] = fieldAbsent
|
states[field] = fieldAbsent
|
||||||
@@ -206,7 +355,7 @@ func captureFields[T any](entries map[string]T, collection string, fields []stri
|
|||||||
states[field] = fieldValue
|
states[field] = fieldValue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[id] = states
|
destination[indexedResourceID(namespace, id)] = states
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,10 +376,10 @@ func normalizeString(value **string, path configkit.Path, resolution *configkit.
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveBackends(backends map[string]backendConfig, resolution *configkit.Resolution) {
|
func resolveBackends(backends map[string]backendConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||||
for _, id := range sortedKeys(backends) {
|
for _, id := range sortedKeys(backends) {
|
||||||
backend := backends[id]
|
backend := backends[id]
|
||||||
base := configkit.Keys("backends", id)
|
base := basePath.Key(id)
|
||||||
if backend.Endpoint != nil {
|
if backend.Endpoint != nil {
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -242,6 +391,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
|
|||||||
if backend.APIKeyEnv != nil {
|
if backend.APIKeyEnv != nil {
|
||||||
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
|
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
|
||||||
}
|
}
|
||||||
|
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||||
|
smokePath := base.Key("smoke_tests")
|
||||||
|
if smokeTests.Reachability != nil {
|
||||||
|
normalizeDiagnosticEndpoint(&smokeTests.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
|
||||||
|
}
|
||||||
|
if smokeTests.Authentication != nil {
|
||||||
|
if smokeTests.Authentication.Kind != nil {
|
||||||
|
normalizeString(&smokeTests.Authentication.Kind,
|
||||||
|
smokePath.Key("authentication").Key("kind"), resolution,
|
||||||
|
strings.ToLower(strings.TrimSpace(*smokeTests.Authentication.Kind)))
|
||||||
|
}
|
||||||
|
normalizeDiagnosticEndpoint(&smokeTests.Authentication.Endpoint,
|
||||||
|
smokePath.Key("authentication").Key("endpoint"), resolution)
|
||||||
|
}
|
||||||
|
}
|
||||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
||||||
capacity := 1024
|
capacity := 1024
|
||||||
if resolution.IsExplicit(base.Key("queue_capacity")) {
|
if resolution.IsExplicit(base.Key("queue_capacity")) {
|
||||||
@@ -254,10 +418,25 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Resolution) {
|
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
|
||||||
|
if value == nil || *value == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
normalized := strings.TrimSpace(**value)
|
||||||
|
request, err := http.NewRequest(http.MethodGet, normalized, nil)
|
||||||
|
if normalized == "" || err != nil || request.URL == nil ||
|
||||||
|
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
|
||||||
|
request.URL.Hostname() == "" {
|
||||||
|
resolution.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
normalizeString(value, path, resolution, normalized)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveProfiles(profiles map[string]profileConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||||
for _, id := range sortedKeys(profiles) {
|
for _, id := range sortedKeys(profiles) {
|
||||||
profile := profiles[id]
|
profile := profiles[id]
|
||||||
base := configkit.Keys("profiles", id)
|
base := basePath.Key(id)
|
||||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -283,11 +462,22 @@ func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Re
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateEffective(config *document, validation *configkit.Validation) {
|
func validateEffective(config *document, validation *configkit.Validation) {
|
||||||
for _, id := range sortedKeys(config.Backends) {
|
sets := resourceSets(config)
|
||||||
backend := config.Backends[id]
|
backendIDs := make(map[string]struct{})
|
||||||
base := configkit.Keys("backends", id)
|
for _, resources := range sets {
|
||||||
if strings.TrimSpace(id) == "" {
|
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
|
||||||
validation.Error(base, "backend ID must not be blank")
|
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
|
||||||
|
}
|
||||||
|
for localKey := range resources.backends {
|
||||||
|
backendIDs[indexedResourceID(resources.namespace, localKey)] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, resources := range sets {
|
||||||
|
for _, id := range sortedKeys(resources.backends) {
|
||||||
|
backend := resources.backends[id]
|
||||||
|
base := resources.bases.backends.Key(id)
|
||||||
|
if !domain.IsValidLocalResourceKey(id) {
|
||||||
|
validation.Error(base, "backend key must be a nonblank local key without a slash")
|
||||||
}
|
}
|
||||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||||
@@ -309,15 +499,35 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||||
|
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||||
|
smokePath := base.Key("smoke_tests")
|
||||||
|
if smokeTests.DefaultProfile != nil {
|
||||||
|
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||||
|
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
for _, id := range sortedKeys(config.Profiles) {
|
}
|
||||||
profile := config.Profiles[id]
|
if authentication := smokeTests.Authentication; authentication != nil {
|
||||||
base := configkit.Keys("profiles", id)
|
kindPath := smokePath.Key("authentication").Key("kind")
|
||||||
if strings.TrimSpace(id) == "" {
|
if authentication.Kind == nil {
|
||||||
validation.Error(base, "profile ID must not be blank")
|
validation.Error(kindPath, "authentication kind is required")
|
||||||
|
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||||
|
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range sortedKeys(resources.profiles) {
|
||||||
|
profile := resources.profiles[id]
|
||||||
|
base := resources.bases.profiles.Key(id)
|
||||||
|
if !domain.IsValidLocalResourceKey(id) {
|
||||||
|
validation.Error(base, "profile key must be a nonblank local key without a slash")
|
||||||
}
|
}
|
||||||
if profile.Backend != nil && *profile.Backend != "" {
|
if profile.Backend != nil && *profile.Backend != "" {
|
||||||
if _, exists := config.Backends[*profile.Backend]; !exists {
|
if !domain.IsValidResourceID(*profile.Backend) {
|
||||||
|
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||||
|
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||||
validation.Error(base.Key("backend"), "backend is not defined")
|
validation.Error(base.Key("backend"), "backend is not defined")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -346,6 +556,17 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
|||||||
}
|
}
|
||||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||||
}
|
}
|
||||||
|
for _, id := range sortedKeys(resources.prompts) {
|
||||||
|
prompt := resources.prompts[id]
|
||||||
|
base := resources.bases.prompts.Key(id)
|
||||||
|
if !domain.IsValidLocalResourceKey(id) {
|
||||||
|
validation.Error(base, "prompt key must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||||
|
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateExtraParams(values map[string]any, path configkit.Path, backend bool, validation *configkit.Validation) {
|
func validateExtraParams(values map[string]any, path configkit.Path, backend bool, validation *configkit.Validation) {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
@@ -11,40 +12,128 @@ func mapping(label string, values map[string]any) configkit.Input {
|
|||||||
return configkit.Input{Source: configkit.Mapping(label, values)}
|
return configkit.Input{Source: configkit.Mapping(label, values)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
version any
|
||||||
|
include bool
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{name: "string one", version: "1", include: true, valid: true},
|
||||||
|
{name: "integer one", version: int64(1), include: true, valid: true},
|
||||||
|
{name: "omitted", valid: true},
|
||||||
|
{name: "explicit null", include: true},
|
||||||
|
{name: "unsupported string", version: "2", include: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
values := map[string]any{}
|
||||||
|
if test.include {
|
||||||
|
values["version"] = test.version
|
||||||
|
}
|
||||||
|
loaded, err := load(mapping("main", values), nil)
|
||||||
|
if test.valid {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load accepted version: %v", err)
|
||||||
|
}
|
||||||
|
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1") {
|
||||||
|
t.Fatalf("typed version = %v, want 1", loaded.result.Config.Version)
|
||||||
|
}
|
||||||
|
if !test.include && loaded.result.Config.Version != nil {
|
||||||
|
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if err == nil || !errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("load error = %v, want ConfigKit load error", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "2") {
|
||||||
|
t.Fatalf("load error disclosed the unsupported value: %v", err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, diagnostic := range loadErr.Diagnostics() {
|
||||||
|
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) {
|
||||||
|
underlay := mapping("declared underlay", map[string]any{
|
||||||
|
"version": int64(1),
|
||||||
|
})
|
||||||
|
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load accepted non-main declaration: %v", err)
|
||||||
|
}
|
||||||
|
if loaded.result.Config.Version != nil {
|
||||||
|
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidFragment := mapping("headerless prompt fragment", map[string]any{
|
||||||
|
"prompts": map[string]any{"incomplete": map[string]any{"description": "still validated"}},
|
||||||
|
})
|
||||||
|
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
|
||||||
|
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
||||||
base := mapping("defaults", map[string]any{
|
base := mapping("defaults", map[string]any{
|
||||||
|
"version": int64(1),
|
||||||
"backends": map[string]any{
|
"backends": map[string]any{
|
||||||
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 2},
|
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 2},
|
||||||
},
|
},
|
||||||
"profiles": map[string]any{
|
"collections": map[string]any{"base": map[string]any{
|
||||||
"base/one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
|
"release_version": "",
|
||||||
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
|
"backends": map[string]any{
|
||||||
"child.v1": map[string]any{"extends": "base/one"},
|
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 3},
|
||||||
},
|
},
|
||||||
"prompts": map[string]any{"invoice/summary": map[string]any{"v1.0": map[string]any{
|
"profiles": map[string]any{
|
||||||
|
"one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
|
||||||
|
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
|
||||||
|
"child.v1": map[string]any{"extends": "one"},
|
||||||
|
"namespaced": map[string]any{"backend": "base/local.v1", "model": "collection-model"},
|
||||||
|
},
|
||||||
|
}, "invoice": map[string]any{
|
||||||
|
"release_version": "r7",
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"summary": map[string]any{
|
||||||
|
"version": "v1.0",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
|
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}},
|
},
|
||||||
|
"version": map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Nested ID."}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
})
|
})
|
||||||
overlay := mapping("operator", map[string]any{
|
overlay := mapping("operator", map[string]any{
|
||||||
"backends": map[string]any{"local.v1": map[string]any{"queue_capacity": 0}},
|
"backends": map[string]any{"local.v1": map[string]any{"queue_capacity": 0}},
|
||||||
"profiles": map[string]any{"child.v1": map[string]any{
|
"collections": map[string]any{"base": map[string]any{"profiles": map[string]any{"child.v1": map[string]any{
|
||||||
"temperature": 0, "top_p": nil, "api_key_required": false,
|
"temperature": 0, "top_p": nil, "api_key_required": false,
|
||||||
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
|
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
|
||||||
}},
|
}}}},
|
||||||
})
|
})
|
||||||
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
|
loaded, err := load(mapping("main", map[string]any{"version": "1"}), []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
|
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profile := loaded.result.Config.Profiles["child.v1"]
|
profile := loaded.result.Config.Collections["base"].Profiles["child.v1"]
|
||||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||||
t.Fatalf("effective values were not retained: %+v", profile)
|
t.Fatalf("effective values were not retained: %+v", profile)
|
||||||
}
|
}
|
||||||
if loaded.profiles["child.v1"]["temperature"] != fieldValue || loaded.profiles["child.v1"]["top_p"] != fieldNull || loaded.profiles["child.v1"]["api_key_required"] != fieldValue {
|
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
|
||||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["child.v1"])
|
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
|
||||||
}
|
}
|
||||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||||
@@ -54,9 +143,69 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
|||||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
||||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||||
}
|
}
|
||||||
if loaded.result.Config.Prompts["invoice/summary"]["v1.0"].Messages[0].Content == nil {
|
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||||
|
t.Fatalf("collection backend was not indexed by its full ID: %v", err)
|
||||||
|
}
|
||||||
|
collectionProfile, exists := loaded.indexedProfiles["base/namespaced"]
|
||||||
|
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
|
||||||
|
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
|
||||||
|
}
|
||||||
|
if loaded.result.Config.Collections["invoice"].Prompts["summary"].Messages[0].Content == nil || loaded.result.Config.Collections["invoice"].Prompts["summary"].Version == nil || *loaded.result.Config.Collections["invoice"].Prompts["summary"].Version != "v1.0" {
|
||||||
t.Fatal("literal prompt ID and version were not decoded")
|
t.Fatal("literal prompt ID and version were not decoded")
|
||||||
}
|
}
|
||||||
|
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1" ||
|
||||||
|
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" ||
|
||||||
|
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
|
||||||
|
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
|
||||||
|
}
|
||||||
|
if _, _, err := loaded.prompt("invoice/version"); err != nil {
|
||||||
|
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidCollectionAndResourceKeys(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
values map[string]any
|
||||||
|
message string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "namespace with separator",
|
||||||
|
values: map[string]any{"collections": map[string]any{"bad/name": map[string]any{}}},
|
||||||
|
message: "collection namespace must be nonblank and contain no slash",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "root resource with separator",
|
||||||
|
values: map[string]any{"backends": map[string]any{
|
||||||
|
"bad/name": map[string]any{"endpoint": "https://example.test/v1"},
|
||||||
|
}},
|
||||||
|
message: "backend key must be a nonblank local key without a slash",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "collection resource with separator",
|
||||||
|
values: map[string]any{"collections": map[string]any{"good": map[string]any{
|
||||||
|
"prompts": map[string]any{"bad/name": map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}},
|
||||||
|
}}},
|
||||||
|
message: "prompt key must be a nonblank local key without a slash",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
_, err := load(mapping("invalid names", test.values), nil)
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if err == nil || !errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("load error = %v, want ConfigKit validation error", err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range loadErr.Diagnostics() {
|
||||||
|
if diagnostic.Message == test.message {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("load diagnostics = %+v, want message %q", loadErr.Diagnostics(), test.message)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
|
func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
|
||||||
@@ -78,6 +227,159 @@ func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
smokeTests any
|
||||||
|
include bool
|
||||||
|
inherited bool
|
||||||
|
}{
|
||||||
|
{name: "omitted"},
|
||||||
|
{name: "null", smokeTests: nil, include: true, inherited: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
backend := map[string]any{"endpoint": "https://backend.example/v1"}
|
||||||
|
if test.include {
|
||||||
|
backend["smoke_tests"] = test.smokeTests
|
||||||
|
}
|
||||||
|
main := mapping("backend", map[string]any{
|
||||||
|
"backends": map[string]any{"local": backend},
|
||||||
|
})
|
||||||
|
var underlays []configkit.Underlay
|
||||||
|
if test.inherited {
|
||||||
|
underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"endpoint": "https://backend.example/v1",
|
||||||
|
"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
|
||||||
|
}},
|
||||||
|
})}}}
|
||||||
|
}
|
||||||
|
loaded, err := load(main, underlays)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := loaded.result.Config.Backends["local"].SmokeTests; got != nil {
|
||||||
|
t.Fatalf("smoke_tests = %+v, want nil", got)
|
||||||
|
}
|
||||||
|
selected, err := loaded.registry.GetBackend("local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if selected.SmokeTests != nil {
|
||||||
|
t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
|
||||||
|
base := mapping("defaults", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"endpoint": "https://backend.example/v1",
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"default_profile": "catalog-profile",
|
||||||
|
"reachability": map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
|
||||||
|
"authentication": map[string]any{
|
||||||
|
"kind": " OPENROUTER_KEY ",
|
||||||
|
"endpoint": "https://probe.example/key?scope=current",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
overlay := mapping("operator", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"default_profile": "not-yet-listed",
|
||||||
|
"authentication": map[string]any{"kind": " NONE ", "endpoint": nil},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{base}},
|
||||||
|
{Inputs: []configkit.Input{overlay}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load effective diagnostic metadata: %v", err)
|
||||||
|
}
|
||||||
|
selected, err := loaded.registry.GetBackend("local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
metadata := selected.SmokeTests
|
||||||
|
if selected.Endpoint != "https://backend.example/v1" {
|
||||||
|
t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
|
||||||
|
}
|
||||||
|
if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
|
||||||
|
t.Fatalf("default profile overlay was not retained: %+v", metadata)
|
||||||
|
}
|
||||||
|
if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
|
||||||
|
*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
|
||||||
|
t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
|
||||||
|
}
|
||||||
|
if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
|
||||||
|
metadata.Authentication.Endpoint != nil {
|
||||||
|
t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
data map[string]any
|
||||||
|
path string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "blank default profile",
|
||||||
|
data: map[string]any{"default_profile": " \t "},
|
||||||
|
path: "/backends/local/smoke_tests/default_profile",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "authentication kind required",
|
||||||
|
data: map[string]any{"authentication": map[string]any{}},
|
||||||
|
path: "/backends/local/smoke_tests/authentication/kind",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown authentication kind",
|
||||||
|
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
|
||||||
|
path: "/backends/local/smoke_tests/authentication/kind",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-http reachability URL",
|
||||||
|
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
|
||||||
|
path: "/backends/local/smoke_tests/reachability/endpoint",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank authentication URL",
|
||||||
|
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
|
||||||
|
path: "/backends/local/smoke_tests/authentication/endpoint",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "relative authentication URL",
|
||||||
|
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
|
||||||
|
path: "/backends/local/smoke_tests/authentication/endpoint",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
_, err := load(mapping("backend", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"endpoint": "https://backend.example/v1",
|
||||||
|
"smoke_tests": test.data,
|
||||||
|
}},
|
||||||
|
}), nil)
|
||||||
|
var loadError *configkit.LoadError
|
||||||
|
if !errors.As(err, &loadError) {
|
||||||
|
t.Fatalf("expected final-value validation error, got %v", err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range loadError.Diagnostics() {
|
||||||
|
if diagnostic.Path.String() == test.path {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
||||||
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
||||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
|
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
|
||||||
@@ -128,6 +430,40 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
|
||||||
|
underlay := mapping("defaults", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"endpoint": "https://backend.example/v1",
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"default_profile": " ",
|
||||||
|
"reachability": map[string]any{"endpoint": "ftp://invalid.example/"},
|
||||||
|
"authentication": map[string]any{"kind": "unsupported"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
main := mapping("operator", map[string]any{
|
||||||
|
"backends": map[string]any{"local": map[string]any{
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"default_profile": "dangling-profile",
|
||||||
|
"reachability": map[string]any{"endpoint": nil},
|
||||||
|
"authentication": map[string]any{"kind": "none"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("replaced lower metadata should not fail: %v", err)
|
||||||
|
}
|
||||||
|
selected, err := loaded.registry.GetBackend("local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
|
||||||
|
*selected.SmokeTests.DefaultProfile != "dangling-profile" {
|
||||||
|
t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
|
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
|
||||||
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
|
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
|
||||||
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
|
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
errPromptNotFound = errors.New("prompt definition not found")
|
errPromptNotFound = errors.New("prompt definition not found")
|
||||||
errPromptAmbiguous = errors.New("prompt version is ambiguous")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptEntry struct {
|
type promptEntry struct {
|
||||||
@@ -29,11 +28,10 @@ type promptEntry struct {
|
|||||||
messages []*template.Template
|
messages []*template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
func resolvePrompts(prompts map[string]promptConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||||
for _, id := range sortedKeys(prompts) {
|
for _, id := range sortedKeys(prompts) {
|
||||||
for _, version := range sortedKeys(prompts[id]) {
|
definition := prompts[id]
|
||||||
definition := prompts[id][version]
|
base := basePath.Key(id)
|
||||||
base := configkit.Keys("prompts", id, version)
|
|
||||||
for _, field := range []struct {
|
for _, field := range []struct {
|
||||||
name string
|
name string
|
||||||
value **string
|
value **string
|
||||||
@@ -79,14 +77,12 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
|
|||||||
if definition.Output != nil && definition.Output.SchemaPath != nil {
|
if definition.Output != nil && definition.Output.SchemaPath != nil {
|
||||||
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
|
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
|
||||||
}
|
}
|
||||||
prompts[id][version] = definition
|
prompts[id] = definition
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) materializePrompts() error {
|
func (state *loaded) materializePrompts() error {
|
||||||
state.prompts = make(map[string]map[string]*promptEntry, len(state.result.Config.Prompts))
|
state.prompts = make(map[string]*promptEntry)
|
||||||
state.ambiguous = make(map[string]bool)
|
|
||||||
var metadata *configkit.Metadata
|
var metadata *configkit.Metadata
|
||||||
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
|
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
|
||||||
if metadata == nil {
|
if metadata == nil {
|
||||||
@@ -95,36 +91,29 @@ func (state *loaded) materializePrompts() error {
|
|||||||
}
|
}
|
||||||
return metadata.Lookup(path)
|
return metadata.Lookup(path)
|
||||||
}
|
}
|
||||||
for _, id := range sortedKeys(state.result.Config.Prompts) {
|
for _, resources := range resourceSets(&state.result.Config) {
|
||||||
|
for _, id := range sortedKeys(resources.prompts) {
|
||||||
|
promptBase := resources.bases.prompts.Key(id)
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return fmt.Errorf("%s: prompt ID must not be blank", configkit.Keys("prompts", id))
|
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
|
||||||
}
|
}
|
||||||
versions := state.result.Config.Prompts[id]
|
fullID := indexedResourceID(resources.namespace, id)
|
||||||
if versions == nil {
|
raw := resources.prompts[id]
|
||||||
return fmt.Errorf("%s: prompt collection must be a mapping", configkit.Keys("prompts", id))
|
entry, err := state.materializePrompt(fullID, raw.Version, raw, promptBase, lookupOrigin)
|
||||||
}
|
|
||||||
state.prompts[id] = make(map[string]*promptEntry, len(versions))
|
|
||||||
state.ambiguous[id] = len(versions) > 1
|
|
||||||
for _, version := range sortedKeys(versions) {
|
|
||||||
if strings.TrimSpace(version) == "" {
|
|
||||||
return fmt.Errorf("%s: prompt version must not be blank", configkit.Keys("prompts", id, version))
|
|
||||||
}
|
|
||||||
entry, err := state.materializePrompt(id, version, versions[version], lookupOrigin)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
state.prompts[id][version] = entry
|
state.prompts[fullID] = entry
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) materializePrompt(id, version string, raw promptConfig, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
func (state *loaded) materializePrompt(id string, version *string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
||||||
base := configkit.Keys("prompts", id, version)
|
|
||||||
if len(raw.Messages) == 0 {
|
if len(raw.Messages) == 0 {
|
||||||
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
||||||
}
|
}
|
||||||
definition := domain.PromptDefinition{ID: id, Version: version}
|
definition := domain.PromptDefinition{ID: id, Version: stringPointerValue(version)}
|
||||||
if raw.DefaultProfile != nil {
|
if raw.DefaultProfile != nil {
|
||||||
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
||||||
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
||||||
@@ -163,7 +152,7 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
|||||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||||
})
|
})
|
||||||
presence := state.promptMetadata[id][version][index]
|
presence := state.promptMetadata[id][index]
|
||||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||||
if presence.contentType == fieldValue {
|
if presence.contentType == fieldValue {
|
||||||
metadata.ContentType = stringPointer(contentType)
|
metadata.ContentType = stringPointer(contentType)
|
||||||
@@ -253,7 +242,7 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
|||||||
entry.definition = definition
|
entry.definition = definition
|
||||||
entry.hash = hash
|
entry.hash = hash
|
||||||
entry.inspection = domain.PromptInspection{
|
entry.inspection = domain.PromptInspection{
|
||||||
PromptID: id, PromptVersion: version, PromptHash: hash,
|
PromptID: id, PromptVersion: stringPointerValue(version), PromptHash: hash,
|
||||||
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||||
OutputContract: contract,
|
OutputContract: contract,
|
||||||
}
|
}
|
||||||
@@ -265,6 +254,13 @@ func stringPointer(value string) *string {
|
|||||||
return ©
|
return ©
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stringPointerValue(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return stringPointer(*value)
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||||
@@ -297,20 +293,11 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
|
|||||||
return string(content), nil
|
return string(content), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error) {
|
func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
|
||||||
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
if state == nil || strings.TrimSpace(id) == "" {
|
||||||
return nil, errPromptNotFound
|
return nil, errPromptNotFound
|
||||||
}
|
}
|
||||||
versions := state.prompts[id]
|
entry := state.prompts[id]
|
||||||
if version == "" {
|
|
||||||
if state.ambiguous[id] {
|
|
||||||
return nil, errPromptAmbiguous
|
|
||||||
}
|
|
||||||
for selected := range versions {
|
|
||||||
version = selected
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entry := versions[version]
|
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
return nil, errPromptNotFound
|
return nil, errPromptNotFound
|
||||||
}
|
}
|
||||||
@@ -318,6 +305,7 @@ func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||||
|
definition.Version = stringPointerValue(definition.Version)
|
||||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||||
for index := range definition.Templates {
|
for index := range definition.Templates {
|
||||||
@@ -330,6 +318,7 @@ func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||||
|
inspection.PromptVersion = stringPointerValue(inspection.PromptVersion)
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
return &inspection
|
return &inspection
|
||||||
}
|
}
|
||||||
@@ -341,8 +330,8 @@ func promptLookupError(err error) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
func (state *loaded) prompt(id string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||||
entry, err := state.lookupPromptEntry(id, version)
|
entry, err := state.lookupPromptEntry(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, domain.PromptInspection{}, err
|
return nil, domain.PromptInspection{}, err
|
||||||
}
|
}
|
||||||
@@ -350,12 +339,12 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||||
// construction-time hash. Lookup retains literal ID and version keys.
|
// construction-time hash.
|
||||||
func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string) (*domain.PromptDefinition, string, error) {
|
func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
entry, err := state.lookupPromptEntry(id, version)
|
entry, err := state.lookupPromptEntry(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", promptLookupError(err)
|
return nil, "", promptLookupError(err)
|
||||||
}
|
}
|
||||||
@@ -364,11 +353,11 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string)
|
|||||||
|
|
||||||
// GetPromptInspection returns an independently owned inspection of one
|
// GetPromptInspection returns an independently owned inspection of one
|
||||||
// captured prompt without copying its execution templates.
|
// captured prompt without copying its execution templates.
|
||||||
func (state *loaded) GetPromptInspection(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
entry, err := state.lookupPromptEntry(id, version)
|
entry, err := state.lookupPromptEntry(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, promptLookupError(err)
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
@@ -376,17 +365,17 @@ func (state *loaded) GetPromptInspection(ctx context.Context, id, version string
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id, version string) (*domain.PromptDefinition, error) {
|
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
||||||
definition, _, err := state.GetPromptSnapshot(ctx, id, version)
|
definition, _, err := state.GetPromptSnapshot(ctx, id)
|
||||||
return definition, err
|
return definition, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCompiledTemplates returns construction-time parses for one exact prompt.
|
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
||||||
func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTemplates, bool) {
|
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
return prompt.CompiledTemplates{}, false
|
return prompt.CompiledTemplates{}, false
|
||||||
}
|
}
|
||||||
entry := state.prompts[id][version]
|
entry := state.prompts[id]
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
return prompt.CompiledTemplates{}, false
|
return prompt.CompiledTemplates{}, false
|
||||||
}
|
}
|
||||||
@@ -394,13 +383,13 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetRequirementSnapshot returns captured templates and input metadata for
|
// GetRequirementSnapshot returns captured templates and input metadata for
|
||||||
// one exact prompt identity. Returned metadata and slices are independently
|
// one prompt ID. Returned metadata and slices are independently
|
||||||
// owned; the compiled templates are shared immutable construction-time parses.
|
// owned; the compiled templates are shared immutable construction-time parses.
|
||||||
func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) {
|
func (state *loaded) GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
return prompt.RequirementSnapshot{}, false
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
entry := state.prompts[id][version]
|
entry := state.prompts[id]
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
return prompt.RequirementSnapshot{}, false
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ import (
|
|||||||
|
|
||||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\nprompts:\n child:\n extends: base\n")},
|
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\n - path: ../other/definition.yaml\ncollections:\n docs:\n prompts:\n child:\n extends: base\n")},
|
||||||
"catalog/base.yaml": {Data: []byte("prompts:\n base:\n v1:\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n base:\n version: \"v1\"\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
||||||
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
||||||
|
"other/definition.yaml": {Data: []byte("collections:\n other:\n prompts:\n child:\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
|
||||||
|
"other/body.md": {Data: []byte("Other collection {{.name}}")},
|
||||||
}
|
}
|
||||||
namespace := configkit.NewFilesystem("embedded", files)
|
namespace := configkit.NewFilesystem("embedded", files)
|
||||||
state, err := load(configkit.Input{
|
state, err := load(configkit.Input{
|
||||||
@@ -32,7 +34,7 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition, inspection, err := state.prompt("child", "")
|
definition, inspection, err := state.prompt("docs/child")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -43,7 +45,7 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
|
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
|
||||||
t.Fatalf("message metadata was not normalized: %+v", message)
|
t.Fatalf("message metadata was not normalized: %+v", message)
|
||||||
}
|
}
|
||||||
field, ok := state.result.Metadata().Lookup(configkit.Keys("prompts", "child", "v1", "messages").Index(0).Key("content_file"))
|
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts", "child", "messages").Index(0).Key("content_file"))
|
||||||
if !ok || len(field.Origin.Locations) != 1 {
|
if !ok || len(field.Origin.Locations) != 1 {
|
||||||
t.Fatalf("content origin was lost during normalization: %+v", field)
|
t.Fatalf("content origin was lost during normalization: %+v", field)
|
||||||
}
|
}
|
||||||
@@ -56,16 +58,25 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
if originPath != "catalog/base.yaml" {
|
if originPath != "catalog/base.yaml" {
|
||||||
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
|
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
|
||||||
}
|
}
|
||||||
|
other, otherInspection, err := state.prompt("other/child")
|
||||||
|
if err != nil || other.Templates[0].Content != "Other collection {{.name}}" {
|
||||||
|
t.Fatalf("other collection did not retain its source context: %+v, %v", other, err)
|
||||||
|
}
|
||||||
files["catalog/body.md"].Data = []byte("Changed")
|
files["catalog/body.md"].Data = []byte("Changed")
|
||||||
again, againInspection, err := state.prompt("child", "v1")
|
files["other/body.md"].Data = []byte("Changed other collection")
|
||||||
|
again, againInspection, err := state.prompt("docs/child")
|
||||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||||
}
|
}
|
||||||
definition.Templates[0].Content = "caller mutation"
|
definition.Templates[0].Content = "caller mutation"
|
||||||
third, _, err := state.prompt("child", "v1")
|
third, _, err := state.prompt("docs/child")
|
||||||
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
|
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
|
||||||
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
|
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
|
||||||
}
|
}
|
||||||
|
otherAgain, otherAgainInspection, err := state.prompt("other/child")
|
||||||
|
if err != nil || otherAgain.Templates[0].Content != "Other collection {{.name}}" || otherAgainInspection.PromptHash != otherInspection.PromptHash {
|
||||||
|
t.Fatalf("other captured definition changed after source mutation: %+v, %+v, %v", otherAgain, otherAgainInspection, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
|
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
|
||||||
@@ -81,12 +92,12 @@ func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition, _, err := state.prompt("example", "v1")
|
definition, _, err := state.prompt("example")
|
||||||
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
|
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
|
||||||
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
|
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
|
||||||
}
|
}
|
||||||
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
|
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
|
||||||
if err == nil || !strings.Contains(err.Error(), "/prompts/example/v1/messages/0/content_file") {
|
if err == nil || !strings.Contains(err.Error(), "/prompts/example/messages/0/content_file") {
|
||||||
t.Fatalf("expected semantic path for contextless resource, got %v", err)
|
t.Fatalf("expected semantic path for contextless resource, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -95,7 +106,7 @@ func TestLoadCapturesNativeContentFile(t *testing.T) {
|
|||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
definitionPath := filepath.Join(root, "definition.yaml")
|
definitionPath := filepath.Join(root, "definition.yaml")
|
||||||
contentPath := filepath.Join(root, "body.md")
|
contentPath := filepath.Join(root, "body.md")
|
||||||
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n v1:\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
|
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n version: \"v1\"\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
|
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
|
||||||
@@ -108,31 +119,132 @@ func TestLoadCapturesNativeContentFile(t *testing.T) {
|
|||||||
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
|
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition, _, err := state.prompt("example", "v1")
|
definition, _, err := state.prompt("example")
|
||||||
if err != nil || definition.Templates[0].Content != "Native body" {
|
if err != nil || definition.Templates[0].Content != "Native body" {
|
||||||
t.Fatalf("native content was not captured: %+v, %v", definition, err)
|
t.Fatalf("native content was not captured: %+v, %v", definition, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
|
func TestLoadRejectsMalformedStaticSyntaxAndCapturesOptionalVersion(t *testing.T) {
|
||||||
_, err := load(mapping("bad-template", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
_, err := load(mapping("bad-template", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||||
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
|
"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}}), nil)
|
}}}), nil)
|
||||||
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
|
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
|
||||||
t.Fatalf("expected construction-time syntax error, got %v", err)
|
t.Fatalf("expected construction-time syntax error, got %v", err)
|
||||||
}
|
}
|
||||||
state, err := load(mapping("versions", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
state, err := load(mapping("version metadata", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||||
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "one"}},
|
"version": "release label",
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
"messages": []any{map[string]any{"role": "user", "content": "one"}},
|
||||||
"v2": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "two"}},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
|
||||||
}}}), nil)
|
}}}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, _, err := state.prompt("example", ""); !errors.Is(err, errPromptAmbiguous) {
|
definition, inspection, err := state.prompt("example")
|
||||||
t.Fatalf("expected ambiguous version selection, got %v", err)
|
if err != nil || definition.Version == nil || *definition.Version != "release label" ||
|
||||||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != "release label" {
|
||||||
|
t.Fatalf("optional version metadata was not retained: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptVersionLabelPresenceAndOwnership(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
version any
|
||||||
|
include bool
|
||||||
|
wantPresent bool
|
||||||
|
wantValue string
|
||||||
|
}{
|
||||||
|
{name: "absent"},
|
||||||
|
{name: "null", include: true},
|
||||||
|
{name: "empty", version: "", include: true, wantPresent: true},
|
||||||
|
{name: "exact text", version: " Release/v1 ", include: true, wantPresent: true, wantValue: " Release/v1 "},
|
||||||
|
}
|
||||||
|
|
||||||
|
hashes := make(map[string]string, len(tests))
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
definition := map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
if test.include {
|
||||||
|
definition["version"] = test.version
|
||||||
|
}
|
||||||
|
state, err := load(mapping("prompt version label", map[string]any{
|
||||||
|
"prompts": map[string]any{"example": definition},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
captured, inspection, err := state.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
identities, err := state.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(identities) != 1 {
|
||||||
|
t.Fatalf("list prompt labels: %#v, %v", identities, err)
|
||||||
|
}
|
||||||
|
check := func(name string, value *string) {
|
||||||
|
t.Helper()
|
||||||
|
if !test.wantPresent {
|
||||||
|
if value != nil {
|
||||||
|
t.Errorf("%s label = %q, want absent", name, *value)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if value == nil || *value != test.wantValue {
|
||||||
|
t.Errorf("%s label = %v, want %q", name, value, test.wantValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check("definition", captured.Version)
|
||||||
|
check("inspection", inspection.PromptVersion)
|
||||||
|
check("listing", identities[0].Version)
|
||||||
|
if captured.Version != nil && (captured.Version == inspection.PromptVersion || captured.Version == identities[0].Version || inspection.PromptVersion == identities[0].Version) {
|
||||||
|
t.Fatal("returned prompt labels share mutable pointers")
|
||||||
|
}
|
||||||
|
hashes[test.name] = inspection.PromptHash
|
||||||
|
|
||||||
|
if captured.Version != nil {
|
||||||
|
*captured.Version = "mutated definition"
|
||||||
|
}
|
||||||
|
if inspection.PromptVersion != nil {
|
||||||
|
*inspection.PromptVersion = "mutated inspection"
|
||||||
|
}
|
||||||
|
if identities[0].Version != nil {
|
||||||
|
*identities[0].Version = "mutated listing"
|
||||||
|
}
|
||||||
|
again, againInspection, err := state.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check("captured definition after mutation", again.Version)
|
||||||
|
check("captured inspection after mutation", againInspection.PromptVersion)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if hashes["absent"] == hashes["empty"] {
|
||||||
|
t.Fatal("absent and declared-empty labels produced the same definition hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptVersionNullClearsInheritedLabel(t *testing.T) {
|
||||||
|
prompt := map[string]any{
|
||||||
|
"version": "base-label",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
state, err := load(
|
||||||
|
mapping("clear inherited label", map[string]any{"prompts": map[string]any{"example": map[string]any{"version": nil}}}),
|
||||||
|
[]configkit.Underlay{{Inputs: []configkit.Input{mapping("base prompt", map[string]any{"prompts": map[string]any{"example": prompt}})}}},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition, inspection, err := state.prompt("example")
|
||||||
|
if err != nil || definition.Version != nil || inspection.PromptVersion != nil {
|
||||||
|
t.Fatalf("null overlay retained inherited label: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,32 +260,29 @@ func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.
|
|||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
selectedPrompt := makePrompt("selected")
|
||||||
|
selectedPrompt["version"] = "v2"
|
||||||
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
||||||
"profiles": map[string]any{"partial": map[string]any{}},
|
"profiles": map[string]any{"partial": map[string]any{}},
|
||||||
"prompts": map[string]any{
|
"prompts": map[string]any{
|
||||||
"multi": map[string]any{"v1": makePrompt("first"), "v2": makePrompt("second")},
|
"selected": selectedPrompt,
|
||||||
"single": map[string]any{"v1": makePrompt("only")},
|
|
||||||
},
|
},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, selection := range []struct {
|
{
|
||||||
id, version string
|
inspection, err := state.GetPromptInspection(context.Background(), "selected")
|
||||||
}{
|
|
||||||
{id: "multi", version: "v2"},
|
|
||||||
{id: "single"},
|
|
||||||
} {
|
|
||||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect %q/%q: %v", selection.id, selection.version, err)
|
t.Fatalf("inspect prompt ID: %v", err)
|
||||||
}
|
}
|
||||||
definition, hash, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
definition, hash, err := state.GetPromptSnapshot(context.Background(), "selected")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("snapshot %q/%q: %v", selection.id, selection.version, err)
|
t.Fatalf("snapshot prompt ID: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
|
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
|
||||||
|
*inspection.PromptVersion != *definition.Version ||
|
||||||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||||
@@ -184,53 +293,42 @@ func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
owned, err := state.GetPromptInspection(context.Background(), "single", "")
|
owned, err := state.GetPromptInspection(context.Background(), "selected")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
owned.Inputs[0].Name = "caller mutation"
|
owned.Inputs[0].Name = "caller mutation"
|
||||||
owned.Inputs[0].Description = "caller mutation"
|
owned.Inputs[0].Description = "caller mutation"
|
||||||
again, err := state.GetPromptInspection(context.Background(), "single", "v1")
|
again, err := state.GetPromptInspection(context.Background(), "selected")
|
||||||
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
||||||
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, selection := range []struct {
|
for _, id := range []string{"missing", " "} {
|
||||||
id, version string
|
inspection, err := state.GetPromptInspection(context.Background(), id)
|
||||||
}{
|
|
||||||
{id: "missing", version: "v1"},
|
|
||||||
{id: "multi", version: "V2"},
|
|
||||||
{id: " ", version: "v1"},
|
|
||||||
} {
|
|
||||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
|
||||||
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
t.Errorf("inspection selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, inspection, err)
|
t.Errorf("inspection lookup %q=(%+v, %v), want not found", id, inspection, err)
|
||||||
}
|
}
|
||||||
definition, _, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
definition, _, err := state.GetPromptSnapshot(context.Background(), id)
|
||||||
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
t.Errorf("snapshot selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, definition, err)
|
t.Errorf("snapshot lookup %q=(%+v, %v), want not found", id, definition, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := state.GetPromptInspection(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
|
||||||
t.Errorf("ambiguous inspection selection error=%v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := state.GetPromptSnapshot(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
|
||||||
t.Errorf("ambiguous snapshot selection error=%v", err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
if _, err := state.GetPromptInspection(ctx, "single", "v1"); !errors.Is(err, context.Canceled) {
|
if _, err := state.GetPromptInspection(ctx, "selected"); !errors.Is(err, context.Canceled) {
|
||||||
t.Errorf("canceled inspection selection error=%v", err)
|
t.Errorf("canceled inspection selection error=%v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||||
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
|
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
|
||||||
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
|
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}}}), nil)
|
}}}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var loadError *configkit.LoadError
|
var loadError *configkit.LoadError
|
||||||
if errors.As(err, &loadError) {
|
if errors.As(err, &loadError) {
|
||||||
@@ -238,18 +336,19 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
|
|||||||
}
|
}
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition, _, err := state.prompt(" exact.id ", "v1")
|
definition, _, err := state.prompt(" exact.id ")
|
||||||
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
|
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
|
||||||
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
|
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
|
||||||
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
|
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
|
||||||
}
|
}
|
||||||
if _, _, err := state.prompt("exact.id", "v1"); !errors.Is(err, errPromptNotFound) {
|
if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) {
|
||||||
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
|
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
|
||||||
}
|
}
|
||||||
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
|
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}}}), nil)
|
}}}), nil)
|
||||||
var loadError *configkit.LoadError
|
var loadError *configkit.LoadError
|
||||||
if !errors.As(err, &loadError) {
|
if !errors.As(err, &loadError) {
|
||||||
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
|
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
|
||||||
@@ -262,7 +361,8 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
|
|||||||
|
|
||||||
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
||||||
state, err := load(mapping("requirement-metadata", map[string]any{
|
state, err := load(mapping("requirement-metadata", map[string]any{
|
||||||
"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
"prompts": map[string]any{"example": map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"session_id": "{{.session}}",
|
"session_id": "{{.session}}",
|
||||||
"inputs": []any{
|
"inputs": []any{
|
||||||
map[string]any{"name": "absent"},
|
map[string]any{"name": "absent"},
|
||||||
@@ -272,7 +372,7 @@ func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testi
|
|||||||
},
|
},
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}},
|
}},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var loadError *configkit.LoadError
|
var loadError *configkit.LoadError
|
||||||
@@ -282,7 +382,7 @@ func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testi
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshot, ok := state.GetRequirementSnapshot("example", "v1")
|
snapshot, ok := state.GetRequirementSnapshot("example")
|
||||||
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
||||||
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
||||||
}
|
}
|
||||||
@@ -300,7 +400,7 @@ func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testi
|
|||||||
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
||||||
}
|
}
|
||||||
|
|
||||||
definition, inspection, err := state.prompt("example", "v1")
|
definition, inspection, err := state.prompt("example")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -312,13 +412,10 @@ func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testi
|
|||||||
*snapshot.Inputs[2].ContentType = "caller mutation"
|
*snapshot.Inputs[2].ContentType = "caller mutation"
|
||||||
snapshot.Inputs[3].Name = "caller mutation"
|
snapshot.Inputs[3].Name = "caller mutation"
|
||||||
snapshot.Templates.Messages[0] = nil
|
snapshot.Templates.Messages[0] = nil
|
||||||
again, ok := state.GetRequirementSnapshot("example", "v1")
|
again, ok := state.GetRequirementSnapshot("example")
|
||||||
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
||||||
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
||||||
}
|
}
|
||||||
if _, ok := state.GetRequirementSnapshot("example", "V1"); ok {
|
|
||||||
t.Fatal("snapshot lookup unexpectedly changed exact version matching")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
||||||
@@ -338,11 +435,12 @@ func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T)
|
|||||||
input[name] = value
|
input[name] = value
|
||||||
}
|
}
|
||||||
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
||||||
"prompts": map[string]any{"same": map[string]any{"v1": map[string]any{
|
"prompts": map[string]any{"same": map[string]any{
|
||||||
|
"version": "v1",
|
||||||
"inputs": []any{input},
|
"inputs": []any{input},
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}},
|
}},
|
||||||
}), nil)
|
}), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var loadError *configkit.LoadError
|
var loadError *configkit.LoadError
|
||||||
@@ -351,7 +449,7 @@ func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T)
|
|||||||
}
|
}
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, inspection, err := state.prompt("same", "v1")
|
_, inspection, err := state.prompt("same")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -368,10 +466,10 @@ func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func promptWithFile(name string) map[string]any {
|
func promptWithFile(name string) map[string]any {
|
||||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
return map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
}}}}
|
}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
type countingFS struct {
|
type countingFS struct {
|
||||||
|
|||||||
@@ -12,14 +12,25 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (state *loaded) buildExecutionProfiles() error {
|
func (state *loaded) buildExecutionProfiles() error {
|
||||||
definitions := make(map[string]domain.Backend, len(state.result.Config.Backends))
|
backendConfigs := indexedBackends(&state.result.Config)
|
||||||
for _, id := range sortedKeys(state.result.Config.Backends) {
|
profileConfigs := indexedProfiles(&state.result.Config)
|
||||||
config := state.result.Config.Backends[id]
|
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||||
params, err := jsonvalue.CopyMap(config.ExtraParams)
|
for _, id := range sortedKeys(backendConfigs) {
|
||||||
if err != nil {
|
config := backendConfigs[id]
|
||||||
return fmt.Errorf("backend %q extra parameters: %w", id, err)
|
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||||
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: config.SmokeTests.Authentication.Kind,
|
||||||
|
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
definition.SmokeTests = smokeTests
|
||||||
}
|
}
|
||||||
definition := domain.Backend{ID: id, ExtraParams: params}
|
|
||||||
if config.Endpoint != nil {
|
if config.Endpoint != nil {
|
||||||
definition.Endpoint = *config.Endpoint
|
definition.Endpoint = *config.Endpoint
|
||||||
}
|
}
|
||||||
@@ -40,9 +51,9 @@ func (state *loaded) buildExecutionProfiles() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
state.registry = registry
|
state.registry = registry
|
||||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(state.result.Config.Profiles))
|
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||||
for _, id := range sortedKeys(state.result.Config.Profiles) {
|
for _, id := range sortedKeys(profileConfigs) {
|
||||||
config := state.result.Config.Profiles[id]
|
config := profileConfigs[id]
|
||||||
fields := state.profiles[id]
|
fields := state.profiles[id]
|
||||||
target := defaults.ExecutionTargetDefault()
|
target := defaults.ExecutionTargetDefault()
|
||||||
if config.Backend != nil && *config.Backend != "" {
|
if config.Backend != nil && *config.Backend != "" {
|
||||||
|
|||||||
@@ -61,7 +61,6 @@ type CacheControl struct {
|
|||||||
// RunRequest represents a request to generate a single artifact.
|
// RunRequest represents a request to generate a single artifact.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
|
||||||
ProfileID string
|
ProfileID string
|
||||||
SessionID string
|
SessionID string
|
||||||
APIKey string `json:"-" yaml:"-"`
|
APIKey string `json:"-" yaml:"-"`
|
||||||
@@ -81,7 +80,7 @@ type RunResult struct {
|
|||||||
RawOutput string
|
RawOutput string
|
||||||
Validation ValidationResult
|
Validation ValidationResult
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
SessionID string
|
SessionID string
|
||||||
RenderedPromptHash string
|
RenderedPromptHash string
|
||||||
@@ -101,7 +100,7 @@ type RunResult struct {
|
|||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
SelectedBackendID string
|
SelectedBackendID string
|
||||||
@@ -135,16 +134,16 @@ type Artifact struct {
|
|||||||
Hash string
|
Hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptIdentity identifies one configured prompt version.
|
// PromptIdentity identifies one configured prompt and its descriptive version label.
|
||||||
type PromptIdentity struct {
|
type PromptIdentity struct {
|
||||||
ID string
|
ID string
|
||||||
Version string
|
Version *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptDefinition represents a configured prompt execution definition.
|
// PromptDefinition represents a configured prompt execution definition.
|
||||||
type PromptDefinition struct {
|
type PromptDefinition struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
Version string `yaml:"version"`
|
Version *string `yaml:"version"`
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
DefaultProfile string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
@@ -157,7 +156,7 @@ type PromptDefinition struct {
|
|||||||
// PromptInspection is the resolved result of exact prompt inspection.
|
// PromptInspection is the resolved result of exact prompt inspection.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
DefaultProfileID string
|
DefaultProfileID string
|
||||||
Inputs []PromptInput
|
Inputs []PromptInput
|
||||||
@@ -180,12 +179,14 @@ type PromptMessageTemplate struct {
|
|||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backend describes reusable OpenAI-compatible connection defaults.
|
// Backend describes reusable OpenAI-compatible connection defaults and optional
|
||||||
|
// diagnostic metadata.
|
||||||
type Backend struct {
|
type Backend struct {
|
||||||
ID string
|
ID string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
|
SmokeTests *BackendSmokeTestConfig
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
QueueCapacity int
|
QueueCapacity int
|
||||||
QueueCapacitySet bool
|
QueueCapacitySet bool
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ type ExecutionEvent struct {
|
|||||||
RunID string
|
RunID string
|
||||||
Time time.Time
|
Time time.Time
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
BackendID string
|
BackendID string
|
||||||
Model string
|
Model string
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ type RequirementIssue struct {
|
|||||||
// RequirementInspection contains the inventories discovered for one prompt.
|
// RequirementInspection contains the inventories discovered for one prompt.
|
||||||
type RequirementInspection struct {
|
type RequirementInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
Inputs []InputRequirement
|
Inputs []InputRequirement
|
||||||
Variables []VariableRequirement
|
Variables []VariableRequirement
|
||||||
|
|||||||
70
internal/domain/resource_identity.go
Normal file
70
internal/domain/resource_identity.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
func validResourceID(id string) bool {
|
||||||
|
if !strings.Contains(id, "/") {
|
||||||
|
return strings.TrimSpace(id) != ""
|
||||||
|
}
|
||||||
|
if strings.Count(id, "/") != 1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
namespace, localID, _ := strings.Cut(id, "/")
|
||||||
|
return namespace != "" && localID != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func validResourceNamespace(namespace string) bool {
|
||||||
|
return strings.TrimSpace(namespace) != "" && !strings.Contains(namespace, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func validLocalResourceKey(key string) bool {
|
||||||
|
return strings.TrimSpace(key) != "" && !strings.Contains(key, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func qualifyResourceID(id, namespace string) (string, bool) {
|
||||||
|
if !validResourceID(id) || (namespace != "" && !validResourceNamespace(namespace)) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if strings.Contains(id, "/") || namespace == "" {
|
||||||
|
return id, true
|
||||||
|
}
|
||||||
|
return namespace + "/" + id, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourceNamespaceFromID(id string) (string, bool) {
|
||||||
|
if !validResourceID(id) || !strings.Contains(id, "/") {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
namespace, _, _ := strings.Cut(id, "/")
|
||||||
|
return namespace, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
||||||
|
// valid namespace/local-ID pair.
|
||||||
|
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
||||||
|
|
||||||
|
// QualifyResourceID validates id and applies namespace to an unqualified ID.
|
||||||
|
// An already-qualified ID is returned unchanged.
|
||||||
|
func QualifyResourceID(id, namespace string) (string, bool) {
|
||||||
|
return qualifyResourceID(id, namespace)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsValidResourceNamespace reports whether namespace is nonblank and contains
|
||||||
|
// no namespace separator.
|
||||||
|
func IsValidResourceNamespace(namespace string) bool { return validResourceNamespace(namespace) }
|
||||||
|
|
||||||
|
// IsValidLocalResourceKey reports whether key is a nonblank key for a
|
||||||
|
// namespace-local resource mapping.
|
||||||
|
func IsValidLocalResourceKey(key string) bool { return validLocalResourceKey(key) }
|
||||||
|
|
||||||
|
// QualifyLocalResourceID combines a validated namespace and local mapping key.
|
||||||
|
// An empty namespace leaves a valid unqualified key unchanged.
|
||||||
|
func QualifyLocalResourceID(namespace, key string) (string, bool) {
|
||||||
|
if !validLocalResourceKey(key) || (namespace != "" && !validResourceNamespace(namespace)) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if namespace == "" {
|
||||||
|
return key, true
|
||||||
|
}
|
||||||
|
return namespace + "/" + key, true
|
||||||
|
}
|
||||||
110
internal/domain/resource_identity_test.go
Normal file
110
internal/domain/resource_identity_test.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestValidResourceID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
id string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "ordinary", id: "draft", want: true},
|
||||||
|
{name: "dotted", id: "group.prompt.v2", want: true},
|
||||||
|
{name: "preserved surrounding whitespace", id: " draft ", want: true},
|
||||||
|
{name: "qualified", id: "group/draft", want: true},
|
||||||
|
{name: "qualified components preserve spelling", id: " Group / draft ", want: true},
|
||||||
|
{name: "empty", id: "", want: false},
|
||||||
|
{name: "blank", id: " \t", want: false},
|
||||||
|
{name: "missing namespace", id: "/draft", want: false},
|
||||||
|
{name: "missing local id", id: "group/", want: false},
|
||||||
|
{name: "multiple separators", id: "group/subgroup/draft", want: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := validResourceID(test.id); got != test.want {
|
||||||
|
t.Fatalf("validResourceID(%q) = %t, want %t", test.id, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidResourceNamespace(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
namespace string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "ordinary", namespace: "group", want: true},
|
||||||
|
{name: "dotted", namespace: "group.v2", want: true},
|
||||||
|
{name: "preserved whitespace", namespace: " group ", want: true},
|
||||||
|
{name: "empty", namespace: "", want: false},
|
||||||
|
{name: "blank", namespace: " \t", want: false},
|
||||||
|
{name: "slash", namespace: "group/subgroup", want: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := validResourceNamespace(test.namespace); got != test.want {
|
||||||
|
t.Fatalf("validResourceNamespace(%q) = %t, want %t", test.namespace, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidLocalResourceKey(t *testing.T) {
|
||||||
|
if !validLocalResourceKey("draft.v2") || !validLocalResourceKey(" draft ") {
|
||||||
|
t.Fatal("expected nonblank slash-free keys to be valid")
|
||||||
|
}
|
||||||
|
for _, key := range []string{"", " \t", "group/draft"} {
|
||||||
|
if validLocalResourceKey(key) {
|
||||||
|
t.Errorf("validLocalResourceKey(%q) = true, want false", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQualifyResourceID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
id string
|
||||||
|
namespace string
|
||||||
|
want string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{name: "without scope", id: "draft", want: "draft", wantOK: true},
|
||||||
|
{name: "with scope", id: "draft", namespace: "group", want: "group/draft", wantOK: true},
|
||||||
|
{name: "qualified bypass", id: "other/draft", namespace: "group", want: "other/draft", wantOK: true},
|
||||||
|
{name: "preserve spelling", id: " draft ", namespace: " Group ", want: " Group / draft ", wantOK: true},
|
||||||
|
{name: "omission remains caller policy", id: "", wantOK: false},
|
||||||
|
{name: "malformed ID", id: "group/draft/extra", namespace: "group", wantOK: false},
|
||||||
|
{name: "malformed scope", id: "draft", namespace: "group/subgroup", wantOK: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, ok := qualifyResourceID(test.id, test.namespace)
|
||||||
|
if got != test.want || ok != test.wantOK {
|
||||||
|
t.Fatalf("qualifyResourceID(%q, %q) = (%q, %t), want (%q, %t)", test.id, test.namespace, got, ok, test.want, test.wantOK)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResourceNamespaceFromID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
id string
|
||||||
|
namespace string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
|
||||||
|
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
|
||||||
|
{name: "unqualified", id: "item", wantOK: false},
|
||||||
|
{name: "malformed", id: "team/sub/item", wantOK: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, ok := resourceNamespaceFromID(test.id)
|
||||||
|
if got != test.namespace || ok != test.wantOK {
|
||||||
|
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
263
internal/domain/smoke_diagnostics.go
Normal file
263
internal/domain/smoke_diagnostics.go
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BackendSmokeTestConfig describes optional backend-owned diagnostic defaults.
|
||||||
|
type BackendSmokeTestConfig struct {
|
||||||
|
DefaultProfile *string
|
||||||
|
Reachability *ReachabilityProbeConfig
|
||||||
|
Authentication *AuthenticationProbeConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachabilityProbeConfig describes an optional endpoint for reachability tests.
|
||||||
|
type ReachabilityProbeConfig struct {
|
||||||
|
Endpoint *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuthenticationProbeConfig describes an optional authentication diagnostic.
|
||||||
|
type AuthenticationProbeConfig struct {
|
||||||
|
Kind *string
|
||||||
|
Endpoint *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestKind identifies one built-in diagnostic operation.
|
||||||
|
type SmokeTestKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
SmokeTestReachability SmokeTestKind = "reachability"
|
||||||
|
SmokeTestAuthentication SmokeTestKind = "authentication"
|
||||||
|
SmokeTestTextInference SmokeTestKind = "text_inference"
|
||||||
|
SmokeTestStructuredOutput SmokeTestKind = "structured_output"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SmokeTestRequest selects one diagnostic and its execution target.
|
||||||
|
type SmokeTestRequest struct {
|
||||||
|
Kind SmokeTestKind
|
||||||
|
BackendID string
|
||||||
|
PromptID string
|
||||||
|
ProfileID string
|
||||||
|
Execution *ExecutionTargetOverride
|
||||||
|
APIKey string `json:"-" yaml:"-"`
|
||||||
|
TransientRetries *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestReport contains factual diagnostic results, not generated content.
|
||||||
|
type SmokeTestReport struct {
|
||||||
|
Kind SmokeTestKind
|
||||||
|
Supported bool
|
||||||
|
Applicable *bool
|
||||||
|
Completed bool
|
||||||
|
Reason SmokeTestReason
|
||||||
|
Target SmokeTestTarget
|
||||||
|
Facts SmokeTestFacts
|
||||||
|
Timing SmokeTestTiming
|
||||||
|
Attempts []SmokeTestAttempt
|
||||||
|
Usage ObservedTokenUsage
|
||||||
|
Failure ExecutionFailureKind
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestTarget identifies the selected diagnostic target and its settings.
|
||||||
|
type SmokeTestTarget struct {
|
||||||
|
BackendID string
|
||||||
|
ProfileID string
|
||||||
|
PromptID string
|
||||||
|
PromptVersion *string
|
||||||
|
DefaultSourceBackendID string
|
||||||
|
SelectionSource string
|
||||||
|
Endpoint string
|
||||||
|
EffectiveSettings ExecutionTarget
|
||||||
|
SettingsPresence ExecutionTargetPresence
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestFacts contains optional factual observations from a diagnostic.
|
||||||
|
type SmokeTestFacts struct {
|
||||||
|
CredentialsSupplied *bool
|
||||||
|
EndpointReachable *bool
|
||||||
|
CredentialsAccepted *bool
|
||||||
|
ResponseReceived *bool
|
||||||
|
GenerationCompleted *bool
|
||||||
|
TextReturned *bool
|
||||||
|
ExpectedTextMatched *bool
|
||||||
|
StructuredOutputRequested *bool
|
||||||
|
ValidJSON *bool
|
||||||
|
SchemaMatched *bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestTiming records elapsed diagnostic and phase durations.
|
||||||
|
type SmokeTestTiming struct {
|
||||||
|
StartedAt time.Time
|
||||||
|
Duration time.Duration
|
||||||
|
CapacityWait *time.Duration
|
||||||
|
RequestDuration *time.Duration
|
||||||
|
RetryDelay *time.Duration
|
||||||
|
EvaluationDuration *time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestAttempt records one dispatched attempt or an interrupted acquisition.
|
||||||
|
type SmokeTestAttempt struct {
|
||||||
|
Dispatched bool
|
||||||
|
InitialEndpoint string
|
||||||
|
Endpoint string
|
||||||
|
HTTPStatus *int
|
||||||
|
Facts SmokeTestFacts
|
||||||
|
Timing SmokeTestTiming
|
||||||
|
Failure ExecutionFailureKind
|
||||||
|
Usage ObservedTokenUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// SmokeTestReason identifies additional factual context for a diagnostic.
|
||||||
|
type SmokeTestReason string
|
||||||
|
|
||||||
|
const (
|
||||||
|
SmokeTestReasonNone SmokeTestReason = ""
|
||||||
|
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||||
|
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||||
|
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||||
|
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||||
|
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CloneBackend returns a copy of a backend, including its mutable settings.
|
||||||
|
func CloneBackend(backend Backend) (Backend, error) {
|
||||||
|
cloned := backend
|
||||||
|
var err error
|
||||||
|
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return Backend{}, err
|
||||||
|
}
|
||||||
|
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
||||||
|
return cloned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
||||||
|
func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTestConfig {
|
||||||
|
if config == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *config
|
||||||
|
cloned.DefaultProfile = cloneString(config.DefaultProfile)
|
||||||
|
if config.Reachability != nil {
|
||||||
|
reachability := *config.Reachability
|
||||||
|
reachability.Endpoint = cloneString(config.Reachability.Endpoint)
|
||||||
|
cloned.Reachability = &reachability
|
||||||
|
}
|
||||||
|
if config.Authentication != nil {
|
||||||
|
authentication := *config.Authentication
|
||||||
|
authentication.Kind = cloneString(config.Authentication.Kind)
|
||||||
|
authentication.Endpoint = cloneString(config.Authentication.Endpoint)
|
||||||
|
cloned.Authentication = &authentication
|
||||||
|
}
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneSmokeTestRequest returns a copy that owns its optional settings.
|
||||||
|
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
||||||
|
cloned := request
|
||||||
|
cloned.TransientRetries = cloneInt(request.TransientRetries)
|
||||||
|
if request.Execution == nil {
|
||||||
|
return cloned, nil
|
||||||
|
}
|
||||||
|
execution := *request.Execution
|
||||||
|
execution.Temperature = cloneFloat64(request.Execution.Temperature)
|
||||||
|
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
|
||||||
|
execution.TopP = cloneFloat64(request.Execution.TopP)
|
||||||
|
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
|
||||||
|
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
|
||||||
|
var err error
|
||||||
|
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return SmokeTestRequest{}, err
|
||||||
|
}
|
||||||
|
cloned.Execution = &execution
|
||||||
|
return cloned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
|
||||||
|
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
||||||
|
cloned := report
|
||||||
|
cloned.Applicable = cloneBool(report.Applicable)
|
||||||
|
cloned.Target.PromptVersion = cloneString(report.Target.PromptVersion)
|
||||||
|
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
||||||
|
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
||||||
|
var err error
|
||||||
|
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return SmokeTestReport{}, err
|
||||||
|
}
|
||||||
|
if report.Attempts != nil {
|
||||||
|
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
|
||||||
|
for index, attempt := range report.Attempts {
|
||||||
|
cloned.Attempts[index] = attempt
|
||||||
|
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
|
||||||
|
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
|
||||||
|
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
|
||||||
|
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
|
||||||
|
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
|
||||||
|
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
|
||||||
|
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
|
||||||
|
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
|
||||||
|
facts.TextReturned = cloneBool(facts.TextReturned)
|
||||||
|
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
|
||||||
|
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
|
||||||
|
facts.ValidJSON = cloneBool(facts.ValidJSON)
|
||||||
|
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
|
||||||
|
return facts
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
|
||||||
|
timing.CapacityWait = cloneDuration(timing.CapacityWait)
|
||||||
|
timing.RequestDuration = cloneDuration(timing.RequestDuration)
|
||||||
|
timing.RetryDelay = cloneDuration(timing.RetryDelay)
|
||||||
|
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
|
||||||
|
return timing
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneBool(value *bool) *bool {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneDuration(value *time.Duration) *time.Duration {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneFloat64(value *float64) *float64 {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneInt(value *int) *int {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneString(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
115
internal/domain/smoke_diagnostics_test.go
Normal file
115
internal/domain/smoke_diagnostics_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
||||||
|
profile := "profile"
|
||||||
|
reachabilityEndpoint := "https://probe.example/health"
|
||||||
|
authKind := "openrouter_key"
|
||||||
|
authEndpoint := "https://probe.example/key"
|
||||||
|
backend := Backend{
|
||||||
|
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
|
SmokeTests: &BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &profile,
|
||||||
|
Reachability: &ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &AuthenticationProbeConfig{
|
||||||
|
Kind: &authKind,
|
||||||
|
Endpoint: &authEndpoint,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cloned, err := CloneBackend(backend)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
*cloned.SmokeTests.DefaultProfile = "changed"
|
||||||
|
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
||||||
|
*cloned.SmokeTests.Authentication.Kind = "changed"
|
||||||
|
*cloned.SmokeTests.Authentication.Endpoint = "changed"
|
||||||
|
cloned.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
|
||||||
|
if profile != "profile" || reachabilityEndpoint != "https://probe.example/health" ||
|
||||||
|
authKind != "openrouter_key" || authEndpoint != "https://probe.example/key" {
|
||||||
|
t.Fatal("cloned diagnostic metadata shared source pointers")
|
||||||
|
}
|
||||||
|
if got := backend.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||||
|
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
|
||||||
|
temperature := 0.25
|
||||||
|
retries := 2
|
||||||
|
reasoning := "low"
|
||||||
|
request := SmokeTestRequest{
|
||||||
|
APIKey: "secret",
|
||||||
|
TransientRetries: &retries,
|
||||||
|
Execution: &ExecutionTargetOverride{
|
||||||
|
Temperature: &temperature,
|
||||||
|
ReasoningEffort: &reasoning,
|
||||||
|
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cloned, err := CloneSmokeTestRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
*cloned.TransientRetries = 3
|
||||||
|
*cloned.Execution.Temperature = 0.5
|
||||||
|
*cloned.Execution.ReasoningEffort = "high"
|
||||||
|
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
|
||||||
|
if retries != 2 || temperature != 0.25 || reasoning != "low" {
|
||||||
|
t.Fatal("cloned request shared optional scalar pointers")
|
||||||
|
}
|
||||||
|
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||||
|
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
||||||
|
applicable := true
|
||||||
|
reachable := true
|
||||||
|
status := 200
|
||||||
|
wait := 3 * time.Millisecond
|
||||||
|
promptVersion := "v1"
|
||||||
|
report := SmokeTestReport{
|
||||||
|
Applicable: &applicable,
|
||||||
|
Target: SmokeTestTarget{
|
||||||
|
PromptVersion: &promptVersion,
|
||||||
|
EffectiveSettings: ExecutionTarget{
|
||||||
|
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
||||||
|
Attempts: []SmokeTestAttempt{{
|
||||||
|
HTTPStatus: &status,
|
||||||
|
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
||||||
|
Timing: SmokeTestTiming{CapacityWait: &wait},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
cloned, err := CloneSmokeTestReport(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
*cloned.Applicable = false
|
||||||
|
*cloned.Facts.EndpointReachable = false
|
||||||
|
*cloned.Attempts[0].HTTPStatus = 503
|
||||||
|
*cloned.Attempts[0].Facts.EndpointReachable = false
|
||||||
|
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
||||||
|
*cloned.Target.PromptVersion = "changed"
|
||||||
|
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
|
||||||
|
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond || promptVersion != "v1" {
|
||||||
|
t.Fatal("cloned report shared optional scalar pointers")
|
||||||
|
}
|
||||||
|
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||||
|
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
@@ -82,13 +81,40 @@ func (o *Options) SetSchemaFile(path string) error {
|
|||||||
// Engine forwards domain operations to the single assembled use-case runner.
|
// Engine forwards domain operations to the single assembled use-case runner.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
|
smokeTester *usecase.SmokeTester
|
||||||
inspection configuration.Inspection
|
inspection configuration.Inspection
|
||||||
listing identityInventory
|
listing identityInventory
|
||||||
|
namespaces resourceNamespaces
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceNamespaces struct {
|
||||||
|
prompt string
|
||||||
|
profile string
|
||||||
|
backend string
|
||||||
}
|
}
|
||||||
|
|
||||||
type identityInventory interface {
|
type identityInventory interface {
|
||||||
|
ListBackends(context.Context) ([]string, error)
|
||||||
ListProfiles(context.Context) ([]string, error)
|
ListProfiles(context.Context) ([]string, error)
|
||||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
||||||
|
ListNamespaces(context.Context, []string) ([]string, error)
|
||||||
|
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListingFilterMode selects namespace membership for one identity listing.
|
||||||
|
type ListingFilterMode uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
ListingDefault ListingFilterMode = iota
|
||||||
|
ListingNamed
|
||||||
|
ListingUnqualified
|
||||||
|
ListingAll
|
||||||
|
)
|
||||||
|
|
||||||
|
// ListingFilter optionally overrides the namespace selected by an engine view.
|
||||||
|
type ListingFilter struct {
|
||||||
|
Mode ListingFilterMode
|
||||||
|
Name string
|
||||||
}
|
}
|
||||||
|
|
||||||
// New assembles an engine from facade-selected configuration and collaborators.
|
// New assembles an engine from facade-selected configuration and collaborators.
|
||||||
@@ -97,11 +123,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
underlays, err := catalog.Underlays()
|
underlays := cfg.Sources.Underlays
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("select maintained configuration: %w", err)
|
|
||||||
}
|
|
||||||
underlays = append(underlays, cfg.Sources.Underlays...)
|
|
||||||
main := configkit.Input{Source: configkit.NoSource()}
|
main := configkit.Input{Source: configkit.NoSource()}
|
||||||
if cfg.Sources.Main != nil {
|
if cfg.Sources.Main != nil {
|
||||||
main = *cfg.Sources.Main
|
main = *cfg.Sources.Main
|
||||||
@@ -124,11 +146,13 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
validator = validate.NewStandardValidator(schemaDir)
|
validator = validate.NewStandardValidator(schemaDir)
|
||||||
}
|
}
|
||||||
var client llm.Client
|
var client llm.Client
|
||||||
|
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
||||||
if options.llmClient == nil {
|
if options.llmClient == nil {
|
||||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||||
if newErr != nil {
|
if newErr != nil {
|
||||||
return nil, newErr
|
return nil, newErr
|
||||||
}
|
}
|
||||||
|
diagnosticHTTPClient = builtIn
|
||||||
client = generation.NewBuiltIn(manager, builtIn)
|
client = generation.NewBuiltIn(manager, builtIn)
|
||||||
} else {
|
} else {
|
||||||
client = generation.NewInjected(manager, options.llmClient)
|
client = generation.NewInjected(manager, options.llmClient)
|
||||||
@@ -138,13 +162,25 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
reader = artifact.NewCompositeReader()
|
reader = artifact.NewCompositeReader()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)
|
||||||
return &Engine{
|
return &Engine{
|
||||||
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
runner: runner,
|
||||||
|
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
||||||
inspection: snapshot.Inspection(),
|
inspection: snapshot.Inspection(),
|
||||||
listing: snapshot,
|
listing: snapshot,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithNamespaces returns an immutable view sharing this engine's assembled
|
||||||
|
// collaborators and configuration snapshot. Caller-supplied selection IDs are
|
||||||
|
// qualified at each operation boundary; captured configuration references are
|
||||||
|
// not rewritten.
|
||||||
|
func (e *Engine) WithNamespaces(prompt, profile, backend string) *Engine {
|
||||||
|
view := *e
|
||||||
|
view.namespaces = resourceNamespaces{prompt: prompt, profile: profile, backend: backend}
|
||||||
|
return &view
|
||||||
|
}
|
||||||
|
|
||||||
// InspectConfiguration returns read-only operations over the captured result.
|
// InspectConfiguration returns read-only operations over the captured result.
|
||||||
func (e *Engine) InspectConfiguration() configuration.Inspection {
|
func (e *Engine) InspectConfiguration() configuration.Inspection {
|
||||||
if e == nil {
|
if e == nil {
|
||||||
@@ -182,35 +218,213 @@ func fileSource(name string) (fs.FS, string, string, error) {
|
|||||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
return e.runner.InspectPrompt(ctx, id, version)
|
var err error
|
||||||
|
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.runner.InspectPrompt(ctx, id)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectRequirements(
|
func (e *Engine) InspectRequirements(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
request usecase.RequirementInspectionRequest,
|
request usecase.RequirementInspectionRequest,
|
||||||
) (*domain.RequirementInspection, error) {
|
) (*domain.RequirementInspection, error) {
|
||||||
|
var err error
|
||||||
|
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return e.runner.InspectRequirements(ctx, request)
|
return e.runner.InspectRequirements(ctx, request)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||||
|
var err error
|
||||||
|
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return e.runner.InspectProfile(ctx, id)
|
return e.runner.InspectProfile(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||||
return e.listing.ListProfiles(ctx)
|
return filterIdentityList(ctx, filters, e.namespaces.backend, e.listing.ListBackends)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||||
return e.listing.ListPrompts(ctx)
|
return filterIdentityList(ctx, filters, e.namespaces.profile, e.listing.ListProfiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
||||||
|
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prompts, err := e.listing.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var filtered []domain.PromptIdentity
|
||||||
|
for _, prompt := range prompts {
|
||||||
|
if matchesListingNamespace(prompt.ID, mode, namespace) {
|
||||||
|
filtered = append(filtered, prompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return filtered, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
|
for _, kind := range kinds {
|
||||||
|
switch kind {
|
||||||
|
case "prompt", "profile", "backend":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: unknown resource kind %q", usecase.ErrInvalidRequest, kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return e.listing.ListNamespaces(ctx, kinds)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ListCollections(ctx context.Context) ([]configuration.CollectionIdentity, error) {
|
||||||
|
return e.listing.ListCollections(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterIdentityList(
|
||||||
|
ctx context.Context,
|
||||||
|
filters []ListingFilter,
|
||||||
|
scope string,
|
||||||
|
list func(context.Context) ([]string, error),
|
||||||
|
) ([]string, error) {
|
||||||
|
mode, namespace, err := resolveListingFilter(filters, scope)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ids, err := list(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var filtered []string
|
||||||
|
for _, id := range ids {
|
||||||
|
if matchesListingNamespace(id, mode, namespace) {
|
||||||
|
filtered = append(filtered, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return filtered, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
||||||
|
if len(filters) > 1 {
|
||||||
|
return 0, "", fmt.Errorf("%w: at most one listing filter is allowed", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
filter := ListingFilter{}
|
||||||
|
if len(filters) == 1 {
|
||||||
|
filter = filters[0]
|
||||||
|
}
|
||||||
|
switch filter.Mode {
|
||||||
|
case ListingDefault:
|
||||||
|
if scope == "" {
|
||||||
|
return ListingAll, "", nil
|
||||||
|
}
|
||||||
|
return ListingNamed, scope, nil
|
||||||
|
case ListingNamed:
|
||||||
|
if !domain.IsValidResourceNamespace(filter.Name) {
|
||||||
|
return 0, "", fmt.Errorf("%w: invalid listing namespace", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
return ListingNamed, filter.Name, nil
|
||||||
|
case ListingUnqualified, ListingAll:
|
||||||
|
return filter.Mode, "", nil
|
||||||
|
default:
|
||||||
|
return 0, "", fmt.Errorf("%w: unknown listing filter mode", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
|
||||||
|
switch mode {
|
||||||
|
case ListingNamed:
|
||||||
|
idNamespace, _, qualified := strings.Cut(id, "/")
|
||||||
|
return qualified && idNamespace == namespace
|
||||||
|
case ListingUnqualified:
|
||||||
|
return !strings.Contains(id, "/")
|
||||||
|
case ListingAll:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||||
|
request, err := e.qualifyRunRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return e.runner.Prepare(ctx, request)
|
return e.runner.Prepare(ctx, request)
|
||||||
}
|
}
|
||||||
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
|
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
|
||||||
|
request, err := e.qualifyRunRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return e.runner.PrepareExecution(ctx, request)
|
return e.runner.PrepareExecution(ctx, request)
|
||||||
}
|
}
|
||||||
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
|
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
|
||||||
|
request, err := e.qualifyRunRequest(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return e.runner.Run(ctx, request)
|
return e.runner.Run(ctx, request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *Engine) qualifyRunRequest(request domain.RunRequest) (domain.RunRequest, error) {
|
||||||
|
var err error
|
||||||
|
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||||
|
if err != nil {
|
||||||
|
return domain.RunRequest{}, err
|
||||||
|
}
|
||||||
|
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||||
|
if err != nil {
|
||||||
|
return domain.RunRequest{}, err
|
||||||
|
}
|
||||||
|
return request, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) qualifyResourceID(id, namespace, kind string) (string, error) {
|
||||||
|
if id == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
qualified, ok := domain.QualifyResourceID(id, namespace)
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("%w: invalid %s ID", usecase.ErrInvalidRequest, kind)
|
||||||
|
}
|
||||||
|
return qualified, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
|
||||||
return e.runner.RunPrepared(ctx, prepared)
|
return e.runner.RunPrepared(ctx, prepared)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
|
||||||
|
if e == nil || e.smokeTester == nil {
|
||||||
|
return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
request.BackendID, err = e.qualifyResourceID(request.BackendID, e.namespaces.backend, "backend")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.smokeTester.Run(ctx, request)
|
||||||
|
}
|
||||||
|
|||||||
124
internal/engine/smoke_test.go
Normal file
124
internal/engine/smoke_test.go
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
package engine
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func engineSmokeBackendInput() configkit.Input {
|
||||||
|
return configkit.Input{Source: configkit.Mapping("smoke-test backend", map[string]any{
|
||||||
|
"backends": map[string]any{"test-backend": map[string]any{
|
||||||
|
"endpoint": "https://test-backend.example/v1",
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"default_profile": "test-profile",
|
||||||
|
"authentication": map[string]any{"kind": "none"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
"profiles": map[string]any{"test-profile": map[string]any{
|
||||||
|
"backend": "test-backend",
|
||||||
|
"model": "synthetic-model",
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
options := Options{}
|
||||||
|
options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
calls++
|
||||||
|
if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" {
|
||||||
|
t.Fatalf("injected generation request = %+v", request)
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{Content: "engine response"}, nil
|
||||||
|
}))
|
||||||
|
input := engineSmokeBackendInput()
|
||||||
|
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}}, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||||
|
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable ||
|
||||||
|
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
||||||
|
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
|
||||||
|
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
|
||||||
|
report.Facts.EndpointReachable != nil || report.Target.BackendID != "test-backend" {
|
||||||
|
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
|
||||||
|
}
|
||||||
|
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
|
||||||
|
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
|
||||||
|
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
|
||||||
|
}
|
||||||
|
|
||||||
|
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||||
|
Kind: domain.SmokeTestAuthentication, BackendID: "test-backend",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable ||
|
||||||
|
authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 {
|
||||||
|
t.Fatalf("injected authentication report = %+v", authentication)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) {
|
||||||
|
var observedURL string
|
||||||
|
var authorization string
|
||||||
|
options := Options{}
|
||||||
|
input := engineSmokeBackendInput()
|
||||||
|
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
observedURL = request.URL.String()
|
||||||
|
authorization = request.Header.Get("Authorization")
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
|
||||||
|
}, nil
|
||||||
|
})}}, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||||
|
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if observedURL == "" || authorization != "" {
|
||||||
|
t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization)
|
||||||
|
}
|
||||||
|
if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
||||||
|
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
||||||
|
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
||||||
|
report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied ||
|
||||||
|
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
|
||||||
|
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 ||
|
||||||
|
report.Attempts[0].Endpoint != observedURL {
|
||||||
|
t.Fatalf("built-in smoke report = %+v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
return f(ctx, request)
|
||||||
|
}
|
||||||
|
|
||||||
|
type engineRoundTripperFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
return f(request)
|
||||||
|
}
|
||||||
@@ -72,14 +72,43 @@ func (e *ExhaustedError) Attempts() uint64 {
|
|||||||
// Generate executes either one opaque client call or a frozen built-in HTTP
|
// Generate executes either one opaque client call or a frozen built-in HTTP
|
||||||
// sequence with one permit per actual dispatch.
|
// sequence with one permit per actual dispatch.
|
||||||
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
response, err, _ := c.generate(ctx, req, false)
|
||||||
|
return response, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateWithEvidence runs the built-in retry path and returns its factual
|
||||||
|
// attempt evidence alongside any response retained before cancellation.
|
||||||
|
func (c *Coordinator) GenerateWithEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
|
return c.generate(ctx, req, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Coordinator) generate(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
captureEvidence bool,
|
||||||
|
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
if c.injected != nil {
|
if c.injected != nil {
|
||||||
return c.generateInjected(ctx, req)
|
return c.generateInjected(ctx, req, captureEvidence)
|
||||||
}
|
}
|
||||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err, GenerationEvidence{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
evidence := GenerationEvidence{}
|
||||||
|
if captureEvidence {
|
||||||
|
evidence.InitialEndpoint = prepared.InitialEndpoint()
|
||||||
|
}
|
||||||
|
var usage observation.UsageAccumulator
|
||||||
|
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
|
if captureEvidence {
|
||||||
|
evidence.Usage = usage.Observed()
|
||||||
|
}
|
||||||
|
return response, err, evidence
|
||||||
|
}
|
||||||
var attempts, retriesUsed uint64
|
var attempts, retriesUsed uint64
|
||||||
recorder := observation.FromContext(ctx)
|
recorder := observation.FromContext(ctx)
|
||||||
generation := req.Generation
|
generation := req.Generation
|
||||||
@@ -88,56 +117,102 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
|
|||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
ordinal := attempts + 1
|
ordinal := attempts + 1
|
||||||
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
|
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
||||||
if dispatched {
|
if dispatched {
|
||||||
attempts++
|
attempts++
|
||||||
}
|
}
|
||||||
|
if captureEvidence {
|
||||||
|
attempt := AttemptEvidence{
|
||||||
|
StartedAt: startedAt.UTC(),
|
||||||
|
Dispatched: dispatched,
|
||||||
|
ResponseReceived: outcome.Evidence.ResponseReceived,
|
||||||
|
HTTP: cloneHTTPEvidence(outcome.Evidence),
|
||||||
|
CapacityWait: capacityWait,
|
||||||
|
RequestDuration: generationDuration,
|
||||||
|
Failure: ClassifyFailure(outcome.Err),
|
||||||
|
ResponseDecoded: outcome.ResponseDecoded,
|
||||||
|
GenerationCompleted: outcome.GenerationCompleted,
|
||||||
|
}
|
||||||
|
if outcome.StructuredOutputRequested != nil {
|
||||||
|
requested := *outcome.StructuredOutputRequested
|
||||||
|
attempt.StructuredOutputRequested = &requested
|
||||||
|
}
|
||||||
|
if dispatched {
|
||||||
|
usage.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
var attemptUsage observation.UsageAccumulator
|
||||||
|
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
attempt.Usage = attemptUsage.Observed()
|
||||||
|
}
|
||||||
|
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||||
|
}
|
||||||
if recorder != nil {
|
if recorder != nil {
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
||||||
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
|
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
|
||||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||||
}
|
}
|
||||||
if outcome.Err == nil {
|
if outcome.Err == nil {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
if captureEvidence {
|
||||||
|
return finish(outcome.Response, err)
|
||||||
}
|
}
|
||||||
return outcome.Response, nil
|
return finish(nil, err)
|
||||||
|
}
|
||||||
|
return finish(outcome.Response, nil)
|
||||||
}
|
}
|
||||||
if !dispatched || !eligibleStatus(outcome) {
|
if !dispatched || !eligibleStatus(outcome) {
|
||||||
return nil, outcome.Err
|
if captureEvidence {
|
||||||
|
return finish(outcome.Response, outcome.Err)
|
||||||
|
}
|
||||||
|
return finish(nil, outcome.Err)
|
||||||
}
|
}
|
||||||
if retriesUsed == req.TransientRetries {
|
if retriesUsed == req.TransientRetries {
|
||||||
if retriesUsed == 0 {
|
if retriesUsed == 0 {
|
||||||
return nil, outcome.Err
|
return finish(nil, outcome.Err)
|
||||||
}
|
}
|
||||||
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
|
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
retryOrdinal := retriesUsed + 1
|
retryOrdinal := retriesUsed + 1
|
||||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
||||||
waitStart := c.now()
|
waitStart := c.now()
|
||||||
waitErr := c.wait(ctx, delay)
|
waitErr := c.wait(ctx, delay)
|
||||||
|
waitDuration := elapsed(waitStart, c.now())
|
||||||
|
if captureEvidence {
|
||||||
|
evidence.RetryDelays = append(evidence.RetryDelays, waitDuration)
|
||||||
|
}
|
||||||
if recorder != nil {
|
if recorder != nil {
|
||||||
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
|
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
|
||||||
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
|
Elapsed: waitDuration, Failure: ClassifyFailure(waitErr)})
|
||||||
}
|
}
|
||||||
if waitErr != nil {
|
if waitErr != nil {
|
||||||
return nil, waitErr
|
return finish(nil, waitErr)
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
retriesUsed++
|
retriesUsed++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *Coordinator) generateInjected(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
captureEvidence bool,
|
||||||
|
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
|
evidence := GenerationEvidence{}
|
||||||
|
var usage observation.UsageAccumulator
|
||||||
|
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
|
if captureEvidence {
|
||||||
|
evidence.Usage = usage.Observed()
|
||||||
|
}
|
||||||
|
return response, err, evidence
|
||||||
|
}
|
||||||
recorder := observation.FromContext(ctx)
|
recorder := observation.FromContext(ctx)
|
||||||
generation := req.Generation
|
generation := req.Generation
|
||||||
if generation == 0 {
|
if generation == 0 {
|
||||||
@@ -147,18 +222,32 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
|||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if recorder != nil {
|
if captureEvidence {
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
||||||
|
StartedAt: start.UTC(),
|
||||||
|
CapacityWait: capacityWait,
|
||||||
|
Failure: ClassifyFailure(err),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return nil, err
|
if recorder != nil {
|
||||||
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
||||||
|
}
|
||||||
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
release()
|
release()
|
||||||
if recorder != nil {
|
if captureEvidence {
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
||||||
|
StartedAt: start.UTC(),
|
||||||
|
CapacityWait: capacityWait,
|
||||||
|
Failure: ClassifyFailure(err),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return nil, err
|
if recorder != nil {
|
||||||
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
||||||
|
}
|
||||||
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
dispatchStart := c.now()
|
dispatchStart := c.now()
|
||||||
response, err := c.injected.Generate(ctx, req)
|
response, err := c.injected.Generate(ctx, req)
|
||||||
@@ -166,10 +255,33 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
|||||||
if err == nil && response == nil {
|
if err == nil && response == nil {
|
||||||
err = llm.ErrMalformedResponse
|
err = llm.ErrMalformedResponse
|
||||||
}
|
}
|
||||||
|
if captureEvidence {
|
||||||
|
attempt := AttemptEvidence{
|
||||||
|
StartedAt: start.UTC(),
|
||||||
|
Dispatched: true,
|
||||||
|
ResponseReceived: response != nil,
|
||||||
|
RequestDuration: duration,
|
||||||
|
Failure: ClassifyFailure(err),
|
||||||
|
GenerationCompleted: err == nil && response != nil,
|
||||||
|
}
|
||||||
|
requested := req.StructuredOutput != nil
|
||||||
|
attempt.StructuredOutputRequested = &requested
|
||||||
|
var reportedUsage domain.TokenUsage
|
||||||
|
var reportedPresence *domain.TokenUsagePresence
|
||||||
|
if err == nil && response != nil {
|
||||||
|
reportedUsage = response.Usage
|
||||||
|
reportedPresence = response.UsagePresence
|
||||||
|
}
|
||||||
|
usage.Add(reportedUsage, reportedPresence)
|
||||||
|
var attemptUsage observation.UsageAccumulator
|
||||||
|
attemptUsage.Add(reportedUsage, reportedPresence)
|
||||||
|
attempt.Usage = attemptUsage.Observed()
|
||||||
|
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||||
|
}
|
||||||
release()
|
release()
|
||||||
if recorder != nil {
|
if recorder != nil {
|
||||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
||||||
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
|
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
data.Usage = response.Usage
|
data.Usage = response.Usage
|
||||||
data.UsagePresence = response.UsagePresence
|
data.UsagePresence = response.UsagePresence
|
||||||
@@ -177,28 +289,38 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
|||||||
recorder.Attempt(data)
|
recorder.Attempt(data)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
if captureEvidence {
|
||||||
|
return finish(response, err)
|
||||||
}
|
}
|
||||||
return response, nil
|
return finish(nil, err)
|
||||||
|
}
|
||||||
|
return finish(response, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
|
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
|
||||||
start := c.now()
|
start := c.now()
|
||||||
|
startedAt = start
|
||||||
release, err := c.manager.Acquire(ctx, backendID)
|
release, err := c.manager.Acquire(ctx, backendID)
|
||||||
capacityWait = c.capacityWait(backendID, start)
|
capacityWait = c.capacityWait(backendID, start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
||||||
}
|
}
|
||||||
dispatchStart := c.now()
|
dispatchStart := c.now()
|
||||||
|
if captureEvidence {
|
||||||
|
outcome = prepared.ExecuteWithEvidence(ctx)
|
||||||
|
dispatched = outcome.Evidence.DispatchInvoked
|
||||||
|
} else {
|
||||||
outcome = prepared.Execute(ctx)
|
outcome = prepared.Execute(ctx)
|
||||||
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
|
dispatched = true
|
||||||
|
}
|
||||||
|
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||||
@@ -222,7 +344,8 @@ func attemptOutcome(err error) domain.ExecutionOutcome {
|
|||||||
return domain.ExecutionOutcomeError
|
return domain.ExecutionOutcomeError
|
||||||
}
|
}
|
||||||
|
|
||||||
func attemptFailure(err error) domain.ExecutionFailureKind {
|
// ClassifyFailure maps an execution error to the shared content-free category.
|
||||||
|
func ClassifyFailure(err error) domain.ExecutionFailureKind {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return domain.ExecutionFailureNone
|
return domain.ExecutionFailureNone
|
||||||
}
|
}
|
||||||
|
|||||||
43
internal/generation/evidence.go
Normal file
43
internal/generation/evidence.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GenerationEvidence contains invocation-local request evidence without raw
|
||||||
|
// provider bodies, headers, credentials, or generated content.
|
||||||
|
type GenerationEvidence struct {
|
||||||
|
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||||
|
// Injected generation leaves it unknown.
|
||||||
|
InitialEndpoint string
|
||||||
|
Attempts []AttemptEvidence
|
||||||
|
RetryDelays []time.Duration
|
||||||
|
Usage domain.ObservedTokenUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// AttemptEvidence describes one capacity acquisition and any dispatch it
|
||||||
|
// reached. Generated content remains on the separate response return value.
|
||||||
|
type AttemptEvidence struct {
|
||||||
|
StartedAt time.Time
|
||||||
|
Dispatched bool
|
||||||
|
ResponseReceived bool
|
||||||
|
HTTP llm.HTTPExecutionEvidence
|
||||||
|
CapacityWait time.Duration
|
||||||
|
RequestDuration time.Duration
|
||||||
|
Failure domain.ExecutionFailureKind
|
||||||
|
Usage domain.ObservedTokenUsage
|
||||||
|
ResponseDecoded bool
|
||||||
|
GenerationCompleted bool
|
||||||
|
StructuredOutputRequested *bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneHTTPEvidence(evidence llm.HTTPExecutionEvidence) llm.HTTPExecutionEvidence {
|
||||||
|
if evidence.HTTPStatus != nil {
|
||||||
|
status := *evidence.HTTPStatus
|
||||||
|
evidence.HTTPStatus = &status
|
||||||
|
}
|
||||||
|
return evidence
|
||||||
|
}
|
||||||
321
internal/generation/evidence_test.go
Normal file
321
internal/generation/evidence_test.go
Normal file
@@ -0,0 +1,321 @@
|
|||||||
|
package generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceRetainsAttemptsAndUsage(t *testing.T) {
|
||||||
|
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||||
|
calls := 0
|
||||||
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
now = now.Add(4 * time.Millisecond)
|
||||||
|
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||||
|
}
|
||||||
|
now = now.Add(7 * time.Millisecond)
|
||||||
|
return response(http.StatusOK, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":3}}`), nil
|
||||||
|
})
|
||||||
|
coordinator.now = func() time.Time { return now }
|
||||||
|
coordinator.wait = func(context.Context, time.Duration) error {
|
||||||
|
now = now.Add(19 * time.Millisecond)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
|
||||||
|
if err != nil || result == nil || result.Content != "ok" || calls != 2 {
|
||||||
|
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 19*time.Millisecond {
|
||||||
|
t.Fatalf("attempts/delays = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
|
||||||
|
}
|
||||||
|
first, second := evidence.Attempts[0], evidence.Attempts[1]
|
||||||
|
if !first.Dispatched || !first.HTTP.RequestObserved || !first.HTTP.ResponseReceived || first.HTTP.HTTPStatus == nil || *first.HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("first attempt evidence = %+v", first)
|
||||||
|
}
|
||||||
|
if first.Failure != domain.ExecutionFailureProviderStatus || first.RequestDuration != 4*time.Millisecond {
|
||||||
|
t.Fatalf("first attempt failure/timing = %v / %v", first.Failure, first.RequestDuration)
|
||||||
|
}
|
||||||
|
if !second.Dispatched || !second.GenerationCompleted || !second.ResponseDecoded || second.HTTP.HTTPStatus == nil || *second.HTTP.HTTPStatus != http.StatusOK {
|
||||||
|
t.Fatalf("second attempt evidence = %+v", second)
|
||||||
|
}
|
||||||
|
if second.Failure != domain.ExecutionFailureNone || second.RequestDuration != 7*time.Millisecond {
|
||||||
|
t.Fatalf("second attempt failure/timing = %v / %v", second.Failure, second.RequestDuration)
|
||||||
|
}
|
||||||
|
if !second.Usage.PromptTokens.Known || !second.Usage.PromptTokens.Complete || second.Usage.PromptTokens.Value != 3 {
|
||||||
|
t.Fatalf("per-attempt usage = %+v", second.Usage.PromptTokens)
|
||||||
|
}
|
||||||
|
if !evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 3 {
|
||||||
|
t.Fatalf("aggregate usage = %+v", evidence.Usage.PromptTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceRetainsExhaustedAttempts(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||||
|
})
|
||||||
|
coordinator.wait = func(context.Context, time.Duration) error { return nil }
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
|
||||||
|
var exhausted *ExhaustedError
|
||||||
|
if result != nil || !errors.As(err, &exhausted) || calls != 2 || exhausted.Attempts() != 2 {
|
||||||
|
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 {
|
||||||
|
t.Fatalf("attempt/delay evidence = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
|
||||||
|
}
|
||||||
|
for _, attempt := range evidence.Attempts {
|
||||||
|
if !attempt.Dispatched || attempt.Failure != domain.ExecutionFailureProviderStatus || attempt.HTTP.HTTPStatus == nil || *attempt.HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("exhausted attempt evidence = %+v", attempt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete {
|
||||||
|
t.Fatalf("missing provider usage was claimed: %+v", evidence.Usage.PromptTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceRetainsInterruptedBackoffWithoutAttempt(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||||
|
calls := 0
|
||||||
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
now = now.Add(3 * time.Millisecond)
|
||||||
|
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||||
|
})
|
||||||
|
coordinator.now = func() time.Time { return now }
|
||||||
|
coordinator.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
now = now.Add(11 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(2))
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
|
||||||
|
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if len(evidence.Attempts) != 1 || !evidence.Attempts[0].Dispatched || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 11*time.Millisecond {
|
||||||
|
t.Fatalf("interrupted evidence = %#v", evidence)
|
||||||
|
}
|
||||||
|
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("last dispatched response was lost: %+v", evidence.Attempts[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceRecordsCanceledReacquisition(t *testing.T) {
|
||||||
|
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||||
|
calls := 0
|
||||||
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
now = now.Add(2 * time.Millisecond)
|
||||||
|
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||||
|
})
|
||||||
|
coordinator.now = func() time.Time { return now }
|
||||||
|
baseContext, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
contextWithCancelOnWait := &cancelOnDoneContext{Context: baseContext, cancel: cancel}
|
||||||
|
var releaseHeld func()
|
||||||
|
coordinator.wait = func(context.Context, time.Duration) error {
|
||||||
|
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hold permit: %v", err)
|
||||||
|
}
|
||||||
|
releaseHeld = release
|
||||||
|
contextWithCancelOnWait.armed.Store(true)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signal cancellation only when the second acquisition has queued behind
|
||||||
|
// the held permit, avoiding scheduler- or sleep-based synchronization.
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(contextWithCancelOnWait, testRequest(2))
|
||||||
|
if releaseHeld != nil {
|
||||||
|
releaseHeld()
|
||||||
|
}
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
|
||||||
|
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if len(evidence.Attempts) != 2 || !evidence.Attempts[0].Dispatched || evidence.Attempts[1].Dispatched {
|
||||||
|
t.Fatalf("acquisition evidence = %#v", evidence.Attempts)
|
||||||
|
}
|
||||||
|
if evidence.Attempts[1].Failure != domain.ExecutionFailureCanceled {
|
||||||
|
t.Fatalf("canceled acquisition failure = %v", evidence.Attempts[1].Failure)
|
||||||
|
}
|
||||||
|
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("last dispatched evidence was lost: %+v", evidence.Attempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceNoDispatchHasCompleteZeroUsage(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
t.Fatal("canceled generation dispatched")
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(0))
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) || len(evidence.Attempts) != 0 {
|
||||||
|
t.Fatalf("result=%+v error=%v evidence=%+v", result, err, evidence)
|
||||||
|
}
|
||||||
|
if !evidence.Usage.PromptTokens.Known || !evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 0 {
|
||||||
|
t.Fatalf("no-dispatch usage = %+v", evidence.Usage.PromptTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceForInjectedClient(t *testing.T) {
|
||||||
|
opaqueFailure := errors.New("injected client failed")
|
||||||
|
presence := &domain.TokenUsagePresence{PromptTokens: true}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
response *domain.GenerateResponse
|
||||||
|
err error
|
||||||
|
wantResponse bool
|
||||||
|
wantGeneration bool
|
||||||
|
wantFailure domain.ExecutionFailureKind
|
||||||
|
wantUsage bool
|
||||||
|
wantStructured bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "success",
|
||||||
|
response: &domain.GenerateResponse{Content: "private output", Usage: domain.TokenUsage{PromptTokens: 4}, UsagePresence: presence},
|
||||||
|
wantResponse: true,
|
||||||
|
wantGeneration: true,
|
||||||
|
wantUsage: true,
|
||||||
|
wantStructured: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nil response",
|
||||||
|
wantFailure: domain.ExecutionFailureMalformedResponse,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "opaque failure",
|
||||||
|
err: opaqueFailure,
|
||||||
|
wantFailure: domain.ExecutionFailureGeneration,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "response with error",
|
||||||
|
response: &domain.GenerateResponse{Content: "must not be evaluated", Usage: domain.TokenUsage{PromptTokens: 9}, UsagePresence: presence},
|
||||||
|
err: opaqueFailure,
|
||||||
|
wantResponse: true,
|
||||||
|
wantFailure: domain.ExecutionFailureGeneration,
|
||||||
|
wantStructured: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &evidenceInjectedClient{response: tc.response, err: tc.err}
|
||||||
|
coordinator := newEvidenceInjectedCoordinator(t, client)
|
||||||
|
req := testRequest(7)
|
||||||
|
if tc.wantStructured {
|
||||||
|
req.StructuredOutput = &domain.StructuredOutputSpec{}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), req)
|
||||||
|
if client.calls != 1 || len(evidence.Attempts) != 1 || len(evidence.RetryDelays) != 0 {
|
||||||
|
t.Fatalf("calls=%d attempts=%+v retry delays=%v", client.calls, evidence.Attempts, evidence.RetryDelays)
|
||||||
|
}
|
||||||
|
attempt := evidence.Attempts[0]
|
||||||
|
if !attempt.Dispatched || attempt.ResponseReceived != tc.wantResponse || attempt.GenerationCompleted != tc.wantGeneration || attempt.Failure != tc.wantFailure {
|
||||||
|
t.Fatalf("attempt evidence = %+v", attempt)
|
||||||
|
}
|
||||||
|
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
|
||||||
|
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
|
||||||
|
}
|
||||||
|
if attempt.StructuredOutputRequested == nil || *attempt.StructuredOutputRequested != tc.wantStructured {
|
||||||
|
t.Fatalf("structured-output request fact = %v", attempt.StructuredOutputRequested)
|
||||||
|
}
|
||||||
|
if tc.wantFailure == domain.ExecutionFailureNone {
|
||||||
|
if err != nil || result == nil || result.Content != "private output" {
|
||||||
|
t.Fatalf("result=%+v error=%v", result, err)
|
||||||
|
}
|
||||||
|
} else if result != nil || err == nil {
|
||||||
|
t.Fatalf("result=%+v error=%v; expected failed call", result, err)
|
||||||
|
}
|
||||||
|
if attempt.Usage.PromptTokens.Known != tc.wantUsage || evidence.Usage.PromptTokens.Known != tc.wantUsage {
|
||||||
|
t.Fatalf("attempt/aggregate usage = %+v / %+v", attempt.Usage, evidence.Usage)
|
||||||
|
}
|
||||||
|
if tc.wantUsage && (attempt.Usage.PromptTokens.Value != 4 || evidence.Usage.PromptTokens.Value != 4) {
|
||||||
|
t.Fatalf("reported usage was not retained: %+v / %+v", attempt.Usage, evidence.Usage)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateWithEvidenceRetainsInjectedSuccessAfterCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
client := &evidenceInjectedClient{
|
||||||
|
response: &domain.GenerateResponse{Content: "private output"},
|
||||||
|
afterGenerate: func() { cancel() },
|
||||||
|
}
|
||||||
|
coordinator := newEvidenceInjectedCoordinator(t, client)
|
||||||
|
req := testRequest(5)
|
||||||
|
req.StructuredOutput = &domain.StructuredOutputSpec{}
|
||||||
|
|
||||||
|
result, err, evidence := coordinator.GenerateWithEvidence(ctx, req)
|
||||||
|
if !errors.Is(err, context.Canceled) || result == nil || result.Content != "private output" {
|
||||||
|
t.Fatalf("result=%+v error=%v", result, err)
|
||||||
|
}
|
||||||
|
if client.calls != 1 || len(evidence.Attempts) != 1 {
|
||||||
|
t.Fatalf("calls=%d attempts=%+v", client.calls, evidence.Attempts)
|
||||||
|
}
|
||||||
|
attempt := evidence.Attempts[0]
|
||||||
|
if !attempt.ResponseReceived || !attempt.GenerationCompleted || attempt.Failure != domain.ExecutionFailureNone {
|
||||||
|
t.Fatalf("successful generation evidence was lost: %+v", attempt)
|
||||||
|
}
|
||||||
|
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
|
||||||
|
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type evidenceInjectedClient struct {
|
||||||
|
calls int
|
||||||
|
response *domain.GenerateResponse
|
||||||
|
err error
|
||||||
|
afterGenerate func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *evidenceInjectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
c.calls++
|
||||||
|
if c.afterGenerate != nil {
|
||||||
|
c.afterGenerate()
|
||||||
|
}
|
||||||
|
return c.response, c.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func newEvidenceInjectedCoordinator(t *testing.T, client llm.Client) *Coordinator {
|
||||||
|
t.Helper()
|
||||||
|
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return NewInjected(manager, client)
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelOnDoneContext struct {
|
||||||
|
context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
once sync.Once
|
||||||
|
armed atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cancelOnDoneContext) Done() <-chan struct{} {
|
||||||
|
if c.armed.Load() {
|
||||||
|
c.once.Do(c.cancel)
|
||||||
|
}
|
||||||
|
return c.Context.Done()
|
||||||
|
}
|
||||||
191
internal/llm/http_evidence.go
Normal file
191
internal/llm/http_evidence.go
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTTPExecutionEvidence contains only facts observable at the configured
|
||||||
|
// RoundTripper boundary. It intentionally excludes request headers and bodies.
|
||||||
|
type HTTPExecutionEvidence struct {
|
||||||
|
DispatchInvoked bool
|
||||||
|
RequestObserved bool
|
||||||
|
Endpoint string
|
||||||
|
CredentialsSupplied bool
|
||||||
|
ResponseReceived bool
|
||||||
|
HTTPStatus *int
|
||||||
|
}
|
||||||
|
|
||||||
|
type httpEvidenceCollector struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
evidence HTTPExecutionEvidence
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *httpEvidenceCollector) observeRequest(request *http.Request) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
c.evidence.RequestObserved = true
|
||||||
|
c.evidence.Endpoint = endpointWithoutUserInfo(request)
|
||||||
|
c.evidence.CredentialsSupplied = requestHasCredentials(request)
|
||||||
|
c.evidence.ResponseReceived = false
|
||||||
|
c.evidence.HTTPStatus = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *httpEvidenceCollector) observeResponse(response *http.Response) {
|
||||||
|
if response == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
status := response.StatusCode
|
||||||
|
c.evidence.ResponseReceived = true
|
||||||
|
c.evidence.HTTPStatus = &status
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *httpEvidenceCollector) snapshot() HTTPExecutionEvidence {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
evidence := c.evidence
|
||||||
|
if c.evidence.HTTPStatus != nil {
|
||||||
|
status := *c.evidence.HTTPStatus
|
||||||
|
evidence.HTTPStatus = &status
|
||||||
|
}
|
||||||
|
return evidence
|
||||||
|
}
|
||||||
|
|
||||||
|
type observingRoundTripper struct {
|
||||||
|
next http.RoundTripper
|
||||||
|
evidence *httpEvidenceCollector
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t observingRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
t.evidence.observeRequest(request)
|
||||||
|
response, err := t.next.RoundTrip(request)
|
||||||
|
t.evidence.observeResponse(response)
|
||||||
|
return response, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// doWithEvidence executes one request with a shallow copy of client. The copy
|
||||||
|
// preserves timeout, redirect, and cookie-jar behavior without changing client.
|
||||||
|
func doWithEvidence(client *http.Client, request *http.Request) (*http.Response, error, HTTPExecutionEvidence) {
|
||||||
|
if client == nil {
|
||||||
|
client = &http.Client{}
|
||||||
|
}
|
||||||
|
|
||||||
|
collector := &httpEvidenceCollector{}
|
||||||
|
clientCopy := *client
|
||||||
|
transport := clientCopy.Transport
|
||||||
|
if transport == nil {
|
||||||
|
transport = http.DefaultTransport
|
||||||
|
}
|
||||||
|
clientCopy.Transport = observingRoundTripper{next: transport, evidence: collector}
|
||||||
|
|
||||||
|
response, err := clientCopy.Do(request)
|
||||||
|
observed := collector.snapshot()
|
||||||
|
observed.DispatchInvoked = true
|
||||||
|
return response, err, observed
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWithEvidence performs a plain GET with the configured built-in HTTP
|
||||||
|
// client. A positive timeoutSeconds applies to the request through response
|
||||||
|
// body closure, matching the whole-request behavior of http.Client.Timeout.
|
||||||
|
func (c *OpenAICompatibleClient) GetWithEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
endpoint string,
|
||||||
|
timeoutSeconds int,
|
||||||
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
|
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWithAuthenticationEvidence performs a plain GET with credentials resolved
|
||||||
|
// from the selected target. Missing credentials are allowed for diagnostic
|
||||||
|
// authentication probes even when generation requires a key.
|
||||||
|
func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
endpoint string,
|
||||||
|
timeoutSeconds int,
|
||||||
|
target domain.ExecutionTarget,
|
||||||
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
|
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *OpenAICompatibleClient) getWithEvidence(
|
||||||
|
ctx context.Context,
|
||||||
|
endpoint string,
|
||||||
|
timeoutSeconds int,
|
||||||
|
apiKey string,
|
||||||
|
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||||
|
requestContext := ctx
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
if timeoutSeconds > 0 {
|
||||||
|
requestContext, cancel = context.WithTimeout(ctx, secondsDuration(timeoutSeconds))
|
||||||
|
}
|
||||||
|
|
||||||
|
request, err := http.NewRequestWithContext(requestContext, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
if cancel != nil {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return nil, HTTPExecutionEvidence{}, err
|
||||||
|
}
|
||||||
|
if apiKey != "" {
|
||||||
|
request.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(c.httpClient, request)
|
||||||
|
if response == nil || response.Body == nil {
|
||||||
|
if cancel != nil {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return response, evidence, err
|
||||||
|
}
|
||||||
|
if cancel != nil {
|
||||||
|
response.Body = &cancelOnCloseReadCloser{ReadCloser: response.Body, cancel: cancel}
|
||||||
|
}
|
||||||
|
return response, evidence, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func secondsDuration(seconds int) time.Duration {
|
||||||
|
return time.Duration(seconds) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelOnCloseReadCloser struct {
|
||||||
|
io.ReadCloser
|
||||||
|
cancel context.CancelFunc
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *cancelOnCloseReadCloser) Close() error {
|
||||||
|
r.once.Do(r.cancel)
|
||||||
|
return r.ReadCloser.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func endpointWithoutUserInfo(request *http.Request) string {
|
||||||
|
if request == nil || request.URL == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
endpoint := *request.URL
|
||||||
|
endpoint.User = nil
|
||||||
|
return endpoint.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestHasCredentials(request *http.Request) bool {
|
||||||
|
if request == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if request.Header.Get("Authorization") != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return request.URL != nil && request.URL.User != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ http.RoundTripper = observingRoundTripper{}
|
||||||
|
var _ io.ReadCloser = (*cancelOnCloseReadCloser)(nil)
|
||||||
340
internal/llm/http_evidence_test.go
Normal file
340
internal/llm/http_evidence_test.go
Normal file
@@ -0,0 +1,340 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cookiejar"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Header.Get("Authorization") == "" {
|
||||||
|
t.Error("request did not carry URL user credentials")
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct client: %v", err)
|
||||||
|
}
|
||||||
|
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
||||||
|
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET endpoint: %v", err)
|
||||||
|
}
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("GET returned no response")
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
|
||||||
|
if !evidence.DispatchInvoked || !evidence.RequestObserved || !evidence.ResponseReceived {
|
||||||
|
t.Fatalf("incomplete response evidence: %+v", evidence)
|
||||||
|
}
|
||||||
|
if evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusNoContent {
|
||||||
|
t.Fatalf("status evidence = %v, want %d", evidence.HTTPStatus, http.StatusNoContent)
|
||||||
|
}
|
||||||
|
if !evidence.CredentialsSupplied {
|
||||||
|
t.Fatal("evidence did not record URL credentials")
|
||||||
|
}
|
||||||
|
if strings.Contains(evidence.Endpoint, "user") || strings.Contains(evidence.Endpoint, "secret") {
|
||||||
|
t.Fatalf("exposed endpoint contains URL user info: %q", evidence.Endpoint)
|
||||||
|
}
|
||||||
|
if want := server.URL + "/health"; evidence.Endpoint != want {
|
||||||
|
t.Fatalf("endpoint = %q, want %q", evidence.Endpoint, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
target domain.ExecutionTarget
|
||||||
|
environment string
|
||||||
|
wantAuth string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "direct key takes precedence",
|
||||||
|
target: domain.ExecutionTarget{
|
||||||
|
APIKey: " direct-key ",
|
||||||
|
APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY",
|
||||||
|
},
|
||||||
|
environment: "environment-key",
|
||||||
|
wantAuth: "Bearer direct-key",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "environment key is used",
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY"},
|
||||||
|
environment: " environment-key ",
|
||||||
|
wantAuth: "Bearer environment-key",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing required key is optional here",
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", APIKeyRequired: true},
|
||||||
|
environment: " ",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
||||||
|
var authorization string
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
authorization = request.Header.Get("Authorization")
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusUnauthorized,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: io.NopCloser(strings.NewReader("")),
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, evidence, err := client.GetWithAuthenticationEvidence(
|
||||||
|
context.Background(), "https://example.invalid/key", 0, tc.target,
|
||||||
|
)
|
||||||
|
if err != nil || response == nil {
|
||||||
|
t.Fatalf("response=%v error=%v", response, err)
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
if authorization != tc.wantAuth || evidence.CredentialsSupplied != (tc.wantAuth != "") {
|
||||||
|
t.Fatalf("authorization=%q evidence=%+v", authorization, evidence)
|
||||||
|
}
|
||||||
|
if evidence.Endpoint != "https://example.invalid/key" || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("HTTP evidence = %+v", evidence)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) {
|
||||||
|
transportErr := errors.New("transport unavailable")
|
||||||
|
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return nil, transportErr
|
||||||
|
})}
|
||||||
|
request, err := http.NewRequest(http.MethodGet, "https://diagnostic.example/status", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(client, request)
|
||||||
|
if response != nil || !errors.Is(err, transportErr) {
|
||||||
|
t.Fatalf("response/error = %v/%v, want nil/transport error", response, err)
|
||||||
|
}
|
||||||
|
if !evidence.DispatchInvoked || !evidence.RequestObserved || evidence.ResponseReceived {
|
||||||
|
t.Fatalf("unexpected request evidence: %+v", evidence)
|
||||||
|
}
|
||||||
|
if evidence.HTTPStatus != nil || evidence.Endpoint != request.URL.String() || evidence.CredentialsSupplied {
|
||||||
|
t.Fatalf("unexpected evidence values: %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoWithEvidenceRetainsResponseReturnedAlongsideError(t *testing.T) {
|
||||||
|
transportErr := errors.New("transport reported an error with a response")
|
||||||
|
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
return responseFor(request, http.StatusBadGateway, ""), transportErr
|
||||||
|
})}
|
||||||
|
request, err := http.NewRequest(http.MethodGet, "https://diagnostic.example/status", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(client, request)
|
||||||
|
if !errors.Is(err, transportErr) {
|
||||||
|
t.Fatalf("error = %v, want transport error", err)
|
||||||
|
}
|
||||||
|
if response != nil {
|
||||||
|
response.Body.Close()
|
||||||
|
}
|
||||||
|
if !evidence.RequestObserved || !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusBadGateway {
|
||||||
|
t.Fatalf("observed response was lost alongside error: %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoWithEvidenceUsesLastRedirectRequest(t *testing.T) {
|
||||||
|
var observedAuthorization []string
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
observedAuthorization = append(observedAuthorization, request.Header.Get("Authorization"))
|
||||||
|
if request.URL.Host == "source.example" {
|
||||||
|
return responseFor(request, http.StatusFound, "https://other.example/final"), nil
|
||||||
|
}
|
||||||
|
return responseFor(request, http.StatusUnauthorized, ""), nil
|
||||||
|
})
|
||||||
|
client := &http.Client{Transport: transport}
|
||||||
|
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create request: %v", err)
|
||||||
|
}
|
||||||
|
request.Header.Set("Authorization", "Bearer private-token")
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(client, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("follow redirect: %v", err)
|
||||||
|
}
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("redirect returned no response")
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
if len(observedAuthorization) != 2 || observedAuthorization[0] == "" || observedAuthorization[1] != "" {
|
||||||
|
t.Fatalf("authorization across redirect = %#v", observedAuthorization)
|
||||||
|
}
|
||||||
|
if evidence.Endpoint != "https://other.example/final" || evidence.CredentialsSupplied {
|
||||||
|
t.Fatalf("evidence did not describe credential-stripped last hop: %+v", evidence)
|
||||||
|
}
|
||||||
|
if !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("last-hop response evidence = %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoWithEvidenceClearsPriorResponseOnFailedRedirect(t *testing.T) {
|
||||||
|
transportErr := errors.New("redirect target unavailable")
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.URL.Host == "source.example" {
|
||||||
|
return responseFor(request, http.StatusTemporaryRedirect, "https://target.example/unavailable"), nil
|
||||||
|
}
|
||||||
|
return nil, transportErr
|
||||||
|
})
|
||||||
|
client := &http.Client{Transport: transport}
|
||||||
|
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(client, request)
|
||||||
|
if response != nil || !errors.Is(err, transportErr) {
|
||||||
|
t.Fatalf("response/error = %v/%v, want nil/transport error", response, err)
|
||||||
|
}
|
||||||
|
if !evidence.RequestObserved || evidence.Endpoint != "https://target.example/unavailable" {
|
||||||
|
t.Fatalf("evidence did not retain failed last request: %+v", evidence)
|
||||||
|
}
|
||||||
|
if evidence.ResponseReceived || evidence.HTTPStatus != nil {
|
||||||
|
t.Fatalf("prior redirect response was not cleared: %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoWithEvidenceRetainsResponseWhenRedirectPolicyStops(t *testing.T) {
|
||||||
|
redirectErr := errors.New("redirect rejected")
|
||||||
|
transportCalls := 0
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
transportCalls++
|
||||||
|
return responseFor(request, http.StatusFound, "https://other.example/never-requested"), nil
|
||||||
|
}),
|
||||||
|
CheckRedirect: func(*http.Request, []*http.Request) error { return redirectErr },
|
||||||
|
}
|
||||||
|
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err, evidence := doWithEvidence(client, request)
|
||||||
|
if !errors.Is(err, redirectErr) {
|
||||||
|
t.Fatalf("redirect error = %v, want policy error", err)
|
||||||
|
}
|
||||||
|
if response == nil || response.StatusCode != http.StatusFound {
|
||||||
|
t.Fatalf("redirect response = %v, want original response", response)
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
if transportCalls != 1 || evidence.Endpoint != "https://source.example/start" {
|
||||||
|
t.Fatalf("redirect policy inferred or dispatched another request: calls=%d evidence=%+v", transportCalls, evidence)
|
||||||
|
}
|
||||||
|
if !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusFound {
|
||||||
|
t.Fatalf("original response evidence was lost: %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
redirect := func(*http.Request, []*http.Request) error { return nil }
|
||||||
|
jar, err := cookiejar.New(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct cookie jar: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
transport := http.DefaultTransport
|
||||||
|
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const calls = 8
|
||||||
|
evidences := make([]HTTPExecutionEvidence, calls)
|
||||||
|
errs := make([]error, calls)
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for index := 0; index < calls; index++ {
|
||||||
|
wait.Add(1)
|
||||||
|
go func(index int) {
|
||||||
|
defer wait.Done()
|
||||||
|
endpoint := server.URL + "/call/" + strconv.Itoa(index)
|
||||||
|
response, evidence, getErr := client.GetWithEvidence(context.Background(), endpoint, 0)
|
||||||
|
if response != nil {
|
||||||
|
response.Body.Close()
|
||||||
|
}
|
||||||
|
evidences[index], errs[index] = evidence, getErr
|
||||||
|
}(index)
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
|
||||||
|
if supplied.Transport != transport || supplied.Timeout != 7*time.Second || supplied.CheckRedirect == nil || supplied.Jar != jar {
|
||||||
|
t.Fatal("diagnostic GET mutated the supplied HTTP client")
|
||||||
|
}
|
||||||
|
for index, evidence := range evidences {
|
||||||
|
if errs[index] != nil {
|
||||||
|
t.Fatalf("GET %d: %v", index, errs[index])
|
||||||
|
}
|
||||||
|
want := server.URL + "/call/" + strconv.Itoa(index)
|
||||||
|
if !evidence.RequestObserved || evidence.Endpoint != want {
|
||||||
|
t.Fatalf("GET %d evidence = %+v, want endpoint %q", index, evidence, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
||||||
|
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
<-request.Context().Done()
|
||||||
|
return nil, request.Context().Err()
|
||||||
|
}),
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
||||||
|
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("response/error = %v/%v, want nil/deadline exceeded", response, err)
|
||||||
|
}
|
||||||
|
if !evidence.DispatchInvoked || !evidence.RequestObserved || evidence.ResponseReceived {
|
||||||
|
t.Fatalf("timeout evidence = %+v", evidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func responseFor(request *http.Request, status int, location string) *http.Response {
|
||||||
|
header := make(http.Header)
|
||||||
|
if location != "" {
|
||||||
|
header.Set("Location", location)
|
||||||
|
}
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: status,
|
||||||
|
Header: header,
|
||||||
|
Body: io.NopCloser(strings.NewReader("")),
|
||||||
|
Request: request,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -101,13 +101,14 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PreparedGeneration holds one encoded request and resolved authentication for
|
// PreparedGeneration holds one encoded request and resolved authentication for
|
||||||
// repeated attempts. Each Execute call creates a fresh HTTP request and context.
|
// repeated attempts. Each execution creates a fresh HTTP request and context.
|
||||||
type PreparedGeneration struct {
|
type PreparedGeneration struct {
|
||||||
endpoint string
|
endpoint string
|
||||||
payload []byte
|
payload []byte
|
||||||
apiKey string
|
apiKey string
|
||||||
timeoutSeconds int
|
timeoutSeconds int
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
|
structuredOutputRequested bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// AttemptOutcome describes one HTTP execution without exposing response bodies
|
// AttemptOutcome describes one HTTP execution without exposing response bodies
|
||||||
@@ -119,6 +120,10 @@ type AttemptOutcome struct {
|
|||||||
Usage domain.TokenUsage
|
Usage domain.TokenUsage
|
||||||
UsagePresence *domain.TokenUsagePresence
|
UsagePresence *domain.TokenUsagePresence
|
||||||
RetryNotBefore time.Time
|
RetryNotBefore time.Time
|
||||||
|
Evidence HTTPExecutionEvidence
|
||||||
|
StructuredOutputRequested *bool
|
||||||
|
ResponseDecoded bool
|
||||||
|
GenerationCompleted bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrepareGeneration validates and freezes one built-in request before dispatch.
|
// PrepareGeneration validates and freezes one built-in request before dispatch.
|
||||||
@@ -159,11 +164,8 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
apiKey := resolveAPIKey(req.Target)
|
||||||
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||||
if apiKey == "" && envName != "" {
|
|
||||||
apiKey = strings.TrimSpace(os.Getenv(envName))
|
|
||||||
}
|
|
||||||
if apiKey == "" && req.Target.APIKeyRequired {
|
if apiKey == "" && req.Target.APIKeyRequired {
|
||||||
if envName != "" {
|
if envName != "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
@@ -177,12 +179,36 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return &PreparedGeneration{
|
return &PreparedGeneration{
|
||||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||||
|
structuredOutputRequested: wireReq.ResponseFormat != nil,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveAPIKey(target domain.ExecutionTarget) string {
|
||||||
|
apiKey := strings.TrimSpace(target.APIKey)
|
||||||
|
if apiKey == "" {
|
||||||
|
if envName := strings.TrimSpace(target.APIKeyEnv); envName != "" {
|
||||||
|
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return apiKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitialEndpoint returns the frozen request URL before redirects.
|
||||||
|
func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint }
|
||||||
|
|
||||||
// Execute dispatches one attempt. The returned response is non-nil only when
|
// Execute dispatches one attempt. The returned response is non-nil only when
|
||||||
// the successful envelope contains a usable first choice.
|
// the successful envelope contains a usable first choice.
|
||||||
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
||||||
|
return p.execute(ctx, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecuteWithEvidence performs one prepared attempt and retains HTTP and
|
||||||
|
// decoded-generation facts even when a later context check returns an error.
|
||||||
|
func (p *PreparedGeneration) ExecuteWithEvidence(ctx context.Context) AttemptOutcome {
|
||||||
|
return p.execute(ctx, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) AttemptOutcome {
|
||||||
requestContext := ctx
|
requestContext := ctx
|
||||||
if p.timeoutSeconds > 0 {
|
if p.timeoutSeconds > 0 {
|
||||||
var cancel context.CancelFunc
|
var cancel context.CancelFunc
|
||||||
@@ -198,13 +224,29 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
|||||||
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
|
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
httpResp, err := p.httpClient.Do(httpReq)
|
var (
|
||||||
|
httpResp *http.Response
|
||||||
|
outcome AttemptOutcome
|
||||||
|
)
|
||||||
|
if captureEvidence {
|
||||||
|
httpResp, err, outcome.Evidence = doWithEvidence(p.httpClient, httpReq)
|
||||||
|
if outcome.Evidence.DispatchInvoked {
|
||||||
|
requested := p.structuredOutputRequested
|
||||||
|
outcome.StructuredOutputRequested = &requested
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
httpResp, err = p.httpClient.Do(httpReq)
|
||||||
|
}
|
||||||
|
if httpResp != nil && (captureEvidence || err == nil) {
|
||||||
|
outcome.StatusCode = httpResp.StatusCode
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AttemptOutcome{Err: &requestFailedError{cause: err}}
|
outcome.Err = &requestFailedError{cause: err}
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
receivedAt := time.Now()
|
receivedAt := time.Now()
|
||||||
defer httpResp.Body.Close()
|
defer httpResp.Body.Close()
|
||||||
outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
|
outcome.StatusCode = httpResp.StatusCode
|
||||||
|
|
||||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||||
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
|
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
|
||||||
@@ -220,6 +262,11 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
|||||||
}
|
}
|
||||||
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
||||||
outcome.Err = openAIChatResponseTooLargeError()
|
outcome.Err = openAIChatResponseTooLargeError()
|
||||||
|
if captureEvidence {
|
||||||
|
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||||
|
outcome.Err = contextErr
|
||||||
|
}
|
||||||
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,6 +278,7 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
|||||||
}
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
|
outcome.ResponseDecoded = true
|
||||||
outcome.Usage = domain.TokenUsage{
|
outcome.Usage = domain.TokenUsage{
|
||||||
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
||||||
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
||||||
@@ -244,14 +292,30 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
|||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
if len(wireResp.Choices) == 0 {
|
||||||
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||||
|
if captureEvidence {
|
||||||
|
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||||
|
outcome.Err = contextErr
|
||||||
|
}
|
||||||
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
content := wireResp.Choices[0].Message.Content
|
content := wireResp.Choices[0].Message.Content
|
||||||
if content == nil {
|
if content == nil {
|
||||||
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||||
|
if captureEvidence {
|
||||||
|
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||||
|
outcome.Err = contextErr
|
||||||
|
}
|
||||||
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
|
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
|
||||||
|
outcome.GenerationCompleted = true
|
||||||
|
if captureEvidence {
|
||||||
|
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
|
||||||
|
outcome.Err = contextErr
|
||||||
|
}
|
||||||
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
222
internal/llm/prepared_generation_evidence_test.go
Normal file
222
internal/llm/prepared_generation_evidence_test.go
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
structured bool
|
||||||
|
wantDecoded bool
|
||||||
|
wantGeneration bool
|
||||||
|
wantResponse bool
|
||||||
|
wantContent string
|
||||||
|
wantFailure error
|
||||||
|
wantUsagePresence bool
|
||||||
|
wantStructuredFact bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid content",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`,
|
||||||
|
structured: true,
|
||||||
|
wantDecoded: true,
|
||||||
|
wantGeneration: true,
|
||||||
|
wantResponse: true,
|
||||||
|
wantContent: "ok",
|
||||||
|
wantUsagePresence: true,
|
||||||
|
wantStructuredFact: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty content",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: `{"choices":[{"message":{"content":""}}]}`,
|
||||||
|
wantDecoded: true,
|
||||||
|
wantGeneration: true,
|
||||||
|
wantResponse: true,
|
||||||
|
wantStructuredFact: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "decoded envelope without generated content",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: `{"choices":[],"usage":{"prompt_tokens":0}}`,
|
||||||
|
wantDecoded: true,
|
||||||
|
wantUsagePresence: true,
|
||||||
|
wantFailure: ErrMalformedResponse,
|
||||||
|
wantStructuredFact: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed envelope",
|
||||||
|
status: http.StatusOK,
|
||||||
|
body: `not json`,
|
||||||
|
wantFailure: ErrMalformedResponse,
|
||||||
|
wantStructuredFact: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "provider status",
|
||||||
|
status: http.StatusTooManyRequests,
|
||||||
|
body: `{"error":{"message":"try again"}}`,
|
||||||
|
wantFailure: ErrUnexpectedStatus,
|
||||||
|
wantStructuredFact: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://provider.example/v1",
|
||||||
|
Model: "model",
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
response := responseFor(request, test.status, "")
|
||||||
|
response.Body = io.NopCloser(strings.NewReader(test.body))
|
||||||
|
response.ContentLength = int64(len(test.body))
|
||||||
|
return response, nil
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := ordinaryGenerateRequest()
|
||||||
|
if test.structured {
|
||||||
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "smoke",
|
||||||
|
Schema: map[string]any{"type": "object"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prepared, err := client.PrepareGeneration(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome := prepared.ExecuteWithEvidence(context.Background())
|
||||||
|
if (outcome.Err != nil) != (test.wantFailure != nil) || (test.wantFailure != nil && !errors.Is(outcome.Err, test.wantFailure)) {
|
||||||
|
t.Fatalf("error = %v, want failure %v", outcome.Err, test.wantFailure)
|
||||||
|
}
|
||||||
|
if !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved || !outcome.Evidence.ResponseReceived {
|
||||||
|
t.Fatalf("response evidence = %+v", outcome.Evidence)
|
||||||
|
}
|
||||||
|
if outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != test.status {
|
||||||
|
t.Fatalf("HTTP evidence status = %v, want %d", outcome.Evidence.HTTPStatus, test.status)
|
||||||
|
}
|
||||||
|
if outcome.GenerationCompleted != test.wantGeneration {
|
||||||
|
t.Fatalf("generation completed = %t, want %t", outcome.GenerationCompleted, test.wantGeneration)
|
||||||
|
}
|
||||||
|
if outcome.ResponseDecoded != test.wantDecoded {
|
||||||
|
t.Fatalf("response decoded = %t, want %t", outcome.ResponseDecoded, test.wantDecoded)
|
||||||
|
}
|
||||||
|
if (outcome.Response != nil) != test.wantResponse {
|
||||||
|
t.Fatalf("response presence = %t, want %t", outcome.Response != nil, test.wantResponse)
|
||||||
|
}
|
||||||
|
if test.wantResponse && outcome.Response.Content != test.wantContent {
|
||||||
|
t.Fatalf("generated content = %q, want %q", outcome.Response.Content, test.wantContent)
|
||||||
|
}
|
||||||
|
if (outcome.UsagePresence != nil && outcome.UsagePresence.PromptTokens) != test.wantUsagePresence {
|
||||||
|
t.Fatalf("usage presence = %+v, want prompt token presence %t", outcome.UsagePresence, test.wantUsagePresence)
|
||||||
|
}
|
||||||
|
if outcome.StructuredOutputRequested == nil || *outcome.StructuredOutputRequested != test.wantStructuredFact {
|
||||||
|
t.Fatalf("structured-output evidence = %v, want %t", outcome.StructuredOutputRequested, test.wantStructuredFact)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://provider.example/v1",
|
||||||
|
Model: "model",
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: &cancelAfterReadBody{Reader: strings.NewReader(body), cancel: cancel},
|
||||||
|
ContentLength: int64(len(body)),
|
||||||
|
Request: request,
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome := prepared.ExecuteWithEvidence(ctx)
|
||||||
|
if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) {
|
||||||
|
t.Fatalf("canceled outcome = %+v", outcome)
|
||||||
|
}
|
||||||
|
if !outcome.Evidence.ResponseReceived || outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != http.StatusOK {
|
||||||
|
t.Fatalf("response evidence was lost after cancellation: %+v", outcome.Evidence)
|
||||||
|
}
|
||||||
|
if !outcome.ResponseDecoded || !outcome.GenerationCompleted || outcome.Response == nil || outcome.Response.Content != "ok" {
|
||||||
|
t.Fatalf("decoded generation was lost after cancellation: %+v", outcome)
|
||||||
|
}
|
||||||
|
if outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 {
|
||||||
|
t.Fatalf("decoded usage was lost after cancellation: %+v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
||||||
|
transportErr := errors.New("provider unavailable")
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://provider.example/v1",
|
||||||
|
Model: "model",
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return nil, transportErr
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := ordinaryGenerateRequest()
|
||||||
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "smoke",
|
||||||
|
Schema: map[string]any{"type": "object"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
prepared, err := client.PrepareGeneration(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome := prepared.ExecuteWithEvidence(context.Background())
|
||||||
|
if !errors.Is(outcome.Err, transportErr) || !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved {
|
||||||
|
t.Fatalf("dispatch outcome = %+v", outcome)
|
||||||
|
}
|
||||||
|
if outcome.Evidence.ResponseReceived || outcome.StructuredOutputRequested == nil || !*outcome.StructuredOutputRequested {
|
||||||
|
t.Fatalf("structured request evidence = %+v, request=%v", outcome.Evidence, outcome.StructuredOutputRequested)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelAfterReadBody struct {
|
||||||
|
io.Reader
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelAfterReadBody) Read(buffer []byte) (int, error) {
|
||||||
|
n, err := b.Reader.Read(buffer)
|
||||||
|
if n > 0 {
|
||||||
|
b.cancel()
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelAfterReadBody) Close() error { return nil }
|
||||||
@@ -28,13 +28,13 @@ type Recorder struct {
|
|||||||
capacityWait time.Duration
|
capacityWait time.Duration
|
||||||
generationDuration time.Duration
|
generationDuration time.Duration
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
usage usageTotals
|
usage UsageAccumulator
|
||||||
}
|
}
|
||||||
|
|
||||||
// Selection contains resolved identities only. Unresolved fields stay empty.
|
// Selection contains resolved identities only. Unresolved fields stay empty.
|
||||||
type Selection struct {
|
type Selection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
BackendID string
|
BackendID string
|
||||||
Model string
|
Model string
|
||||||
@@ -79,8 +79,7 @@ func New(observer domain.ExecutionObserver) (*Recorder, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newRecorder(runID string, now func() time.Time, observer domain.ExecutionObserver) *Recorder {
|
func newRecorder(runID string, now func() time.Time, observer domain.ExecutionObserver) *Recorder {
|
||||||
return &Recorder{runID: runID, startedAt: now(), now: now, observer: observer,
|
return &Recorder{runID: runID, startedAt: now(), now: now, observer: observer}
|
||||||
usage: newUsageTotals()}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRunID creates the identity shared by observations and successful results.
|
// NewRunID creates the identity shared by observations and successful results.
|
||||||
@@ -104,7 +103,10 @@ func (r *Recorder) Start() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// UpdateSelection replaces the known resolved identities for later events.
|
// UpdateSelection replaces the known resolved identities for later events.
|
||||||
func (r *Recorder) UpdateSelection(selection Selection) { r.selection = selection }
|
func (r *Recorder) UpdateSelection(selection Selection) {
|
||||||
|
selection.PromptVersion = copyStringPointer(selection.PromptVersion)
|
||||||
|
r.selection = selection
|
||||||
|
}
|
||||||
|
|
||||||
// Attempt records one completed acquisition and optional dispatch.
|
// Attempt records one completed acquisition and optional dispatch.
|
||||||
func (r *Recorder) Attempt(data AttemptData) {
|
func (r *Recorder) Attempt(data AttemptData) {
|
||||||
@@ -119,7 +121,7 @@ func (r *Recorder) Attempt(data AttemptData) {
|
|||||||
r.lastRepairGeneration = data.Generation
|
r.lastRepairGeneration = data.Generation
|
||||||
}
|
}
|
||||||
r.generationDuration = addDuration(r.generationDuration, data.GenerationDuration)
|
r.generationDuration = addDuration(r.generationDuration, data.GenerationDuration)
|
||||||
r.usage.add(data.Usage, data.UsagePresence)
|
r.usage.Add(data.Usage, data.UsagePresence)
|
||||||
}
|
}
|
||||||
event := r.event(domain.ExecutionAttempt)
|
event := r.event(domain.ExecutionAttempt)
|
||||||
event.Generation = data.Generation
|
event.Generation = data.Generation
|
||||||
@@ -171,18 +173,26 @@ func (r *Recorder) Finish(data FinishData) {
|
|||||||
if event.Duration < 0 {
|
if event.Duration < 0 {
|
||||||
event.Duration = 0
|
event.Duration = 0
|
||||||
}
|
}
|
||||||
event.Usage = r.usage.snapshot(r.attempts)
|
event.Usage = r.usage.Observed()
|
||||||
r.deliver(event)
|
r.deliver(event)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
||||||
return domain.ExecutionEvent{
|
return domain.ExecutionEvent{
|
||||||
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
||||||
PromptVersion: r.selection.PromptVersion, ProfileID: r.selection.ProfileID,
|
PromptVersion: copyStringPointer(r.selection.PromptVersion), ProfileID: r.selection.ProfileID,
|
||||||
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPointer(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
||||||
if r.observer != nil {
|
if r.observer != nil {
|
||||||
r.observer(event)
|
r.observer(event)
|
||||||
@@ -243,7 +253,29 @@ type usageTotals struct {
|
|||||||
prompt, completion, total, cached, cacheWrite usageField
|
prompt, completion, total, cached, cacheWrite usageField
|
||||||
}
|
}
|
||||||
|
|
||||||
func newUsageTotals() usageTotals {
|
// UsageAccumulator combines presence-aware usage across dispatched attempts.
|
||||||
|
// Its zero value represents no dispatched attempts.
|
||||||
|
type UsageAccumulator struct {
|
||||||
|
totals usageTotals
|
||||||
|
attempts uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add includes the usage reported by one dispatched attempt.
|
||||||
|
func (u *UsageAccumulator) Add(usage domain.TokenUsage, presence *domain.TokenUsagePresence) {
|
||||||
|
if u.attempts == 0 {
|
||||||
|
u.totals = completeUsageTotals()
|
||||||
|
}
|
||||||
|
u.attempts++
|
||||||
|
u.totals.add(usage, presence)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Observed returns aggregate counts, marking every field known and complete
|
||||||
|
// when no attempt has been added.
|
||||||
|
func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
|
||||||
|
return u.totals.snapshot(u.attempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func completeUsageTotals() usageTotals {
|
||||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
||||||
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ func TestRecorderAccountsDispatchedAttemptsAndReportedUsage(t *testing.T) {
|
|||||||
var events []domain.ExecutionEvent
|
var events []domain.ExecutionEvent
|
||||||
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||||
recorder.Start()
|
recorder.Start()
|
||||||
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: "v1", ProfileID: "profile", BackendID: "backend", Model: "model"})
|
version := "v1"
|
||||||
|
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: &version, ProfileID: "profile", BackendID: "backend", Model: "model"})
|
||||||
recorder.Attempt(AttemptData{
|
recorder.Attempt(AttemptData{
|
||||||
Generation: 1, Attempt: 1, Dispatched: true,
|
Generation: 1, Attempt: 1, Dispatched: true,
|
||||||
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,
|
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,
|
||||||
|
|||||||
27
internal/observation/usage_accumulator_test.go
Normal file
27
internal/observation/usage_accumulator_test.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package observation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
|
||||||
|
var empty UsageAccumulator
|
||||||
|
zero := empty.Observed()
|
||||||
|
if !zero.PromptTokens.Known || !zero.PromptTokens.Complete || zero.PromptTokens.Value != 0 ||
|
||||||
|
!zero.CacheWriteTokens.Known || !zero.CacheWriteTokens.Complete || zero.CacheWriteTokens.Value != 0 {
|
||||||
|
t.Fatalf("zero-attempt usage = %+v", zero)
|
||||||
|
}
|
||||||
|
|
||||||
|
var usage UsageAccumulator
|
||||||
|
usage.Add(domain.TokenUsage{PromptTokens: 0}, &domain.TokenUsagePresence{PromptTokens: true})
|
||||||
|
usage.Add(domain.TokenUsage{PromptTokens: 3}, &domain.TokenUsagePresence{PromptTokens: true})
|
||||||
|
observed := usage.Observed()
|
||||||
|
if observed.PromptTokens.Value != 3 || !observed.PromptTokens.Known || !observed.PromptTokens.Complete {
|
||||||
|
t.Fatalf("reported prompt usage = %+v", observed.PromptTokens)
|
||||||
|
}
|
||||||
|
if observed.CompletionTokens.Known || observed.CompletionTokens.Complete {
|
||||||
|
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -20,7 +20,6 @@ var (
|
|||||||
|
|
||||||
type rendererBenchmarkIdentity struct {
|
type rendererBenchmarkIdentity struct {
|
||||||
id string
|
id string
|
||||||
version string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type rendererBenchmarkLookup struct {
|
type rendererBenchmarkLookup struct {
|
||||||
@@ -28,8 +27,8 @@ type rendererBenchmarkLookup struct {
|
|||||||
templates CompiledTemplates
|
templates CompiledTemplates
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
||||||
if id != lookup.identity.id || version != lookup.identity.version {
|
if id != lookup.identity.id {
|
||||||
return CompiledTemplates{}, false
|
return CompiledTemplates{}, false
|
||||||
}
|
}
|
||||||
return lookup.templates, true
|
return lookup.templates, true
|
||||||
@@ -53,7 +52,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
small := newCompiledRenderingBenchmarkCase(b,
|
small := newCompiledRenderingBenchmarkCase(b,
|
||||||
"small_message", "", []string{"Summary for {{.audience}}."},
|
"small_message", "", []string{"Summary for {{.audience}}."},
|
||||||
&domain.PromptDefinition{
|
&domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1",
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
||||||
}, nil, map[string]string{"audience": "project leads"},
|
}, nil, map[string]string{"audience": "project leads"},
|
||||||
len("Summary for project leads."),
|
len("Summary for project leads."),
|
||||||
@@ -69,7 +68,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
}
|
}
|
||||||
largeContent := `{{input "document"}}`
|
largeContent := `{{input "document"}}`
|
||||||
largeDefinition := &domain.PromptDefinition{
|
largeDefinition := &domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1",
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
||||||
}
|
}
|
||||||
@@ -92,7 +91,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
`{{input "document"}}|{{input "document"}}`,
|
`{{input "document"}}|{{input "document"}}`,
|
||||||
}
|
}
|
||||||
repeatedDefinition := &domain.PromptDefinition{
|
repeatedDefinition := &domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"), SessionID: repeatedSession,
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "system", Content: repeatedMessages[0]},
|
{Role: "system", Content: repeatedMessages[0]},
|
||||||
@@ -142,13 +141,13 @@ func BenchmarkRetainedRenderedOutput(b *testing.B) {
|
|||||||
bodyText := string(body)
|
bodyText := string(body)
|
||||||
content := `{{input "document"}}`
|
content := `{{input "document"}}`
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: "retained-render-benchmark", Version: "v1",
|
ID: "retained-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
||||||
}
|
}
|
||||||
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
|
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
|
||||||
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
|
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
|
||||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
identity: rendererBenchmarkIdentity{id: definition.ID},
|
||||||
templates: compiled,
|
templates: compiled,
|
||||||
})
|
})
|
||||||
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
|
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
|
||||||
@@ -228,7 +227,7 @@ func newCompiledRenderingBenchmarkCase(
|
|||||||
return compiledRenderingBenchmarkCase{
|
return compiledRenderingBenchmarkCase{
|
||||||
name: name,
|
name: name,
|
||||||
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
|
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
|
||||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
identity: rendererBenchmarkIdentity{id: definition.ID},
|
||||||
templates: compiled,
|
templates: compiled,
|
||||||
}),
|
}),
|
||||||
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
|
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
var compiled CompiledTemplates
|
var compiled CompiledTemplates
|
||||||
if r.templates != nil {
|
if r.templates != nil {
|
||||||
var found bool
|
var found bool
|
||||||
compiled, found = r.templates.GetCompiledTemplates(definition.ID, definition.Version)
|
compiled, found = r.templates.GetCompiledTemplates(definition.ID)
|
||||||
if !found || len(compiled.Messages) != len(definition.Templates) {
|
if !found || len(compiled.Messages) != len(definition.Templates) {
|
||||||
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -519,14 +519,13 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
renderer := NewCompiledRenderer(compiledRendererTestLookup{
|
renderer := NewCompiledRenderer(compiledRendererTestLookup{
|
||||||
id: promptID,
|
id: promptID,
|
||||||
version: version,
|
|
||||||
templates: CompiledTemplates{
|
templates: CompiledTemplates{
|
||||||
Session: sessionTemplate,
|
Session: sessionTemplate,
|
||||||
Messages: []*template.Template{messageTemplate},
|
Messages: []*template.Template{messageTemplate},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: promptID, Version: version, SessionID: "session-{{.request}}",
|
ID: promptID, Version: stringPointerForTest(version), SessionID: "session-{{.request}}",
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
||||||
}
|
}
|
||||||
@@ -562,12 +561,12 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type compiledRendererTestLookup struct {
|
type compiledRendererTestLookup struct {
|
||||||
id, version string
|
id string
|
||||||
templates CompiledTemplates
|
templates CompiledTemplates
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
||||||
if id != lookup.id || version != lookup.version {
|
if id != lookup.id {
|
||||||
return CompiledTemplates{}, false
|
return CompiledTemplates{}, false
|
||||||
}
|
}
|
||||||
return lookup.templates, true
|
return lookup.templates, true
|
||||||
|
|||||||
@@ -384,7 +384,7 @@ func AnalyzeRequirements(
|
|||||||
result := domain.RequirementInspection{Complete: true}
|
result := domain.RequirementInspection{Complete: true}
|
||||||
if definition != nil {
|
if definition != nil {
|
||||||
result.PromptID = definition.ID
|
result.PromptID = definition.ID
|
||||||
result.PromptVersion = definition.Version
|
result.PromptVersion = copyString(definition.Version)
|
||||||
}
|
}
|
||||||
walker := &requirementWalker{
|
walker := &requirementWalker{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
||||||
description := "retained metadata"
|
description := "retained metadata"
|
||||||
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
definition := &domain.PromptDefinition{ID: "capture", Version: stringPointerForTest("v1")}
|
||||||
snapshot := RequirementSnapshot{
|
snapshot := RequirementSnapshot{
|
||||||
Inputs: []InputMetadata{
|
Inputs: []InputMetadata{
|
||||||
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
||||||
@@ -123,7 +123,7 @@ func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
if result.PromptID != "capture" || result.PromptVersion == nil || *result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
||||||
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
||||||
}
|
}
|
||||||
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ type RequirementSnapshot struct {
|
|||||||
|
|
||||||
// TemplateLookup provides parsed templates for a captured prompt identity.
|
// TemplateLookup provides parsed templates for a captured prompt identity.
|
||||||
type TemplateLookup interface {
|
type TemplateLookup interface {
|
||||||
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)
|
GetCompiledTemplates(id string) (CompiledTemplates, bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompileTemplate checks static syntax with the same grammar and function
|
// CompileTemplate checks static syntax with the same grammar and function
|
||||||
|
|||||||
@@ -12,5 +12,5 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
|
|
||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -229,6 +229,7 @@ func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
copied := *source
|
copied := *source
|
||||||
|
copied.PromptVersion = copyStringPointer(source.PromptVersion)
|
||||||
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -662,6 +662,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
"properties": map[string]any{"value": map[string]any{"type": "string"}},
|
"properties": map[string]any{"value": map[string]any{"type": "string"}},
|
||||||
}
|
}
|
||||||
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
wantPromptVersion := *definition.Version
|
||||||
definition.Validation.SchemaPath = "schema.json"
|
definition.Validation.SchemaPath = "schema.json"
|
||||||
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||||
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||||
@@ -680,7 +681,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
reader, renderer, llmClient, validator, nil,
|
reader, renderer, llmClient, validator, nil,
|
||||||
)
|
)
|
||||||
request := domain.RunRequest{
|
request := domain.RunRequest{
|
||||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
PromptID: "p", ProfileID: "exec",
|
||||||
Inputs: map[string]domain.ArtifactRef{
|
Inputs: map[string]domain.ArtifactRef{
|
||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
|
||||||
},
|
},
|
||||||
@@ -697,6 +698,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
mutateExtra(profileExtras, "mutated profile source")
|
mutateExtra(profileExtras, "mutated profile source")
|
||||||
mutateExtra(requestExtras, "mutated request source")
|
mutateExtra(requestExtras, "mutated request source")
|
||||||
|
*definition.Version = "mutated definition label"
|
||||||
rendered.Messages[0].Content = "mutated renderer source"
|
rendered.Messages[0].Content = "mutated renderer source"
|
||||||
control.TTL = "mutated renderer cache metadata"
|
control.TTL = "mutated renderer cache metadata"
|
||||||
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
|
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
|
||||||
@@ -711,11 +713,21 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||||
}
|
}
|
||||||
|
if details.PromptVersion == nil || *details.PromptVersion != wantPromptVersion {
|
||||||
|
t.Fatalf("prepared details did not preserve the captured version: %+v", details)
|
||||||
|
}
|
||||||
|
*details.PromptVersion = "mutated details label"
|
||||||
|
if again := prepared.Details(); again == nil || again.PromptVersion == nil || *again.PromptVersion != wantPromptVersion {
|
||||||
|
t.Fatalf("details version mutation escaped into the retained snapshot: %+v", again)
|
||||||
|
}
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||||
}
|
}
|
||||||
|
if result.PromptVersion == nil || *result.PromptVersion != wantPromptVersion {
|
||||||
|
t.Fatalf("details version mutation escaped into execution metadata: %+v", result)
|
||||||
|
}
|
||||||
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
|
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
|
||||||
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
|
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
|
||||||
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
|
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
|
||||||
@@ -725,6 +737,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
|
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
|
||||||
}
|
}
|
||||||
if completedDetails := prepared.Details(); completedDetails == nil ||
|
if completedDetails := prepared.Details(); completedDetails == nil ||
|
||||||
|
completedDetails.PromptVersion == nil || *completedDetails.PromptVersion != wantPromptVersion ||
|
||||||
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
|
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
|
||||||
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
|
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
|
||||||
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
|
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ func newPreparedSnapshotBenchmarkRunner(
|
|||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
request := domain.RunRequest{
|
request := domain.RunRequest{
|
||||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
PromptID: "p", ProfileID: "exec",
|
||||||
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
|
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
|
||||||
AppendedMessages: []domain.RenderedMessage{{
|
AppendedMessages: []domain.RenderedMessage{{
|
||||||
Role: domain.RoleAssistant, Content: "appended message",
|
Role: domain.RoleAssistant, Content: "appended message",
|
||||||
|
|||||||
@@ -76,14 +76,22 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
|
target, err := normalizeResolvedExecutionSettings(target)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionTarget{}, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(target.Model) == "" {
|
||||||
|
return domain.ExecutionTarget{}, errors.New("execution model is required")
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeResolvedExecutionSettings(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.ExecutionTarget{}, fmt.Errorf("execution endpoint: %w", err)
|
return domain.ExecutionTarget{}, fmt.Errorf("execution endpoint: %w", err)
|
||||||
}
|
}
|
||||||
target.Endpoint = endpoint
|
target.Endpoint = endpoint
|
||||||
if strings.TrimSpace(target.Model) == "" {
|
|
||||||
return domain.ExecutionTarget{}, errors.New("execution model is required")
|
|
||||||
}
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(target); err != nil {
|
if err := domain.ValidateExecutionTargetSettings(target); err != nil {
|
||||||
return domain.ExecutionTarget{}, err
|
return domain.ExecutionTarget{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,14 +14,10 @@ type resolvedPromptDefinition struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptInspectionRepository interface {
|
type promptInspectionRepository interface {
|
||||||
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
|
GetPromptInspection(context.Context, string) (*domain.PromptInspection, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) resolvePromptDefinition(
|
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
||||||
ctx context.Context,
|
|
||||||
promptID string,
|
|
||||||
promptVersion string,
|
|
||||||
) (*resolvedPromptDefinition, error) {
|
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
@@ -33,11 +29,11 @@ func (r *Runner) resolvePromptDefinition(
|
|||||||
var hash string
|
var hash string
|
||||||
var err error
|
var err error
|
||||||
if indexed, ok := r.promptDefs.(interface {
|
if indexed, ok := r.promptDefs.(interface {
|
||||||
GetPromptSnapshot(context.Context, string, string) (*domain.PromptDefinition, string, error)
|
GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error)
|
||||||
}); ok {
|
}); ok {
|
||||||
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID, promptVersion)
|
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID)
|
||||||
} else {
|
} else {
|
||||||
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
|
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
@@ -55,11 +51,7 @@ func (r *Runner) resolvePromptDefinition(
|
|||||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) resolvePromptInspection(
|
func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (*domain.PromptInspection, error) {
|
||||||
ctx context.Context,
|
|
||||||
promptID string,
|
|
||||||
promptVersion string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
@@ -67,7 +59,7 @@ func (r *Runner) resolvePromptInspection(
|
|||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
||||||
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
|
inspection, err := indexed.GetPromptInspection(ctx, promptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
}
|
}
|
||||||
@@ -77,14 +69,14 @@ func (r *Runner) resolvePromptInspection(
|
|||||||
return inspection, nil
|
return inspection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
selection, err := r.resolvePromptDefinition(ctx, promptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
definition := selection.definition
|
definition := selection.definition
|
||||||
return &domain.PromptInspection{
|
return &domain.PromptInspection{
|
||||||
PromptID: definition.ID,
|
PromptID: definition.ID,
|
||||||
PromptVersion: definition.Version,
|
PromptVersion: copyStringPointer(definition.Version),
|
||||||
PromptHash: selection.hash,
|
PromptHash: selection.hash,
|
||||||
DefaultProfileID: definition.DefaultProfile,
|
DefaultProfileID: definition.DefaultProfile,
|
||||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||||
@@ -93,11 +85,7 @@ func (r *Runner) resolvePromptInspection(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
// InspectPrompt resolves one explicit prompt without execution work.
|
||||||
func (r *Runner) InspectPrompt(
|
func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.PromptInspection, error) {
|
||||||
ctx context.Context,
|
|
||||||
promptID string,
|
|
||||||
promptVersion string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
@@ -107,5 +95,5 @@ func (r *Runner) InspectPrompt(
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
return r.resolvePromptInspection(ctx, promptID, promptVersion)
|
return r.resolvePromptInspection(ctx, promptID)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,41 +15,35 @@ type inspectionPromptRepository struct {
|
|||||||
err error
|
err error
|
||||||
calls int
|
calls int
|
||||||
id string
|
id string
|
||||||
version string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptMetadataRepository struct {
|
type promptMetadataRepository struct {
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
inspectionErr error
|
inspectionErr error
|
||||||
lookupID string
|
lookupID string
|
||||||
lookupVersion string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
version string,
|
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, error) {
|
||||||
r.calls++
|
r.calls++
|
||||||
r.id = id
|
r.id = id
|
||||||
r.version = version
|
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
return r.definition, nil
|
return r.definition, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
|
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string) (*domain.PromptDefinition, error) {
|
||||||
return nil, errors.New("full prompt definition should not be requested")
|
return nil, errors.New("full prompt definition should not be requested")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptInspection(
|
func (r *promptMetadataRepository) GetPromptInspection(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
version string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
) (*domain.PromptInspection, error) {
|
||||||
r.lookupID = id
|
r.lookupID = id
|
||||||
r.lookupVersion = version
|
|
||||||
inspection := r.inspection
|
inspection := r.inspection
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
return &inspection, r.inspectionErr
|
return &inspection, r.inspectionErr
|
||||||
@@ -58,7 +52,7 @@ func (r *promptMetadataRepository) GetPromptInspection(
|
|||||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: "normalized.prompt",
|
ID: "normalized.prompt",
|
||||||
Version: "1.2.3",
|
Version: stringPtr("1.2.3"),
|
||||||
DefaultProfile: "not-resolved",
|
DefaultProfile: "not-resolved",
|
||||||
Inputs: []domain.PromptInput{
|
Inputs: []domain.PromptInput{
|
||||||
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
||||||
@@ -74,7 +68,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
repository := &inspectionPromptRepository{definition: definition}
|
repository := &inspectionPromptRepository{definition: definition}
|
||||||
runner := &Runner{promptDefs: repository}
|
runner := &Runner{promptDefs: repository}
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
}
|
}
|
||||||
@@ -82,11 +76,11 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("hash prompt definition: %v", err)
|
t.Fatalf("hash prompt definition: %v", err)
|
||||||
}
|
}
|
||||||
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
|
if repository.calls != 1 || repository.id != " prompt-id " {
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
|
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID ||
|
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
|
||||||
inspection.PromptVersion != definition.Version ||
|
*inspection.PromptVersion != *definition.Version ||
|
||||||
inspection.PromptHash != wantHash ||
|
inspection.PromptHash != wantHash ||
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||||
@@ -95,7 +89,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
}
|
}
|
||||||
|
|
||||||
inspection.Inputs[0].Name = "changed"
|
inspection.Inputs[0].Name = "changed"
|
||||||
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
second, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect prompt again: %v", err)
|
t.Fatalf("inspect prompt again: %v", err)
|
||||||
}
|
}
|
||||||
@@ -106,22 +100,22 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||||
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
|
PromptID: "captured.prompt", PromptVersion: stringPtr("v2"), PromptHash: "cached-prompt-hash",
|
||||||
DefaultProfileID: "partial-profile",
|
DefaultProfileID: "partial-profile",
|
||||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
||||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
||||||
}}
|
}}
|
||||||
runner := &Runner{promptDefs: repository}
|
runner := &Runner{promptDefs: repository}
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
|
if err != nil || repository.lookupID != " requested-id " ||
|
||||||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
|
inspection.PromptID != "captured.prompt" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v2" ||
|
||||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
||||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
|
t.Fatalf("metadata inspection=(%+v, %v), lookup=%q", inspection, err, repository.lookupID)
|
||||||
}
|
}
|
||||||
inspection.Inputs[0].Name = "caller mutation"
|
inspection.Inputs[0].Name = "caller mutation"
|
||||||
inspection.OutputContract.SchemaPath = "caller mutation"
|
inspection.OutputContract.SchemaPath = "caller mutation"
|
||||||
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
|
second, err := runner.InspectPrompt(context.Background(), "requested-id")
|
||||||
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
||||||
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
||||||
}
|
}
|
||||||
@@ -130,11 +124,11 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
|
|||||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{
|
definition: &domain.PromptDefinition{
|
||||||
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
|
ID: "captured.prompt", Version: stringPtr("v1"), Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
|
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v1" ||
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +137,7 @@ func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *te
|
|||||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||||
t.Run("blank ID", func(t *testing.T) {
|
t.Run("blank ID", func(t *testing.T) {
|
||||||
repository := &inspectionPromptRepository{}
|
repository := &inspectionPromptRepository{}
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ")
|
||||||
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
||||||
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
||||||
}
|
}
|
||||||
@@ -153,7 +147,7 @@ func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancella
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
repository := &inspectionPromptRepository{}
|
repository := &inspectionPromptRepository{}
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt")
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
||||||
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
||||||
}
|
}
|
||||||
@@ -161,7 +155,7 @@ func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancella
|
|||||||
|
|
||||||
t.Run("missing prompt", func(t *testing.T) {
|
t.Run("missing prompt", func(t *testing.T) {
|
||||||
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing")
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +173,7 @@ func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancella
|
|||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
|
_, err := tc.runner.InspectPrompt(context.Background(), "prompt")
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("inspection error=%v, want prompt load", err)
|
t.Fatalf("inspection error=%v, want prompt load", err)
|
||||||
}
|
}
|
||||||
@@ -202,11 +196,11 @@ func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
|
|||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
|
inspection, err := runner.InspectPrompt(context.Background(), definition.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
}
|
}
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, ProfileID: "exec"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare prompt: %v", err)
|
t.Fatalf("prepare prompt: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,14 +13,13 @@ import (
|
|||||||
// context used for dependency classification.
|
// context used for dependency classification.
|
||||||
type RequirementInspectionRequest struct {
|
type RequirementInspectionRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
|
||||||
ProfileID string
|
ProfileID string
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
SessionID string
|
SessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
type requirementSnapshotRepository interface {
|
type requirementSnapshotRepository interface {
|
||||||
GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool)
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectRequirements discovers prompt inputs and caller variables without
|
// InspectRequirements discovers prompt inputs and caller variables without
|
||||||
@@ -40,7 +39,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
|
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -66,11 +65,11 @@ func (r *Runner) InspectRequirements(
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
|
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
|
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: copyStringPointer(selection.PromptVersion)}
|
||||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
|||||||
@@ -19,16 +19,13 @@ type requirementInspectionPromptRepository struct {
|
|||||||
lookupCalls int
|
lookupCalls int
|
||||||
snapshotCalls int
|
snapshotCalls int
|
||||||
lookupID string
|
lookupID string
|
||||||
lookupVersion string
|
|
||||||
snapshotID string
|
snapshotID string
|
||||||
snapshotVersion string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type requirementPromptMetadataRepository struct {
|
type requirementPromptMetadataRepository struct {
|
||||||
*requirementInspectionPromptRepository
|
*requirementInspectionPromptRepository
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
lookupID string
|
lookupID string
|
||||||
lookupVersion string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type inspectionProfileIdentityRepository struct {
|
type inspectionProfileIdentityRepository struct {
|
||||||
@@ -51,7 +48,6 @@ func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
|||||||
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
||||||
context.Context,
|
context.Context,
|
||||||
string,
|
string,
|
||||||
string,
|
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, error) {
|
||||||
return repository.definition, repository.lookupErr
|
return repository.definition, repository.lookupErr
|
||||||
}
|
}
|
||||||
@@ -59,10 +55,8 @@ func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
|||||||
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
version string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
) (*domain.PromptInspection, error) {
|
||||||
repository.lookupID = id
|
repository.lookupID = id
|
||||||
repository.lookupVersion = version
|
|
||||||
inspection := repository.inspection
|
inspection := repository.inspection
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
return &inspection, nil
|
return &inspection, nil
|
||||||
@@ -71,21 +65,17 @@ func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
|||||||
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
version string,
|
|
||||||
) (*domain.PromptDefinition, string, error) {
|
) (*domain.PromptDefinition, string, error) {
|
||||||
repository.lookupCalls++
|
repository.lookupCalls++
|
||||||
repository.lookupID = id
|
repository.lookupID = id
|
||||||
repository.lookupVersion = version
|
|
||||||
return repository.definition, "captured-hash", repository.lookupErr
|
return repository.definition, "captured-hash", repository.lookupErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||||
id string,
|
id string,
|
||||||
version string,
|
|
||||||
) (prompt.RequirementSnapshot, bool) {
|
) (prompt.RequirementSnapshot, bool) {
|
||||||
repository.snapshotCalls++
|
repository.snapshotCalls++
|
||||||
repository.snapshotID = id
|
repository.snapshotID = id
|
||||||
repository.snapshotVersion = version
|
|
||||||
return repository.snapshot, repository.snapshotFound
|
return repository.snapshot, repository.snapshotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,7 +88,7 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v2"), DefaultProfile: "default-profile"}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: definition,
|
definition: definition,
|
||||||
snapshotFound: true,
|
snapshotFound: true,
|
||||||
@@ -111,19 +101,19 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t
|
|||||||
}}
|
}}
|
||||||
runner := &Runner{promptDefs: repository, profiles: profiles}
|
runner := &Runner{promptDefs: repository, profiles: profiles}
|
||||||
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
|
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||||
PromptID: " requested ", PromptVersion: " exact-version ", ProfileID: "partial-profile",
|
PromptID: " requested ", ProfileID: "partial-profile",
|
||||||
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
|
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect requirements: %v", err)
|
t.Fatalf("inspect requirements: %v", err)
|
||||||
}
|
}
|
||||||
if repository.lookupCalls != 1 || repository.lookupID != " requested " || repository.lookupVersion != " exact-version " ||
|
if repository.lookupCalls != 1 || repository.lookupID != " requested " ||
|
||||||
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID || repository.snapshotVersion != definition.Version {
|
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID {
|
||||||
t.Fatalf("selection lookup=(%d,%q,%q), snapshot=(%d,%q,%q)", repository.lookupCalls, repository.lookupID,
|
t.Fatalf("selection lookup=(%d,%q), snapshot=(%d,%q)", repository.lookupCalls, repository.lookupID,
|
||||||
repository.lookupVersion, repository.snapshotCalls, repository.snapshotID, repository.snapshotVersion)
|
repository.snapshotCalls, repository.snapshotID)
|
||||||
}
|
}
|
||||||
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
||||||
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != *definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
||||||
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
||||||
@@ -148,7 +138,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
promptRepository := &requirementPromptMetadataRepository{
|
promptRepository := &requirementPromptMetadataRepository{
|
||||||
requirementInspectionPromptRepository: baseRepository,
|
requirementInspectionPromptRepository: baseRepository,
|
||||||
inspection: domain.PromptInspection{
|
inspection: domain.PromptInspection{
|
||||||
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
|
PromptID: "captured-id", PromptVersion: stringPtr("captured-version"), PromptHash: "captured-hash",
|
||||||
DefaultProfileID: "default-profile",
|
DefaultProfileID: "default-profile",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -158,19 +148,18 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
}
|
}
|
||||||
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
|
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
|
||||||
context.Background(), RequirementInspectionRequest{
|
context.Background(), RequirementInspectionRequest{
|
||||||
PromptID: " requested-id ", PromptVersion: " requested-version ",
|
PromptID: " requested-id ",
|
||||||
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
|
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
|
if err != nil || promptRepository.lookupID != " requested-id " ||
|
||||||
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
|
baseRepository.snapshotID != "captured-id" ||
|
||||||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
||||||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
||||||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
||||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
|
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=%q, snapshot=%q, profile=%q",
|
||||||
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
|
inspection, err, promptRepository.lookupID, baseRepository.snapshotID, profiles.identityID)
|
||||||
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,7 +168,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1")}
|
||||||
makePromptRepository := func() *requirementInspectionPromptRepository {
|
makePromptRepository := func() *requirementInspectionPromptRepository {
|
||||||
return &requirementInspectionPromptRepository{
|
return &requirementInspectionPromptRepository{
|
||||||
definition: definition, snapshotFound: true,
|
definition: definition, snapshotFound: true,
|
||||||
@@ -245,7 +234,7 @@ func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1"), DefaultProfile: "default-profile"}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: definition, snapshotFound: true,
|
definition: definition, snapshotFound: true,
|
||||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
||||||
@@ -271,7 +260,7 @@ func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
|
definition: &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1"), DefaultProfile: "profile"},
|
||||||
snapshotFound: true,
|
snapshotFound: true,
|
||||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
||||||
}
|
}
|
||||||
@@ -322,12 +311,12 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("ambiguous prompt selection", func(t *testing.T) {
|
t.Run("prompt lookup failure", func(t *testing.T) {
|
||||||
ambiguous := errors.New("ambiguous omitted version")
|
lookupFailure := errors.New("prompt lookup failed")
|
||||||
repository := &requirementInspectionPromptRepository{lookupErr: ambiguous}
|
repository := &requirementInspectionPromptRepository{lookupErr: lookupFailure}
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, ambiguous) || repository.lookupVersion != "" {
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, lookupFailure) {
|
||||||
t.Fatalf("ambiguous selection error=%v version=%q", err, repository.lookupVersion)
|
t.Fatalf("prompt lookup error=%v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
|||||||
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
||||||
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
||||||
runner := &Runner{
|
runner := &Runner{
|
||||||
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: "v1", Validation: domain.OutputContract{
|
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: stringPtr("v1"), Validation: domain.OutputContract{
|
||||||
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
||||||
}}},
|
}}},
|
||||||
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
||||||
@@ -63,7 +63,7 @@ func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
|||||||
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
||||||
}}
|
}}
|
||||||
runner := &Runner{
|
runner := &Runner{
|
||||||
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: "v1", Validation: domain.OutputContract{
|
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: stringPtr("v1"), Validation: domain.OutputContract{
|
||||||
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
||||||
}}},
|
}}},
|
||||||
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
||||||
|
|||||||
@@ -252,7 +252,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
RawOutput: genResp.Content,
|
RawOutput: genResp.Content,
|
||||||
Validation: validationResult,
|
Validation: validationResult,
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: copyStringPointer(prepared.PromptVersion),
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
@@ -315,13 +315,13 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
def := promptSelection.definition
|
def := promptSelection.definition
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version})
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version)})
|
||||||
}
|
}
|
||||||
promptDefinitionHash := promptSelection.hash
|
promptDefinitionHash := promptSelection.hash
|
||||||
effectiveContract, err := resolveOutputContract(def, req.Validation)
|
effectiveContract, err := resolveOutputContract(def, req.Validation)
|
||||||
@@ -342,7 +342,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version, ProfileID: selection.id})
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version), ProfileID: selection.id})
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution)
|
effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution)
|
||||||
@@ -355,7 +355,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version,
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version),
|
||||||
ProfileID: selection.id, BackendID: effectiveModel.BackendID, Model: effectiveModel.Model})
|
ProfileID: selection.id, BackendID: effectiveModel.BackendID, Model: effectiveModel.Model})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -481,7 +481,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
effectiveModel.APIKey = ""
|
effectiveModel.APIKey = ""
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
PromptID: state.definition.ID,
|
PromptID: state.definition.ID,
|
||||||
PromptVersion: state.definition.Version,
|
PromptVersion: copyStringPointer(state.definition.Version),
|
||||||
PromptHash: state.promptDefinitionHash,
|
PromptHash: state.promptDefinitionHash,
|
||||||
SelectedProfileID: state.selectedProfileID,
|
SelectedProfileID: state.selectedProfileID,
|
||||||
SelectedBackendID: state.effectiveModel.BackendID,
|
SelectedBackendID: state.effectiveModel.BackendID,
|
||||||
@@ -517,22 +517,15 @@ func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *dom
|
|||||||
return &domain.StructuredOutputSpec{
|
return &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
Name: deriveStructuredSchemaName(def.ID),
|
||||||
Strict: true,
|
Strict: true,
|
||||||
Schema: schemaDocument,
|
Schema: schemaDocument,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
func deriveStructuredSchemaName(promptID string) string {
|
||||||
raw := strings.TrimSpace(promptID)
|
raw := strings.TrimSpace(promptID)
|
||||||
if v := strings.TrimSpace(promptVersion); v != "" {
|
|
||||||
if raw == "" {
|
|
||||||
raw = v
|
|
||||||
} else {
|
|
||||||
raw = raw + "_" + v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
for _, r := range raw {
|
for _, r := range raw {
|
||||||
@@ -720,3 +713,11 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
|
|||||||
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
||||||
return promptdef.HashDefinition(def)
|
return promptdef.HashDefinition(def)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPointer(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user