Add session and reasoning controls
This commit is contained in:
11
docs/api.md
11
docs/api.md
@@ -45,6 +45,7 @@ copyable shape. At the HTTP adapter boundary, the smallest valid shape is:
|
|||||||
| `prompt_id` | yes | Non-blank prompt ID. |
|
| `prompt_id` | yes | Non-blank prompt ID. |
|
||||||
| `prompt_version` | no | Prompt version filter. |
|
| `prompt_version` | no | Prompt version filter. |
|
||||||
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
||||||
|
| `session_id` | no | Optional direct, non-secret session identifier. |
|
||||||
| `inputs` | no | Optional object mapping input names to references. Promptkit decides whether the selected definition needs them. |
|
| `inputs` | no | Optional object mapping input names to references. Promptkit decides whether the selected definition needs them. |
|
||||||
| `vars` | no | Object mapping template-variable names to strings. |
|
| `vars` | no | Object mapping template-variable names to strings. |
|
||||||
| `model` | no | Runtime model-override object. |
|
| `model` | no | Runtime model-override object. |
|
||||||
@@ -77,6 +78,12 @@ field cause `400 invalid_json`.
|
|||||||
inherits the selected profile, a non-empty string replaces its value, and an
|
inherits the selected profile, a non-empty string replaces its value, and an
|
||||||
empty string explicitly clears it. JSON `null` is treated as omission.
|
empty string explicitly clears it. JSON `null` is treated as omission.
|
||||||
|
|
||||||
|
`session_id` is passed directly to Promptkit. A nonblank value replaces a
|
||||||
|
definition-rendered session ID; omission or a blank value lets the definition
|
||||||
|
provide one. Promptkit trims direct values and limits them to 256 Unicode code
|
||||||
|
points. Session IDs are not credentials and may be included in prepared data,
|
||||||
|
results, and provider-facing requests, so use stable non-sensitive identifiers.
|
||||||
|
|
||||||
### Strict JSON
|
### Strict JSON
|
||||||
|
|
||||||
Request decoding rejects malformed JSON, unknown fields at every request level,
|
Request decoding rejects malformed JSON, unknown fields at every request level,
|
||||||
@@ -104,6 +111,10 @@ validation contract. The response contains:
|
|||||||
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
||||||
cache usage is reported as zero.
|
cache usage is reported as zero.
|
||||||
|
|
||||||
|
When Promptkit resolves a direct or definition-rendered session ID,
|
||||||
|
`metadata.session_id` contains that effective result value. It is omitted when
|
||||||
|
no effective session ID exists.
|
||||||
|
|
||||||
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
||||||
and any available diagnostic errors, while still returning the artifact and
|
and any available diagnostic errors, while still returning the artifact and
|
||||||
metadata.
|
metadata.
|
||||||
|
|||||||
18
docs/cli.md
18
docs/cli.md
@@ -50,6 +50,7 @@ Optional flags:
|
|||||||
| `--schema-dir <dir>` | Schema base-directory override. |
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
| `--prompt-version <version>` | Optional prompt-definition version selector. |
|
| `--prompt-version <version>` | Optional prompt-definition version selector. |
|
||||||
| `--profile <id>` | Execution-profile override. |
|
| `--profile <id>` | Execution-profile override. |
|
||||||
|
| `--session-id <id>` | Optional direct session identifier. |
|
||||||
| `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. |
|
| `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. |
|
||||||
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
||||||
| `--out <path>` | Write generated content to this file instead of stdout. |
|
| `--out <path>` | Write generated content to this file instead of stdout. |
|
||||||
@@ -59,6 +60,7 @@ Optional flags:
|
|||||||
| `--temperature <float>` | Runtime temperature override. |
|
| `--temperature <float>` | Runtime temperature override. |
|
||||||
| `--max-tokens <int>` | Runtime maximum-token override. |
|
| `--max-tokens <int>` | Runtime maximum-token override. |
|
||||||
| `--top-p <float>` | Runtime top-p override. |
|
| `--top-p <float>` | Runtime top-p override. |
|
||||||
|
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
||||||
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||||
|
|
||||||
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||||
@@ -79,6 +81,18 @@ selected prompt ID must have exactly one available version. `--input` is
|
|||||||
optional at the CLI boundary: Promptkit decides whether the selected definition
|
optional at the CLI boundary: Promptkit decides whether the selected definition
|
||||||
requires declared inputs or template-referenced values.
|
requires declared inputs or template-referenced values.
|
||||||
|
|
||||||
|
`--session-id` supplies a direct, non-secret session identifier. A nonblank
|
||||||
|
value replaces a definition-rendered session ID; an omitted or blank value lets
|
||||||
|
the definition supply one. Promptkit trims direct values and limits them to 256
|
||||||
|
Unicode code points. Use stable, non-sensitive identifiers because effective
|
||||||
|
session IDs may appear in prepared output, run metadata, and provider-facing
|
||||||
|
requests.
|
||||||
|
|
||||||
|
`--reasoning-effort` is presence-aware: omitting it inherits the selected
|
||||||
|
profile value, a nonblank value replaces that value, and
|
||||||
|
`--reasoning-effort=` explicitly clears inherited reasoning. Promptkit treats
|
||||||
|
nonblank values as provider-specific opaque strings.
|
||||||
|
|
||||||
## `scriptorium render`
|
## `scriptorium render`
|
||||||
|
|
||||||
```text
|
```text
|
||||||
@@ -89,11 +103,13 @@ scriptorium render [flags]
|
|||||||
`--config`, `--prompt-dir`, `--profile-dir`, `--prompt-version`, `--profile`,
|
`--config`, `--prompt-dir`, `--profile-dir`, `--prompt-version`, `--profile`,
|
||||||
`--input`, `--var`, `--out`, `--llm-base-url`,
|
`--input`, `--var`, `--out`, `--llm-base-url`,
|
||||||
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
||||||
`--timeout`, and `--format text|json`. Their meanings match the corresponding
|
`--reasoning-effort`, `--session-id`, `--timeout`, and `--format text|json`. Their meanings match the corresponding
|
||||||
`run` flags; `--format` selects prepared-run output and otherwise uses
|
`run` flags; `--format` selects prepared-run output and otherwise uses
|
||||||
`defaults.render_format`.
|
`defaults.render_format`.
|
||||||
|
|
||||||
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
||||||
|
The same session and reasoning inheritance, replacement, and clearing behavior
|
||||||
|
also applies.
|
||||||
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
||||||
configuration file. It resolves profiles and schemas as part of preparation but
|
configuration file. It resolves profiles and schemas as part of preparation but
|
||||||
does not call an LLM.
|
does not call an LLM.
|
||||||
|
|||||||
@@ -225,6 +225,8 @@ built-in, or endpoint-only execution targets.
|
|||||||
|
|
||||||
## Stage 4: Expose Session And Presence-Aware Reasoning Controls
|
## Stage 4: Expose Session And Presence-Aware Reasoning Controls
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
Complete the request mapping for direct session identifiers and the v0.9.0
|
Complete the request mapping for direct session identifiers and the v0.9.0
|
||||||
reasoning override semantics.
|
reasoning override semantics.
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ type runConfig struct {
|
|||||||
promptID string
|
promptID string
|
||||||
promptVersion string
|
promptVersion string
|
||||||
profileID string
|
profileID string
|
||||||
|
sessionID string
|
||||||
inputRaw listFlag
|
inputRaw listFlag
|
||||||
varRaw listFlag
|
varRaw listFlag
|
||||||
outputPath string
|
outputPath string
|
||||||
@@ -46,6 +47,7 @@ type runConfig struct {
|
|||||||
temperature float64
|
temperature float64
|
||||||
maxTokens int
|
maxTokens int
|
||||||
topP float64
|
topP float64
|
||||||
|
reasoningEffort string
|
||||||
schemaDir string
|
schemaDir string
|
||||||
backends []appconfig.BackendSettings
|
backends []appconfig.BackendSettings
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
@@ -58,6 +60,7 @@ type runConfig struct {
|
|||||||
temperatureSet bool
|
temperatureSet bool
|
||||||
maxTokensSet bool
|
maxTokensSet bool
|
||||||
topPSet bool
|
topPSet bool
|
||||||
|
reasoningEffortSet bool
|
||||||
timeoutSet bool
|
timeoutSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -352,6 +355,7 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
|||||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
||||||
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
|
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
|
||||||
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
||||||
|
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
|
||||||
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
||||||
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
||||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||||
@@ -361,6 +365,7 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
|||||||
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
||||||
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
||||||
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
||||||
|
fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override")
|
||||||
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
||||||
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
||||||
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
||||||
@@ -405,6 +410,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
||||||
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
||||||
cfg.topPSet = flagWasSet(fs, "top-p")
|
cfg.topPSet = flagWasSet(fs, "top-p")
|
||||||
|
cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort")
|
||||||
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -605,7 +611,7 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var modelOverride *promptkit.ExecutionTargetOverride
|
var modelOverride *promptkit.ExecutionTargetOverride
|
||||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.reasoningEffortSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||||
modelOverride = &promptkit.ExecutionTargetOverride{
|
modelOverride = &promptkit.ExecutionTargetOverride{
|
||||||
Endpoint: cfg.llmBaseURL,
|
Endpoint: cfg.llmBaseURL,
|
||||||
Model: cfg.model,
|
Model: cfg.model,
|
||||||
@@ -620,6 +626,9 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
|||||||
if cfg.topPSet {
|
if cfg.topPSet {
|
||||||
modelOverride.TopP = &cfg.topP
|
modelOverride.TopP = &cfg.topP
|
||||||
}
|
}
|
||||||
|
if cfg.reasoningEffortSet {
|
||||||
|
modelOverride.ReasoningEffort = &cfg.reasoningEffort
|
||||||
|
}
|
||||||
if cfg.timeoutSet {
|
if cfg.timeoutSet {
|
||||||
timeoutSeconds := int(cfg.timeout.Seconds())
|
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||||
modelOverride.TimeoutSeconds = &timeoutSeconds
|
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||||
@@ -630,6 +639,7 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
|||||||
PromptID: cfg.promptID,
|
PromptID: cfg.promptID,
|
||||||
PromptVersion: cfg.promptVersion,
|
PromptVersion: cfg.promptVersion,
|
||||||
ProfileID: cfg.profileID,
|
ProfileID: cfg.profileID,
|
||||||
|
SessionID: cfg.sessionID,
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
Vars: varMappings,
|
Vars: varMappings,
|
||||||
Execution: modelOverride,
|
Execution: modelOverride,
|
||||||
@@ -727,7 +737,7 @@ func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
|||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func printUsage(w io.Writer) {
|
||||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
||||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -103,12 +103,14 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
"--prompt", "prompt.a",
|
"--prompt", "prompt.a",
|
||||||
"--prompt-version", "2",
|
"--prompt-version", "2",
|
||||||
"--profile", "profile.a",
|
"--profile", "profile.a",
|
||||||
|
"--session-id", "session-1",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--llm-base-url", "http://x/v1",
|
"--llm-base-url", "http://x/v1",
|
||||||
"--model", "m",
|
"--model", "m",
|
||||||
"--temperature", "0.7",
|
"--temperature", "0.7",
|
||||||
"--max-tokens", "111",
|
"--max-tokens", "111",
|
||||||
"--top-p", "0.8",
|
"--top-p", "0.8",
|
||||||
|
"--reasoning-effort", "medium",
|
||||||
"--timeout", "30s",
|
"--timeout", "30s",
|
||||||
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
||||||
})
|
})
|
||||||
@@ -121,6 +123,9 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
||||||
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
||||||
}
|
}
|
||||||
|
if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet {
|
||||||
|
t.Fatalf("unexpected session or reasoning configuration: %+v", cfg)
|
||||||
|
}
|
||||||
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
||||||
t.Fatalf("expected override flags set, got %+v", cfg)
|
t.Fatalf("expected override flags set, got %+v", cfg)
|
||||||
}
|
}
|
||||||
@@ -206,6 +211,8 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
|||||||
usage := stderr.String()
|
usage := stderr.String()
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"--prompt-version VERSION",
|
"--prompt-version VERSION",
|
||||||
|
"--session-id ID",
|
||||||
|
"--reasoning-effort VALUE",
|
||||||
"--artifact-root",
|
"--artifact-root",
|
||||||
"--max-request-bytes",
|
"--max-request-bytes",
|
||||||
"--max-artifact-bytes",
|
"--max-artifact-bytes",
|
||||||
@@ -694,6 +701,39 @@ func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.T) {
|
||||||
|
omitted, err := buildRunRequestFromConfig(&runConfig{promptID: "prompt-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected omitted request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if omitted.Execution != nil {
|
||||||
|
t.Fatalf("expected omitted reasoning flag to leave execution nil, got %#v", omitted.Execution)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{name: "replacement", value: "high"},
|
||||||
|
{name: "clear", value: ""},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
req, err := buildRunRequestFromConfig(&runConfig{
|
||||||
|
promptID: "prompt-1",
|
||||||
|
sessionID: "session-1",
|
||||||
|
reasoningEffort: tc.value,
|
||||||
|
reasoningEffortSet: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if req.SessionID != "session-1" || req.Execution == nil || req.Execution.ReasoningEffort == nil || *req.Execution.ReasoningEffort != tc.value {
|
||||||
|
t.Fatalf("unexpected mapped request: %#v", req)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||||
req, err := buildRunRequestFromConfig(&runConfig{
|
req, err := buildRunRequestFromConfig(&runConfig{
|
||||||
promptID: "prompt-1",
|
promptID: "prompt-1",
|
||||||
@@ -1002,6 +1042,118 @@ backends:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
session_id: definition-session
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
model: local-model
|
||||||
|
reasoning_effort: low
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
wantReasoning string
|
||||||
|
wantSessionID string
|
||||||
|
absentReasoning bool
|
||||||
|
}{
|
||||||
|
{name: "omitted reasoning inherits profile", wantReasoning: "low", wantSessionID: "definition-session"},
|
||||||
|
{name: "nonblank reasoning replaces profile", args: []string{"--reasoning-effort", "high"}, wantReasoning: "high", wantSessionID: "definition-session"},
|
||||||
|
{name: "empty reasoning clears profile", args: []string{"--reasoning-effort="}, wantSessionID: "definition-session", absentReasoning: true},
|
||||||
|
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"}
|
||||||
|
args = append(args, tc.args...)
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "session_id: "+tc.wantSessionID) {
|
||||||
|
t.Fatalf("expected session ID %q, got:\n%s", tc.wantSessionID, stdout)
|
||||||
|
}
|
||||||
|
hasReasoning := strings.Contains(stdout, "reasoning_effort:")
|
||||||
|
if tc.absentReasoning {
|
||||||
|
if hasReasoning {
|
||||||
|
t.Fatalf("expected cleared reasoning to be omitted, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "reasoning_effort: "+tc.wantReasoning) {
|
||||||
|
t.Fatalf("expected reasoning effort %q, got:\n%s", tc.wantReasoning, stdout)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "plain",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout, "session_id:") {
|
||||||
|
t.Fatalf("expected no effective session ID, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
|
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "session",
|
||||||
|
"--session-id", strings.Repeat("x", 257),
|
||||||
|
})
|
||||||
|
if code != ExitRuntimeError {
|
||||||
|
t.Fatalf("expected runtime error, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr, "invalid run request") {
|
||||||
|
t.Fatalf("expected invalid-request context, got %q", stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ type runRequestDTO struct {
|
|||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Inputs map[string]inputRefDTO `json:"inputs"`
|
Inputs map[string]inputRefDTO `json:"inputs"`
|
||||||
Vars map[string]string `json:"vars,omitempty"`
|
Vars map[string]string `json:"vars,omitempty"`
|
||||||
Model *modelOverrideRequestDTO `json:"model,omitempty"`
|
Model *modelOverrideRequestDTO `json:"model,omitempty"`
|
||||||
@@ -56,6 +57,7 @@ type metadataDTO struct {
|
|||||||
PromptHash string `json:"prompt_hash"`
|
PromptHash string `json:"prompt_hash"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
ModelParams modelParamsDTO `json:"model_params"`
|
ModelParams modelParamsDTO `json:"model_params"`
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
|
SessionID: req.SessionID,
|
||||||
Inputs: mappedInputs,
|
Inputs: mappedInputs,
|
||||||
Vars: req.Vars,
|
Vars: req.Vars,
|
||||||
Execution: model,
|
Execution: model,
|
||||||
@@ -124,6 +125,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
PromptHash: res.PromptHash,
|
PromptHash: res.PromptHash,
|
||||||
RenderedPromptHash: res.RenderedPromptHash,
|
RenderedPromptHash: res.RenderedPromptHash,
|
||||||
SelectedProfileID: res.SelectedProfileID,
|
SelectedProfileID: res.SelectedProfileID,
|
||||||
|
SessionID: res.SessionID,
|
||||||
ModelName: res.ModelName,
|
ModelName: res.ModelName,
|
||||||
Endpoint: res.Endpoint,
|
Endpoint: res.Endpoint,
|
||||||
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
||||||
|
|||||||
@@ -514,7 +514,9 @@ func TestHandlerModelOverridePreservesReasoningEffortPresence(t *testing.T) {
|
|||||||
wantValue string
|
wantValue string
|
||||||
}{
|
}{
|
||||||
{name: "omitted", model: `{}`},
|
{name: "omitted", model: `{}`},
|
||||||
|
{name: "nonblank", model: `{"reasoning_effort":"high"}`, wantPresent: true, wantValue: "high"},
|
||||||
{name: "explicit empty", model: `{"reasoning_effort":""}`, wantPresent: true},
|
{name: "explicit empty", model: `{"reasoning_effort":""}`, wantPresent: true},
|
||||||
|
{name: "null", model: `{"reasoning_effort":null}`},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -546,6 +548,136 @@ func TestHandlerModelOverridePreservesReasoningEffortPresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerMapsSessionIDAndReportsEffectiveResultSessionID(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
SessionID: "effective-session",
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","session_id":"request-session"}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.SessionID != "request-session" {
|
||||||
|
t.Fatalf("expected session ID in run request, got %q", r.last.SessionID)
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if metadata["session_id"] != "effective-session" {
|
||||||
|
t.Fatalf("expected effective session ID in response, got %#v", metadata["session_id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1"}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if _, ok := metadata["session_id"]; ok {
|
||||||
|
t.Fatalf("expected empty effective session ID to be omitted, got %#v", metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
definitionSession string
|
||||||
|
requestSession string
|
||||||
|
wantSession string
|
||||||
|
}{
|
||||||
|
{name: "definition session", definitionSession: "definition-session", wantSession: "definition-session"},
|
||||||
|
{name: "direct session", definitionSession: "definition-session", requestSession: "direct-session", wantSession: "direct-session"},
|
||||||
|
{name: "no effective session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
promptDir := t.TempDir()
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
sessionLine := ""
|
||||||
|
if tc.definitionSession != "" {
|
||||||
|
sessionLine = "session_id: " + tc.definitionSession + "\n"
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: exec\n"+sessionLine+`messages:
|
||||||
|
- role: user
|
||||||
|
content: "hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: exec\nendpoint: http://127.0.0.1:1/v1\nmodel: test\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := `{"prompt_id":"p"}`
|
||||||
|
if tc.requestSession != "" {
|
||||||
|
body = `{"prompt_id":"p","session_id":"` + tc.requestSession + `"}`
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if tc.wantSession == "" {
|
||||||
|
if _, ok := metadata["session_id"]; ok {
|
||||||
|
t.Fatalf("expected session_id to be omitted, got %#v", metadata)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if metadata["session_id"] != tc.wantSession {
|
||||||
|
t.Fatalf("expected effective session ID %q, got %#v", tc.wantSession, metadata["session_id"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRejectsOverlongSessionIDAndNonStringReasoningEffort(t *testing.T) {
|
||||||
|
engine := newHandlerEngine(t)
|
||||||
|
for _, body := range []string{
|
||||||
|
`{"prompt_id":"p","session_id":"` + strings.Repeat("x", 257) + `"}`,
|
||||||
|
`{"prompt_id":"p","model":{"reasoning_effort":1}}`,
|
||||||
|
} {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||||
r := &fakeRunner{result: &promptkit.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
|||||||
Reference in New Issue
Block a user