Add PreparedRun model and render formatters (text/json) with tests
This commit is contained in:
@@ -75,6 +75,22 @@ type RunResult struct {
|
|||||||
Error error
|
Error error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
|
type PreparedRun struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
// ArtifactRef represents a reference to an input artifact.
|
||||||
type ArtifactRef struct {
|
type ArtifactRef struct {
|
||||||
Type ArtifactRefType
|
Type ArtifactRefType
|
||||||
@@ -156,13 +172,13 @@ type OutputContract struct {
|
|||||||
|
|
||||||
// RenderedPrompt represents the prompt after template application.
|
// RenderedPrompt represents the prompt after template application.
|
||||||
type RenderedPrompt struct {
|
type RenderedPrompt struct {
|
||||||
Messages []RenderedMessage
|
Messages []RenderedMessage `json:"messages"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenderedMessage is a single message in a rendered prompt.
|
// RenderedMessage is a single message in a rendered prompt.
|
||||||
type RenderedMessage struct {
|
type RenderedMessage struct {
|
||||||
Role string
|
Role string `json:"role"`
|
||||||
Content string
|
Content string `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
// GenerateRequest is the internal request passed to the LLM client.
|
||||||
|
|||||||
55
internal/domain/prepared_run_test.go
Normal file
55
internal/domain/prepared_run_test.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||||
|
const envName = "SCRIPTORIUM_TEST_API_KEY"
|
||||||
|
const secret = "super-secret-value"
|
||||||
|
t.Setenv(envName, secret)
|
||||||
|
|
||||||
|
prepared := PreparedRun{
|
||||||
|
PromptID: "prompt.id",
|
||||||
|
PromptVersion: "v1",
|
||||||
|
PromptHash: "prompt-hash",
|
||||||
|
SelectedProfileID: "local-fast",
|
||||||
|
EffectiveModelParams: ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
APIKeyEnv: envName,
|
||||||
|
},
|
||||||
|
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||||
|
RenderedPromptHash: "rendered-hash",
|
||||||
|
Messages: []RenderedMessage{
|
||||||
|
{Role: "system", Content: "You are helpful."},
|
||||||
|
{Role: "user", Content: "Summarize this."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := string(b)
|
||||||
|
if strings.Contains(out, secret) {
|
||||||
|
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
||||||
|
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
var top map[string]any
|
||||||
|
if err := json.Unmarshal(b, &top); err != nil {
|
||||||
|
t.Fatalf("unmarshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
||||||
|
if _, ok := top[forbidden]; ok {
|
||||||
|
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
144
internal/format/prepared_run.go
Normal file
144
internal/format/prepared_run.go
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
// Package format formats already-prepared domain data for adapters.
|
||||||
|
package format
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||||
|
|
||||||
|
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||||
|
type PreparedRunOutputFormat string
|
||||||
|
|
||||||
|
const (
|
||||||
|
PreparedRunFormatText PreparedRunOutputFormat = "text"
|
||||||
|
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
|
||||||
|
|
||||||
|
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||||
|
type PreparedRunFormatter interface {
|
||||||
|
Format(prepared *domain.PreparedRun) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePreparedRunOutputFormat parses a format name.
|
||||||
|
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
||||||
|
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||||
|
case "":
|
||||||
|
return DefaultPreparedRunOutputFormat, nil
|
||||||
|
case PreparedRunFormatText:
|
||||||
|
return PreparedRunFormatText, nil
|
||||||
|
case PreparedRunFormatJSON:
|
||||||
|
return PreparedRunFormatJSON, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatterForPreparedRun returns a formatter strategy for the chosen format.
|
||||||
|
func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) {
|
||||||
|
switch outputFormat {
|
||||||
|
case PreparedRunFormatText:
|
||||||
|
return textPreparedRunFormatter{}, nil
|
||||||
|
case PreparedRunFormatJSON:
|
||||||
|
return jsonPreparedRunFormatter{}, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatPreparedRun formats a prepared run using the selected format.
|
||||||
|
func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||||
|
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return formatter.Format(prepared)
|
||||||
|
}
|
||||||
|
|
||||||
|
type jsonPreparedRunFormatter struct{}
|
||||||
|
|
||||||
|
func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||||
|
if prepared == nil {
|
||||||
|
return nil, errors.New("prepared run is nil")
|
||||||
|
}
|
||||||
|
return json.MarshalIndent(prepared, "", " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
type textPreparedRunFormatter struct{}
|
||||||
|
|
||||||
|
func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||||
|
if prepared == nil {
|
||||||
|
return nil, errors.New("prepared run is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
||||||
|
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
|
||||||
|
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
||||||
|
if prepared.PromptHash != "" {
|
||||||
|
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||||
|
|
||||||
|
target := prepared.EffectiveModelParams
|
||||||
|
fmt.Fprintln(&b, "effective_model_params:")
|
||||||
|
fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint)
|
||||||
|
fmt.Fprintf(&b, " model: %s\n", target.Model)
|
||||||
|
fmt.Fprintf(&b, " temperature: %g\n", target.Temperature)
|
||||||
|
fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens)
|
||||||
|
fmt.Fprintf(&b, " top_p: %g\n", target.TopP)
|
||||||
|
fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds)
|
||||||
|
if target.ReasoningEffort != "" {
|
||||||
|
fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if target.APIKeyEnv != "" {
|
||||||
|
fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv)
|
||||||
|
}
|
||||||
|
if len(target.ExtraParams) > 0 {
|
||||||
|
fmt.Fprintln(&b, " extra_params:")
|
||||||
|
keys := make([]string, 0, len(target.ExtraParams))
|
||||||
|
for k := range target.ExtraParams {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(&b, "input_hashes:")
|
||||||
|
inputKeys := make([]string, 0, len(prepared.InputHashes))
|
||||||
|
for k := range prepared.InputHashes {
|
||||||
|
inputKeys = append(inputKeys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(inputKeys)
|
||||||
|
for _, k := range inputKeys {
|
||||||
|
fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k])
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(&b, "messages:")
|
||||||
|
for i, msg := range prepared.Messages {
|
||||||
|
fmt.Fprintf(&b, " - index: %d\n", i)
|
||||||
|
fmt.Fprintf(&b, " role: %s\n", msg.Role)
|
||||||
|
fmt.Fprintln(&b, " content: |")
|
||||||
|
content := msg.Content
|
||||||
|
if content == "" {
|
||||||
|
fmt.Fprintln(&b, " ")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(content, "\n") {
|
||||||
|
fmt.Fprintf(&b, " %s\n", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.Bytes(), nil
|
||||||
|
}
|
||||||
41
internal/format/prepared_run_test.go
Normal file
41
internal/format/prepared_run_test.go
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
package format
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
want PreparedRunOutputFormat
|
||||||
|
}{
|
||||||
|
{name: "default empty", input: "", want: DefaultPreparedRunOutputFormat},
|
||||||
|
{name: "text", input: "text", want: PreparedRunFormatText},
|
||||||
|
{name: "json", input: "json", want: PreparedRunFormatJSON},
|
||||||
|
{name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := ParsePreparedRunOutputFormat(tc.input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) {
|
||||||
|
_, err := ParsePreparedRunOutputFormat("yaml")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for unknown format")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
||||||
|
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user