Add PreparedRun model and render formatters (text/json) with tests
This commit is contained in:
@@ -75,6 +75,22 @@ type RunResult struct {
|
||||
Error error
|
||||
}
|
||||
|
||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||
// It must never include resolved API key values, model output, or validation data.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to an input artifact.
|
||||
type ArtifactRef struct {
|
||||
Type ArtifactRefType
|
||||
@@ -156,13 +172,13 @@ type OutputContract struct {
|
||||
|
||||
// RenderedPrompt represents the prompt after template application.
|
||||
type RenderedPrompt struct {
|
||||
Messages []RenderedMessage
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a single message in a rendered prompt.
|
||||
type RenderedMessage struct {
|
||||
Role string
|
||||
Content string
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// GenerateRequest is the internal request passed to the LLM client.
|
||||
|
||||
55
internal/domain/prepared_run_test.go
Normal file
55
internal/domain/prepared_run_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_TEST_API_KEY"
|
||||
const secret = "super-secret-value"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
PromptVersion: "v1",
|
||||
PromptHash: "prompt-hash",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
APIKeyEnv: envName,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{Role: "system", Content: "You are helpful."},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
out := string(b)
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
||||
}
|
||||
|
||||
var top map[string]any
|
||||
if err := json.Unmarshal(b, &top); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
||||
if _, ok := top[forbidden]; ok {
|
||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
144
internal/format/prepared_run.go
Normal file
144
internal/format/prepared_run.go
Normal file
@@ -0,0 +1,144 @@
|
||||
// Package format formats already-prepared domain data for adapters.
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||
|
||||
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||
type PreparedRunOutputFormat string
|
||||
|
||||
const (
|
||||
PreparedRunFormatText PreparedRunOutputFormat = "text"
|
||||
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
|
||||
|
||||
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
|
||||
)
|
||||
|
||||
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||
type PreparedRunFormatter interface {
|
||||
Format(prepared *domain.PreparedRun) ([]byte, error)
|
||||
}
|
||||
|
||||
// ParsePreparedRunOutputFormat parses a format name.
|
||||
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
||||
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||
case "":
|
||||
return DefaultPreparedRunOutputFormat, nil
|
||||
case PreparedRunFormatText:
|
||||
return PreparedRunFormatText, nil
|
||||
case PreparedRunFormatJSON:
|
||||
return PreparedRunFormatJSON, nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatterForPreparedRun returns a formatter strategy for the chosen format.
|
||||
func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) {
|
||||
switch outputFormat {
|
||||
case PreparedRunFormatText:
|
||||
return textPreparedRunFormatter{}, nil
|
||||
case PreparedRunFormatJSON:
|
||||
return jsonPreparedRunFormatter{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatPreparedRun formats a prepared run using the selected format.
|
||||
func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return formatter.Format(prepared)
|
||||
}
|
||||
|
||||
type jsonPreparedRunFormatter struct{}
|
||||
|
||||
func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
return json.MarshalIndent(prepared, "", " ")
|
||||
}
|
||||
|
||||
type textPreparedRunFormatter struct{}
|
||||
|
||||
func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
||||
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
|
||||
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
||||
if prepared.PromptHash != "" {
|
||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||
}
|
||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||
|
||||
target := prepared.EffectiveModelParams
|
||||
fmt.Fprintln(&b, "effective_model_params:")
|
||||
fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint)
|
||||
fmt.Fprintf(&b, " model: %s\n", target.Model)
|
||||
fmt.Fprintf(&b, " temperature: %g\n", target.Temperature)
|
||||
fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens)
|
||||
fmt.Fprintf(&b, " top_p: %g\n", target.TopP)
|
||||
fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds)
|
||||
if target.ReasoningEffort != "" {
|
||||
fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort)
|
||||
}
|
||||
if target.APIKeyEnv != "" {
|
||||
fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv)
|
||||
}
|
||||
if len(target.ExtraParams) > 0 {
|
||||
fmt.Fprintln(&b, " extra_params:")
|
||||
keys := make([]string, 0, len(target.ExtraParams))
|
||||
for k := range target.ExtraParams {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintln(&b, "input_hashes:")
|
||||
inputKeys := make([]string, 0, len(prepared.InputHashes))
|
||||
for k := range prepared.InputHashes {
|
||||
inputKeys = append(inputKeys, k)
|
||||
}
|
||||
sort.Strings(inputKeys)
|
||||
for _, k := range inputKeys {
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k])
|
||||
}
|
||||
|
||||
fmt.Fprintln(&b, "messages:")
|
||||
for i, msg := range prepared.Messages {
|
||||
fmt.Fprintf(&b, " - index: %d\n", i)
|
||||
fmt.Fprintf(&b, " role: %s\n", msg.Role)
|
||||
fmt.Fprintln(&b, " content: |")
|
||||
content := msg.Content
|
||||
if content == "" {
|
||||
fmt.Fprintln(&b, " ")
|
||||
continue
|
||||
}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
fmt.Fprintf(&b, " %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
41
internal/format/prepared_run_test.go
Normal file
41
internal/format/prepared_run_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want PreparedRunOutputFormat
|
||||
}{
|
||||
{name: "default empty", input: "", want: DefaultPreparedRunOutputFormat},
|
||||
{name: "text", input: "text", want: PreparedRunFormatText},
|
||||
{name: "json", input: "json", want: PreparedRunFormatJSON},
|
||||
{name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := ParsePreparedRunOutputFormat(tc.input)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) {
|
||||
_, err := ParsePreparedRunOutputFormat("yaml")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unknown format")
|
||||
}
|
||||
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
||||
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user