Add PreparedRun model and render formatters (text/json) with tests

This commit is contained in:
2026-05-06 14:27:38 +00:00
parent 7fbc6cd9fe
commit 48c06218dc
4 changed files with 259 additions and 3 deletions

View File

@@ -75,6 +75,22 @@ type RunResult struct {
Error error
}
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data.
type PreparedRun struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
}
// ArtifactRef represents a reference to an input artifact.
type ArtifactRef struct {
Type ArtifactRefType
@@ -156,13 +172,13 @@ type OutputContract struct {
// RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct {
Messages []RenderedMessage
Messages []RenderedMessage `json:"messages"`
}
// RenderedMessage is a single message in a rendered prompt.
type RenderedMessage struct {
Role string
Content string
Role string `json:"role"`
Content string `json:"content"`
}
// GenerateRequest is the internal request passed to the LLM client.

View File

@@ -0,0 +1,55 @@
package domain
import (
"encoding/json"
"strings"
"testing"
)
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
const envName = "SCRIPTORIUM_TEST_API_KEY"
const secret = "super-secret-value"
t.Setenv(envName, secret)
prepared := PreparedRun{
PromptID: "prompt.id",
PromptVersion: "v1",
PromptHash: "prompt-hash",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
APIKeyEnv: envName,
},
InputHashes: map[string]string{"transcript": "hash-1"},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{Role: "system", Content: "You are helpful."},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
out := string(b)
if strings.Contains(out, secret) {
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
}
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
}
var top map[string]any
if err := json.Unmarshal(b, &top); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
if _, ok := top[forbidden]; ok {
t.Fatalf("prepared run JSON should not include %q", forbidden)
}
}
}

View File

@@ -0,0 +1,144 @@
// Package format formats already-prepared domain data for adapters.
package format
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
// PreparedRunOutputFormat is the output format for prepared render data.
type PreparedRunOutputFormat string
const (
PreparedRunFormatText PreparedRunOutputFormat = "text"
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
)
// PreparedRunFormatter serializes a prepared run without performing use case work.
type PreparedRunFormatter interface {
Format(prepared *domain.PreparedRun) ([]byte, error)
}
// ParsePreparedRunOutputFormat parses a format name.
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
case "":
return DefaultPreparedRunOutputFormat, nil
case PreparedRunFormatText:
return PreparedRunFormatText, nil
case PreparedRunFormatJSON:
return PreparedRunFormatJSON, nil
default:
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
}
}
// FormatterForPreparedRun returns a formatter strategy for the chosen format.
func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) {
switch outputFormat {
case PreparedRunFormatText:
return textPreparedRunFormatter{}, nil
case PreparedRunFormatJSON:
return jsonPreparedRunFormatter{}, nil
default:
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
}
}
// FormatPreparedRun formats a prepared run using the selected format.
func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
formatter, err := FormatterForPreparedRun(outputFormat)
if err != nil {
return nil, err
}
return formatter.Format(prepared)
}
type jsonPreparedRunFormatter struct{}
func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
if prepared == nil {
return nil, errors.New("prepared run is nil")
}
return json.MarshalIndent(prepared, "", " ")
}
type textPreparedRunFormatter struct{}
func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
if prepared == nil {
return nil, errors.New("prepared run is nil")
}
var b bytes.Buffer
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
if prepared.PromptHash != "" {
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
}
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
target := prepared.EffectiveModelParams
fmt.Fprintln(&b, "effective_model_params:")
fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint)
fmt.Fprintf(&b, " model: %s\n", target.Model)
fmt.Fprintf(&b, " temperature: %g\n", target.Temperature)
fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens)
fmt.Fprintf(&b, " top_p: %g\n", target.TopP)
fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds)
if target.ReasoningEffort != "" {
fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort)
}
if target.APIKeyEnv != "" {
fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv)
}
if len(target.ExtraParams) > 0 {
fmt.Fprintln(&b, " extra_params:")
keys := make([]string, 0, len(target.ExtraParams))
for k := range target.ExtraParams {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
}
}
fmt.Fprintln(&b, "input_hashes:")
inputKeys := make([]string, 0, len(prepared.InputHashes))
for k := range prepared.InputHashes {
inputKeys = append(inputKeys, k)
}
sort.Strings(inputKeys)
for _, k := range inputKeys {
fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k])
}
fmt.Fprintln(&b, "messages:")
for i, msg := range prepared.Messages {
fmt.Fprintf(&b, " - index: %d\n", i)
fmt.Fprintf(&b, " role: %s\n", msg.Role)
fmt.Fprintln(&b, " content: |")
content := msg.Content
if content == "" {
fmt.Fprintln(&b, " ")
continue
}
for _, line := range strings.Split(content, "\n") {
fmt.Fprintf(&b, " %s\n", line)
}
}
return b.Bytes(), nil
}

View File

@@ -0,0 +1,41 @@
package format
import (
"errors"
"testing"
)
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
tests := []struct {
name string
input string
want PreparedRunOutputFormat
}{
{name: "default empty", input: "", want: DefaultPreparedRunOutputFormat},
{name: "text", input: "text", want: PreparedRunFormatText},
{name: "json", input: "json", want: PreparedRunFormatJSON},
{name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := ParsePreparedRunOutputFormat(tc.input)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) {
_, err := ParsePreparedRunOutputFormat("yaml")
if err == nil {
t.Fatal("expected error for unknown format")
}
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
}
}