372 lines
11 KiB
Go
372 lines
11 KiB
Go
package format
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
s := string(out)
|
|
|
|
for _, want := range []string{
|
|
"prompt: prompt.id",
|
|
"prompt_version: v1",
|
|
"selected_profile_id: local-fast",
|
|
"endpoint: http://llm/v1",
|
|
"model: gpt-test",
|
|
"temperature: 0.4",
|
|
"max_tokens: 256",
|
|
"top_p: 0.8",
|
|
"timeout_seconds: 45",
|
|
"service_tier: priority",
|
|
"reasoning_effort: medium",
|
|
"api_key_env: SCRIPTORIUM_API_KEY",
|
|
"prompt_hash: prompt-hash",
|
|
"rendered_prompt_hash: rendered-hash",
|
|
"glossary: hash-glossary",
|
|
"transcript: hash-transcript",
|
|
"messages:",
|
|
" system:",
|
|
" user:",
|
|
"System guidance.",
|
|
"Summarize the transcript.",
|
|
"Include key entities.",
|
|
"Second user message.",
|
|
} {
|
|
if !strings.Contains(s, want) {
|
|
t.Fatalf("expected text output to include %q, got:\n%s", want, s)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
|
"z_string": "enabled",
|
|
"b_number": 42,
|
|
"a_object": map[string]any{
|
|
"nested": "value",
|
|
"count": 2,
|
|
},
|
|
"c_array": []any{"first", 3, false},
|
|
}
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
s := string(out)
|
|
|
|
want := strings.Join([]string{
|
|
" extra_params:",
|
|
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
|
" b_number: 42",
|
|
" c_array: [\"first\",3,false]",
|
|
" z_string: enabled",
|
|
}, "\n")
|
|
if !strings.Contains(s, want) {
|
|
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|
const secret = "super-secret-api-key"
|
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
|
|
|
out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if strings.Contains(string(out), secret) {
|
|
t.Fatalf("text output should not include resolved api key value: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
|
const directKey = "direct-format-key"
|
|
// PreparedRun intentionally has no field for direct API keys.
|
|
prepared := samplePreparedRun()
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if strings.Contains(string(out), directKey) {
|
|
t.Fatalf("text output should not include direct api key value: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.Messages = []promptkit.RenderedMessage{
|
|
{
|
|
Role: "system",
|
|
Content: "System guidance.",
|
|
CacheControl: &promptkit.CacheControl{
|
|
Type: promptkit.CacheControlEphemeral,
|
|
TTL: "1h",
|
|
},
|
|
},
|
|
{Role: "user", Content: "Summarize the transcript."},
|
|
}
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
s := string(out)
|
|
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
|
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
|
}
|
|
if strings.Count(s, "cache_control:") != 1 {
|
|
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.SessionID = "session-123"
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if !strings.Contains(string(out), "session_id: session-123\n") {
|
|
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.Messages = []promptkit.RenderedMessage{
|
|
{
|
|
Role: "system",
|
|
Content: "System guidance.",
|
|
CacheControl: &promptkit.CacheControl{
|
|
Type: promptkit.CacheControlEphemeral,
|
|
},
|
|
},
|
|
}
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
s := string(out)
|
|
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
|
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
|
}
|
|
if strings.Contains(s, "ttl=") {
|
|
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
|
}
|
|
}
|
|
|
|
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.SessionID = "session-123"
|
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
|
"number": 42,
|
|
"nested": map[string]any{
|
|
"enabled": true,
|
|
},
|
|
}
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
|
|
var decoded map[string]any
|
|
if err := json.Unmarshal(out, &decoded); err != nil {
|
|
t.Fatalf("expected valid json output, got %v", err)
|
|
}
|
|
|
|
if decoded["prompt_id"] != "prompt.id" {
|
|
t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"])
|
|
}
|
|
if decoded["prompt_version"] != "v1" {
|
|
t.Fatalf("expected prompt_version in json output, got %#v", decoded["prompt_version"])
|
|
}
|
|
if decoded["selected_profile_id"] != "local-fast" {
|
|
t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"])
|
|
}
|
|
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
|
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
|
}
|
|
if decoded["session_id"] != "session-123" {
|
|
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
|
}
|
|
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
|
}
|
|
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
|
}
|
|
if extraParams["number"] != float64(42) {
|
|
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
|
}
|
|
nested, ok := extraParams["nested"].(map[string]any)
|
|
if !ok || nested["enabled"] != true {
|
|
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
|
}
|
|
if _, ok := decoded["input_hashes"]; !ok {
|
|
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
|
}
|
|
if _, ok := decoded["messages"]; !ok {
|
|
t.Fatalf("expected messages in json output, got %#v", decoded)
|
|
}
|
|
}
|
|
|
|
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
|
prepared := samplePreparedRun()
|
|
prepared.Messages = []promptkit.RenderedMessage{
|
|
{
|
|
Role: "system",
|
|
Content: "System guidance.",
|
|
CacheControl: &promptkit.CacheControl{
|
|
Type: promptkit.CacheControlEphemeral,
|
|
TTL: "1h",
|
|
},
|
|
},
|
|
{Role: "user", Content: "Summarize the transcript."},
|
|
}
|
|
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
|
|
var decoded struct {
|
|
Messages []map[string]any `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(out, &decoded); err != nil {
|
|
t.Fatalf("expected valid json output, got %v", err)
|
|
}
|
|
if len(decoded.Messages) != 2 {
|
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
|
}
|
|
|
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
|
}
|
|
if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
|
}
|
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
|
}
|
|
}
|
|
|
|
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|
const secret = "super-secret-api-key"
|
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
|
|
|
out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatJSON)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if strings.Contains(string(out), secret) {
|
|
t.Fatalf("json output should not include resolved api key value: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
|
const directKey = "direct-format-key"
|
|
// PreparedRun intentionally has no field for direct API keys.
|
|
prepared := samplePreparedRun()
|
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if strings.Contains(string(out), directKey) {
|
|
t.Fatalf("json output should not include direct api key value: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want PreparedRunOutputFormat
|
|
}{
|
|
{name: "default empty", input: "", want: DefaultPreparedRunOutputFormat},
|
|
{name: "text", input: "text", want: PreparedRunFormatText},
|
|
{name: "json", input: "json", want: PreparedRunFormatJSON},
|
|
{name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := ParsePreparedRunOutputFormat(tc.input)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if got != tc.want {
|
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) {
|
|
_, err := ParsePreparedRunOutputFormat("yaml")
|
|
if err == nil {
|
|
t.Fatal("expected error for unknown format")
|
|
}
|
|
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
|
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
|
|
_, err := FormatPreparedRunByName(samplePreparedRun(), "yaml")
|
|
if err == nil {
|
|
t.Fatal("expected unknown format error")
|
|
}
|
|
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
|
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
|
}
|
|
}
|
|
|
|
func samplePreparedRun() *promptkit.PreparedRun {
|
|
return &promptkit.PreparedRun{
|
|
PromptID: "prompt.id",
|
|
PromptVersion: "v1",
|
|
PromptHash: "prompt-hash",
|
|
SelectedProfileID: "local-fast",
|
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
|
Endpoint: "http://llm/v1",
|
|
Model: "gpt-test",
|
|
Temperature: 0.4,
|
|
MaxTokens: 256,
|
|
TopP: 0.8,
|
|
TimeoutSeconds: 45,
|
|
ServiceTier: "priority",
|
|
ReasoningEffort: "medium",
|
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
|
},
|
|
InputHashes: map[string]string{
|
|
"transcript": "hash-transcript",
|
|
"glossary": "hash-glossary",
|
|
},
|
|
RenderedPromptHash: "rendered-hash",
|
|
Messages: []promptkit.RenderedMessage{
|
|
{Role: "system", Content: "System guidance."},
|
|
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
|
{Role: "user", Content: "Second user message."},
|
|
},
|
|
}
|
|
}
|