Resolve public documentation contract questions
This commit is contained in:
323
public_contract_test.go
Normal file
323
public_contract_test.go
Normal file
@@ -0,0 +1,323 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
||||
payload, err := json.Marshal(promptkit.PreparedRun{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared run: %v", err)
|
||||
}
|
||||
for _, field := range []string{"start_time", "end_time", "duration_ms"} {
|
||||
if strings.Contains(string(payload), `"`+field+`"`) {
|
||||
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
||||
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
||||
result := promptkit.RunResult{
|
||||
RunID: "opaque-run-id",
|
||||
Artifact: promptkit.Artifact{Name: "output", ContentType: "text/plain", Body: []byte("ok")},
|
||||
StartTime: start,
|
||||
EndTime: start.Add(1500 * time.Millisecond),
|
||||
Duration: 1500 * time.Millisecond,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal run result: %v", err)
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(payload, &object); err != nil {
|
||||
t.Fatalf("decode run result JSON: %v", err)
|
||||
}
|
||||
if got := object["duration_ms"]; got != float64(1500) {
|
||||
t.Fatalf("expected duration_ms=1500, got %#v in %s", got, payload)
|
||||
}
|
||||
if _, exists := object["duration"]; exists {
|
||||
t.Fatalf("unexpected nanosecond duration field in %s", payload)
|
||||
}
|
||||
artifact, ok := object["artifact"].(map[string]any)
|
||||
if !ok || artifact["content_type"] != "text/plain" {
|
||||
t.Fatalf("expected stable artifact JSON fields, got %#v", object["artifact"])
|
||||
}
|
||||
|
||||
var decoded promptkit.RunResult
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal run result: %v", err)
|
||||
}
|
||||
if decoded.Duration != result.Duration || !decoded.StartTime.Equal(result.StartTime) || !decoded.EndTime.Equal(result.EndTime) {
|
||||
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, result)
|
||||
}
|
||||
|
||||
payload, err = json.Marshal(promptkit.RunResult{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal zero run result: %v", err)
|
||||
}
|
||||
for _, field := range []string{"start_time", "end_time", "duration_ms"} {
|
||||
if strings.Contains(string(payload), `"`+field+`"`) {
|
||||
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
||||
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
|
||||
|
||||
t.Run("prompt source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("first", "profile", "first"), "."),
|
||||
promptkit.WithPromptFS(contractPromptFS("second", "profile", "second"), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "second"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last prompt source: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "second" {
|
||||
t.Fatalf("expected last prompt source, got %#v", prepared.Messages)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfileFS(contractProfileFS("profile", "first-model"), "."),
|
||||
promptkit.WithProfileFS(contractProfileFS("profile", "second-model"), "."),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last profile source: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "second-model" {
|
||||
t.Fatalf("expected last profile source, got %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("in-memory profiles", func(t *testing.T) {
|
||||
first := profile
|
||||
first.Model = "first-model"
|
||||
second := profile
|
||||
second.Model = "second-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(first),
|
||||
promptkit.WithProfiles(second),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last in-memory profile option: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "second-model" {
|
||||
t.Fatalf("expected last in-memory profiles, got %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("schema source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractSchemaPromptFS(), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithSchemaFS(contractSchemaFS("first"), "."),
|
||||
promptkit.WithSchemaFS(contractSchemaFS("second"), "."),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "schema-prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last schema source: %v", err)
|
||||
}
|
||||
schema := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||
if schema["title"] != "second" {
|
||||
t.Fatalf("expected last schema source, got %#v", schema)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("model client", func(t *testing.T) {
|
||||
var firstCalls, secondCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatalf("run with last model client: %v", err)
|
||||
}
|
||||
if firstCalls.Load() != 0 || secondCalls.Load() != 1 {
|
||||
t.Fatalf("expected only last client call, got first=%d second=%d", firstCalls.Load(), secondCalls.Load())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("artifact reader", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractInputPromptFS(), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithArtifactReader(fixedArtifactReader("first")),
|
||||
promptkit.WithArtifactReader(fixedArtifactReader("second")),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "input-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.Inline("ignored")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with last artifact reader: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "second" {
|
||||
t.Fatalf("expected last artifact reader, got %#v", prepared.Messages)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://example.test/v1",
|
||||
Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(countingLLMClient{}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
const calls = 40
|
||||
errs := make(chan error, calls)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < calls; i++ {
|
||||
wg.Add(1)
|
||||
go func(run bool) {
|
||||
defer wg.Done()
|
||||
request := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if run {
|
||||
_, err := engine.Run(context.Background(), request)
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
_, err := engine.Prepare(context.Background(), request)
|
||||
errs <- err
|
||||
}(i%2 == 0)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("concurrent call failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type countingLLMClient struct {
|
||||
calls *atomic.Int64
|
||||
}
|
||||
|
||||
func (c countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if c.calls != nil {
|
||||
c.calls.Add(1)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type fixedArtifactReader string
|
||||
|
||||
func (r fixedArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
return &promptkit.Artifact{Body: []byte(r)}, nil
|
||||
}
|
||||
|
||||
func contractPromptFS(id, profileID, message string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
|
||||
version: "1"
|
||||
default_profile: %s
|
||||
messages:
|
||||
- role: user
|
||||
content: %q
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`, id, profileID, message))},
|
||||
}
|
||||
}
|
||||
|
||||
func contractInputPromptFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: input-prompt
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
inputs:
|
||||
- name: input
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "input"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func contractProfileFS(id, model string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
|
||||
endpoint: http://example.test/v1
|
||||
model: %s
|
||||
`, id, model))},
|
||||
}
|
||||
}
|
||||
|
||||
func contractSchemaPromptFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: schema-prompt
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
messages:
|
||||
- role: user
|
||||
content: message
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schema.json
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func contractSchemaFS(title string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: []byte(fmt.Sprintf(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": %q,
|
||||
"type": "object"
|
||||
}`, title))},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user