Replace the Scriptorium adapter with PromptKit

This commit is contained in:
2026-07-28 16:31:36 +00:00
parent 7cfab8ada0
commit 53a330587b
20 changed files with 374 additions and 277 deletions

View File

@@ -12,7 +12,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
type AssetSource struct {
@@ -72,7 +72,7 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
@@ -81,9 +81,9 @@ func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
}
return []scriptorium.Option{
scriptorium.WithPromptFS(promptFS, "."),
scriptorium.WithSchemaFS(schemaFS, "."),
return []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}, nil
}

View File

@@ -7,7 +7,7 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
@@ -22,12 +22,12 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
engine := newAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err != nil {
@@ -50,12 +50,12 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
@@ -74,12 +74,12 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
@@ -145,18 +145,18 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
}
}
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}

View File

@@ -13,23 +13,23 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
const scriptoriumProviderName = "scriptorium"
const promptKitProviderName = "promptkit"
type ScriptoriumClientConfig struct {
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []scriptorium.Option
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
}
type ScriptoriumClient struct {
engine *scriptorium.Engine
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
}
@@ -38,49 +38,49 @@ type LLMProfileRecorder struct {
profiles map[string]artifacts.LLMProfileManifest
}
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("scriptorium client assets must not be nil")
return nil, fmt.Errorf("PromptKit client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
}
options, err := cfg.Assets.ScriptoriumOptions()
options, err := cfg.Assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, scriptorium.WithProfileFile(profileFile))
options = append(options, promptkit.WithProfileFile(profileFile))
}
options = append(options, cfg.EngineOptions...)
engine, err := scriptorium.NewEngine(scriptorium.Config{
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
if err != nil {
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
return nil, fmt.Errorf("create PromptKit engine: %w", err)
}
recorder := cfg.Recorder
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
return &ScriptoriumClient{
return &PromptKitClient{
engine: engine,
recorder: recorder,
}, nil
}
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client must not be nil")
}
if c.engine == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client engine must not be nil")
}
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
@@ -90,52 +90,52 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
runReq := scriptorium.RunRequest{
runReq := promptkit.RunRequest{
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
Inputs: scriptoriumInputs(req.Inputs),
Vars: scriptoriumVars(req),
Metadata: scriptoriumMetadata(req),
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req),
Metadata: promptKitMetadata(req),
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
result, err := c.engine.Run(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
if len(strings.TrimSpace(string(response.Content))) == 0 {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
return response, fmt.Errorf("run PromptKit prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
}
if err := json.Unmarshal(response.Content, out); err != nil {
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
return response, fmt.Errorf("decode PromptKit structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
}
return response, nil
}
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body
if len(content) == 0 {
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: scriptoriumProviderName,
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
}
if c.recorder != nil {
@@ -149,17 +149,17 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, pr
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
Debug: scriptoriumDebugMaterial(prepared, result),
Debug: promptKitDebugMaterial(prepared, result),
}
}
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
func promptKitDebugMaterial(prepared *promptkit.PreparedRun, result *promptkit.RunResult) *contracts.LLMDebugMaterial {
material := &contracts.LLMDebugMaterial{}
if prepared != nil {
material.Prompt = scriptoriumDebugPrompt(prepared)
material.Prompt = promptKitDebugPrompt(prepared)
}
if result != nil {
material.Response = scriptoriumDebugResponse(result)
material.Response = promptKitDebugResponse(result)
}
if material.Prompt == nil && material.Response == nil {
return nil
@@ -167,7 +167,7 @@ func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scripto
return material
}
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPrompt {
if prepared == nil {
return nil
}
@@ -194,7 +194,7 @@ func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDeb
}
}
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
func promptKitDebugResponse(result *promptkit.RunResult) *contracts.LLMDebugResponse {
if result == nil {
return nil
}
@@ -254,7 +254,7 @@ func cloneStringMap(values map[string]string) map[string]string {
return out
}
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
func (c *PromptKitClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.recorder == nil {
return nil
}
@@ -302,11 +302,11 @@ func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
return out
}
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
func promptKitInputs(inputs contracts.LLMInputSet) map[string]promptkit.ArtifactRef {
if len(inputs) == 0 {
return nil
}
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
out := make(map[string]promptkit.ArtifactRef, len(inputs))
for key, material := range inputs {
name := strings.TrimSpace(key)
if name == "" {
@@ -320,15 +320,15 @@ func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.Arti
body = " "
}
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
out[name] = scriptorium.InlineWithURI(origin, body)
out[name] = promptkit.InlineWithURI(origin, body)
} else {
out[name] = scriptorium.Inline(body)
out[name] = promptkit.Inline(body)
}
}
return out
}
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitVars(req contracts.StructuredCompletionRequest) map[string]string {
vars := make(map[string]string, len(req.Vars)+1)
for key, value := range req.Vars {
name := strings.TrimSpace(key)
@@ -346,7 +346,7 @@ func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]strin
return vars
}
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitMetadata(req contracts.StructuredCompletionRequest) map[string]string {
metadata := map[string]string{}
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
metadata["stage_name"] = stageName
@@ -359,7 +359,7 @@ func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]s
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
func redactScriptoriumError(err error) error {
func redactPromptKitError(err error) error {
if err == nil {
return nil
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
@@ -12,12 +14,12 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out struct {
OK bool `json:"ok"`
@@ -39,7 +41,7 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if !out.OK {
t.Fatalf("decoded output OK = false, want true")
}
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
if resp.Provider != promptKitProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
t.Fatalf("response metadata = %#v", resp)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
@@ -57,6 +59,9 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
@@ -71,21 +76,52 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
if len(gotReq.Prompt.Messages) != 1 ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages)
}
if gotReq.StructuredOutput == nil {
t.Fatalf("structured output = nil, want JSON schema")
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != promptKitProviderName ||
manifests[0].Model != "explicit-model" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
registry := NewAssetRegistry()
for _, root := range []string{"one", "two"} {
if err := registry.RegisterPromptFS(fstest.MapFS{
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
}, root); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
}
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
!strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
}
})
t.Run("engine", func(t *testing.T) {
if _, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
}
})
}
func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -102,8 +138,8 @@ func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *tes
}
}
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -127,8 +163,8 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out []any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -138,7 +174,7 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") {
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
}
if got := string(resp.Content); got != `{"ok":true}` {
@@ -149,8 +185,8 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -166,7 +202,7 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
}
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) {
t.Fatalf("error = %q, want operation context", err.Error())
}
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
@@ -177,10 +213,10 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
@@ -194,8 +230,57 @@ func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
}
}
func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true})
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
var remaining time.Duration
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
deadline, ok := req.Context().Deadline()
if !ok {
t.Fatal("outbound request context has no deadline")
}
remaining = time.Until(deadline)
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(
`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`,
)),
Request: req,
}, nil
})
const configuredTimeout = 2 * time.Second
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
Timeout: configuredTimeout,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://promptkit.test/v1",
Model: "default-model",
})),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out); err != nil {
t.Fatalf("CompleteStructured() error = %v", err)
}
if remaining < configuredTimeout-time.Second || remaining > configuredTimeout {
t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout)
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -210,12 +295,12 @@ func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
}
}
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakeScriptoriumLLM{
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakePromptKitLLM{
content: `{"ok":true}`,
block: make(chan struct{}),
}
client := newTestScriptoriumClient(t, fake)
client := newTestPromptKitClient(t, fake)
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
@@ -249,8 +334,8 @@ func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
wg.Wait()
}
func TestScriptoriumClientValidatesRequest(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientValidatesRequest(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
@@ -260,7 +345,34 @@ func TestScriptoriumClientValidatesRequest(t *testing.T) {
}
}
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
t.Helper()
registry := newTestPromptKitAssets(t)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
promptkit.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
return client
}
func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterPromptFS(fstest.MapFS{
@@ -274,7 +386,7 @@ inputs:
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }}"
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
output:
format: json
validation_mode: json_schema
@@ -289,43 +401,22 @@ output:
}, "."); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v", err)
}
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
Assets: registry,
EngineOptions: []scriptorium.Option{
scriptorium.WithProfiles(
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
scriptorium.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
}
return client
return registry
}
type fakeScriptoriumLLM struct {
type fakePromptKitLLM struct {
content string
allowEmpty bool
err error
block chan struct{}
mu sync.Mutex
last scriptorium.GenerateRequest
last promptkit.GenerateRequest
calls int32
inFlight int32
maxInFlight int32
}
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.mu.Lock()
f.last = req
f.mu.Unlock()
@@ -355,17 +446,25 @@ func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.Gener
if !f.allowEmpty && !json.Valid([]byte(content)) {
return nil, errors.New("test fake must return JSON content")
}
return &scriptorium.GenerateResponse{
return &promptkit.GenerateResponse{
Content: content,
Usage: scriptorium.TokenUsage{
Usage: promptkit.TokenUsage{
PromptTokens: 11,
CompletionTokens: 7,
TotalTokens: 18,
CachedTokens: 5,
CacheWriteTokens: 3,
},
}, nil
}
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
f.mu.Lock()
defer f.mu.Unlock()
return f.last