Replace the Scriptorium adapter with PromptKit

This commit is contained in:
2026-07-28 16:31:36 +00:00
parent 7cfab8ada0
commit 53a330587b
20 changed files with 374 additions and 277 deletions

View File

@@ -168,14 +168,14 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
return nil, nil, fmt.Errorf("production asset registry must not be nil")
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.Scriptorium.ProfileDir,
ProfileFile: cfg.Scriptorium.ProfileFile,
Assets: assets,
Recorder: recorder,
})
if err != nil {
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
}
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
if err != nil {

View File

@@ -7,7 +7,7 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
const profileCheckPromptID = "notarius.profile.check"
@@ -38,14 +38,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
return fmt.Errorf("load Scriptorium profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
if _, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("profile check"),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, scriptorium.ErrProfileNotFound) {
if errors.Is(err, promptkit.ErrProfileNotFound) {
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
}
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
@@ -54,14 +54,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
return nil
}
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
opts := []scriptorium.Option{
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
opts := []promptkit.Option{
promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
}
if cfg.Scriptorium.ProfileFile != "" {
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
opts = append(opts, promptkit.WithProfileFile(cfg.Scriptorium.ProfileFile))
}
return scriptorium.NewEngine(scriptorium.Config{
return promptkit.NewEngine(promptkit.Config{
PromptDir: "unused",
ProfileDir: cfg.Scriptorium.ProfileDir,
}, opts...)

View File

@@ -12,7 +12,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
type AssetSource struct {
@@ -72,7 +72,7 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
@@ -81,9 +81,9 @@ func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
}
return []scriptorium.Option{
scriptorium.WithPromptFS(promptFS, "."),
scriptorium.WithSchemaFS(schemaFS, "."),
return []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}, nil
}

View File

@@ -7,7 +7,7 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
@@ -22,12 +22,12 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
engine := newAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err != nil {
@@ -50,12 +50,12 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
@@ -74,12 +74,12 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
@@ -145,18 +145,18 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
}
}
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}

View File

@@ -13,23 +13,23 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
const scriptoriumProviderName = "scriptorium"
const promptKitProviderName = "promptkit"
type ScriptoriumClientConfig struct {
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []scriptorium.Option
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
}
type ScriptoriumClient struct {
engine *scriptorium.Engine
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
}
@@ -38,49 +38,49 @@ type LLMProfileRecorder struct {
profiles map[string]artifacts.LLMProfileManifest
}
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("scriptorium client assets must not be nil")
return nil, fmt.Errorf("PromptKit client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
}
options, err := cfg.Assets.ScriptoriumOptions()
options, err := cfg.Assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, scriptorium.WithProfileFile(profileFile))
options = append(options, promptkit.WithProfileFile(profileFile))
}
options = append(options, cfg.EngineOptions...)
engine, err := scriptorium.NewEngine(scriptorium.Config{
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
if err != nil {
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
return nil, fmt.Errorf("create PromptKit engine: %w", err)
}
recorder := cfg.Recorder
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
return &ScriptoriumClient{
return &PromptKitClient{
engine: engine,
recorder: recorder,
}, nil
}
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client must not be nil")
}
if c.engine == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client engine must not be nil")
}
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
@@ -90,52 +90,52 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
runReq := scriptorium.RunRequest{
runReq := promptkit.RunRequest{
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
Inputs: scriptoriumInputs(req.Inputs),
Vars: scriptoriumVars(req),
Metadata: scriptoriumMetadata(req),
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req),
Metadata: promptKitMetadata(req),
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
result, err := c.engine.Run(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
if len(strings.TrimSpace(string(response.Content))) == 0 {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
return response, fmt.Errorf("run PromptKit prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
}
if err := json.Unmarshal(response.Content, out); err != nil {
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
return response, fmt.Errorf("decode PromptKit structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
}
return response, nil
}
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body
if len(content) == 0 {
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: scriptoriumProviderName,
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
}
if c.recorder != nil {
@@ -149,17 +149,17 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, pr
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
Debug: scriptoriumDebugMaterial(prepared, result),
Debug: promptKitDebugMaterial(prepared, result),
}
}
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
func promptKitDebugMaterial(prepared *promptkit.PreparedRun, result *promptkit.RunResult) *contracts.LLMDebugMaterial {
material := &contracts.LLMDebugMaterial{}
if prepared != nil {
material.Prompt = scriptoriumDebugPrompt(prepared)
material.Prompt = promptKitDebugPrompt(prepared)
}
if result != nil {
material.Response = scriptoriumDebugResponse(result)
material.Response = promptKitDebugResponse(result)
}
if material.Prompt == nil && material.Response == nil {
return nil
@@ -167,7 +167,7 @@ func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scripto
return material
}
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPrompt {
if prepared == nil {
return nil
}
@@ -194,7 +194,7 @@ func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDeb
}
}
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
func promptKitDebugResponse(result *promptkit.RunResult) *contracts.LLMDebugResponse {
if result == nil {
return nil
}
@@ -254,7 +254,7 @@ func cloneStringMap(values map[string]string) map[string]string {
return out
}
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
func (c *PromptKitClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.recorder == nil {
return nil
}
@@ -302,11 +302,11 @@ func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
return out
}
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
func promptKitInputs(inputs contracts.LLMInputSet) map[string]promptkit.ArtifactRef {
if len(inputs) == 0 {
return nil
}
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
out := make(map[string]promptkit.ArtifactRef, len(inputs))
for key, material := range inputs {
name := strings.TrimSpace(key)
if name == "" {
@@ -320,15 +320,15 @@ func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.Arti
body = " "
}
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
out[name] = scriptorium.InlineWithURI(origin, body)
out[name] = promptkit.InlineWithURI(origin, body)
} else {
out[name] = scriptorium.Inline(body)
out[name] = promptkit.Inline(body)
}
}
return out
}
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitVars(req contracts.StructuredCompletionRequest) map[string]string {
vars := make(map[string]string, len(req.Vars)+1)
for key, value := range req.Vars {
name := strings.TrimSpace(key)
@@ -346,7 +346,7 @@ func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]strin
return vars
}
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitMetadata(req contracts.StructuredCompletionRequest) map[string]string {
metadata := map[string]string{}
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
metadata["stage_name"] = stageName
@@ -359,7 +359,7 @@ func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]s
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
func redactScriptoriumError(err error) error {
func redactPromptKitError(err error) error {
if err == nil {
return nil
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
@@ -12,12 +14,12 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out struct {
OK bool `json:"ok"`
@@ -39,7 +41,7 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if !out.OK {
t.Fatalf("decoded output OK = false, want true")
}
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
if resp.Provider != promptKitProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
t.Fatalf("response metadata = %#v", resp)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
@@ -57,6 +59,9 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
@@ -71,21 +76,52 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
if len(gotReq.Prompt.Messages) != 1 ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages)
}
if gotReq.StructuredOutput == nil {
t.Fatalf("structured output = nil, want JSON schema")
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != promptKitProviderName ||
manifests[0].Model != "explicit-model" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
registry := NewAssetRegistry()
for _, root := range []string{"one", "two"} {
if err := registry.RegisterPromptFS(fstest.MapFS{
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
}, root); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
}
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
!strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
}
})
t.Run("engine", func(t *testing.T) {
if _, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
}
})
}
func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -102,8 +138,8 @@ func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *tes
}
}
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -127,8 +163,8 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out []any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -138,7 +174,7 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") {
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
}
if got := string(resp.Content); got != `{"ok":true}` {
@@ -149,8 +185,8 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -166,7 +202,7 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
}
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) {
t.Fatalf("error = %q, want operation context", err.Error())
}
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
@@ -177,10 +213,10 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
@@ -194,8 +230,57 @@ func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
}
}
func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true})
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
var remaining time.Duration
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
deadline, ok := req.Context().Deadline()
if !ok {
t.Fatal("outbound request context has no deadline")
}
remaining = time.Until(deadline)
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(
`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`,
)),
Request: req,
}, nil
})
const configuredTimeout = 2 * time.Second
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
Timeout: configuredTimeout,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://promptkit.test/v1",
Model: "default-model",
})),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out); err != nil {
t.Fatalf("CompleteStructured() error = %v", err)
}
if remaining < configuredTimeout-time.Second || remaining > configuredTimeout {
t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout)
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -210,12 +295,12 @@ func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
}
}
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakeScriptoriumLLM{
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakePromptKitLLM{
content: `{"ok":true}`,
block: make(chan struct{}),
}
client := newTestScriptoriumClient(t, fake)
client := newTestPromptKitClient(t, fake)
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
@@ -249,8 +334,8 @@ func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
wg.Wait()
}
func TestScriptoriumClientValidatesRequest(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientValidatesRequest(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
@@ -260,7 +345,34 @@ func TestScriptoriumClientValidatesRequest(t *testing.T) {
}
}
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
t.Helper()
registry := newTestPromptKitAssets(t)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
promptkit.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
return client
}
func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterPromptFS(fstest.MapFS{
@@ -274,7 +386,7 @@ inputs:
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }}"
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
output:
format: json
validation_mode: json_schema
@@ -289,43 +401,22 @@ output:
}, "."); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v", err)
}
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
Assets: registry,
EngineOptions: []scriptorium.Option{
scriptorium.WithProfiles(
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
scriptorium.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
}
return client
return registry
}
type fakeScriptoriumLLM struct {
type fakePromptKitLLM struct {
content string
allowEmpty bool
err error
block chan struct{}
mu sync.Mutex
last scriptorium.GenerateRequest
last promptkit.GenerateRequest
calls int32
inFlight int32
maxInFlight int32
}
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.mu.Lock()
f.last = req
f.mu.Unlock()
@@ -355,17 +446,25 @@ func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.Gener
if !f.allowEmpty && !json.Valid([]byte(content)) {
return nil, errors.New("test fake must return JSON content")
}
return &scriptorium.GenerateResponse{
return &promptkit.GenerateResponse{
Content: content,
Usage: scriptorium.TokenUsage{
Usage: promptkit.TokenUsage{
PromptTokens: 11,
CompletionTokens: 7,
TotalTokens: 18,
CachedTokens: 5,
CacheWriteTokens: 3,
},
}, nil
}
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
f.mu.Lock()
defer f.mu.Unlock()
return f.last

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
@@ -27,7 +27,7 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
}
}
for _, index := range []int{1, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
@@ -94,22 +94,22 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register scene prompt assets: %v", err)
}
engine := newScenesScriptoriumEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: ResponseSchemaVersion,
ProfileID: "scene-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {
@@ -118,18 +118,18 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
return prepared
}
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "scene-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) {
@@ -34,26 +34,26 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
transcript := `{"units":[1]}`
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
"npcs": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
"npcs": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -9,7 +9,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
@@ -17,24 +17,24 @@ func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "item-events-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-events-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
@@ -23,26 +23,26 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
if _, err := fs.ReadFile(schemaFS, "dnd_npc_interactions_llm.v1.json"); err != nil {
t.Fatalf("response schema asset: %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-interactions-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-interactions-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
transcript := `{"units":[1]}`
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-interactions-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"npcs": scriptorium.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
"players": promptkit.Inline("Dana: Mira"),
"party": promptkit.Inline("Mira: ranger"),
"glossary": promptkit.Inline("Greencloak: title"),
"npcs": promptkit.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
@@ -82,36 +82,36 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register spell prompt assets: %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "spell-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "spell-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: "spell-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": scriptorium.Inline(`{"spell_names":["Cure Wounds"]}`),
"npcs": scriptorium.Inline(`{"npcs":[]}`),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": promptkit.Inline(`{"spell_names":["Cure Wounds"]}`),
"npcs": promptkit.Inline(`{"npcs":[]}`),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
@@ -22,22 +22,22 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v", err)
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "normalize-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "normalize-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "normalize-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"candidates": scriptorium.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
"transcript": scriptorium.Inline(`{"windows":[{"units":[]}]}`),
Inputs: map[string]promptkit.ArtifactRef{
"candidates": promptkit.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
},
})
if err != nil {
@@ -55,7 +55,7 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
}
}
for _, index := range []int{2, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}

View File

@@ -14,7 +14,7 @@ import (
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
@@ -31,17 +31,17 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
t.Fatalf("registerPromptAssets() error = %v", err)
}
engine := newPromptCacheEngine(t, registry)
commonInputs := map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": scriptorium.Inline(playersSentinel),
"party": scriptorium.Inline(partySentinel),
"glossary": scriptorium.Inline(glossarySentinel),
commonInputs := map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": promptkit.Inline(playersSentinel),
"party": promptkit.Inline(partySentinel),
"glossary": promptkit.Inline(glossarySentinel),
}
cases := []struct {
name string
promptID string
promptVersion string
inputs map[string]scriptorium.ArtifactRef
inputs map[string]promptkit.ArtifactRef
npcInput bool
spellCatalogInput bool
}{
@@ -52,8 +52,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "combat turns",
promptID: combatextract.PromptID,
promptVersion: combatextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
@@ -61,8 +61,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "npc interactions",
promptID: interactionextract.PromptID,
promptVersion: interactionextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
@@ -70,19 +70,19 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "spells",
promptID: spellextract.PromptID,
promptVersion: spellextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
}),
npcInput: true,
spellCatalogInput: true,
},
}
var sharedPrefix []scriptorium.RenderedMessage
var sharedPrefix []promptkit.RenderedMessage
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: testCase.promptID,
PromptVersion: testCase.promptVersion,
ProfileID: "prompt-cache-test-profile",
@@ -100,7 +100,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
}
if sharedPrefix == nil {
sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...)
sharedPrefix = append([]promptkit.RenderedMessage(nil), prefix...)
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
}
@@ -114,24 +114,24 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
}
}
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v", err)
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
return engine
}
func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef {
merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras))
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
for name, input := range inputs {
merged[name] = input
}
@@ -141,14 +141,14 @@ func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[str
return merged
}
func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) {
func assertRenderedInputAfter(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, index int) {
t.Helper()
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
}
}
func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int {
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
t.Helper()
index := -1
occurrences := 0