Replace the Scriptorium adapter with PromptKit

This commit is contained in:
2026-07-28 16:31:36 +00:00
parent 7cfab8ada0
commit 53a330587b
20 changed files with 374 additions and 277 deletions

View File

@@ -51,7 +51,7 @@ catalog used for resolution and the concrete constructors used for preparation.
Tests may provide a catalog or registries instead; production code must not
silently merge an injected partial catalog with production registrations.
The production LLM factory builds the Scriptorium-backed client from resolved
The production LLM factory builds the PromptKit-backed client from resolved
configuration, creates one scheduler from the effective global LLM limit, and
wraps the client before it reaches modules. Registration and LLM construction
errors are returned before a pipeline is prepared. Concrete module keys and

View File

@@ -1,7 +1,7 @@
# LLM Runtime Internals
`internal/framework/llm` is Notariuss provider-independent structured
completion boundary. It adapts framework requests to Scriptorium, bounds
completion boundary. It adapts framework requests to PromptKit, bounds
provider calls, assembles registered prompt and schema assets, records selected
profiles, and redacts provider errors. The architectural boundary is defined in
[Architecture](../policy/architecture.md#llm-boundary); profile sources,
@@ -24,22 +24,22 @@ adapter does not own source evidence, artifact conversion, normalization, or
durable schemas. Those responsibilities remain with the module and its
[integration contract](../integrations/).
`ScriptoriumClient` validates the request target and prompt identity, maps each
named material to a Scriptorium inline artifact while preserving its origin URI,
`PromptKitClient` validates the request target and prompt identity, maps each
named material to a PromptKit inline artifact while preserving its origin URI,
forwards session and profile selection, then prepares and runs the prompt. It
returns Scriptoriums validated raw bytes rather than re-encoding the decoded
returns PromptKits validated raw bytes rather than re-encoding the decoded
target. An empty optional material is represented as one space so its named
input is retained by Scriptorium.
input is retained by PromptKit.
An empty request profile lets the prompt select its configured default. The CLI
prepares every explicitly selected binding profile before a run begins, so a
missing explicit profile fails before stage execution. Calls record the profile
actually selected by Scriptorium; the recorder deduplicates non-secret profile
actually selected by PromptKit; the recorder deduplicates non-secret profile
identity, provider, and model values for manifest use.
## Shared Provider-Call Limit
Production construction creates one Scriptorium client and wraps it in one
Production construction creates one PromptKit client and wraps it in one
scheduled client. The scheduler has a fixed, positive permit limit, serves
queued calls in FIFO order, and removes a queued call when its context is
cancelled. A granted permit is released exactly once on every completion path.
@@ -54,7 +54,7 @@ and its effective default are owned by
## Prompt And Schema Assets
An `AssetRegistry` collects prompt and schema filesystems from production module
families. It flattens registered roots into the Scriptorium filesystems and
families. It flattens registered roots into the PromptKit filesystems and
rejects invalid roots, unreadable assets, duplicate paths, and missing prompt
or schema files during preparation. The frameworks `promptfs` helper combines
module-owned prompt files with reusable domain fragments without making the
@@ -95,14 +95,14 @@ meaningful input placement and cache controls of the prompt being changed.
## Validation, Repair, And Retries
Scriptorium performs prompt rendering, provider execution, and the prompts
PromptKit performs prompt rendering, provider execution, and the prompts
structured-output validation. The adapter reports an empty result, validation
failure, empty structured body, or decode failure as
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
material when they exist. Provider failures remain operational errors rather
than output-validation failures.
Prompt-declared repair is executed within Scriptoriums structured-output flow.
Prompt-declared repair is executed within PromptKits structured-output flow.
The current production D&D prompt manifests set repair attempts to zero. That
setting does not replace pipeline retry behavior: a bindings configured retry
count reruns its stage attempt after an error or rejection, and an exhausted

View File

@@ -2,7 +2,9 @@
## Status
Planned.
In progress. The dependency and framework adapter replacement is implemented;
the configuration, module asset terminology, and remaining provenance alignment
are still planned.
## Objective

2
go.mod
View File

@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5
require (
gitea.maximumdirect.net/eric/scriptorium v0.11.1
gitea.maximumdirect.net/eric/promptkit v0.1.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)

8
go.sum
View File

@@ -1,13 +1,9 @@
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
gitea.maximumdirect.net/eric/scriptorium v0.11.1 h1:zBKtB3+fP8FcHGI8DJD99CiTL6crAGitBhWtE+xYJHc=
gitea.maximumdirect.net/eric/scriptorium v0.11.1/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
gitea.maximumdirect.net/eric/promptkit v0.1.0 h1:vuKeBxkiY8E54LRFbLQFjlJJCiOfMvB1++DYBCrD/ug=
gitea.maximumdirect.net/eric/promptkit v0.1.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

View File

@@ -168,14 +168,14 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
return nil, nil, fmt.Errorf("production asset registry must not be nil")
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.Scriptorium.ProfileDir,
ProfileFile: cfg.Scriptorium.ProfileFile,
Assets: assets,
Recorder: recorder,
})
if err != nil {
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
}
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
if err != nil {

View File

@@ -7,7 +7,7 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
const profileCheckPromptID = "notarius.profile.check"
@@ -38,14 +38,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
return fmt.Errorf("load Scriptorium profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
if _, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("profile check"),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, scriptorium.ErrProfileNotFound) {
if errors.Is(err, promptkit.ErrProfileNotFound) {
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
}
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
@@ -54,14 +54,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
return nil
}
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
opts := []scriptorium.Option{
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
opts := []promptkit.Option{
promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
}
if cfg.Scriptorium.ProfileFile != "" {
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
opts = append(opts, promptkit.WithProfileFile(cfg.Scriptorium.ProfileFile))
}
return scriptorium.NewEngine(scriptorium.Config{
return promptkit.NewEngine(promptkit.Config{
PromptDir: "unused",
ProfileDir: cfg.Scriptorium.ProfileDir,
}, opts...)

View File

@@ -12,7 +12,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
type AssetSource struct {
@@ -72,7 +72,7 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
@@ -81,9 +81,9 @@ func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
}
return []scriptorium.Option{
scriptorium.WithPromptFS(promptFS, "."),
scriptorium.WithSchemaFS(schemaFS, "."),
return []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}, nil
}

View File

@@ -7,7 +7,7 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
@@ -22,12 +22,12 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
engine := newAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err != nil {
@@ -50,12 +50,12 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
@@ -74,12 +74,12 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
@@ -145,18 +145,18 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
}
}
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}

View File

@@ -13,23 +13,23 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
const scriptoriumProviderName = "scriptorium"
const promptKitProviderName = "promptkit"
type ScriptoriumClientConfig struct {
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []scriptorium.Option
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
}
type ScriptoriumClient struct {
engine *scriptorium.Engine
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
}
@@ -38,49 +38,49 @@ type LLMProfileRecorder struct {
profiles map[string]artifacts.LLMProfileManifest
}
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("scriptorium client assets must not be nil")
return nil, fmt.Errorf("PromptKit client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
}
options, err := cfg.Assets.ScriptoriumOptions()
options, err := cfg.Assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, scriptorium.WithProfileFile(profileFile))
options = append(options, promptkit.WithProfileFile(profileFile))
}
options = append(options, cfg.EngineOptions...)
engine, err := scriptorium.NewEngine(scriptorium.Config{
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
if err != nil {
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
return nil, fmt.Errorf("create PromptKit engine: %w", err)
}
recorder := cfg.Recorder
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
return &ScriptoriumClient{
return &PromptKitClient{
engine: engine,
recorder: recorder,
}, nil
}
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client must not be nil")
}
if c.engine == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client engine must not be nil")
}
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
@@ -90,52 +90,52 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
runReq := scriptorium.RunRequest{
runReq := promptkit.RunRequest{
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
Inputs: scriptoriumInputs(req.Inputs),
Vars: scriptoriumVars(req),
Metadata: scriptoriumMetadata(req),
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req),
Metadata: promptKitMetadata(req),
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
result, err := c.engine.Run(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
if len(strings.TrimSpace(string(response.Content))) == 0 {
return response, fmt.Errorf("run Scriptorium prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
return response, fmt.Errorf("run PromptKit prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
}
if err := json.Unmarshal(response.Content, out); err != nil {
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
return response, fmt.Errorf("decode PromptKit structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
}
return response, nil
}
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body
if len(content) == 0 {
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: scriptoriumProviderName,
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
}
if c.recorder != nil {
@@ -149,17 +149,17 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, pr
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
Debug: scriptoriumDebugMaterial(prepared, result),
Debug: promptKitDebugMaterial(prepared, result),
}
}
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
func promptKitDebugMaterial(prepared *promptkit.PreparedRun, result *promptkit.RunResult) *contracts.LLMDebugMaterial {
material := &contracts.LLMDebugMaterial{}
if prepared != nil {
material.Prompt = scriptoriumDebugPrompt(prepared)
material.Prompt = promptKitDebugPrompt(prepared)
}
if result != nil {
material.Response = scriptoriumDebugResponse(result)
material.Response = promptKitDebugResponse(result)
}
if material.Prompt == nil && material.Response == nil {
return nil
@@ -167,7 +167,7 @@ func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scripto
return material
}
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPrompt {
if prepared == nil {
return nil
}
@@ -194,7 +194,7 @@ func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDeb
}
}
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
func promptKitDebugResponse(result *promptkit.RunResult) *contracts.LLMDebugResponse {
if result == nil {
return nil
}
@@ -254,7 +254,7 @@ func cloneStringMap(values map[string]string) map[string]string {
return out
}
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
func (c *PromptKitClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.recorder == nil {
return nil
}
@@ -302,11 +302,11 @@ func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
return out
}
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
func promptKitInputs(inputs contracts.LLMInputSet) map[string]promptkit.ArtifactRef {
if len(inputs) == 0 {
return nil
}
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
out := make(map[string]promptkit.ArtifactRef, len(inputs))
for key, material := range inputs {
name := strings.TrimSpace(key)
if name == "" {
@@ -320,15 +320,15 @@ func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.Arti
body = " "
}
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
out[name] = scriptorium.InlineWithURI(origin, body)
out[name] = promptkit.InlineWithURI(origin, body)
} else {
out[name] = scriptorium.Inline(body)
out[name] = promptkit.Inline(body)
}
}
return out
}
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitVars(req contracts.StructuredCompletionRequest) map[string]string {
vars := make(map[string]string, len(req.Vars)+1)
for key, value := range req.Vars {
name := strings.TrimSpace(key)
@@ -346,7 +346,7 @@ func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]strin
return vars
}
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
func promptKitMetadata(req contracts.StructuredCompletionRequest) map[string]string {
metadata := map[string]string{}
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
metadata["stage_name"] = stageName
@@ -359,7 +359,7 @@ func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]s
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
func redactScriptoriumError(err error) error {
func redactPromptKitError(err error) error {
if err == nil {
return nil
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
@@ -12,12 +14,12 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out struct {
OK bool `json:"ok"`
@@ -39,7 +41,7 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if !out.OK {
t.Fatalf("decoded output OK = false, want true")
}
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
if resp.Provider != promptKitProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
t.Fatalf("response metadata = %#v", resp)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
@@ -57,6 +59,9 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
@@ -71,21 +76,52 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
if len(gotReq.Prompt.Messages) != 1 ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages)
}
if gotReq.StructuredOutput == nil {
t.Fatalf("structured output = nil, want JSON schema")
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != promptKitProviderName ||
manifests[0].Model != "explicit-model" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
registry := NewAssetRegistry()
for _, root := range []string{"one", "two"} {
if err := registry.RegisterPromptFS(fstest.MapFS{
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
}, root); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
}
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
!strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
}
})
t.Run("engine", func(t *testing.T) {
if _, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
}
})
}
func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client := newTestPromptKitClient(t, fake)
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -102,8 +138,8 @@ func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *tes
}
}
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -127,8 +163,8 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out []any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -138,7 +174,7 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") {
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
}
if got := string(resp.Content); got != `{"ok":true}` {
@@ -149,8 +185,8 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -166,7 +202,7 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
}
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) {
t.Fatalf("error = %q, want operation context", err.Error())
}
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
@@ -177,10 +213,10 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
@@ -194,8 +230,57 @@ func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
}
}
func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true})
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
var remaining time.Duration
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
deadline, ok := req.Context().Deadline()
if !ok {
t.Fatal("outbound request context has no deadline")
}
remaining = time.Until(deadline)
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(
`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`,
)),
Request: req,
}, nil
})
const configuredTimeout = 2 * time.Second
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
Timeout: configuredTimeout,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://promptkit.test/v1",
Model: "default-model",
})),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out); err != nil {
t.Fatalf("CompleteStructured() error = %v", err)
}
if remaining < configuredTimeout-time.Second || remaining > configuredTimeout {
t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout)
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
@@ -210,12 +295,12 @@ func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
}
}
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakeScriptoriumLLM{
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakePromptKitLLM{
content: `{"ok":true}`,
block: make(chan struct{}),
}
client := newTestScriptoriumClient(t, fake)
client := newTestPromptKitClient(t, fake)
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
@@ -249,8 +334,8 @@ func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
wg.Wait()
}
func TestScriptoriumClientValidatesRequest(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
func TestPromptKitClientValidatesRequest(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
@@ -260,7 +345,34 @@ func TestScriptoriumClientValidatesRequest(t *testing.T) {
}
}
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
t.Helper()
registry := newTestPromptKitAssets(t)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
promptkit.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
return client
}
func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterPromptFS(fstest.MapFS{
@@ -274,7 +386,7 @@ inputs:
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }}"
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
output:
format: json
validation_mode: json_schema
@@ -289,43 +401,22 @@ output:
}, "."); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v", err)
}
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
Assets: registry,
EngineOptions: []scriptorium.Option{
scriptorium.WithProfiles(
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
scriptorium.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
}
return client
return registry
}
type fakeScriptoriumLLM struct {
type fakePromptKitLLM struct {
content string
allowEmpty bool
err error
block chan struct{}
mu sync.Mutex
last scriptorium.GenerateRequest
last promptkit.GenerateRequest
calls int32
inFlight int32
maxInFlight int32
}
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.mu.Lock()
f.last = req
f.mu.Unlock()
@@ -355,17 +446,25 @@ func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.Gener
if !f.allowEmpty && !json.Valid([]byte(content)) {
return nil, errors.New("test fake must return JSON content")
}
return &scriptorium.GenerateResponse{
return &promptkit.GenerateResponse{
Content: content,
Usage: scriptorium.TokenUsage{
Usage: promptkit.TokenUsage{
PromptTokens: 11,
CompletionTokens: 7,
TotalTokens: 18,
CachedTokens: 5,
CacheWriteTokens: 3,
},
}, nil
}
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
f.mu.Lock()
defer f.mu.Unlock()
return f.last

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
@@ -27,7 +27,7 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
}
}
for _, index := range []int{1, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
@@ -94,22 +94,22 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register scene prompt assets: %v", err)
}
engine := newScenesScriptoriumEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: ResponseSchemaVersion,
ProfileID: "scene-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {
@@ -118,18 +118,18 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
return prepared
}
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "scene-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) {
@@ -34,26 +34,26 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
transcript := `{"units":[1]}`
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
"npcs": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
"npcs": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -9,7 +9,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
@@ -17,24 +17,24 @@ func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "item-events-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-events-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
@@ -23,26 +23,26 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
if _, err := fs.ReadFile(schemaFS, "dnd_npc_interactions_llm.v1.json"); err != nil {
t.Fatalf("response schema asset: %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-interactions-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-interactions-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
transcript := `{"units":[1]}`
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-interactions-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"npcs": scriptorium.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
"players": promptkit.Inline("Dana: Mira"),
"party": promptkit.Inline("Mira: ranger"),
"glossary": promptkit.Inline("Greencloak: title"),
"npcs": promptkit.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": promptkit.Inline(" "),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
@@ -82,36 +82,36 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register spell prompt assets: %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "spell-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "spell-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: "spell-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": scriptorium.Inline(`{"spell_names":["Cure Wounds"]}`),
"npcs": scriptorium.Inline(`{"npcs":[]}`),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": promptkit.Inline(`{"spell_names":["Cure Wounds"]}`),
"npcs": promptkit.Inline(`{"npcs":[]}`),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {

View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
@@ -22,22 +22,22 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v", err)
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "normalize-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "normalize-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "normalize-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"candidates": scriptorium.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
"transcript": scriptorium.Inline(`{"windows":[{"units":[]}]}`),
Inputs: map[string]promptkit.ArtifactRef{
"candidates": promptkit.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
},
})
if err != nil {
@@ -55,7 +55,7 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
}
}
for _, index := range []int{2, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}

View File

@@ -14,7 +14,7 @@ import (
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
@@ -31,17 +31,17 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
t.Fatalf("registerPromptAssets() error = %v", err)
}
engine := newPromptCacheEngine(t, registry)
commonInputs := map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": scriptorium.Inline(playersSentinel),
"party": scriptorium.Inline(partySentinel),
"glossary": scriptorium.Inline(glossarySentinel),
commonInputs := map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": promptkit.Inline(playersSentinel),
"party": promptkit.Inline(partySentinel),
"glossary": promptkit.Inline(glossarySentinel),
}
cases := []struct {
name string
promptID string
promptVersion string
inputs map[string]scriptorium.ArtifactRef
inputs map[string]promptkit.ArtifactRef
npcInput bool
spellCatalogInput bool
}{
@@ -52,8 +52,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "combat turns",
promptID: combatextract.PromptID,
promptVersion: combatextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
@@ -61,8 +61,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "npc interactions",
promptID: interactionextract.PromptID,
promptVersion: interactionextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
@@ -70,19 +70,19 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
name: "spells",
promptID: spellextract.PromptID,
promptVersion: spellextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
}),
npcInput: true,
spellCatalogInput: true,
},
}
var sharedPrefix []scriptorium.RenderedMessage
var sharedPrefix []promptkit.RenderedMessage
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: testCase.promptID,
PromptVersion: testCase.promptVersion,
ProfileID: "prompt-cache-test-profile",
@@ -100,7 +100,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
}
if sharedPrefix == nil {
sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...)
sharedPrefix = append([]promptkit.RenderedMessage(nil), prefix...)
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
}
@@ -114,24 +114,24 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
}
}
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v", err)
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
return engine
}
func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef {
merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras))
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
for name, input := range inputs {
merged[name] = input
}
@@ -141,14 +141,14 @@ func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[str
return merged
}
func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) {
func assertRenderedInputAfter(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, index int) {
t.Helper()
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
}
}
func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int {
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
t.Helper()
index := -1
occurrences := 0