Replace the Scriptorium adapter with PromptKit
This commit is contained in:
@@ -51,7 +51,7 @@ catalog used for resolution and the concrete constructors used for preparation.
|
||||
Tests may provide a catalog or registries instead; production code must not
|
||||
silently merge an injected partial catalog with production registrations.
|
||||
|
||||
The production LLM factory builds the Scriptorium-backed client from resolved
|
||||
The production LLM factory builds the PromptKit-backed client from resolved
|
||||
configuration, creates one scheduler from the effective global LLM limit, and
|
||||
wraps the client before it reaches modules. Registration and LLM construction
|
||||
errors are returned before a pipeline is prepared. Concrete module keys and
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# LLM Runtime Internals
|
||||
|
||||
`internal/framework/llm` is Notarius’s provider-independent structured
|
||||
completion boundary. It adapts framework requests to Scriptorium, bounds
|
||||
completion boundary. It adapts framework requests to PromptKit, bounds
|
||||
provider calls, assembles registered prompt and schema assets, records selected
|
||||
profiles, and redacts provider errors. The architectural boundary is defined in
|
||||
[Architecture](../policy/architecture.md#llm-boundary); profile sources,
|
||||
@@ -24,22 +24,22 @@ adapter does not own source evidence, artifact conversion, normalization, or
|
||||
durable schemas. Those responsibilities remain with the module and its
|
||||
[integration contract](../integrations/).
|
||||
|
||||
`ScriptoriumClient` validates the request target and prompt identity, maps each
|
||||
named material to a Scriptorium inline artifact while preserving its origin URI,
|
||||
`PromptKitClient` validates the request target and prompt identity, maps each
|
||||
named material to a PromptKit inline artifact while preserving its origin URI,
|
||||
forwards session and profile selection, then prepares and runs the prompt. It
|
||||
returns Scriptorium’s validated raw bytes rather than re-encoding the decoded
|
||||
returns PromptKit’s validated raw bytes rather than re-encoding the decoded
|
||||
target. An empty optional material is represented as one space so its named
|
||||
input is retained by Scriptorium.
|
||||
input is retained by PromptKit.
|
||||
|
||||
An empty request profile lets the prompt select its configured default. The CLI
|
||||
prepares every explicitly selected binding profile before a run begins, so a
|
||||
missing explicit profile fails before stage execution. Calls record the profile
|
||||
actually selected by Scriptorium; the recorder deduplicates non-secret profile
|
||||
actually selected by PromptKit; the recorder deduplicates non-secret profile
|
||||
identity, provider, and model values for manifest use.
|
||||
|
||||
## Shared Provider-Call Limit
|
||||
|
||||
Production construction creates one Scriptorium client and wraps it in one
|
||||
Production construction creates one PromptKit client and wraps it in one
|
||||
scheduled client. The scheduler has a fixed, positive permit limit, serves
|
||||
queued calls in FIFO order, and removes a queued call when its context is
|
||||
cancelled. A granted permit is released exactly once on every completion path.
|
||||
@@ -54,7 +54,7 @@ and its effective default are owned by
|
||||
## Prompt And Schema Assets
|
||||
|
||||
An `AssetRegistry` collects prompt and schema filesystems from production module
|
||||
families. It flattens registered roots into the Scriptorium filesystems and
|
||||
families. It flattens registered roots into the PromptKit filesystems and
|
||||
rejects invalid roots, unreadable assets, duplicate paths, and missing prompt
|
||||
or schema files during preparation. The framework’s `promptfs` helper combines
|
||||
module-owned prompt files with reusable domain fragments without making the
|
||||
@@ -95,14 +95,14 @@ meaningful input placement and cache controls of the prompt being changed.
|
||||
|
||||
## Validation, Repair, And Retries
|
||||
|
||||
Scriptorium performs prompt rendering, provider execution, and the prompt’s
|
||||
PromptKit performs prompt rendering, provider execution, and the prompt’s
|
||||
structured-output validation. The adapter reports an empty result, validation
|
||||
failure, empty structured body, or decode failure as
|
||||
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
|
||||
material when they exist. Provider failures remain operational errors rather
|
||||
than output-validation failures.
|
||||
|
||||
Prompt-declared repair is executed within Scriptorium’s structured-output flow.
|
||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
||||
The current production D&D prompt manifests set repair attempts to zero. That
|
||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
## Status
|
||||
|
||||
Planned.
|
||||
In progress. The dependency and framework adapter replacement is implemented;
|
||||
the configuration, module asset terminology, and remaining provenance alignment
|
||||
are still planned.
|
||||
|
||||
## Objective
|
||||
|
||||
|
||||
2
go.mod
2
go.mod
@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.1
|
||||
gitea.maximumdirect.net/eric/promptkit v0.1.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@@ -1,13 +1,9 @@
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.1 h1:zBKtB3+fP8FcHGI8DJD99CiTL6crAGitBhWtE+xYJHc=
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.1/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.1.0 h1:vuKeBxkiY8E54LRFbLQFjlJJCiOfMvB1++DYBCrD/ug=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.1.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
|
||||
@@ -168,14 +168,14 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
|
||||
return nil, nil, fmt.Errorf("production asset registry must not be nil")
|
||||
}
|
||||
recorder := llm.NewLLMProfileRecorder()
|
||||
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
|
||||
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
ProfileFile: cfg.Scriptorium.ProfileFile,
|
||||
Assets: assets,
|
||||
Recorder: recorder,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
|
||||
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
|
||||
}
|
||||
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
|
||||
if err != nil {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
const profileCheckPromptID = "notarius.profile.check"
|
||||
@@ -38,14 +38,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
|
||||
return fmt.Errorf("load Scriptorium profiles: %w", err)
|
||||
}
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
if _, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: profileCheckPromptID,
|
||||
ProfileID: profileID,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("profile check"),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("profile check"),
|
||||
},
|
||||
}); err != nil {
|
||||
if errors.Is(err, scriptorium.ErrProfileNotFound) {
|
||||
if errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
|
||||
}
|
||||
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
|
||||
@@ -54,14 +54,14 @@ func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config,
|
||||
return nil
|
||||
}
|
||||
|
||||
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
|
||||
opts := []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
|
||||
func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
|
||||
opts := []promptkit.Option{
|
||||
promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
|
||||
}
|
||||
if cfg.Scriptorium.ProfileFile != "" {
|
||||
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
|
||||
opts = append(opts, promptkit.WithProfileFile(cfg.Scriptorium.ProfileFile))
|
||||
}
|
||||
return scriptorium.NewEngine(scriptorium.Config{
|
||||
return promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "unused",
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
}, opts...)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type AssetSource struct {
|
||||
@@ -72,7 +72,7 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
|
||||
return flattenAssetSources(r.schemas)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
|
||||
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
|
||||
promptFS, err := r.PromptFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare prompt assets: %w", err)
|
||||
@@ -81,9 +81,9 @@ func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||
}
|
||||
return []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(promptFS, "."),
|
||||
scriptorium.WithSchemaFS(schemaFS, "."),
|
||||
return []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
promptkit.WithSchemaFS(schemaFS, "."),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
@@ -22,12 +22,12 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "root")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -50,12 +50,12 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
||||
@@ -74,12 +74,12 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||
@@ -145,18 +145,18 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
|
||||
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "asset-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "asset-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -13,23 +13,23 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
const scriptoriumProviderName = "scriptorium"
|
||||
const promptKitProviderName = "promptkit"
|
||||
|
||||
type ScriptoriumClientConfig struct {
|
||||
type PromptKitClientConfig struct {
|
||||
ProfileDir string
|
||||
ProfileFile string
|
||||
Assets *AssetRegistry
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
EngineOptions []scriptorium.Option
|
||||
EngineOptions []promptkit.Option
|
||||
Recorder *LLMProfileRecorder
|
||||
}
|
||||
|
||||
type ScriptoriumClient struct {
|
||||
engine *scriptorium.Engine
|
||||
type PromptKitClient struct {
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
}
|
||||
|
||||
@@ -38,49 +38,49 @@ type LLMProfileRecorder struct {
|
||||
profiles map[string]artifacts.LLMProfileManifest
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
|
||||
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
|
||||
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
|
||||
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
|
||||
|
||||
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
|
||||
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if cfg.Assets == nil {
|
||||
return nil, fmt.Errorf("scriptorium client assets must not be nil")
|
||||
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
|
||||
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
|
||||
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
|
||||
}
|
||||
options, err := cfg.Assets.ScriptoriumOptions()
|
||||
options, err := cfg.Assets.PromptKitOptions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
|
||||
options = append(options, scriptorium.WithProfileFile(profileFile))
|
||||
options = append(options, promptkit.WithProfileFile(profileFile))
|
||||
}
|
||||
options = append(options, cfg.EngineOptions...)
|
||||
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
|
||||
return nil, fmt.Errorf("create PromptKit engine: %w", err)
|
||||
}
|
||||
recorder := cfg.Recorder
|
||||
if recorder == nil {
|
||||
recorder = NewLLMProfileRecorder()
|
||||
}
|
||||
return &ScriptoriumClient{
|
||||
return &PromptKitClient{
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if c == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client must not be nil")
|
||||
}
|
||||
if c.engine == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("PromptKit client engine must not be nil")
|
||||
}
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
@@ -90,52 +90,52 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||
}
|
||||
|
||||
runReq := scriptorium.RunRequest{
|
||||
runReq := promptkit.RunRequest{
|
||||
PromptID: promptID,
|
||||
PromptVersion: strings.TrimSpace(req.PromptVersion),
|
||||
ProfileID: strings.TrimSpace(req.ProfileID),
|
||||
Inputs: scriptoriumInputs(req.Inputs),
|
||||
Vars: scriptoriumVars(req),
|
||||
Metadata: scriptoriumMetadata(req),
|
||||
Inputs: promptKitInputs(req.Inputs),
|
||||
Vars: promptKitVars(req),
|
||||
Metadata: promptKitMetadata(req),
|
||||
}
|
||||
prepared, err := c.engine.Prepare(ctx, runReq)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
|
||||
}
|
||||
result, err := c.engine.Run(ctx, runReq)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w", promptID, redactPromptKitError(err))
|
||||
}
|
||||
if result == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
|
||||
}
|
||||
response := c.responseFromResult(result, prepared)
|
||||
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
|
||||
return response, fmt.Errorf("run Scriptorium prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
|
||||
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
|
||||
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
|
||||
}
|
||||
if len(strings.TrimSpace(string(response.Content))) == 0 {
|
||||
return response, fmt.Errorf("run Scriptorium prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
|
||||
return response, fmt.Errorf("run PromptKit prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
|
||||
}
|
||||
if err := json.Unmarshal(response.Content, out); err != nil {
|
||||
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
|
||||
return response, fmt.Errorf("decode PromptKit structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
|
||||
func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepared *promptkit.PreparedRun) contracts.StructuredCompletionResponse {
|
||||
content := result.Artifact.Body
|
||||
if len(content) == 0 {
|
||||
content = []byte(result.RawOutput)
|
||||
}
|
||||
profile := artifacts.LLMProfileManifest{
|
||||
ID: strings.TrimSpace(result.SelectedProfileID),
|
||||
Provider: scriptoriumProviderName,
|
||||
Provider: promptKitProviderName,
|
||||
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
|
||||
}
|
||||
if c.recorder != nil {
|
||||
@@ -149,17 +149,17 @@ func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, pr
|
||||
PromptTokens: result.Usage.PromptTokens,
|
||||
CompletionTokens: result.Usage.CompletionTokens,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
Debug: scriptoriumDebugMaterial(prepared, result),
|
||||
Debug: promptKitDebugMaterial(prepared, result),
|
||||
}
|
||||
}
|
||||
|
||||
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
|
||||
func promptKitDebugMaterial(prepared *promptkit.PreparedRun, result *promptkit.RunResult) *contracts.LLMDebugMaterial {
|
||||
material := &contracts.LLMDebugMaterial{}
|
||||
if prepared != nil {
|
||||
material.Prompt = scriptoriumDebugPrompt(prepared)
|
||||
material.Prompt = promptKitDebugPrompt(prepared)
|
||||
}
|
||||
if result != nil {
|
||||
material.Response = scriptoriumDebugResponse(result)
|
||||
material.Response = promptKitDebugResponse(result)
|
||||
}
|
||||
if material.Prompt == nil && material.Response == nil {
|
||||
return nil
|
||||
@@ -167,7 +167,7 @@ func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scripto
|
||||
return material
|
||||
}
|
||||
|
||||
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
|
||||
func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPrompt {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDeb
|
||||
}
|
||||
}
|
||||
|
||||
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
|
||||
func promptKitDebugResponse(result *promptkit.RunResult) *contracts.LLMDebugResponse {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -254,7 +254,7 @@ func cloneStringMap(values map[string]string) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
func (c *PromptKitClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
if c == nil || c.recorder == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -302,11 +302,11 @@ func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
|
||||
func promptKitInputs(inputs contracts.LLMInputSet) map[string]promptkit.ArtifactRef {
|
||||
if len(inputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
|
||||
out := make(map[string]promptkit.ArtifactRef, len(inputs))
|
||||
for key, material := range inputs {
|
||||
name := strings.TrimSpace(key)
|
||||
if name == "" {
|
||||
@@ -320,15 +320,15 @@ func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.Arti
|
||||
body = " "
|
||||
}
|
||||
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
|
||||
out[name] = scriptorium.InlineWithURI(origin, body)
|
||||
out[name] = promptkit.InlineWithURI(origin, body)
|
||||
} else {
|
||||
out[name] = scriptorium.Inline(body)
|
||||
out[name] = promptkit.Inline(body)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
func promptKitVars(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
vars := make(map[string]string, len(req.Vars)+1)
|
||||
for key, value := range req.Vars {
|
||||
name := strings.TrimSpace(key)
|
||||
@@ -346,7 +346,7 @@ func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]strin
|
||||
return vars
|
||||
}
|
||||
|
||||
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
func promptKitMetadata(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
metadata := map[string]string{}
|
||||
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
|
||||
metadata["stage_name"] = stageName
|
||||
@@ -359,7 +359,7 @@ func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]s
|
||||
|
||||
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
|
||||
|
||||
func redactScriptoriumError(err error) error {
|
||||
func redactPromptKitError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -12,12 +14,12 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
|
||||
var out struct {
|
||||
OK bool `json:"ok"`
|
||||
@@ -39,7 +41,7 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
if !out.OK {
|
||||
t.Fatalf("decoded output OK = false, want true")
|
||||
}
|
||||
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
|
||||
if resp.Provider != promptKitProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
|
||||
t.Fatalf("response metadata = %#v", resp)
|
||||
}
|
||||
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
||||
@@ -57,6 +59,9 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
|
||||
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
|
||||
}
|
||||
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
|
||||
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
|
||||
}
|
||||
debugJSON, err := json.Marshal(resp.Debug)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal debug material: %v", err)
|
||||
@@ -71,21 +76,52 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
if gotReq.Target.Model != "explicit-model" {
|
||||
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
|
||||
}
|
||||
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
|
||||
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
|
||||
if len(gotReq.Prompt.Messages) != 1 ||
|
||||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
|
||||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
|
||||
t.Fatalf("rendered messages = %#v, want transcript input and variable content", gotReq.Prompt.Messages)
|
||||
}
|
||||
if gotReq.StructuredOutput == nil {
|
||||
t.Fatalf("structured output = nil, want JSON schema")
|
||||
}
|
||||
manifests := client.LLMProfileManifests()
|
||||
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
|
||||
if len(manifests) != 1 ||
|
||||
manifests[0].ID != "explicit-profile" ||
|
||||
manifests[0].Provider != promptKitProviderName ||
|
||||
manifests[0].Model != "explicit-model" {
|
||||
t.Fatalf("profile manifests = %#v", manifests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
||||
t.Run("assets", func(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
for _, root := range []string{"one", "two"} {
|
||||
if err := registry.RegisterPromptFS(fstest.MapFS{
|
||||
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
|
||||
}, root); err != nil {
|
||||
t.Fatalf("RegisterPromptFS() error = %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
|
||||
!strings.Contains(err.Error(), "duplicate asset path") {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("engine", func(t *testing.T) {
|
||||
if _, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
|
||||
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
@@ -102,8 +138,8 @@ func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
|
||||
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"bad":true}`})
|
||||
|
||||
var out map[string]any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
@@ -127,8 +163,8 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
func TestPromptKitClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
|
||||
var out []any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
@@ -138,7 +174,7 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
|
||||
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode PromptKit structured output") {
|
||||
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
|
||||
}
|
||||
if got := string(resp.Content); got != `{"ok":true}` {
|
||||
@@ -149,8 +185,8 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
|
||||
func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{err: errors.New("provider failed with Bearer secret-token")})
|
||||
|
||||
var out map[string]any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
@@ -166,7 +202,7 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
|
||||
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
|
||||
if !strings.Contains(err.Error(), `run PromptKit prompt "adapter.test"`) {
|
||||
t.Fatalf("error = %q, want operation context", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
|
||||
@@ -177,10 +213,10 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
|
||||
func TestPromptKitClientContextCancellationIsRespected(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
@@ -194,8 +230,57 @@ func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true})
|
||||
func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
|
||||
var remaining time.Duration
|
||||
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
deadline, ok := req.Context().Deadline()
|
||||
if !ok {
|
||||
t.Fatal("outbound request context has no deadline")
|
||||
}
|
||||
remaining = time.Until(deadline)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: "200 OK",
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(
|
||||
`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`,
|
||||
)),
|
||||
Request: req,
|
||||
}, nil
|
||||
})
|
||||
const configuredTimeout = 2 * time.Second
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
Timeout: configuredTimeout,
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
EngineOptions: []promptkit.Option{
|
||||
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "default-profile",
|
||||
Endpoint: "http://promptkit.test/v1",
|
||||
Model: "default-model",
|
||||
})),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out); err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v", err)
|
||||
}
|
||||
if remaining < configuredTimeout-time.Second || remaining > configuredTimeout {
|
||||
t.Fatalf("transport deadline remaining = %v, want near %v", remaining, configuredTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})
|
||||
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
@@ -210,12 +295,12 @@ func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{
|
||||
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{
|
||||
content: `{"ok":true}`,
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
@@ -249,8 +334,8 @@ func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestScriptoriumClientValidatesRequest(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
|
||||
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
|
||||
@@ -260,7 +345,34 @@ func TestScriptoriumClientValidatesRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
|
||||
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
|
||||
t.Helper()
|
||||
registry := newTestPromptKitAssets(t)
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: registry,
|
||||
EngineOptions: []promptkit.Option{
|
||||
promptkit.WithProfiles(
|
||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "default-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "default-model",
|
||||
}),
|
||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "explicit-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "explicit-model",
|
||||
}),
|
||||
),
|
||||
promptkit.WithLLMClient(fake),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
|
||||
t.Helper()
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterPromptFS(fstest.MapFS{
|
||||
@@ -274,7 +386,7 @@ inputs:
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content: "Transcript: {{ input \"transcript\" }}"
|
||||
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
@@ -289,43 +401,22 @@ output:
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("RegisterSchemaFS() error = %v", err)
|
||||
}
|
||||
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
|
||||
Assets: registry,
|
||||
EngineOptions: []scriptorium.Option{
|
||||
scriptorium.WithProfiles(
|
||||
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "default-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "default-model",
|
||||
}),
|
||||
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "explicit-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "explicit-model",
|
||||
}),
|
||||
),
|
||||
scriptorium.WithLLMClient(fake),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
|
||||
}
|
||||
return client
|
||||
return registry
|
||||
}
|
||||
|
||||
type fakeScriptoriumLLM struct {
|
||||
type fakePromptKitLLM struct {
|
||||
content string
|
||||
allowEmpty bool
|
||||
err error
|
||||
block chan struct{}
|
||||
mu sync.Mutex
|
||||
last scriptorium.GenerateRequest
|
||||
last promptkit.GenerateRequest
|
||||
calls int32
|
||||
inFlight int32
|
||||
maxInFlight int32
|
||||
}
|
||||
|
||||
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
func (f *fakePromptKitLLM) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
f.mu.Lock()
|
||||
f.last = req
|
||||
f.mu.Unlock()
|
||||
@@ -355,17 +446,25 @@ func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.Gener
|
||||
if !f.allowEmpty && !json.Valid([]byte(content)) {
|
||||
return nil, errors.New("test fake must return JSON content")
|
||||
}
|
||||
return &scriptorium.GenerateResponse{
|
||||
return &promptkit.GenerateResponse{
|
||||
Content: content,
|
||||
Usage: scriptorium.TokenUsage{
|
||||
Usage: promptkit.TokenUsage{
|
||||
PromptTokens: 11,
|
||||
CompletionTokens: 7,
|
||||
TotalTokens: 18,
|
||||
CachedTokens: 5,
|
||||
CacheWriteTokens: 3,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func (f *fakePromptKitLLM) lastRequest() promptkit.GenerateRequest {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.last
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
|
||||
@@ -27,7 +27,7 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, index := range []int{1, 4} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
@@ -94,22 +94,22 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
|
||||
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register scene prompt assets: %v", err)
|
||||
}
|
||||
engine := newScenesScriptoriumEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: "scene-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"players": scriptorium.Inline(players),
|
||||
"party": scriptorium.Inline(party),
|
||||
"glossary": scriptorium.Inline(glossary),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"players": promptkit.Inline(players),
|
||||
"party": promptkit.Inline(party),
|
||||
"glossary": promptkit.Inline(glossary),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -118,18 +118,18 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
|
||||
return prepared
|
||||
}
|
||||
|
||||
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
|
||||
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "scene-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) {
|
||||
@@ -34,26 +34,26 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
transcript := `{"units":[1]}`
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
"npcs": scriptorium.Inline(" "),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
|
||||
"players": promptkit.Inline(" "),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
"npcs": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
|
||||
@@ -17,24 +17,24 @@ func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "item-events-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-events-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"segments":[1]}`),
|
||||
"players": promptkit.Inline(" "),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
|
||||
@@ -23,26 +23,26 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
|
||||
if _, err := fs.ReadFile(schemaFS, "dnd_npc_interactions_llm.v1.json"); err != nil {
|
||||
t.Fatalf("response schema asset: %v", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-interactions-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-interactions-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcript := `{"units":[1]}`
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-interactions-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
"npcs": scriptorium.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
|
||||
"players": promptkit.Inline("Dana: Mira"),
|
||||
"party": promptkit.Inline("Mira: ranger"),
|
||||
"glossary": promptkit.Inline("Greencloak: title"),
|
||||
"npcs": promptkit.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": promptkit.Inline(" "),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
@@ -16,24 +16,24 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": promptkit.Inline(" "),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
|
||||
@@ -82,36 +82,36 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
|
||||
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register spell prompt assets: %v", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "spell-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "spell-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: "spell-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"spell_catalog": scriptorium.Inline(`{"spell_names":["Cure Wounds"]}`),
|
||||
"npcs": scriptorium.Inline(`{"npcs":[]}`),
|
||||
"players": scriptorium.Inline(players),
|
||||
"party": scriptorium.Inline(party),
|
||||
"glossary": scriptorium.Inline(glossary),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"spell_catalog": promptkit.Inline(`{"spell_names":["Cure Wounds"]}`),
|
||||
"npcs": promptkit.Inline(`{"npcs":[]}`),
|
||||
"players": promptkit.Inline(players),
|
||||
"party": promptkit.Inline(party),
|
||||
"glossary": promptkit.Inline(glossary),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
@@ -22,22 +22,22 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "normalize-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "normalize-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "normalize-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"candidates": scriptorium.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
|
||||
"transcript": scriptorium.Inline(`{"windows":[{"units":[]}]}`),
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"candidates": promptkit.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`),
|
||||
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -55,7 +55,7 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, index := range []int{2, 4} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
|
||||
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
@@ -31,17 +31,17 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
t.Fatalf("registerPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := newPromptCacheEngine(t, registry)
|
||||
commonInputs := map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
|
||||
"players": scriptorium.Inline(playersSentinel),
|
||||
"party": scriptorium.Inline(partySentinel),
|
||||
"glossary": scriptorium.Inline(glossarySentinel),
|
||||
commonInputs := map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
|
||||
"players": promptkit.Inline(playersSentinel),
|
||||
"party": promptkit.Inline(partySentinel),
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]scriptorium.ArtifactRef
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
npcInput bool
|
||||
spellCatalogInput bool
|
||||
}{
|
||||
@@ -52,8 +52,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
name: "combat turns",
|
||||
promptID: combatextract.PromptID,
|
||||
promptVersion: combatextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
},
|
||||
@@ -61,8 +61,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
name: "npc interactions",
|
||||
promptID: interactionextract.PromptID,
|
||||
promptVersion: interactionextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
},
|
||||
@@ -70,19 +70,19 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
name: "spells",
|
||||
promptID: spellextract.PromptID,
|
||||
promptVersion: spellextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
spellCatalogInput: true,
|
||||
},
|
||||
}
|
||||
|
||||
var sharedPrefix []scriptorium.RenderedMessage
|
||||
var sharedPrefix []promptkit.RenderedMessage
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: testCase.promptID,
|
||||
PromptVersion: testCase.promptVersion,
|
||||
ProfileID: "prompt-cache-test-profile",
|
||||
@@ -100,7 +100,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
|
||||
}
|
||||
if sharedPrefix == nil {
|
||||
sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...)
|
||||
sharedPrefix = append([]promptkit.RenderedMessage(nil), prefix...)
|
||||
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
|
||||
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
|
||||
}
|
||||
@@ -114,24 +114,24 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
|
||||
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v", err)
|
||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef {
|
||||
merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras))
|
||||
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
|
||||
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
|
||||
for name, input := range inputs {
|
||||
merged[name] = input
|
||||
}
|
||||
@@ -141,14 +141,14 @@ func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[str
|
||||
return merged
|
||||
}
|
||||
|
||||
func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) {
|
||||
func assertRenderedInputAfter(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, index int) {
|
||||
t.Helper()
|
||||
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
|
||||
}
|
||||
}
|
||||
|
||||
func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int {
|
||||
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
|
||||
Reference in New Issue
Block a user