Refactor: split prompt definition from execution settings and migrate run contracts to prompt_* + execution_target

This commit is contained in:
2026-05-05 10:09:31 -05:00
parent fdfd8641f5
commit a633c67538
28 changed files with 712 additions and 1021 deletions

View File

@@ -30,12 +30,13 @@ const (
type runConfig struct {
profileDir string
promptID string
profileID string
inputRaw listFlag
varRaw listFlag
outputPath string
llmBaseURL string
llmAPIKey string
apiKeyEnv string
model string
temperature float64
maxTokens int
@@ -43,6 +44,7 @@ type runConfig struct {
timeout time.Duration
llmBaseURLSet bool
apiKeyEnvSet bool
modelSet bool
temperatureSet bool
maxTokensSet bool
@@ -53,7 +55,6 @@ type serveConfig struct {
profileDir string
schemaDir string
llmBaseURL string
llmAPIKey string
model string
timeout time.Duration
}
@@ -115,7 +116,6 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
BaseURL: cfg.llmBaseURL,
APIKey: cfg.llmAPIKey,
Model: cfg.model,
Timeout: cfg.timeout,
})
@@ -132,21 +132,24 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
validate.NewStandardValidator(cfg.schemaDir),
)
var modelOverride *domain.ModelTarget
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet {
modelOverride = &domain.ModelTarget{
Endpoint: cfg.llmBaseURL,
Model: cfg.model,
Temperature: cfg.temperature,
MaxTokens: cfg.maxTokens,
var modelOverride *domain.ExecutionTarget
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.apiKeyEnvSet {
modelOverride = &domain.ExecutionTarget{
Endpoint: cfg.llmBaseURL,
Model: cfg.model,
Temperature: cfg.temperature,
MaxTokens: cfg.maxTokens,
TimeoutSeconds: int(cfg.timeout.Seconds()),
APIKeyEnv: cfg.apiKeyEnv,
}
}
res, runErr := runner.Run(context.Background(), domain.RunRequest{
PromptID: cfg.promptID,
ProfileID: cfg.profileID,
Inputs: inputs,
Vars: varMappings,
Model: modelOverride,
Execution: modelOverride,
})
if runErr != nil {
fmt.Fprintf(stderr, "run error: %v\n", runErr)
@@ -171,7 +174,6 @@ func serveCommand(args []string, stderr io.Writer) int {
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
BaseURL: cfg.llmBaseURL,
APIKey: cfg.llmAPIKey,
Model: cfg.model,
Timeout: cfg.timeout,
})
@@ -208,13 +210,14 @@ func parseRunArgs(args []string) (*runConfig, error) {
fs := flag.NewFlagSet("run", flag.ContinueOnError)
fs.SetOutput(io.Discard)
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt profile YAML files")
fs.StringVar(&cfg.profileID, "profile-id", "", "profile ID to run")
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt definition YAML files")
fs.StringVar(&cfg.promptID, "prompt-id", "", "prompt ID to run")
fs.StringVar(&cfg.profileID, "profile-id", "", "optional execution profile ID; if omitted, prompt default_profile is used")
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1")
fs.StringVar(&cfg.llmAPIKey, "llm-api-key", "", "optional API key")
fs.StringVar(&cfg.apiKeyEnv, "api-key-env", "", "environment variable name containing API key")
fs.StringVar(&cfg.model, "model", "", "model name")
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
@@ -231,8 +234,8 @@ func parseRunArgs(args []string) (*runConfig, error) {
if strings.TrimSpace(cfg.profileDir) == "" {
return nil, errors.New("--profile-dir is required")
}
if strings.TrimSpace(cfg.profileID) == "" {
return nil, errors.New("--profile-id is required")
if strings.TrimSpace(cfg.promptID) == "" {
return nil, errors.New("--prompt-id is required")
}
if len(cfg.inputRaw) == 0 {
return nil, errors.New("at least one --input is required")
@@ -243,6 +246,7 @@ func parseRunArgs(args []string) (*runConfig, error) {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url")
cfg.apiKeyEnvSet = flagWasSet(fs, "api-key-env")
cfg.modelSet = flagWasSet(fs, "model")
cfg.temperatureSet = flagWasSet(fs, "temperature")
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
@@ -256,10 +260,9 @@ func parseServeArgs(args []string) (*serveConfig, error) {
fs.SetOutput(io.Discard)
fs.StringVar(&cfg.addr, "addr", ":8080", "HTTP listen address")
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt profile YAML files")
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt definition YAML files")
fs.StringVar(&cfg.schemaDir, "schema-dir", ".", "base directory for validation schemas")
fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1")
fs.StringVar(&cfg.llmAPIKey, "llm-api-key", "", "optional API key")
fs.StringVar(&cfg.model, "model", "", "optional default model")
fs.DurationVar(&cfg.timeout, "timeout", 10*time.Minute, "LLM request timeout")
@@ -351,14 +354,15 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
if res == nil {
return
}
fmt.Fprintf(stderr, "profile=%s@%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
res.ProfileID,
res.ProfileVersion,
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
res.PromptID,
res.PromptVersion,
res.SelectedProfileID,
res.ModelName,
res.Validation.Status,
res.Validation.Mode,
len(res.Validation.Errors),
res.PromptHash,
res.RenderedPromptHash,
len(res.InputHashes),
res.Usage.PromptTokens,
res.Usage.CompletionTokens,
@@ -368,6 +372,6 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|serve> ...")
fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]")
fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --prompt-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--model NAME] [--timeout 10m]")
}

View File

@@ -51,17 +51,17 @@ func TestParseMappingsMalformed(t *testing.T) {
}
func TestParseRunArgsRequiredFlags(t *testing.T) {
_, err := parseRunArgs([]string{"--profile-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"})
_, err := parseRunArgs([]string{"--prompt-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"})
if err == nil {
t.Fatal("expected missing --profile-dir error")
}
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"})
if err == nil {
t.Fatal("expected missing --profile-id error")
t.Fatal("expected missing --prompt-id error")
}
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--llm-base-url", "http://x/v1", "--model", "m"})
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--prompt-id", "p", "--llm-base-url", "http://x/v1", "--model", "m"})
if err == nil {
t.Fatal("expected missing --input error")
}
@@ -70,7 +70,7 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
cfg, err := parseRunArgs([]string{
"--profile-dir", "./profiles",
"--profile-id", "p",
"--prompt-id", "p",
"--input", "a=b",
})
if err != nil {
@@ -107,7 +107,7 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
func TestParseRunArgsTimeout(t *testing.T) {
cfg, err := parseRunArgs([]string{
"--profile-dir", "./profiles",
"--profile-id", "p",
"--prompt-id", "p",
"--input", "a=b",
"--llm-base-url", "http://x/v1",
"--model", "m",
@@ -121,7 +121,7 @@ func TestParseRunArgsTimeout(t *testing.T) {
cfg, err = parseRunArgs([]string{
"--profile-dir", "./profiles",
"--profile-id", "p",
"--prompt-id", "p",
"--input", "a=b",
"--llm-base-url", "http://x/v1",
"--model", "m",
@@ -156,7 +156,7 @@ func TestRunCommandVarsOptional(t *testing.T) {
code := runCommand([]string{
"--profile-dir", "./profiles",
"--profile-id", "p",
"--prompt-id", "p",
"--input", "transcript=./t.md",
"--llm-base-url", "://bad-url",
"--model", "m",
@@ -184,18 +184,19 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
t.Fatalf("unexpected writeOutput error: %v", err)
}
printSummary(&stderr, &domain.RunResult{
ProfileID: "p",
ProfileVersion: "1",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
PromptHash: "h",
InputHashes: map[string]string{"in": "x"},
PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
RenderedPromptHash: "h",
InputHashes: map[string]string{"in": "x"},
})
if stdout.String() != "artifact-body" {
t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "profile=p@1") {
if !strings.Contains(stderr.String(), "prompt=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String())
}
}

View File

@@ -5,11 +5,12 @@ import (
)
type runRequestDTO struct {
ProfileID string `json:"profile_id"`
ProfileVersion string `json:"profile_version,omitempty"`
Inputs map[string]inputRefDTO `json:"inputs"`
Vars map[string]string `json:"vars,omitempty"`
Model *modelOverrideRequestDTO `json:"model,omitempty"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Inputs map[string]inputRefDTO `json:"inputs"`
Vars map[string]string `json:"vars,omitempty"`
Model *modelOverrideRequestDTO `json:"model,omitempty"`
}
type inputRefDTO struct {
@@ -19,12 +20,15 @@ type inputRefDTO struct {
}
type modelOverrideRequestDTO struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
TopP float64 `json:"top_p,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
TopP float64 `json:"top_p,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"`
}
type runResponseDTO struct {
@@ -45,14 +49,15 @@ type artifactDTO struct {
type metadataDTO struct {
RunID string `json:"run_id"`
ProfileID string `json:"profile_id"`
ProfileVersion string `json:"profile_version"`
ProfileHash string `json:"profile_hash"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
PromptHash string `json:"prompt_hash"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
ModelParams modelParamsDTO `json:"model_params"`
InputHashes map[string]string `json:"input_hashes"`
PromptHash string `json:"prompt_hash"`
Usage tokenUsageDTO `json:"usage"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
@@ -63,12 +68,15 @@ type metadataDTO struct {
}
type modelParamsDTO struct {
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"`
}
type tokenUsageDTO struct {

View File

@@ -40,8 +40,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
if strings.TrimSpace(req.ProfileID) == "" {
writeError(w, http.StatusBadRequest, "invalid_request", "profile_id is required")
if strings.TrimSpace(req.PromptID) == "" {
writeError(w, http.StatusBadRequest, "invalid_request", "prompt_id is required")
return
}
if len(req.Inputs) == 0 {
@@ -58,24 +58,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
var model *domain.ModelTarget
var model *domain.ExecutionTarget
if req.Model != nil {
model = &domain.ModelTarget{
Endpoint: req.Model.Endpoint,
Model: req.Model.Model,
Temperature: req.Model.Temperature,
MaxTokens: req.Model.MaxTokens,
TopP: req.Model.TopP,
TimeoutSeconds: req.Model.TimeoutSeconds,
model = &domain.ExecutionTarget{
Endpoint: req.Model.Endpoint,
Model: req.Model.Model,
Temperature: req.Model.Temperature,
MaxTokens: req.Model.MaxTokens,
TopP: req.Model.TopP,
TimeoutSeconds: req.Model.TimeoutSeconds,
ReasoningEffort: req.Model.ReasoningEffort,
APIKeyEnv: req.Model.APIKeyEnv,
ExtraParams: req.Model.ExtraParams,
}
}
res, err := h.runner.Run(r.Context(), domain.RunRequest{
ProfileID: req.ProfileID,
ProfileVersion: req.ProfileVersion,
Inputs: mappedInputs,
Vars: req.Vars,
Model: model,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
Inputs: mappedInputs,
Vars: req.Vars,
Execution: model,
})
if err != nil {
status, code, message := mapRunError(err)
@@ -94,22 +98,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
},
Validation: mapValidation(res.Validation),
Metadata: metadataDTO{
RunID: res.RunID,
ProfileID: res.ProfileID,
ProfileVersion: res.ProfileVersion,
ProfileHash: res.ProfileHash,
ModelName: res.ModelName,
Endpoint: res.Endpoint,
RunID: res.RunID,
PromptID: res.PromptID,
PromptVersion: res.PromptVersion,
PromptHash: res.PromptHash,
RenderedPromptHash: res.RenderedPromptHash,
SelectedProfileID: res.SelectedProfileID,
ModelName: res.ModelName,
Endpoint: res.Endpoint,
ModelParams: modelParamsDTO{
Endpoint: res.ModelParams.Endpoint,
Model: res.ModelParams.Model,
Temperature: res.ModelParams.Temperature,
MaxTokens: res.ModelParams.MaxTokens,
TopP: res.ModelParams.TopP,
TimeoutSeconds: res.ModelParams.TimeoutSeconds,
Endpoint: res.EffectiveModelParams.Endpoint,
Model: res.EffectiveModelParams.Model,
Temperature: res.EffectiveModelParams.Temperature,
MaxTokens: res.EffectiveModelParams.MaxTokens,
TopP: res.EffectiveModelParams.TopP,
TimeoutSeconds: res.EffectiveModelParams.TimeoutSeconds,
ReasoningEffort: res.EffectiveModelParams.ReasoningEffort,
APIKeyEnv: res.EffectiveModelParams.APIKeyEnv,
ExtraParams: res.EffectiveModelParams.ExtraParams,
},
InputHashes: res.InputHashes,
PromptHash: res.PromptHash,
Usage: tokenUsageDTO{
PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens,
@@ -140,11 +148,11 @@ func mapValidation(v domain.ValidationResult) validationDTO {
func mapRunError(err error) (int, string, string) {
switch {
case errors.Is(err, profile.ErrProfileNotFound):
return http.StatusNotFound, "profile_not_found", "profile not found"
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, usecase.ErrProfileLoad):
return http.StatusBadRequest, "profile_load_failed", "failed to load profile"
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, usecase.ErrArtifactLoad):
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
case errors.Is(err, usecase.ErrPromptRender):

View File

@@ -42,13 +42,15 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
Size: 5,
Hash: "abc",
},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
ProfileID: "p1",
ProfileVersion: "1.0.0",
ProfileHash: "phash",
ModelName: "m1",
Endpoint: "http://llm/v1",
ModelParams: domain.ModelTarget{
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
PromptID: "prompt-1",
PromptVersion: "1.0.0",
PromptHash: "phash",
RenderedPromptHash: "rhash",
SelectedProfileID: "exec-default",
ModelName: "m1",
Endpoint: "http://llm/v1",
EffectiveModelParams: domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "m1",
Temperature: 0.2,
@@ -57,7 +59,6 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
TimeoutSeconds: 120,
},
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
@@ -68,7 +69,8 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
h := NewHandler(r)
body := []byte(`{
"profile_id": "p1",
"prompt_id": "prompt-1",
"profile_id": "exec-default",
"inputs": {
"transcript": {"type": "file", "uri": "./t.md"}
},
@@ -101,8 +103,8 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" {
t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"])
}
if metadata["profile_hash"] != "phash" {
t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"])
if metadata["prompt_hash"] != "phash" {
t.Fatalf("unexpected metadata.prompt_hash: %#v", metadata["prompt_hash"])
}
usage := metadata["usage"].(map[string]any)
if usage["total_tokens"] != float64(3) {
@@ -122,14 +124,17 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
}
if r.last.ProfileID != "p1" {
t.Fatalf("expected request profile_id p1, got %q", r.last.ProfileID)
if r.last.PromptID != "prompt-1" {
t.Fatalf("expected request prompt_id prompt-1, got %q", r.last.PromptID)
}
if r.last.Model == nil || r.last.Model.Model != "gpt-x" {
t.Fatalf("expected model override, got %#v", r.last.Model)
if r.last.ProfileID != "exec-default" {
t.Fatalf("expected request profile_id exec-default, got %q", r.last.ProfileID)
}
if r.last.Model.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Model)
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
t.Fatalf("expected model override, got %#v", r.last.Execution)
}
if r.last.Execution.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
}
}
@@ -145,7 +150,7 @@ func TestHandlerInvalidJSON(t *testing.T) {
}
}
func TestHandlerMissingProfileID(t *testing.T) {
func TestHandlerMissingPromptID(t *testing.T) {
h := NewHandler(&fakeRunner{})
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
w := httptest.NewRecorder()
@@ -173,7 +178,7 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
h := NewHandler(&fakeRunner{err: tc.err})
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"profile_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
@@ -210,7 +215,7 @@ func TestHandlerValidationFailureStillSuccess(t *testing.T) {
},
}})
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"profile_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)

View File

@@ -43,34 +43,36 @@ const (
// RunRequest represents a request to generate a single artifact.
type RunRequest struct {
ProfileID string
ProfileVersion string
Inputs map[string]ArtifactRef
Vars map[string]string
Model *ModelTarget
Validation *OutputContract
Metadata map[string]string
PromptID string
PromptVersion string
ProfileID string
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTarget
Validation *OutputContract
Metadata map[string]string
}
// RunResult represents the complete result of a prompt execution run.
type RunResult struct {
RunID string
Artifact Artifact
RawOutput string
Validation ValidationResult
ProfileID string
ProfileVersion string
ProfileHash string
ModelName string
Endpoint string
ModelParams ModelTarget
InputHashes map[string]string
PromptHash string
Usage TokenUsage
StartTime time.Time
EndTime time.Time
Duration time.Duration
Error error
RunID string
Artifact Artifact
RawOutput string
Validation ValidationResult
PromptID string
PromptVersion string
PromptHash string
RenderedPromptHash string
SelectedProfileID string
ModelName string
Endpoint string
EffectiveModelParams ExecutionTarget
InputHashes map[string]string
Usage TokenUsage
StartTime time.Time
EndTime time.Time
Duration time.Duration
Error error
}
// ArtifactRef represents a reference to an input artifact.
@@ -90,32 +92,58 @@ type Artifact struct {
Hash string
}
// PromptProfile represents a configured prompt execution profile.
type PromptProfile struct {
// PromptDefinition represents a configured prompt execution definition.
type PromptDefinition struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"`
ExpectedInputs []string `yaml:"expected_inputs"`
Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"`
ModelDefaults ModelTarget `yaml:"model_defaults"`
OutputFormat OutputFormat `yaml:"output_format"`
Validation OutputContract `yaml:"validation"`
}
// PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct {
Role string `yaml:"role"`
Content string `yaml:"content"`
// PromptInput describes one named input expected by a prompt definition.
type PromptInput struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
}
// ModelTarget represents the LLM endpoint and configuration.
type ModelTarget struct {
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
// PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct {
Role string `yaml:"role"`
Content string `yaml:"content"`
ContentFile string `yaml:"content_file"`
}
// ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct {
ID string `yaml:"id"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params"`
}
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"`
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
TopP float64 `yaml:"top_p" json:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"`
}
// OutputContract defines the requirements for the output artifact.
@@ -140,7 +168,7 @@ type RenderedMessage struct {
// GenerateRequest is the internal request passed to the LLM client.
type GenerateRequest struct {
Prompt RenderedPrompt
Target ModelTarget
Target ExecutionTarget
}
// GenerateResponse is the response received from the LLM client.
@@ -168,19 +196,20 @@ type ValidationResult struct {
// RunMetadata contains auditing information for a run.
type RunMetadata struct {
RunID string
ProfileID string
ProfileVersion string
ProfileHash string
PromptHash string
InputHashes map[string]string
ModelEndpoint string
ModelName string
Params ModelTarget
Timestamp time.Time
Duration time.Duration
Usage TokenUsage
ValidationMode ValidationMode
ValidationStatus ValidationStatus
RepairAttempts int
RunID string
PromptID string
PromptVersion string
PromptHash string
RenderedPromptHash string
SelectedProfileID string
InputHashes map[string]string
ModelEndpoint string
ModelName string
Params ExecutionTarget
Timestamp time.Time
Duration time.Duration
Usage TokenUsage
ValidationMode ValidationMode
ValidationStatus ValidationStatus
RepairAttempts int
}

View File

@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
"os"
"net/http"
"net/url"
"strings"
@@ -25,7 +26,6 @@ var (
type OpenAICompatibleConfig struct {
BaseURL string
APIKey string
Model string
Timeout time.Duration
HTTPClient *http.Client
@@ -33,7 +33,6 @@ type OpenAICompatibleConfig struct {
type OpenAICompatibleClient struct {
baseURL string
apiKey string
defaultModel string
timeout time.Duration
httpClient *http.Client
@@ -64,7 +63,6 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
return &OpenAICompatibleClient{
baseURL: strings.TrimRight(baseURL, "/"),
apiKey: cfg.APIKey,
defaultModel: cfg.Model,
timeout: timeout,
httpClient: client,
@@ -125,8 +123,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
}
httpReq.Header.Set("Content-Type", "application/json")
if strings.TrimSpace(c.apiKey) != "" {
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
apiKey := strings.TrimSpace(os.Getenv(envName))
if apiKey == "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
}
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
}
effectiveTimeout := c.timeout

View File

@@ -44,23 +44,24 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
APIKey: "secret-key",
Timeout: 2 * time.Second,
})
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret-key")
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "You are helpful."},
{Role: "user", Content: "Say hello"},
}},
Target: domain.ModelTarget{
Target: domain.ExecutionTarget{
Model: "gpt-test",
Temperature: 0.4,
MaxTokens: 123,
TopP: 0.7,
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
},
})
if err != nil {
@@ -110,7 +111,7 @@ func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{Model: "model"},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -120,6 +121,29 @@ func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
}
}
func TestOpenAICompatibleClientAPIKeyEnvMissing(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
})
if err == nil {
t.Fatal("expected missing API key env error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) {
gotModel := ""
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -139,7 +163,7 @@ func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) {
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{},
Target: domain.ExecutionTarget{},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -175,7 +199,7 @@ func TestOpenAICompatibleClientEndpointOverride(t *testing.T) {
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{Endpoint: overrideServer.URL + "/v1"},
Target: domain.ExecutionTarget{Endpoint: overrideServer.URL + "/v1"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
@@ -306,7 +330,7 @@ func TestOpenAICompatibleClientRequestTimeoutOverride(t *testing.T) {
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{TimeoutSeconds: 1},
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
})
if err != nil {
t.Fatalf("expected request-level timeout override to succeed, got %v", err)
@@ -327,7 +351,7 @@ func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) {
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{TimeoutSeconds: -1},
Target: domain.ExecutionTarget{TimeoutSeconds: -1},
})
if err == nil {
t.Fatal("expected invalid request error")
@@ -348,7 +372,7 @@ func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) {
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{Endpoint: "http://localhost:9999/v1"},
Target: domain.ExecutionTarget{Endpoint: "http://localhost:9999/v1"},
})
if err == nil {
t.Fatal("expected request failure due to unreachable endpoint")
@@ -369,7 +393,7 @@ func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T)
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{},
Target: domain.ExecutionTarget{},
})
if err == nil {
t.Fatal("expected endpoint-required error")

View File

@@ -13,9 +13,9 @@ import (
)
var (
ErrProfileNotFound = errors.New("prompt profile not found")
ErrProfileNotFound = errors.New("prompt definition not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidProfile = errors.New("invalid profile configuration")
ErrInvalidProfile = errors.New("invalid prompt definition configuration")
)
type filesystemRepository struct {
@@ -26,9 +26,9 @@ func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func (r *filesystemRepository) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) {
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidProfile)
}
files, err := os.ReadDir(r.dir)
@@ -53,7 +53,7 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string, versio
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err)
}
var prof domain.PromptProfile
var prof domain.PromptDefinition
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil {
@@ -77,22 +77,33 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string, versio
return nil, ErrProfileNotFound
}
func validateProfile(p *domain.PromptProfile) error {
func validateProfile(p *domain.PromptDefinition) error {
if p.ID == "" {
return errors.New("profile id is required")
return errors.New("prompt id is required")
}
if p.Version == "" {
return errors.New("profile version is required")
return errors.New("prompt version is required")
}
if len(p.Templates) == 0 {
return errors.New("at least one prompt template message is required")
}
if len(p.Inputs) == 0 {
return errors.New("at least one prompt input is required")
}
for i, input := range p.Inputs {
if strings.TrimSpace(input.Name) == "" {
return fmt.Errorf("input %d has empty name", i)
}
}
for i, t := range p.Templates {
if !isValidMessageRole(t.Role) {
return fmt.Errorf("template message %d has invalid role %q", i, t.Role)
}
if t.Content == "" {
return fmt.Errorf("template message %d is missing content", i)
if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" {
return fmt.Errorf("template message %d must provide content or content_file", i)
}
if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" {
return fmt.Errorf("template message %d cannot set both content and content_file", i)
}
}
if !isValidOutputFormat(p.OutputFormat) {
@@ -107,17 +118,9 @@ func validateProfile(p *domain.PromptProfile) error {
if p.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(p.Validation.SchemaPath) == "" {
return errors.New("validation.schema_path is required when validation_mode is json_schema")
}
if p.ModelDefaults.TimeoutSeconds < 0 {
return errors.New("model_defaults.timeout_seconds must be greater than or equal to 0")
}
if p.Validation.Format != "" && p.Validation.Format != p.OutputFormat {
return fmt.Errorf("validation format %q does not match output format %q", p.Validation.Format, p.OutputFormat)
}
for i, input := range p.ExpectedInputs {
if strings.TrimSpace(input) == "" {
return fmt.Errorf("expected input %d has empty name", i)
}
}
return nil
}

View File

@@ -5,7 +5,9 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
// Repository handles loading and storing prompt profiles.
// Repository is a transitional prompt-definition repository.
// It currently lives in internal/profile until package responsibilities
// are split in a follow-up refactor.
type Repository interface {
GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error)
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
}

View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func TestFilesystemRepository_GetProfile(t *testing.T) {
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "profile_test")
if err != nil {
t.Fatal(err)
@@ -38,19 +38,19 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
repo := NewFilesystemRepository(tmpDir)
ctx := context.Background()
t.Run("valid profile", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "test-profile", "")
t.Run("valid prompt definition", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "test-profile", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p == nil || p.ID != "test-profile" {
t.Errorf("expected profile test-profile, got %v", p)
t.Errorf("expected prompt definition test-profile, got %v", p)
}
if p.Version != "1.0.0" {
t.Fatalf("expected version 1.0.0, got %q", p.Version)
}
if len(p.ExpectedInputs) != 2 || p.ExpectedInputs[0] != "transcript" || p.ExpectedInputs[1] != "glossary" {
t.Fatalf("unexpected expected_inputs: %#v", p.ExpectedInputs)
if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" {
t.Fatalf("unexpected inputs: %#v", p.Inputs)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 templates, got %d", len(p.Templates))
@@ -64,48 +64,41 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
if p.Validation.ValidationMode != domain.ValidationBasic {
t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode)
}
if p.ModelDefaults.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds 120, got %d", p.ModelDefaults.TimeoutSeconds)
if p.DefaultProfile != "test-exec" {
t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile)
}
})
t.Run("invalid YAML", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "invalid_yaml", "")
_, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Errorf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("missing ID", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing-id", "")
_, err := repo.GetPromptDefinition(ctx, "missing-id", "")
if !errors.Is(err, ErrProfileNotFound) {
t.Errorf("expected ErrProfileNotFound for profile with missing ID, got %v", err)
}
})
t.Run("no templates", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "no-templates", "")
_, err := repo.GetPromptDefinition(ctx, "no-templates", "")
if !errors.Is(err, ErrInvalidProfile) {
t.Errorf("expected ErrInvalidProfile for profile with no templates, got %v", err)
}
})
t.Run("json schema mode missing schema path", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "json-schema-missing-path", "")
_, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "")
if !errors.Is(err, ErrInvalidProfile) {
t.Errorf("expected ErrInvalidProfile for json_schema profile without schema_path, got %v", err)
}
})
t.Run("negative timeout seconds", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "negative-timeout", "")
if !errors.Is(err, ErrInvalidProfile) {
t.Errorf("expected ErrInvalidProfile for negative timeout_seconds, got %v", err)
}
})
t.Run("profile not found", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "unknown", "")
t.Run("prompt definition not found", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "unknown", "")
if !errors.Is(err, ErrProfileNotFound) {
t.Errorf("expected ErrProfileNotFound, got %v", err)
}

View File

@@ -1,5 +1,8 @@
id: json-schema-missing-path
version: "1.0.0"
inputs:
- name: transcript
required: true
templates:
- role: user
content: "Return JSON"

View File

@@ -1,5 +1,8 @@
version: 1.0.0
description: Missing ID
inputs:
- name: transcript
required: true
templates:
- role: system
content: Hello

View File

@@ -1,10 +1,11 @@
id: negative-timeout
version: "1.0.0"
inputs:
- name: transcript
required: true
templates:
- role: user
content: "Say hi"
model_defaults:
timeout_seconds: -1
output_format: text
validation:
validation_mode: none

View File

@@ -1,5 +1,8 @@
id: no-templates
version: 1.0.0
inputs:
- name: transcript
required: true
templates: []
output_format: text
validation:

View File

@@ -1,18 +1,19 @@
id: test-profile
version: "1.0.0"
description: A valid test profile
expected_inputs:
- transcript
- glossary
default_profile: test-exec
description: A valid test prompt definition
inputs:
- name: transcript
required: true
content_type: text/markdown
- name: glossary
required: false
content_type: text/yaml
templates:
- role: system
content: "You are a helpful assistant."
- role: user
content: 'Analyze this: {{input "transcript"}}'
model_defaults:
model: gpt-4o
temperature: 0.7
timeout_seconds: 120
output_format: markdown
validation:
validation_mode: basic

View File

@@ -23,16 +23,19 @@ func NewGoRenderer() Renderer {
return &goRenderer{}
}
func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
if profile == nil {
return nil, fmt.Errorf("%w: nil profile", ErrRenderFailure)
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
if definition == nil {
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
}
// 1. Verify required inputs
for _, req := range profile.ExpectedInputs {
art, ok := inputs[req]
for _, in := range definition.Inputs {
if !in.Required {
continue
}
art, ok := inputs[in.Name]
if !ok || art == nil {
return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, req)
return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, in.Name)
}
}
@@ -49,7 +52,7 @@ func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile,
var renderedMessages []domain.RenderedMessage
for i, tmplMsg := range profile.Templates {
for i, tmplMsg := range definition.Templates {
select {
case <-ctx.Done():
return nil, ctx.Err()
@@ -61,6 +64,9 @@ func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile,
}
// Parse and execute template
if tmplMsg.ContentFile != "" {
return nil, fmt.Errorf("%w: message %d: content_file is not implemented yet", ErrRenderFailure, i)
}
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
if err != nil {
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)

View File

@@ -7,5 +7,5 @@ import (
// Renderer renders prompt templates using named artifacts and variables.
type Renderer interface {
Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error)
Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error)
}

View File

@@ -12,9 +12,9 @@ func TestGoRenderer_Render(t *testing.T) {
renderer := NewGoRenderer()
ctx := context.Background()
profile := &domain.PromptProfile{
profile := &domain.PromptDefinition{
ID: "test-profile",
ExpectedInputs: []string{"transcript"},
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are a {{.role}}."},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
@@ -54,7 +54,8 @@ func TestGoRenderer_Render(t *testing.T) {
})
t.Run("unknown input in template", func(t *testing.T) {
profileUnknown := &domain.PromptProfile{
profileUnknown := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Hello {{input \"ghost\"}}"},
},
@@ -69,7 +70,8 @@ func TestGoRenderer_Render(t *testing.T) {
})
t.Run("invalid template syntax", func(t *testing.T) {
profileInvalid := &domain.PromptProfile{
profileInvalid := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Hello {{.unclosed"},
},
@@ -81,7 +83,8 @@ func TestGoRenderer_Render(t *testing.T) {
})
t.Run("empty message role", func(t *testing.T) {
profileNoRole := &domain.PromptProfile{
profileNoRole := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "", Content: "Hello"},
},
@@ -93,7 +96,8 @@ func TestGoRenderer_Render(t *testing.T) {
})
t.Run("missing variable in template", func(t *testing.T) {
profileMissingVar := &domain.PromptProfile{
profileMissingVar := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are {{.missing}}"},
},

View File

@@ -44,7 +44,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "generic.structured_events",
PromptID: "generic.structured_events",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "test-model",
},
Inputs: map[string]domain.ArtifactRef{
"transcript": {
Type: domain.ArtifactRefFile,
@@ -60,17 +65,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
t.Fatalf("expected no error, got %v", err)
}
if res.ProfileID != "generic.structured_events" {
t.Fatalf("unexpected profile id: %q", res.ProfileID)
if res.PromptID != "generic.structured_events" {
t.Fatalf("unexpected prompt id: %q", res.PromptID)
}
if res.RunID == "" {
t.Fatal("expected run id")
}
if res.ProfileHash == "" {
t.Fatal("expected profile hash")
if res.PromptHash == "" {
t.Fatal("expected prompt hash")
}
if res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile version: %q", res.ProfileVersion)
if res.PromptVersion != "1.0.0" {
t.Fatalf("unexpected prompt version: %q", res.PromptVersion)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)

View File

@@ -17,7 +17,7 @@ type OutputRepairer interface {
type RepairRequest struct {
PreviousOutput string
ValidationErrors []string
Target domain.ModelTarget
Target domain.ExecutionTarget
Attempt int
MaxAttempts int
Mode domain.ValidationMode

View File

@@ -21,7 +21,7 @@ import (
var (
ErrInvalidRequest = errors.New("invalid run request")
ErrProfileLoad = errors.New("failed to load profile")
ErrProfileLoad = errors.New("failed to load prompt definition")
ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
@@ -67,8 +67,8 @@ func NewRunnerWithRepairer(
}
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if strings.TrimSpace(req.ProfileID) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
runID, err := newRunID()
@@ -78,17 +78,32 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
profileHash, err := hashProfile(prof)
promptDefinitionHash, err := hashPromptDefinition(def)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err)
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
}
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
effectiveContract := resolveOutputContract(prof, req.Validation)
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
}
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
}
if req.Execution == nil {
return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest)
}
effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution)
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
}
effectiveContract := resolveOutputContract(def, req.Validation)
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
inputHashes := make(map[string]string, len(req.Inputs))
@@ -104,12 +119,12 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
inputHashes[name] = art.Hash
}
renderedPrompt, err := r.renderer.Render(ctx, prof, resolvedInputs, req.Vars)
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
}
promptHash := hashRenderedPrompt(*renderedPrompt)
renderedPromptHash := hashRenderedPrompt(*renderedPrompt)
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: *renderedPrompt,
@@ -158,22 +173,23 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
ProfileID: prof.ID,
ProfileVersion: prof.Version,
ProfileHash: profileHash,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
ModelParams: effectiveModel,
InputHashes: inputHashes,
PromptHash: promptHash,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
PromptID: def.ID,
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
RenderedPromptHash: renderedPromptHash,
SelectedProfileID: selectedProfileID,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
EffectiveModelParams: effectiveModel,
InputHashes: inputHashes,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
@@ -209,7 +225,7 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema
}
func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget {
func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.ExecutionTarget) domain.ExecutionTarget {
if override == nil {
return base
}
@@ -233,13 +249,26 @@ func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) dom
if override.TimeoutSeconds != 0 {
out.TimeoutSeconds = override.TimeoutSeconds
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if len(override.ExtraParams) > 0 {
cp := make(map[string]string, len(override.ExtraParams))
for k, v := range override.ExtraParams {
cp[k] = v
}
out.ExtraParams = cp
}
return out
}
func resolveOutputContract(prof *domain.PromptProfile, override *domain.OutputContract) domain.OutputContract {
contract := prof.Validation
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation
if contract.Format == "" {
contract.Format = prof.OutputFormat
contract.Format = def.OutputFormat
}
if override != nil {
contract = *override
@@ -283,8 +312,8 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
}
}
func hashProfile(prof *domain.PromptProfile) (string, error) {
b, err := json.Marshal(prof)
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
b, err := json.Marshal(def)
if err != nil {
return "", err
}

View File

@@ -5,8 +5,6 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"os"
"path/filepath"
"regexp"
"testing"
@@ -14,20 +12,20 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
type fakeProfileRepo struct {
profile *domain.PromptProfile
type fakePromptRepo struct {
def *domain.PromptDefinition
err error
lastID string
lastVersion string
}
func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) {
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
f.lastID = id
f.lastVersion = version
if f.err != nil {
return nil, f.err
}
return f.profile, nil
return f.def, nil
}
type fakeArtifactReader struct {
@@ -40,8 +38,8 @@ func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (
return nil, err
}
if art, ok := f.artifactsByURI[ref.URI]; ok {
copy := *art
return &copy, nil
cp := *art
return &cp, nil
}
return nil, errors.New("artifact not found")
}
@@ -51,7 +49,7 @@ type fakeRenderer struct {
err error
}
func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
if f.err != nil {
return nil, f.err
}
@@ -73,15 +71,11 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do
}
type fakeValidator struct {
result domain.ValidationResult
err error
called bool
lastContract domain.OutputContract
result domain.ValidationResult
err error
}
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
f.called = true
f.lastContract = contract
if f.err != nil {
return domain.ValidationResult{}, f.err
}
@@ -92,12 +86,10 @@ type fakeRepairer struct {
responses []*domain.GenerateResponse
err error
calls int
lastReq RepairRequest
}
func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) {
f.calls++
f.lastReq = req
if f.err != nil {
return nil, f.err
}
@@ -112,156 +104,83 @@ func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.G
}
func TestRunnerRunSuccessful(t *testing.T) {
repo := &fakeProfileRepo{
profile: &domain.PromptProfile{
ID: "p1",
Version: "1.0.0",
OutputFormat: domain.FormatMarkdown,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep1",
Model: "model-default",
Temperature: 0.4,
MaxTokens: 200,
TopP: 0.9,
TimeoutSeconds: 90,
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationBasic,
},
},
}
repo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")},
"a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "System context"},
{Role: "user", Content: "Please summarize"},
}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{
Content: "# recap\n- item",
Usage: domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 20,
TotalTokens: 30,
},
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(repo, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p1",
ProfileVersion: "1.0.0",
PromptID: "p",
PromptVersion: "1",
ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
},
Model: &domain.ModelTarget{
Model: "model-override",
Temperature: 0,
MaxTokens: 0,
TimeoutSeconds: 0,
},
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
if res.PromptID != "p" || res.PromptVersion != "1" {
t.Fatalf("unexpected prompt metadata: %+v", res)
}
if res.SelectedProfileID != "exec" {
t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID)
}
if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
t.Fatalf("expected UUIDv4 run id, got %q", res.RunID)
t.Fatalf("invalid run id: %q", res.RunID)
}
if res.ProfileHash == "" {
t.Fatal("expected non-empty profile hash")
if res.PromptHash == "" || res.RenderedPromptHash == "" {
t.Fatal("expected prompt hashes")
}
if res.ModelName != "model-override" {
t.Fatalf("expected model override to apply, got %q", res.ModelName)
if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" {
t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams)
}
if res.Endpoint != "ep1" {
t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint)
}
if res.Artifact.ContentType != "text/markdown" {
t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType)
}
if string(res.Artifact.Body) != "# recap\n- item" {
t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body))
}
if res.RawOutput != "# recap\n- item" {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
if res.RawOutput != "# recap" {
t.Fatalf("expected raw output, got %q", res.RawOutput)
}
if res.Validation.Status != domain.ValidationSkipped {
t.Fatalf("expected skipped validation, got %q", res.Validation.Status)
}
if res.Validation.Mode != domain.ValidationBasic {
t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode)
}
if res.PromptHash == "" {
t.Fatal("expected non-empty prompt hash")
}
if res.Usage.TotalTokens != 30 {
t.Fatalf("expected usage to propagate, got %+v", res.Usage)
}
if res.StartTime.IsZero() || res.EndTime.IsZero() {
t.Fatal("expected start and end times")
}
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
t.Fatalf("unexpected transcript hash: %q", got)
}
if got := res.InputHashes["glossary"]; got != hashString("glossary body") {
t.Fatalf("unexpected glossary hash: %q", got)
}
if llmClient.lastReq.Target.Temperature != 0.4 {
t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature)
}
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" {
t.Fatalf("expected effective model params in result, got %+v", res.ModelParams)
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
}
func TestRunnerRunProfileLoadFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{err: errors.New("boom")},
&fakeArtifactReader{},
&fakeRenderer{},
&fakeLLM{},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"})
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
}
func TestMergeModelTargetTimeoutOverride(t *testing.T) {
base := domain.ModelTarget{TimeoutSeconds: 30}
override := &domain.ModelTarget{TimeoutSeconds: 75}
func TestRunnerRunMissingProfileSelection(t *testing.T) {
repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
got := mergeModelTarget(base, override)
if got.TimeoutSeconds != 75 {
t.Fatalf("expected timeout override to apply, got %d", got.TimeoutSeconds)
func TestRunnerRunMissingExecutionOverride(t *testing.T) {
repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected invalid request, got %v", err)
}
}
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -269,10 +188,10 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}},
})
if !errors.Is(err, ErrArtifactLoad) {
t.Fatalf("expected ErrArtifactLoad, got %v", err)
@@ -281,18 +200,17 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrPromptRender) {
t.Fatalf("expected ErrPromptRender, got %v", err)
@@ -301,405 +219,82 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{err: errors.New("llm failed")},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("expected ErrLLMGenerate, got %v", err)
}
}
func TestRunnerRunValidationFailureNonError(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{
Status: domain.ValidationFailed,
Mode: domain.ValidationBasic,
Errors: []string{"bad output"},
IsValid: false,
}}
func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected validation failed result, got %q", res.Validation.Status)
}
if res.RawOutput != "raw output" {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
}
if !validator.called {
t.Fatal("expected validator to be called")
if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" {
t.Fatalf("unexpected validation/raw output: %+v", res)
}
}
func TestRunnerRunValidationRuntimeError(t *testing.T) {
validator := &fakeValidator{err: errors.New("validator unavailable")}
runner := NewRunner(
&fakeProfileRepo{profile: minimalProfile()},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
}
func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) {
tmp := t.TempDir()
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
}
}`), 0644); err != nil {
t.Fatal(err)
}
runner := NewRunner(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
Format: domain.FormatJSON,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
validate.NewStandardValidator(tmp),
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected validation failed, got %q", res.Validation.Status)
}
if res.RawOutput != `{"count":1}` {
t.Fatalf("expected raw output preserved, got %q", res.RawOutput)
}
}
func TestRunnerRunNoRepairWhenDisabled(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"ok":true}`}},
}
func TestRunnerRunRepairBounded(t *testing.T) {
repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 0,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 0 {
t.Fatalf("expected no repair calls, got %d", repairer.calls)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected failed validation, got %q", res.Validation.Status)
}
if res.RawOutput != `{"broken":` {
t.Fatalf("expected original output preserved, got %q", res.RawOutput)
}
if res.Validation.RepairAttempts != 0 {
t.Fatalf("expected repair attempts 0, got %d", res.Validation.RepairAttempts)
}
}
func TestRunnerRunSuccessfulRepairAfterInvalidJSON(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"ok":true}`, Usage: domain.TokenUsage{TotalTokens: 5}}},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`, Usage: domain.TokenUsage{TotalTokens: 3}}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 1 {
t.Fatalf("expected one repair call, got %d", repairer.calls)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"ok":true}` {
t.Fatalf("expected repaired output, got %q", res.RawOutput)
}
}
func TestRunnerRunSuccessfulRepairAfterSchemaFailure(t *testing.T) {
tmp := t.TempDir()
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
}
}`), 0644); err != nil {
t.Fatal(err)
}
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"name":"eris"}`}},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}},
validate.NewStandardValidator(tmp),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationPassed {
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"name":"eris"}` {
t.Fatalf("expected repaired output, got %q", res.RawOutput)
}
}
func TestRunnerRunFailedRepairPreservesRawOutputAndErrors(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{
{Content: `{"repair1":`},
{Content: `{"repair2":`},
},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 2,
},
}},
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
validate.NewStandardValidator(t.TempDir()),
validate.NewStandardValidator("."),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
PromptID: "p",
ProfileID: "exec",
Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"},
Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Validation.Status != domain.ValidationFailed {
t.Fatalf("expected failed validation, got %q", res.Validation.Status)
}
if len(res.Validation.Errors) == 0 {
t.Fatal("expected validation errors after failed repair")
}
if res.Validation.RepairAttempts != 2 {
t.Fatalf("expected repair attempts 2, got %d", res.Validation.RepairAttempts)
}
if res.RawOutput != `{"repair2":` {
t.Fatalf("expected final repaired output preserved, got %q", res.RawOutput)
if repairer.calls != 1 || res.Validation.RepairAttempts != 1 {
t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts)
}
}
func TestRunnerRunRepairAttemptsBounded(t *testing.T) {
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{
{Content: `{"repair1":`},
{Content: `{"repair2":`},
{Content: `{"repair3":`},
},
}
runner := NewRunnerWithRepairer(
&fakeProfileRepo{profile: &domain.PromptProfile{
ID: "p-json",
Version: "1",
OutputFormat: domain.FormatJSON,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
Validation: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
Format: domain.FormatJSON,
RepairAttempts: 1,
},
}},
&fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}},
validate.NewStandardValidator(t.TempDir()),
repairer,
)
res, err := runner.Run(context.Background(), domain.RunRequest{
ProfileID: "p-json",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if repairer.calls != 1 {
t.Fatalf("expected repair calls bounded to 1, got %d", repairer.calls)
}
if res.Validation.RepairAttempts != 1 {
t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts)
}
}
func minimalProfile() *domain.PromptProfile {
return &domain.PromptProfile{
ID: "p",
Version: "1",
OutputFormat: domain.FormatText,
ModelDefaults: domain.ModelTarget{
Endpoint: "ep",
Model: "m",
},
func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition {
return &domain.PromptDefinition{
ID: "p",
Version: "1",
DefaultProfile: "exec",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
OutputFormat: format,
Validation: domain.OutputContract{
ValidationMode: domain.ValidationBasic,
ValidationMode: mode,
RepairAttempts: attempts,
Format: format,
},
}
}