Relaxed CLI requirements when defaults are specified in the profile or application defaults

This commit is contained in:
2026-05-05 08:41:07 -05:00
parent 281202e313
commit ca4d939fdc
13 changed files with 311 additions and 74 deletions

View File

@@ -27,11 +27,12 @@ go run ./cmd/scriptorium run \
--profile-id generic.markdown_summary \ --profile-id generic.markdown_summary \
--input transcript=./examples/fixtures/transcript.md \ --input transcript=./examples/fixtures/transcript.md \
--input glossary=./examples/fixtures/glossary.yml \ --input glossary=./examples/fixtures/glossary.yml \
--llm-base-url http://localhost:8000/v1 \
--model gpt-4o-mini \
--out ./out.md --out ./out.md
``` ```
This relies on `model_defaults.endpoint` and `model_defaults.model` in the selected profile.
You can override either at runtime with `--llm-base-url` and/or `--model`.
For schema-validated JSON output: For schema-validated JSON output:
```bash ```bash
@@ -78,17 +79,23 @@ Response shape:
- `metadata` - `metadata`
- `raw_model_output` - `raw_model_output`
`metadata` includes stable audit fields such as run/profile IDs, profile hash, effective model params, prompt/input hashes, timing, usage, and validation summary.
## Add a New Prompt Profile ## Add a New Prompt Profile
1. Add a YAML file under `profiles/` with: 1. Add a YAML file under `profiles/` with:
- `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation` - `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation`
- optional model timeout via `model_defaults.timeout_seconds` (per-run LLM timeout override) - optional model timeout via `model_defaults.timeout_seconds` (per-run LLM timeout override)
2. Use template helpers such as `{{input "transcript"}}`. 2. Ensure endpoint/model are available from either:
3. For structured JSON output, set: - profile defaults (`model_defaults.endpoint`, `model_defaults.model`), or
- request overrides (`--llm-base-url`, `--model`, or HTTP `model.endpoint`/`model.model`).
3. Use template helpers such as `{{input "transcript"}}` and template vars like `{{.session_date}}`.
4. For structured JSON output, set:
- `output_format: json` - `output_format: json`
- `validation.validation_mode: json_schema` - `validation.validation_mode: json_schema`
- `validation.schema_path: <schema file>` - `validation.schema_path: <schema file>`
4. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve. 5. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve.
6. `validation.repair_attempts` is bounded and applies only to structured modes (`json`, `json_schema`).
## Validation Behavior ## Validation Behavior

View File

@@ -527,33 +527,30 @@ The response should include:
- artifact - artifact
- validation - validation
- metadata - metadata
- raw_model_output, optionally controlled by request or config - raw_model_output
- error details, if applicable - error details, if applicable
The HTTP layer should not contain business logic. The HTTP layer should not contain business logic.
## CLI ## CLI
The CLI should also be thin. The CLI should be thin and call the same core use case as HTTP.
It should support local development and pipeline usage. Current command surface:
Suggested commands:
- scriptorium run - scriptorium run
- scriptorium profiles list - scriptorium serve
- scriptorium profiles inspect
The run command should accept: The `run` command should accept:
- profile ID - profile ID
- input mappings - input mappings
- variable mappings - variable mappings
- output path, optional - output path, optional
- profile directory - profile directory
- config path - optional model/endpoint overrides
The CLI should call the same use case used by the HTTP API. If model/endpoint overrides are omitted, profile model defaults should be used.
## Configuration ## Configuration

View File

@@ -41,6 +41,11 @@ type runConfig struct {
maxTokens int maxTokens int
schemaDir string schemaDir string
timeout time.Duration timeout time.Duration
llmBaseURLSet bool
modelSet bool
temperatureSet bool
maxTokensSet bool
} }
type serveConfig struct { type serveConfig struct {
@@ -127,16 +132,21 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
validate.NewStandardValidator(cfg.schemaDir), validate.NewStandardValidator(cfg.schemaDir),
) )
res, runErr := runner.Run(context.Background(), domain.RunRequest{ var modelOverride *domain.ModelTarget
ProfileID: cfg.profileID, if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet {
Inputs: inputs, modelOverride = &domain.ModelTarget{
Vars: varMappings,
Model: &domain.ModelTarget{
Endpoint: cfg.llmBaseURL, Endpoint: cfg.llmBaseURL,
Model: cfg.model, Model: cfg.model,
Temperature: cfg.temperature, Temperature: cfg.temperature,
MaxTokens: cfg.maxTokens, MaxTokens: cfg.maxTokens,
}, }
}
res, runErr := runner.Run(context.Background(), domain.RunRequest{
ProfileID: cfg.profileID,
Inputs: inputs,
Vars: varMappings,
Model: modelOverride,
}) })
if runErr != nil { if runErr != nil {
fmt.Fprintf(stderr, "run error: %v\n", runErr) fmt.Fprintf(stderr, "run error: %v\n", runErr)
@@ -227,18 +237,15 @@ func parseRunArgs(args []string) (*runConfig, error) {
if len(cfg.inputRaw) == 0 { if len(cfg.inputRaw) == 0 {
return nil, errors.New("at least one --input is required") return nil, errors.New("at least one --input is required")
} }
if strings.TrimSpace(cfg.llmBaseURL) == "" {
return nil, errors.New("--llm-base-url is required")
}
if strings.TrimSpace(cfg.model) == "" {
return nil, errors.New("--model is required")
}
cfg.profileDir = filepath.Clean(cfg.profileDir) cfg.profileDir = filepath.Clean(cfg.profileDir)
cfg.schemaDir = filepath.Clean(cfg.schemaDir) cfg.schemaDir = filepath.Clean(cfg.schemaDir)
if cfg.outputPath != "" { if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath) cfg.outputPath = filepath.Clean(cfg.outputPath)
} }
cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url")
cfg.modelSet = flagWasSet(fs, "model")
cfg.temperatureSet = flagWasSet(fs, "temperature")
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
return cfg, nil return cfg, nil
} }
@@ -312,6 +319,16 @@ func parseMapping(value string) (string, string, error) {
return key, val, nil return key, val, nil
} }
func flagWasSet(fs *flag.FlagSet, name string) bool {
set := false
fs.Visit(func(f *flag.Flag) {
if f.Name == name {
set = true
}
})
return set
}
func writeOutput(stdout io.Writer, outputPath string, body []byte) error { func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
if outputPath == "" { if outputPath == "" {
_, err := stdout.Write(body) _, err := stdout.Write(body)
@@ -351,6 +368,6 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
func printUsage(w io.Writer) { func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|serve> ...") fmt.Fprintln(w, "usage: scriptorium <run|serve> ...")
fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] --llm-base-url URL --model NAME [--var k=v] [--out path] [--timeout 10m]") fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]") fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]")
} }

View File

@@ -65,15 +65,19 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("expected missing --input error") t.Fatal("expected missing --input error")
} }
}
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--model", "m"}) func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
if err == nil { cfg, err := parseRunArgs([]string{
t.Fatal("expected missing --llm-base-url error") "--profile-dir", "./profiles",
"--profile-id", "p",
"--input", "a=b",
})
if err != nil {
t.Fatalf("expected valid args without model/base url, got %v", err)
} }
if cfg.llmBaseURL != "" || cfg.model != "" {
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1"}) t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL)
if err == nil {
t.Fatal("expected missing --model error")
} }
} }
@@ -167,4 +171,31 @@ func TestRunCommandVarsOptional(t *testing.T) {
if !strings.Contains(stderr.String(), "llm client error") { if !strings.Contains(stderr.String(), "llm client error") {
t.Fatalf("expected llm client error after parsing succeeds, got stderr=%q", stderr.String()) t.Fatalf("expected llm client error after parsing succeeds, got stderr=%q", stderr.String())
} }
if stdout.Len() != 0 {
t.Fatalf("expected no stdout output on error, got %q", stdout.String())
}
}
func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
t.Fatalf("unexpected writeOutput error: %v", err)
}
printSummary(&stderr, &domain.RunResult{
ProfileID: "p",
ProfileVersion: "1",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
PromptHash: "h",
InputHashes: map[string]string{"in": "x"},
})
if stdout.String() != "artifact-body" {
t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "profile=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String())
}
} }

View File

@@ -44,15 +44,31 @@ type artifactDTO struct {
} }
type metadataDTO struct { type metadataDTO struct {
ProfileID string `json:"profile_id"` RunID string `json:"run_id"`
ProfileVersion string `json:"profile_version"` ProfileID string `json:"profile_id"`
ModelName string `json:"model_name"` ProfileVersion string `json:"profile_version"`
Endpoint string `json:"endpoint"` ProfileHash string `json:"profile_hash"`
InputHashes map[string]string `json:"input_hashes"` ModelName string `json:"model_name"`
PromptHash string `json:"prompt_hash"` Endpoint string `json:"endpoint"`
Usage tokenUsageDTO `json:"usage"` ModelParams modelParamsDTO `json:"model_params"`
StartTime time.Time `json:"start_time"` InputHashes map[string]string `json:"input_hashes"`
EndTime time.Time `json:"end_time"` PromptHash string `json:"prompt_hash"`
Usage tokenUsageDTO `json:"usage"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
DurationMS int64 `json:"duration_ms"`
ValidationMode string `json:"validation_mode"`
ValidationStatus string `json:"validation_status"`
RepairAttemptsUsed int `json:"repair_attempts_used"`
}
type modelParamsDTO struct {
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
} }
type tokenUsageDTO struct { type tokenUsageDTO struct {

View File

@@ -4,7 +4,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"net/http" "net/http"
"strings" "strings"
@@ -37,7 +36,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var req runRequestDTO var req runRequestDTO
if err := json.NewDecoder(r.Body).Decode(&req); err != nil { if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", fmt.Sprintf("invalid JSON request: %v", err)) writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
return return
} }
@@ -79,8 +78,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Model: model, Model: model,
}) })
if err != nil { if err != nil {
status, code := mapRunError(err) status, code, message := mapRunError(err)
writeError(w, status, code, err.Error()) writeError(w, status, code, message)
return return
} }
@@ -95,19 +94,33 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}, },
Validation: mapValidation(res.Validation), Validation: mapValidation(res.Validation),
Metadata: metadataDTO{ Metadata: metadataDTO{
RunID: res.RunID,
ProfileID: res.ProfileID, ProfileID: res.ProfileID,
ProfileVersion: res.ProfileVersion, ProfileVersion: res.ProfileVersion,
ProfileHash: res.ProfileHash,
ModelName: res.ModelName, ModelName: res.ModelName,
Endpoint: res.Endpoint, Endpoint: res.Endpoint,
InputHashes: res.InputHashes, ModelParams: modelParamsDTO{
PromptHash: res.PromptHash, Endpoint: res.ModelParams.Endpoint,
Model: res.ModelParams.Model,
Temperature: res.ModelParams.Temperature,
MaxTokens: res.ModelParams.MaxTokens,
TopP: res.ModelParams.TopP,
TimeoutSeconds: res.ModelParams.TimeoutSeconds,
},
InputHashes: res.InputHashes,
PromptHash: res.PromptHash,
Usage: tokenUsageDTO{ Usage: tokenUsageDTO{
PromptTokens: res.Usage.PromptTokens, PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens, CompletionTokens: res.Usage.CompletionTokens,
TotalTokens: res.Usage.TotalTokens, TotalTokens: res.Usage.TotalTokens,
}, },
StartTime: res.StartTime, StartTime: res.StartTime,
EndTime: res.EndTime, EndTime: res.EndTime,
DurationMS: res.Duration.Milliseconds(),
ValidationMode: string(res.Validation.Mode),
ValidationStatus: string(res.Validation.Status),
RepairAttemptsUsed: res.Validation.RepairAttempts,
}, },
RawModelOutput: res.RawOutput, RawModelOutput: res.RawOutput,
}) })
@@ -124,24 +137,24 @@ func mapValidation(v domain.ValidationResult) validationDTO {
} }
} }
func mapRunError(err error) (int, string) { func mapRunError(err error) (int, string, string) {
switch { switch {
case errors.Is(err, profile.ErrProfileNotFound): case errors.Is(err, profile.ErrProfileNotFound):
return http.StatusNotFound, "profile_not_found" return http.StatusNotFound, "profile_not_found", "profile not found"
case errors.Is(err, usecase.ErrInvalidRequest): case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request" return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, usecase.ErrProfileLoad): case errors.Is(err, usecase.ErrProfileLoad):
return http.StatusBadRequest, "profile_load_failed" return http.StatusBadRequest, "profile_load_failed", "failed to load profile"
case errors.Is(err, usecase.ErrArtifactLoad): case errors.Is(err, usecase.ErrArtifactLoad):
return http.StatusBadRequest, "artifact_read_failed" return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
case errors.Is(err, usecase.ErrPromptRender): case errors.Is(err, usecase.ErrPromptRender):
return http.StatusBadRequest, "prompt_render_failed" return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
case errors.Is(err, usecase.ErrLLMGenerate): case errors.Is(err, usecase.ErrLLMGenerate):
return http.StatusBadGateway, "llm_failed" return http.StatusBadGateway, "llm_failed", "model generation request failed"
case errors.Is(err, usecase.ErrValidation): case errors.Is(err, usecase.ErrValidation):
return http.StatusInternalServerError, "validation_runtime_failed" return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
default: default:
return http.StatusInternalServerError, "internal_error" return http.StatusInternalServerError, "internal_error", "internal server error"
} }
} }

View File

@@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"time" "time"
@@ -33,6 +34,7 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
start := time.Now().UTC() start := time.Now().UTC()
end := start.Add(2 * time.Second) end := start.Add(2 * time.Second)
r := &fakeRunner{result: &domain.RunResult{ r := &fakeRunner{result: &domain.RunResult{
RunID: "11111111-1111-4111-8111-111111111111",
Artifact: domain.Artifact{ Artifact: domain.Artifact{
Name: "output", Name: "output",
ContentType: "text/plain", ContentType: "text/plain",
@@ -43,14 +45,24 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true}, Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
ProfileID: "p1", ProfileID: "p1",
ProfileVersion: "1.0.0", ProfileVersion: "1.0.0",
ProfileHash: "phash",
ModelName: "m1", ModelName: "m1",
Endpoint: "http://llm/v1", Endpoint: "http://llm/v1",
InputHashes: map[string]string{"transcript": "h1"}, ModelParams: domain.ModelTarget{
PromptHash: "ph", Endpoint: "http://llm/v1",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, Model: "m1",
StartTime: start, Temperature: 0.2,
EndTime: end, MaxTokens: 42,
RawOutput: "hello", TopP: 0.9,
TimeoutSeconds: 120,
},
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}} }}
h := NewHandler(r) h := NewHandler(r)
@@ -86,10 +98,26 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
t.Fatalf("expected validation.status passed, got %#v", validation["status"]) t.Fatalf("expected validation.status passed, got %#v", validation["status"])
} }
metadata := resp["metadata"].(map[string]any) metadata := resp["metadata"].(map[string]any)
if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" {
t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"])
}
if metadata["profile_hash"] != "phash" {
t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"])
}
usage := metadata["usage"].(map[string]any) usage := metadata["usage"].(map[string]any)
if usage["total_tokens"] != float64(3) { if usage["total_tokens"] != float64(3) {
t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"]) t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"])
} }
if metadata["duration_ms"] != float64(2000) {
t.Fatalf("expected duration_ms=2000, got %#v", metadata["duration_ms"])
}
if metadata["validation_mode"] != "basic" || metadata["validation_status"] != "passed" {
t.Fatalf("unexpected validation metadata: mode=%#v status=%#v", metadata["validation_mode"], metadata["validation_status"])
}
modelParams := metadata["model_params"].(map[string]any)
if modelParams["model"] != "m1" {
t.Fatalf("unexpected model_params.model: %#v", modelParams["model"])
}
if resp["raw_model_output"] != "hello" { if resp["raw_model_output"] != "hello" {
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"]) t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
} }
@@ -153,6 +181,20 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
if w.Code != tc.status { if w.Code != tc.status {
t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String()) t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String())
} }
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if _, ok := errBody["code"].(string); !ok {
t.Fatalf("expected error code string, got %#v", errBody["code"])
}
if msg, ok := errBody["message"].(string); !ok || msg == "" {
t.Fatalf("expected non-empty error message, got %#v", errBody["message"])
}
if strings.Contains(w.Body.String(), "read failed") || strings.Contains(w.Body.String(), "render failed") || strings.Contains(w.Body.String(), "llm failed") {
t.Fatalf("expected response to avoid leaking internal cause details, got %s", w.Body.String())
}
}) })
} }
} }

View File

@@ -54,18 +54,22 @@ type RunRequest struct {
// RunResult represents the complete result of a prompt execution run. // RunResult represents the complete result of a prompt execution run.
type RunResult struct { type RunResult struct {
RunID string
Artifact Artifact Artifact Artifact
RawOutput string RawOutput string
Validation ValidationResult Validation ValidationResult
ProfileID string ProfileID string
ProfileVersion string ProfileVersion string
ProfileHash string
ModelName string ModelName string
Endpoint string Endpoint string
ModelParams ModelTarget
InputHashes map[string]string InputHashes map[string]string
PromptHash string PromptHash string
Usage TokenUsage Usage TokenUsage
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
Duration time.Duration
Error error Error error
} }

View File

@@ -40,11 +40,11 @@ type OpenAICompatibleClient struct {
} }
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) { func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
if strings.TrimSpace(cfg.BaseURL) == "" { baseURL := strings.TrimSpace(cfg.BaseURL)
return nil, fmt.Errorf("%w: base URL is required", ErrInvalidConfig) if baseURL != "" {
} if _, err := url.ParseRequestURI(baseURL); err != nil {
if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil { return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err) }
} }
timeout := cfg.Timeout timeout := cfg.Timeout
@@ -63,7 +63,7 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
} }
return &OpenAICompatibleClient{ return &OpenAICompatibleClient{
baseURL: strings.TrimRight(cfg.BaseURL, "/"), baseURL: strings.TrimRight(baseURL, "/"),
apiKey: cfg.APIKey, apiKey: cfg.APIKey,
defaultModel: cfg.Model, defaultModel: cfg.Model,
timeout: timeout, timeout: timeout,
@@ -88,6 +88,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
if endpoint == "" { if endpoint == "" {
endpoint = c.baseURL endpoint = c.baseURL
} }
if endpoint == "" {
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
}
endpoint = strings.TrimRight(endpoint, "/") + "/chat/completions" endpoint = strings.TrimRight(endpoint, "/") + "/chat/completions"
wireReq := openAIChatRequest{ wireReq := openAIChatRequest{

View File

@@ -336,3 +336,45 @@ func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
} }
func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "",
Model: "m",
})
if err != nil {
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{Endpoint: "http://localhost:9999/v1"},
})
if err == nil {
t.Fatal("expected request failure due to unreachable endpoint")
}
if !errors.Is(err, ErrRequestFailed) {
t.Fatalf("expected ErrRequestFailed with request endpoint override, got %v", err)
}
}
func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "",
Model: "m",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{},
})
if err == nil {
t.Fatal("expected endpoint-required error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}

View File

@@ -63,6 +63,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.ProfileID != "generic.structured_events" { if res.ProfileID != "generic.structured_events" {
t.Fatalf("unexpected profile id: %q", res.ProfileID) t.Fatalf("unexpected profile id: %q", res.ProfileID)
} }
if res.RunID == "" {
t.Fatal("expected run id")
}
if res.ProfileHash == "" {
t.Fatal("expected profile hash")
}
if res.ProfileVersion != "1.0.0" { if res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile version: %q", res.ProfileVersion) t.Fatalf("unexpected profile version: %q", res.ProfileVersion)
} }
@@ -96,4 +102,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.EndTime.Before(res.StartTime) { if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
} }
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
} }

View File

@@ -2,8 +2,10 @@ package usecase
import ( import (
"context" "context"
"crypto/rand"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
@@ -69,12 +71,21 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
} }
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
}
start := time.Now().UTC() start := time.Now().UTC()
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion) prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
} }
profileHash, err := hashProfile(prof)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err)
}
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model) effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
effectiveContract := resolveOutputContract(prof, req.Validation) effectiveContract := resolveOutputContract(prof, req.Validation)
@@ -147,18 +158,22 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
end := time.Now().UTC() end := time.Now().UTC()
return &domain.RunResult{ return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact, Artifact: outputArtifact,
RawOutput: genResp.Content, RawOutput: genResp.Content,
Validation: validationResult, Validation: validationResult,
ProfileID: prof.ID, ProfileID: prof.ID,
ProfileVersion: prof.Version, ProfileVersion: prof.Version,
ProfileHash: profileHash,
ModelName: effectiveModel.Model, ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint, Endpoint: effectiveModel.Endpoint,
ModelParams: effectiveModel,
InputHashes: inputHashes, InputHashes: inputHashes,
PromptHash: promptHash, PromptHash: promptHash,
Usage: genResp.Usage, Usage: genResp.Usage,
StartTime: start, StartTime: start,
EndTime: end, EndTime: end,
Duration: end.Sub(start),
}, nil }, nil
} }
@@ -267,3 +282,31 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
Hash: hex.EncodeToString(hash[:]), Hash: hex.EncodeToString(hash[:]),
} }
} }
func hashProfile(prof *domain.PromptProfile) (string, error) {
b, err := json.Marshal(prof)
if err != nil {
return "", err
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:]), nil
}
func newRunID() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
// UUID v4 (RFC 4122 variant).
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
b[0:4],
b[4:6],
b[6:8],
b[8:10],
b[10:16],
), nil
}

View File

@@ -7,6 +7,7 @@ import (
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"testing" "testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -172,6 +173,12 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" { if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion) t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
} }
if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
t.Fatalf("expected UUIDv4 run id, got %q", res.RunID)
}
if res.ProfileHash == "" {
t.Fatal("expected non-empty profile hash")
}
if res.ModelName != "model-override" { if res.ModelName != "model-override" {
t.Fatalf("expected model override to apply, got %q", res.ModelName) t.Fatalf("expected model override to apply, got %q", res.ModelName)
} }
@@ -205,6 +212,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.EndTime.Before(res.StartTime) { if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
} }
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
if got := res.InputHashes["transcript"]; got != hashString("transcript body") { if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
t.Fatalf("unexpected transcript hash: %q", got) t.Fatalf("unexpected transcript hash: %q", got)
@@ -219,6 +229,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 { if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds) t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds)
} }
if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" {
t.Fatalf("expected effective model params in result, got %+v", res.ModelParams)
}
} }
func TestRunnerRunProfileLoadFailure(t *testing.T) { func TestRunnerRunProfileLoadFailure(t *testing.T) {