eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Relaxed CLI requirements when defaults are specified in the profile or application defaults

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-05 08:41:07 -05:00
parent 281202e313
commit ca4d939fdc
13 changed files with 311 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
README.md
View File

@@ -27,11 +27,12 @@ go run ./cmd/scriptorium run \
--profile-id generic.markdown_summary \
--input transcript=./examples/fixtures/transcript.md \
--input glossary=./examples/fixtures/glossary.yml \
--llm-base-url http://localhost:8000/v1 \
--model gpt-4o-mini \
--out ./out.md
```
This relies on `model_defaults.endpoint` and `model_defaults.model` in the selected profile.
You can override either at runtime with `--llm-base-url` and/or `--model`.
For schema-validated JSON output:
```bash
@@ -78,17 +79,23 @@ Response shape:
- `metadata`
- `raw_model_output`
`metadata` includes stable audit fields such as run/profile IDs, profile hash, effective model params, prompt/input hashes, timing, usage, and validation summary.
## Add a New Prompt Profile
1. Add a YAML file under `profiles/` with:
- `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation`
- optional model timeout via `model_defaults.timeout_seconds` (per-run LLM timeout override)
2. Use template helpers such as `{{input "transcript"}}`.
3. For structured JSON output, set:
2. Ensure endpoint/model are available from either:
- profile defaults (`model_defaults.endpoint`, `model_defaults.model`), or
- request overrides (`--llm-base-url`, `--model`, or HTTP `model.endpoint`/`model.model`).
3. Use template helpers such as `{{input "transcript"}}` and template vars like `{{.session_date}}`.
4. For structured JSON output, set:
- `output_format: json`
- `validation.validation_mode: json_schema`
- `validation.schema_path: <schema file>`
4. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve.
5. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve.
6. `validation.repair_attempts` is bounded and applies only to structured modes (`json`, `json_schema`).
## Validation Behavior

17
architecture.md
View File

@@ -527,33 +527,30 @@ The response should include:
- artifact
- validation
- metadata
- raw_model_output, optionally controlled by request or config
- raw_model_output
- error details, if applicable
The HTTP layer should not contain business logic.
## CLI
The CLI should also be thin.
The CLI should be thin and call the same core use case as HTTP.
It should support local development and pipeline usage.
Suggested commands:
Current command surface:
- scriptorium run
- scriptorium profiles list
- scriptorium profiles inspect
- scriptorium serve
The run command should accept:
The `run` command should accept:
- profile ID
- input mappings
- variable mappings
- output path, optional
- profile directory
- config path
- optional model/endpoint overrides
The CLI should call the same use case used by the HTTP API.
If model/endpoint overrides are omitted, profile model defaults should be used.
## Configuration

45
internal/adapter/cli/run.go
View File

@@ -41,6 +41,11 @@ type runConfig struct {
maxTokens int
schemaDir string
timeout time.Duration
llmBaseURLSet bool
modelSet bool
temperatureSet bool
maxTokensSet bool
}
type serveConfig struct {
@@ -127,16 +132,21 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
validate.NewStandardValidator(cfg.schemaDir),
)
res, runErr := runner.Run(context.Background(), domain.RunRequest{
ProfileID: cfg.profileID,
Inputs: inputs,
Vars: varMappings,
Model: &domain.ModelTarget{
var modelOverride *domain.ModelTarget
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet {
modelOverride = &domain.ModelTarget{
Endpoint: cfg.llmBaseURL,
Model: cfg.model,
Temperature: cfg.temperature,
MaxTokens: cfg.maxTokens,
},
}
}
res, runErr := runner.Run(context.Background(), domain.RunRequest{
ProfileID: cfg.profileID,
Inputs: inputs,
Vars: varMappings,
Model: modelOverride,
})
if runErr != nil {
fmt.Fprintf(stderr, "run error: %v\n", runErr)
@@ -227,18 +237,15 @@ func parseRunArgs(args []string) (*runConfig, error) {
if len(cfg.inputRaw) == 0 {
return nil, errors.New("at least one --input is required")
}
if strings.TrimSpace(cfg.llmBaseURL) == "" {
return nil, errors.New("--llm-base-url is required")
}
if strings.TrimSpace(cfg.model) == "" {
return nil, errors.New("--model is required")
}
cfg.profileDir = filepath.Clean(cfg.profileDir)
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url")
cfg.modelSet = flagWasSet(fs, "model")
cfg.temperatureSet = flagWasSet(fs, "temperature")
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
return cfg, nil
}
@@ -312,6 +319,16 @@ func parseMapping(value string) (string, string, error) {
return key, val, nil
}
func flagWasSet(fs *flag.FlagSet, name string) bool {
set := false
fs.Visit(func(f *flag.Flag) {
if f.Name == name {
set = true
}
})
return set
}
func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
if outputPath == "" {
_, err := stdout.Write(body)
@@ -351,6 +368,6 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|serve> ...")
fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] --llm-base-url URL --model NAME [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]")
}

45
internal/adapter/cli/run_test.go
View File

@@ -65,15 +65,19 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
if err == nil {
t.Fatal("expected missing --input error")
}
}
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--model", "m"})
if err == nil {
t.Fatal("expected missing --llm-base-url error")
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
cfg, err := parseRunArgs([]string{
"--profile-dir", "./profiles",
"--profile-id", "p",
"--input", "a=b",
})
if err != nil {
t.Fatalf("expected valid args without model/base url, got %v", err)
}
_, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1"})
if err == nil {
t.Fatal("expected missing --model error")
if cfg.llmBaseURL != "" || cfg.model != "" {
t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL)
}
}
@@ -167,4 +171,31 @@ func TestRunCommandVarsOptional(t *testing.T) {
if !strings.Contains(stderr.String(), "llm client error") {
t.Fatalf("expected llm client error after parsing succeeds, got stderr=%q", stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected no stdout output on error, got %q", stdout.String())
}
}
func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
t.Fatalf("unexpected writeOutput error: %v", err)
}
printSummary(&stderr, &domain.RunResult{
ProfileID: "p",
ProfileVersion: "1",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
PromptHash: "h",
InputHashes: map[string]string{"in": "x"},
})
if stdout.String() != "artifact-body" {
t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "profile=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String())
}
}

34
internal/adapter/http/dto.go
View File

@@ -44,15 +44,31 @@ type artifactDTO struct {
}
type metadataDTO struct {
ProfileID string `json:"profile_id"`
ProfileVersion string `json:"profile_version"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
InputHashes map[string]string `json:"input_hashes"`
PromptHash string `json:"prompt_hash"`
Usage tokenUsageDTO `json:"usage"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
RunID string `json:"run_id"`
ProfileID string `json:"profile_id"`
ProfileVersion string `json:"profile_version"`
ProfileHash string `json:"profile_hash"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
ModelParams modelParamsDTO `json:"model_params"`
InputHashes map[string]string `json:"input_hashes"`
PromptHash string `json:"prompt_hash"`
Usage tokenUsageDTO `json:"usage"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
DurationMS int64 `json:"duration_ms"`
ValidationMode string `json:"validation_mode"`
ValidationStatus string `json:"validation_status"`
RepairAttemptsUsed int `json:"repair_attempts_used"`
}
type modelParamsDTO struct {
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
}
type tokenUsageDTO struct {

47
internal/adapter/http/handler.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
@@ -37,7 +36,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var req runRequestDTO
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", fmt.Sprintf("invalid JSON request: %v", err))
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
return
}
@@ -79,8 +78,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Model: model,
})
if err != nil {
status, code := mapRunError(err)
writeError(w, status, code, err.Error())
status, code, message := mapRunError(err)
writeError(w, status, code, message)
return
}
@@ -95,19 +94,33 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
},
Validation: mapValidation(res.Validation),
Metadata: metadataDTO{
RunID: res.RunID,
ProfileID: res.ProfileID,
ProfileVersion: res.ProfileVersion,
ProfileHash: res.ProfileHash,
ModelName: res.ModelName,
Endpoint: res.Endpoint,
InputHashes: res.InputHashes,
PromptHash: res.PromptHash,
ModelParams: modelParamsDTO{
Endpoint: res.ModelParams.Endpoint,
Model: res.ModelParams.Model,
Temperature: res.ModelParams.Temperature,
MaxTokens: res.ModelParams.MaxTokens,
TopP: res.ModelParams.TopP,
TimeoutSeconds: res.ModelParams.TimeoutSeconds,
},
InputHashes: res.InputHashes,
PromptHash: res.PromptHash,
Usage: tokenUsageDTO{
PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens,
TotalTokens: res.Usage.TotalTokens,
},
StartTime: res.StartTime,
EndTime: res.EndTime,
StartTime: res.StartTime,
EndTime: res.EndTime,
DurationMS: res.Duration.Milliseconds(),
ValidationMode: string(res.Validation.Mode),
ValidationStatus: string(res.Validation.Status),
RepairAttemptsUsed: res.Validation.RepairAttempts,
},
RawModelOutput: res.RawOutput,
})
@@ -124,24 +137,24 @@ func mapValidation(v domain.ValidationResult) validationDTO {
}
}
func mapRunError(err error) (int, string) {
func mapRunError(err error) (int, string, string) {
switch {
case errors.Is(err, profile.ErrProfileNotFound):
return http.StatusNotFound, "profile_not_found"
return http.StatusNotFound, "profile_not_found", "profile not found"
case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request"
return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, usecase.ErrProfileLoad):
return http.StatusBadRequest, "profile_load_failed"
return http.StatusBadRequest, "profile_load_failed", "failed to load profile"
case errors.Is(err, usecase.ErrArtifactLoad):
return http.StatusBadRequest, "artifact_read_failed"
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
case errors.Is(err, usecase.ErrPromptRender):
return http.StatusBadRequest, "prompt_render_failed"
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
case errors.Is(err, usecase.ErrLLMGenerate):
return http.StatusBadGateway, "llm_failed"
return http.StatusBadGateway, "llm_failed", "model generation request failed"
case errors.Is(err, usecase.ErrValidation):
return http.StatusInternalServerError, "validation_runtime_failed"
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
default:
return http.StatusInternalServerError, "internal_error"
return http.StatusInternalServerError, "internal_error", "internal server error"
}
}

54
internal/adapter/http/handler_test.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@@ -33,6 +34,7 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
start := time.Now().UTC()
end := start.Add(2 * time.Second)
r := &fakeRunner{result: &domain.RunResult{
RunID: "11111111-1111-4111-8111-111111111111",
Artifact: domain.Artifact{
Name: "output",
ContentType: "text/plain",
@@ -43,14 +45,24 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
ProfileID: "p1",
ProfileVersion: "1.0.0",
ProfileHash: "phash",
ModelName: "m1",
Endpoint: "http://llm/v1",
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
RawOutput: "hello",
ModelParams: domain.ModelTarget{
Endpoint: "http://llm/v1",
Model: "m1",
Temperature: 0.2,
MaxTokens: 42,
TopP: 0.9,
TimeoutSeconds: 120,
},
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}}
h := NewHandler(r)
@@ -86,10 +98,26 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
t.Fatalf("expected validation.status passed, got %#v", validation["status"])
}
metadata := resp["metadata"].(map[string]any)
if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" {
t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"])
}
if metadata["profile_hash"] != "phash" {
t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"])
}
usage := metadata["usage"].(map[string]any)
if usage["total_tokens"] != float64(3) {
t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"])
}
if metadata["duration_ms"] != float64(2000) {
t.Fatalf("expected duration_ms=2000, got %#v", metadata["duration_ms"])
}
if metadata["validation_mode"] != "basic" || metadata["validation_status"] != "passed" {
t.Fatalf("unexpected validation metadata: mode=%#v status=%#v", metadata["validation_mode"], metadata["validation_status"])
}
modelParams := metadata["model_params"].(map[string]any)
if modelParams["model"] != "m1" {
t.Fatalf("unexpected model_params.model: %#v", modelParams["model"])
}
if resp["raw_model_output"] != "hello" {
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
}
@@ -153,6 +181,20 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
if w.Code != tc.status {
t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if _, ok := errBody["code"].(string); !ok {
t.Fatalf("expected error code string, got %#v", errBody["code"])
}
if msg, ok := errBody["message"].(string); !ok || msg == "" {
t.Fatalf("expected non-empty error message, got %#v", errBody["message"])
}
if strings.Contains(w.Body.String(), "read failed") || strings.Contains(w.Body.String(), "render failed") || strings.Contains(w.Body.String(), "llm failed") {
t.Fatalf("expected response to avoid leaking internal cause details, got %s", w.Body.String())
}
})
}
}

4
internal/domain/domain.go
View File

@@ -54,18 +54,22 @@ type RunRequest struct {
// RunResult represents the complete result of a prompt execution run.
type RunResult struct {
RunID string
Artifact Artifact
RawOutput string
Validation ValidationResult
ProfileID string
ProfileVersion string
ProfileHash string
ModelName string
Endpoint string
ModelParams ModelTarget
InputHashes map[string]string
PromptHash string
Usage TokenUsage
StartTime time.Time
EndTime time.Time
Duration time.Duration
Error error
}

15
internal/llm/openai_compatible_client.go
View File

@@ -40,11 +40,11 @@ type OpenAICompatibleClient struct {
}
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
if strings.TrimSpace(cfg.BaseURL) == "" {
return nil, fmt.Errorf("%w: base URL is required", ErrInvalidConfig)
}
if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil {
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
baseURL := strings.TrimSpace(cfg.BaseURL)
if baseURL != "" {
if _, err := url.ParseRequestURI(baseURL); err != nil {
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
}
}
timeout := cfg.Timeout
@@ -63,7 +63,7 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
}
return &OpenAICompatibleClient{
baseURL: strings.TrimRight(cfg.BaseURL, "/"),
baseURL: strings.TrimRight(baseURL, "/"),
apiKey: cfg.APIKey,
defaultModel: cfg.Model,
timeout: timeout,
@@ -88,6 +88,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
if endpoint == "" {
endpoint = c.baseURL
}
if endpoint == "" {
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
}
endpoint = strings.TrimRight(endpoint, "/") + "/chat/completions"
wireReq := openAIChatRequest{

42
internal/llm/openai_compatible_client_test.go
View File

@@ -336,3 +336,45 @@ func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "",
Model: "m",
})
if err != nil {
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{Endpoint: "http://localhost:9999/v1"},
})
if err == nil {
t.Fatal("expected request failure due to unreachable endpoint")
}
if !errors.Is(err, ErrRequestFailed) {
t.Fatalf("expected ErrRequestFailed with request endpoint override, got %v", err)
}
}
func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "",
Model: "m",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ModelTarget{},
})
if err == nil {
t.Fatal("expected endpoint-required error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}

9
internal/usecase/integration_test.go
View File

@@ -63,6 +63,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.ProfileID != "generic.structured_events" {
t.Fatalf("unexpected profile id: %q", res.ProfileID)
}
if res.RunID == "" {
t.Fatal("expected run id")
}
if res.ProfileHash == "" {
t.Fatal("expected profile hash")
}
if res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile version: %q", res.ProfileVersion)
}
@@ -96,4 +102,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
}

43
internal/usecase/runner.go
View File

@@ -2,8 +2,10 @@ package usecase
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -69,12 +71,21 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
}
start := time.Now().UTC()
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
profileHash, err := hashProfile(prof)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err)
}
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
effectiveContract := resolveOutputContract(prof, req.Validation)
@@ -147,18 +158,22 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
ProfileID: prof.ID,
ProfileVersion: prof.Version,
ProfileHash: profileHash,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
ModelParams: effectiveModel,
InputHashes: inputHashes,
PromptHash: promptHash,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
@@ -267,3 +282,31 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
Hash: hex.EncodeToString(hash[:]),
}
}
func hashProfile(prof *domain.PromptProfile) (string, error) {
b, err := json.Marshal(prof)
if err != nil {
return "", err
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:]), nil
}
func newRunID() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
// UUID v4 (RFC 4122 variant).
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
b[0:4],
b[4:6],
b[6:8],
b[8:10],
b[10:16],
), nil
}

13
internal/usecase/runner_test.go
View File

@@ -7,6 +7,7 @@ import (
"errors"
"os"
"path/filepath"
"regexp"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -172,6 +173,12 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
}
if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
t.Fatalf("expected UUIDv4 run id, got %q", res.RunID)
}
if res.ProfileHash == "" {
t.Fatal("expected non-empty profile hash")
}
if res.ModelName != "model-override" {
t.Fatalf("expected model override to apply, got %q", res.ModelName)
}
@@ -205,6 +212,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
t.Fatalf("unexpected transcript hash: %q", got)
@@ -219,6 +229,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" {
t.Fatalf("expected effective model params in result, got %+v", res.ModelParams)
}
}
func TestRunnerRunProfileLoadFailure(t *testing.T) {