diff --git a/README.md b/README.md index 51ad38d..fe7e44c 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,12 @@ go run ./cmd/scriptorium run \ --profile-id generic.markdown_summary \ --input transcript=./examples/fixtures/transcript.md \ --input glossary=./examples/fixtures/glossary.yml \ - --llm-base-url http://localhost:8000/v1 \ - --model gpt-4o-mini \ --out ./out.md ``` +This relies on `model_defaults.endpoint` and `model_defaults.model` in the selected profile. +You can override either at runtime with `--llm-base-url` and/or `--model`. + For schema-validated JSON output: ```bash @@ -78,17 +79,23 @@ Response shape: - `metadata` - `raw_model_output` +`metadata` includes stable audit fields such as run/profile IDs, profile hash, effective model params, prompt/input hashes, timing, usage, and validation summary. + ## Add a New Prompt Profile 1. Add a YAML file under `profiles/` with: - `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation` - optional model timeout via `model_defaults.timeout_seconds` (per-run LLM timeout override) -2. Use template helpers such as `{{input "transcript"}}`. -3. For structured JSON output, set: +2. Ensure endpoint/model are available from either: + - profile defaults (`model_defaults.endpoint`, `model_defaults.model`), or + - request overrides (`--llm-base-url`, `--model`, or HTTP `model.endpoint`/`model.model`). +3. Use template helpers such as `{{input "transcript"}}` and template vars like `{{.session_date}}`. +4. For structured JSON output, set: - `output_format: json` - `validation.validation_mode: json_schema` - `validation.schema_path: ` -4. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve. +5. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve. +6. `validation.repair_attempts` is bounded and applies only to structured modes (`json`, `json_schema`). ## Validation Behavior diff --git a/architecture.md b/architecture.md index 9396f0e..5c81868 100644 --- a/architecture.md +++ b/architecture.md @@ -527,33 +527,30 @@ The response should include: - artifact - validation - metadata -- raw_model_output, optionally controlled by request or config +- raw_model_output - error details, if applicable The HTTP layer should not contain business logic. ## CLI -The CLI should also be thin. +The CLI should be thin and call the same core use case as HTTP. -It should support local development and pipeline usage. - -Suggested commands: +Current command surface: - scriptorium run -- scriptorium profiles list -- scriptorium profiles inspect +- scriptorium serve -The run command should accept: +The `run` command should accept: - profile ID - input mappings - variable mappings - output path, optional - profile directory -- config path +- optional model/endpoint overrides -The CLI should call the same use case used by the HTTP API. +If model/endpoint overrides are omitted, profile model defaults should be used. ## Configuration diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index 7bbbfeb..eef43dd 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -41,6 +41,11 @@ type runConfig struct { maxTokens int schemaDir string timeout time.Duration + + llmBaseURLSet bool + modelSet bool + temperatureSet bool + maxTokensSet bool } type serveConfig struct { @@ -127,16 +132,21 @@ func runCommand(args []string, stdout, stderr io.Writer) int { validate.NewStandardValidator(cfg.schemaDir), ) - res, runErr := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: cfg.profileID, - Inputs: inputs, - Vars: varMappings, - Model: &domain.ModelTarget{ + var modelOverride *domain.ModelTarget + if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet { + modelOverride = &domain.ModelTarget{ Endpoint: cfg.llmBaseURL, Model: cfg.model, Temperature: cfg.temperature, MaxTokens: cfg.maxTokens, - }, + } + } + + res, runErr := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: cfg.profileID, + Inputs: inputs, + Vars: varMappings, + Model: modelOverride, }) if runErr != nil { fmt.Fprintf(stderr, "run error: %v\n", runErr) @@ -227,18 +237,15 @@ func parseRunArgs(args []string) (*runConfig, error) { if len(cfg.inputRaw) == 0 { return nil, errors.New("at least one --input is required") } - if strings.TrimSpace(cfg.llmBaseURL) == "" { - return nil, errors.New("--llm-base-url is required") - } - if strings.TrimSpace(cfg.model) == "" { - return nil, errors.New("--model is required") - } - cfg.profileDir = filepath.Clean(cfg.profileDir) cfg.schemaDir = filepath.Clean(cfg.schemaDir) if cfg.outputPath != "" { cfg.outputPath = filepath.Clean(cfg.outputPath) } + cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url") + cfg.modelSet = flagWasSet(fs, "model") + cfg.temperatureSet = flagWasSet(fs, "temperature") + cfg.maxTokensSet = flagWasSet(fs, "max-tokens") return cfg, nil } @@ -312,6 +319,16 @@ func parseMapping(value string) (string, string, error) { return key, val, nil } +func flagWasSet(fs *flag.FlagSet, name string) bool { + set := false + fs.Visit(func(f *flag.Flag) { + if f.Name == name { + set = true + } + }) + return set +} + func writeOutput(stdout io.Writer, outputPath string, body []byte) error { if outputPath == "" { _, err := stdout.Write(body) @@ -351,6 +368,6 @@ func printSummary(stderr io.Writer, res *domain.RunResult) { func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] --llm-base-url URL --model NAME [--var k=v] [--out path] [--timeout 10m]") + fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--var k=v] [--out path] [--timeout 10m]") fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]") } diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index bc5bab6..e181ddb 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -65,15 +65,19 @@ func TestParseRunArgsRequiredFlags(t *testing.T) { if err == nil { t.Fatal("expected missing --input error") } +} - _, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--model", "m"}) - if err == nil { - t.Fatal("expected missing --llm-base-url error") +func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { + cfg, err := parseRunArgs([]string{ + "--profile-dir", "./profiles", + "--profile-id", "p", + "--input", "a=b", + }) + if err != nil { + t.Fatalf("expected valid args without model/base url, got %v", err) } - - _, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1"}) - if err == nil { - t.Fatal("expected missing --model error") + if cfg.llmBaseURL != "" || cfg.model != "" { + t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL) } } @@ -167,4 +171,31 @@ func TestRunCommandVarsOptional(t *testing.T) { if !strings.Contains(stderr.String(), "llm client error") { t.Fatalf("expected llm client error after parsing succeeds, got stderr=%q", stderr.String()) } + if stdout.Len() != 0 { + t.Fatalf("expected no stdout output on error, got %q", stdout.String()) + } +} + +func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil { + t.Fatalf("unexpected writeOutput error: %v", err) + } + printSummary(&stderr, &domain.RunResult{ + ProfileID: "p", + ProfileVersion: "1", + ModelName: "m", + Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic}, + PromptHash: "h", + InputHashes: map[string]string{"in": "x"}, + }) + + if stdout.String() != "artifact-body" { + t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "profile=p@1") { + t.Fatalf("expected summary on stderr, got %q", stderr.String()) + } } diff --git a/internal/adapter/http/dto.go b/internal/adapter/http/dto.go index 32f51db..a681fe4 100644 --- a/internal/adapter/http/dto.go +++ b/internal/adapter/http/dto.go @@ -44,15 +44,31 @@ type artifactDTO struct { } type metadataDTO struct { - ProfileID string `json:"profile_id"` - ProfileVersion string `json:"profile_version"` - ModelName string `json:"model_name"` - Endpoint string `json:"endpoint"` - InputHashes map[string]string `json:"input_hashes"` - PromptHash string `json:"prompt_hash"` - Usage tokenUsageDTO `json:"usage"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` + RunID string `json:"run_id"` + ProfileID string `json:"profile_id"` + ProfileVersion string `json:"profile_version"` + ProfileHash string `json:"profile_hash"` + ModelName string `json:"model_name"` + Endpoint string `json:"endpoint"` + ModelParams modelParamsDTO `json:"model_params"` + InputHashes map[string]string `json:"input_hashes"` + PromptHash string `json:"prompt_hash"` + Usage tokenUsageDTO `json:"usage"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + DurationMS int64 `json:"duration_ms"` + ValidationMode string `json:"validation_mode"` + ValidationStatus string `json:"validation_status"` + RepairAttemptsUsed int `json:"repair_attempts_used"` +} + +type modelParamsDTO struct { + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` } type tokenUsageDTO struct { diff --git a/internal/adapter/http/handler.go b/internal/adapter/http/handler.go index 5a48755..8e3a23e 100644 --- a/internal/adapter/http/handler.go +++ b/internal/adapter/http/handler.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "strings" @@ -37,7 +36,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var req runRequestDTO if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid_json", fmt.Sprintf("invalid JSON request: %v", err)) + writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body") return } @@ -79,8 +78,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Model: model, }) if err != nil { - status, code := mapRunError(err) - writeError(w, status, code, err.Error()) + status, code, message := mapRunError(err) + writeError(w, status, code, message) return } @@ -95,19 +94,33 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }, Validation: mapValidation(res.Validation), Metadata: metadataDTO{ + RunID: res.RunID, ProfileID: res.ProfileID, ProfileVersion: res.ProfileVersion, + ProfileHash: res.ProfileHash, ModelName: res.ModelName, Endpoint: res.Endpoint, - InputHashes: res.InputHashes, - PromptHash: res.PromptHash, + ModelParams: modelParamsDTO{ + Endpoint: res.ModelParams.Endpoint, + Model: res.ModelParams.Model, + Temperature: res.ModelParams.Temperature, + MaxTokens: res.ModelParams.MaxTokens, + TopP: res.ModelParams.TopP, + TimeoutSeconds: res.ModelParams.TimeoutSeconds, + }, + InputHashes: res.InputHashes, + PromptHash: res.PromptHash, Usage: tokenUsageDTO{ PromptTokens: res.Usage.PromptTokens, CompletionTokens: res.Usage.CompletionTokens, TotalTokens: res.Usage.TotalTokens, }, - StartTime: res.StartTime, - EndTime: res.EndTime, + StartTime: res.StartTime, + EndTime: res.EndTime, + DurationMS: res.Duration.Milliseconds(), + ValidationMode: string(res.Validation.Mode), + ValidationStatus: string(res.Validation.Status), + RepairAttemptsUsed: res.Validation.RepairAttempts, }, RawModelOutput: res.RawOutput, }) @@ -124,24 +137,24 @@ func mapValidation(v domain.ValidationResult) validationDTO { } } -func mapRunError(err error) (int, string) { +func mapRunError(err error) (int, string, string) { switch { case errors.Is(err, profile.ErrProfileNotFound): - return http.StatusNotFound, "profile_not_found" + return http.StatusNotFound, "profile_not_found", "profile not found" case errors.Is(err, usecase.ErrInvalidRequest): - return http.StatusBadRequest, "invalid_request" + return http.StatusBadRequest, "invalid_request", "invalid run request" case errors.Is(err, usecase.ErrProfileLoad): - return http.StatusBadRequest, "profile_load_failed" + return http.StatusBadRequest, "profile_load_failed", "failed to load profile" case errors.Is(err, usecase.ErrArtifactLoad): - return http.StatusBadRequest, "artifact_read_failed" + return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact" case errors.Is(err, usecase.ErrPromptRender): - return http.StatusBadRequest, "prompt_render_failed" + return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt" case errors.Is(err, usecase.ErrLLMGenerate): - return http.StatusBadGateway, "llm_failed" + return http.StatusBadGateway, "llm_failed", "model generation request failed" case errors.Is(err, usecase.ErrValidation): - return http.StatusInternalServerError, "validation_runtime_failed" + return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed" default: - return http.StatusInternalServerError, "internal_error" + return http.StatusInternalServerError, "internal_error", "internal server error" } } diff --git a/internal/adapter/http/handler_test.go b/internal/adapter/http/handler_test.go index d40756c..be608d4 100644 --- a/internal/adapter/http/handler_test.go +++ b/internal/adapter/http/handler_test.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -33,6 +34,7 @@ func TestHandlerPostRunsSuccess(t *testing.T) { start := time.Now().UTC() end := start.Add(2 * time.Second) r := &fakeRunner{result: &domain.RunResult{ + RunID: "11111111-1111-4111-8111-111111111111", Artifact: domain.Artifact{ Name: "output", ContentType: "text/plain", @@ -43,14 +45,24 @@ func TestHandlerPostRunsSuccess(t *testing.T) { Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true}, ProfileID: "p1", ProfileVersion: "1.0.0", + ProfileHash: "phash", ModelName: "m1", Endpoint: "http://llm/v1", - InputHashes: map[string]string{"transcript": "h1"}, - PromptHash: "ph", - Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, - StartTime: start, - EndTime: end, - RawOutput: "hello", + ModelParams: domain.ModelTarget{ + Endpoint: "http://llm/v1", + Model: "m1", + Temperature: 0.2, + MaxTokens: 42, + TopP: 0.9, + TimeoutSeconds: 120, + }, + InputHashes: map[string]string{"transcript": "h1"}, + PromptHash: "ph", + Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, + StartTime: start, + EndTime: end, + Duration: 2 * time.Second, + RawOutput: "hello", }} h := NewHandler(r) @@ -86,10 +98,26 @@ func TestHandlerPostRunsSuccess(t *testing.T) { t.Fatalf("expected validation.status passed, got %#v", validation["status"]) } metadata := resp["metadata"].(map[string]any) + if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" { + t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"]) + } + if metadata["profile_hash"] != "phash" { + t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"]) + } usage := metadata["usage"].(map[string]any) if usage["total_tokens"] != float64(3) { t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"]) } + if metadata["duration_ms"] != float64(2000) { + t.Fatalf("expected duration_ms=2000, got %#v", metadata["duration_ms"]) + } + if metadata["validation_mode"] != "basic" || metadata["validation_status"] != "passed" { + t.Fatalf("unexpected validation metadata: mode=%#v status=%#v", metadata["validation_mode"], metadata["validation_status"]) + } + modelParams := metadata["model_params"].(map[string]any) + if modelParams["model"] != "m1" { + t.Fatalf("unexpected model_params.model: %#v", modelParams["model"]) + } if resp["raw_model_output"] != "hello" { t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"]) } @@ -153,6 +181,20 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) { if w.Code != tc.status { t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String()) } + var resp map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("invalid JSON response: %v", err) + } + errBody := resp["error"].(map[string]any) + if _, ok := errBody["code"].(string); !ok { + t.Fatalf("expected error code string, got %#v", errBody["code"]) + } + if msg, ok := errBody["message"].(string); !ok || msg == "" { + t.Fatalf("expected non-empty error message, got %#v", errBody["message"]) + } + if strings.Contains(w.Body.String(), "read failed") || strings.Contains(w.Body.String(), "render failed") || strings.Contains(w.Body.String(), "llm failed") { + t.Fatalf("expected response to avoid leaking internal cause details, got %s", w.Body.String()) + } }) } } diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 97038c1..1914c89 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -54,18 +54,22 @@ type RunRequest struct { // RunResult represents the complete result of a prompt execution run. type RunResult struct { + RunID string Artifact Artifact RawOutput string Validation ValidationResult ProfileID string ProfileVersion string + ProfileHash string ModelName string Endpoint string + ModelParams ModelTarget InputHashes map[string]string PromptHash string Usage TokenUsage StartTime time.Time EndTime time.Time + Duration time.Duration Error error } diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index 09fd329..260be58 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -40,11 +40,11 @@ type OpenAICompatibleClient struct { } func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) { - if strings.TrimSpace(cfg.BaseURL) == "" { - return nil, fmt.Errorf("%w: base URL is required", ErrInvalidConfig) - } - if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil { - return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err) + baseURL := strings.TrimSpace(cfg.BaseURL) + if baseURL != "" { + if _, err := url.ParseRequestURI(baseURL); err != nil { + return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err) + } } timeout := cfg.Timeout @@ -63,7 +63,7 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli } return &OpenAICompatibleClient{ - baseURL: strings.TrimRight(cfg.BaseURL, "/"), + baseURL: strings.TrimRight(baseURL, "/"), apiKey: cfg.APIKey, defaultModel: cfg.Model, timeout: timeout, @@ -88,6 +88,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera if endpoint == "" { endpoint = c.baseURL } + if endpoint == "" { + return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest) + } endpoint = strings.TrimRight(endpoint, "/") + "/chat/completions" wireReq := openAIChatRequest{ diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index b028975..36e2182 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -336,3 +336,45 @@ func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) { t.Fatalf("expected ErrInvalidRequest, got %v", err) } } + +func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) { + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "", + Model: "m", + }) + if err != nil { + t.Fatalf("expected empty configured base URL to be allowed, got %v", err) + } + + _, err = client.Generate(context.Background(), domain.GenerateRequest{ + Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, + Target: domain.ModelTarget{Endpoint: "http://localhost:9999/v1"}, + }) + if err == nil { + t.Fatal("expected request failure due to unreachable endpoint") + } + if !errors.Is(err, ErrRequestFailed) { + t.Fatalf("expected ErrRequestFailed with request endpoint override, got %v", err) + } +} + +func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) { + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "", + Model: "m", + }) + if err != nil { + t.Fatal(err) + } + + _, err = client.Generate(context.Background(), domain.GenerateRequest{ + Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, + Target: domain.ModelTarget{}, + }) + if err == nil { + t.Fatal("expected endpoint-required error") + } + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } +} diff --git a/internal/usecase/integration_test.go b/internal/usecase/integration_test.go index 97cfd0d..455e975 100644 --- a/internal/usecase/integration_test.go +++ b/internal/usecase/integration_test.go @@ -63,6 +63,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) { if res.ProfileID != "generic.structured_events" { t.Fatalf("unexpected profile id: %q", res.ProfileID) } + if res.RunID == "" { + t.Fatal("expected run id") + } + if res.ProfileHash == "" { + t.Fatal("expected profile hash") + } if res.ProfileVersion != "1.0.0" { t.Fatalf("unexpected profile version: %q", res.ProfileVersion) } @@ -96,4 +102,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) { if res.EndTime.Before(res.StartTime) { t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) } + if res.Duration < 0 { + t.Fatalf("expected non-negative duration, got %s", res.Duration) + } } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 08df6d0..2a7cc6b 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -2,8 +2,10 @@ package usecase import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "strings" @@ -69,12 +71,21 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) } + runID, err := newRunID() + if err != nil { + return nil, fmt.Errorf("failed to create run id: %w", err) + } + start := time.Now().UTC() prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) } + profileHash, err := hashProfile(prof) + if err != nil { + return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err) + } effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model) effectiveContract := resolveOutputContract(prof, req.Validation) @@ -147,18 +158,22 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes end := time.Now().UTC() return &domain.RunResult{ + RunID: runID, Artifact: outputArtifact, RawOutput: genResp.Content, Validation: validationResult, ProfileID: prof.ID, ProfileVersion: prof.Version, + ProfileHash: profileHash, ModelName: effectiveModel.Model, Endpoint: effectiveModel.Endpoint, + ModelParams: effectiveModel, InputHashes: inputHashes, PromptHash: promptHash, Usage: genResp.Usage, StartTime: start, EndTime: end, + Duration: end.Sub(start), }, nil } @@ -267,3 +282,31 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti Hash: hex.EncodeToString(hash[:]), } } + +func hashProfile(prof *domain.PromptProfile) (string, error) { + b, err := json.Marshal(prof) + if err != nil { + return "", err + } + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +func newRunID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + + // UUID v4 (RFC 4122 variant). + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + b[0:4], + b[4:6], + b[6:8], + b[8:10], + b[10:16], + ), nil +} diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index c9a9ddd..199f39d 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -7,6 +7,7 @@ import ( "errors" "os" "path/filepath" + "regexp" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" @@ -172,6 +173,12 @@ func TestRunnerRunSuccessful(t *testing.T) { if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" { t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion) } + if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok { + t.Fatalf("expected UUIDv4 run id, got %q", res.RunID) + } + if res.ProfileHash == "" { + t.Fatal("expected non-empty profile hash") + } if res.ModelName != "model-override" { t.Fatalf("expected model override to apply, got %q", res.ModelName) } @@ -205,6 +212,9 @@ func TestRunnerRunSuccessful(t *testing.T) { if res.EndTime.Before(res.StartTime) { t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) } + if res.Duration < 0 { + t.Fatalf("expected non-negative duration, got %s", res.Duration) + } if got := res.InputHashes["transcript"]; got != hashString("transcript body") { t.Fatalf("unexpected transcript hash: %q", got) @@ -219,6 +229,9 @@ func TestRunnerRunSuccessful(t *testing.T) { if llmClient.lastReq.Target.TimeoutSeconds != 90 { t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds) } + if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" { + t.Fatalf("expected effective model params in result, got %+v", res.ModelParams) + } } func TestRunnerRunProfileLoadFailure(t *testing.T) {