Relaxed CLI requirements when defaults are specified in the profile or application defaults

This commit is contained in:
2026-05-05 08:41:07 -05:00
parent 281202e313
commit ca4d939fdc
13 changed files with 311 additions and 74 deletions

View File

@@ -63,6 +63,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.ProfileID != "generic.structured_events" {
t.Fatalf("unexpected profile id: %q", res.ProfileID)
}
if res.RunID == "" {
t.Fatal("expected run id")
}
if res.ProfileHash == "" {
t.Fatal("expected profile hash")
}
if res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile version: %q", res.ProfileVersion)
}
@@ -96,4 +102,7 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) {
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
}

View File

@@ -2,8 +2,10 @@ package usecase
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -69,12 +71,21 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
}
start := time.Now().UTC()
prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
profileHash, err := hashProfile(prof)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err)
}
effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model)
effectiveContract := resolveOutputContract(prof, req.Validation)
@@ -147,18 +158,22 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
end := time.Now().UTC()
return &domain.RunResult{
RunID: runID,
Artifact: outputArtifact,
RawOutput: genResp.Content,
Validation: validationResult,
ProfileID: prof.ID,
ProfileVersion: prof.Version,
ProfileHash: profileHash,
ModelName: effectiveModel.Model,
Endpoint: effectiveModel.Endpoint,
ModelParams: effectiveModel,
InputHashes: inputHashes,
PromptHash: promptHash,
Usage: genResp.Usage,
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
@@ -267,3 +282,31 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
Hash: hex.EncodeToString(hash[:]),
}
}
func hashProfile(prof *domain.PromptProfile) (string, error) {
b, err := json.Marshal(prof)
if err != nil {
return "", err
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:]), nil
}
func newRunID() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
// UUID v4 (RFC 4122 variant).
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
b[0:4],
b[4:6],
b[6:8],
b[8:10],
b[10:16],
), nil
}

View File

@@ -7,6 +7,7 @@ import (
"errors"
"os"
"path/filepath"
"regexp"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -172,6 +173,12 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" {
t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion)
}
if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok {
t.Fatalf("expected UUIDv4 run id, got %q", res.RunID)
}
if res.ProfileHash == "" {
t.Fatal("expected non-empty profile hash")
}
if res.ModelName != "model-override" {
t.Fatalf("expected model override to apply, got %q", res.ModelName)
}
@@ -205,6 +212,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if res.EndTime.Before(res.StartTime) {
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
}
if res.Duration < 0 {
t.Fatalf("expected non-negative duration, got %s", res.Duration)
}
if got := res.InputHashes["transcript"]; got != hashString("transcript body") {
t.Fatalf("unexpected transcript hash: %q", got)
@@ -219,6 +229,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds)
}
if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" {
t.Fatalf("expected effective model params in result, got %+v", res.ModelParams)
}
}
func TestRunnerRunProfileLoadFailure(t *testing.T) {