diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 744c2c2..4b0d4a5 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -75,6 +75,22 @@ type RunResult struct { Error error } +// PreparedRun contains pre-LLM execution state from the prepare/render phase. +// It must never include resolved API key values, model output, or validation data. +type PreparedRun struct { + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + PromptHash string `json:"prompt_hash,omitempty"` + SelectedProfileID string `json:"selected_profile_id"` + EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + InputHashes map[string]string `json:"input_hashes,omitempty"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + Messages []RenderedMessage `json:"messages"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` +} + // ArtifactRef represents a reference to an input artifact. type ArtifactRef struct { Type ArtifactRefType @@ -156,13 +172,13 @@ type OutputContract struct { // RenderedPrompt represents the prompt after template application. type RenderedPrompt struct { - Messages []RenderedMessage + Messages []RenderedMessage `json:"messages"` } // RenderedMessage is a single message in a rendered prompt. type RenderedMessage struct { - Role string - Content string + Role string `json:"role"` + Content string `json:"content"` } // GenerateRequest is the internal request passed to the LLM client. diff --git a/internal/domain/prepared_run_test.go b/internal/domain/prepared_run_test.go new file mode 100644 index 0000000..7d90883 --- /dev/null +++ b/internal/domain/prepared_run_test.go @@ -0,0 +1,55 @@ +package domain + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) { + const envName = "SCRIPTORIUM_TEST_API_KEY" + const secret = "super-secret-value" + t.Setenv(envName, secret) + + prepared := PreparedRun{ + PromptID: "prompt.id", + PromptVersion: "v1", + PromptHash: "prompt-hash", + SelectedProfileID: "local-fast", + EffectiveModelParams: ExecutionTarget{ + Endpoint: "http://llm/v1", + Model: "gpt-test", + APIKeyEnv: envName, + }, + InputHashes: map[string]string{"transcript": "hash-1"}, + RenderedPromptHash: "rendered-hash", + Messages: []RenderedMessage{ + {Role: "system", Content: "You are helpful."}, + {Role: "user", Content: "Summarize this."}, + }, + } + + b, err := json.Marshal(prepared) + if err != nil { + t.Fatalf("marshal failed: %v", err) + } + + out := string(b) + if strings.Contains(out, secret) { + t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out) + } + if !strings.Contains(out, `"api_key_env":"`+envName+`"`) { + t.Fatalf("prepared run JSON should include api_key_env name: %s", out) + } + + var top map[string]any + if err := json.Unmarshal(b, &top); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + for _, forbidden := range []string{"raw_output", "validation", "artifact"} { + if _, ok := top[forbidden]; ok { + t.Fatalf("prepared run JSON should not include %q", forbidden) + } + } +} diff --git a/internal/format/prepared_run.go b/internal/format/prepared_run.go new file mode 100644 index 0000000..d276a6c --- /dev/null +++ b/internal/format/prepared_run.go @@ -0,0 +1,144 @@ +// Package format formats already-prepared domain data for adapters. +package format + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" +) + +var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format") + +// PreparedRunOutputFormat is the output format for prepared render data. +type PreparedRunOutputFormat string + +const ( + PreparedRunFormatText PreparedRunOutputFormat = "text" + PreparedRunFormatJSON PreparedRunOutputFormat = "json" + + DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText +) + +// PreparedRunFormatter serializes a prepared run without performing use case work. +type PreparedRunFormatter interface { + Format(prepared *domain.PreparedRun) ([]byte, error) +} + +// ParsePreparedRunOutputFormat parses a format name. +func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) { + switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) { + case "": + return DefaultPreparedRunOutputFormat, nil + case PreparedRunFormatText: + return PreparedRunFormatText, nil + case PreparedRunFormatJSON: + return PreparedRunFormatJSON, nil + default: + return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON) + } +} + +// FormatterForPreparedRun returns a formatter strategy for the chosen format. +func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) { + switch outputFormat { + case PreparedRunFormatText: + return textPreparedRunFormatter{}, nil + case PreparedRunFormatJSON: + return jsonPreparedRunFormatter{}, nil + default: + return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) + } +} + +// FormatPreparedRun formats a prepared run using the selected format. +func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) { + formatter, err := FormatterForPreparedRun(outputFormat) + if err != nil { + return nil, err + } + return formatter.Format(prepared) +} + +type jsonPreparedRunFormatter struct{} + +func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) { + if prepared == nil { + return nil, errors.New("prepared run is nil") + } + return json.MarshalIndent(prepared, "", " ") +} + +type textPreparedRunFormatter struct{} + +func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) { + if prepared == nil { + return nil, errors.New("prepared run is nil") + } + + var b bytes.Buffer + fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID) + fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion) + fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID) + if prepared.PromptHash != "" { + fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) + } + fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash) + + target := prepared.EffectiveModelParams + fmt.Fprintln(&b, "effective_model_params:") + fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint) + fmt.Fprintf(&b, " model: %s\n", target.Model) + fmt.Fprintf(&b, " temperature: %g\n", target.Temperature) + fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens) + fmt.Fprintf(&b, " top_p: %g\n", target.TopP) + fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds) + if target.ReasoningEffort != "" { + fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort) + } + if target.APIKeyEnv != "" { + fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv) + } + if len(target.ExtraParams) > 0 { + fmt.Fprintln(&b, " extra_params:") + keys := make([]string, 0, len(target.ExtraParams)) + for k := range target.ExtraParams { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k]) + } + } + + fmt.Fprintln(&b, "input_hashes:") + inputKeys := make([]string, 0, len(prepared.InputHashes)) + for k := range prepared.InputHashes { + inputKeys = append(inputKeys, k) + } + sort.Strings(inputKeys) + for _, k := range inputKeys { + fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k]) + } + + fmt.Fprintln(&b, "messages:") + for i, msg := range prepared.Messages { + fmt.Fprintf(&b, " - index: %d\n", i) + fmt.Fprintf(&b, " role: %s\n", msg.Role) + fmt.Fprintln(&b, " content: |") + content := msg.Content + if content == "" { + fmt.Fprintln(&b, " ") + continue + } + for _, line := range strings.Split(content, "\n") { + fmt.Fprintf(&b, " %s\n", line) + } + } + + return b.Bytes(), nil +} diff --git a/internal/format/prepared_run_test.go b/internal/format/prepared_run_test.go new file mode 100644 index 0000000..048e93f --- /dev/null +++ b/internal/format/prepared_run_test.go @@ -0,0 +1,41 @@ +package format + +import ( + "errors" + "testing" +) + +func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) { + tests := []struct { + name string + input string + want PreparedRunOutputFormat + }{ + {name: "default empty", input: "", want: DefaultPreparedRunOutputFormat}, + {name: "text", input: "text", want: PreparedRunFormatText}, + {name: "json", input: "json", want: PreparedRunFormatJSON}, + {name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ParsePreparedRunOutputFormat(tc.input) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tc.want { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} + +func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) { + _, err := ParsePreparedRunOutputFormat("yaml") + if err == nil { + t.Fatal("expected error for unknown format") + } + if !errors.Is(err, ErrUnknownPreparedRunFormat) { + t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err) + } +}